Browse Source

add "shut pkg bump" with most logic from shore

shut-new-model
Niklas Rosenstein 9 months ago
parent
commit
ffd539d8b7
No known key found for this signature in database GPG Key ID: 6D269B33D25F6C6
12 changed files with 396 additions and 96 deletions
  1. + 1
    - 1
      src/shut/checks/__init__.py
  2. + 2
    - 2
      src/shut/checks/generic.py
  3. + 211
    - 0
      src/shut/commands/commons/bump.py
  4. + 1
    - 0
      src/shut/commands/commons/checks.py
  5. + 1
    - 0
      src/shut/commands/pkg/__init__.py
  6. + 70
    - 0
      src/shut/commands/pkg/bump.py
  7. + 21
    - 16
      src/shut/commands/pkg/checks.py
  8. + 0
    - 63
      src/shut/render/core.py
  9. + 4
    - 2
      src/shut/update/__init__.py
  10. + 23
    - 1
      src/shut/update/core.py
  11. + 26
    - 1
      src/shut/update/generic.py
  12. + 36
    - 10
      src/shut/update/setuptools.py

+ 1
- 1
src/shut/checks/__init__.py

@ -32,4 +32,4 @@ from .core import (
get_checks,
)
from . import basic, monorepo, package
from . import generic, monorepo, package

src/shut/checks/basic.py → src/shut/checks/generic.py

@ -24,7 +24,7 @@ from shut.model import AbstractProjectModel, Project
from typing import Iterable, Optional, Union
class BasicChecker(Checker):
class GenericChecker(Checker):
@check('unknown-config')
def _check_unknown_keys(
@ -37,5 +37,5 @@ class BasicChecker(Checker):
', '.join(map(str, obj.unknown_keys)) if obj.unknown_keys else None)
register_checker(AbstractProjectModel, BasicChecker)
register_checker(AbstractProjectModel, GenericChecker)

+ 211
- 0
src/shut/commands/commons/bump.py

@ -0,0 +1,211 @@
# -*- coding: utf8 -*-
# Copyright (c) 2020 Niklas Rosenstein
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
import abc
import logging
import os
import sys
from typing import Iterable, Generic, Optional, T, Type
import click
from databind.core import datamodel
from nr.utils.git import Git
from shut.commands import project
from shut.model import AbstractProjectModel, Project
from shut.model.version import bump_version, parse_version, Version
from shut.update import get_version_refs, VersionRef
logger = logging.getLogger(__name__)
@datamodel
class Args:
version: Optional[Version]
major: bool
minor: bool
patch: bool
post: bool
snapshot: bool
tag: bool
push: bool
dry: bool
warnings_as_errors: bool
skip_checks: bool
skip_update: bool
force: bool
allow_lower: bool = False
class VersionBumpData(Generic[T], metaclass=abc.ABCMeta):
def __init__(self, args: Args, project: Project, obj: T) -> None:
self.args = args
self.project = project
self.obj = obj
def post_init(self) -> None:
pass
@abc.abstractmethod
def run_checks(self) -> int:
pass
@abc.abstractmethod
def get_snapshot_version(self) -> Version:
pass
@abc.abstractmethod
def update(self) -> None:
pass
def make_bump_command(
data_class: Type[VersionBumpData[AbstractProjectModel]],
model_type: Type[AbstractProjectModel],
) -> click.Command:
@click.argument('version', type=parse_version, required=False)
@click.option('--major', is_flag=True, help='bump the major number')
@click.option('--minor', is_flag=True, help='bump the minor number')
@click.option('--patch', is_flag=True, help='bump the patch number')
@click.option('--post', is_flag=True, help='bump the post-release number')
@click.option('--snapshot', is_flag=True, help='update the version number by appending the Git commit distance and shasum (note: not compatible with publishing to PyPI)')
@click.option('--tag', is_flag=True, help='create a commit and Git tag after bumping the version')
@click.option('--push', is_flag=True, help='push the new commit and tag to the Git "origin" remote')
@click.option('--dry', is_flag=True, help='do not write changes to disk')
@click.option('-w', '--warnings-as-errors', is_flag=True, help='treat check warnings as errors')
@click.option('--skip-checks', is_flag=True, help='skip running checks before bumping')
@click.option('--skip-update', is_flag=True, help='skip update after bumping')
@click.option('--force', '-f', is_flag=True, help='force target version (allowing you to "bump" '
'the same or a lower version). the flag will also result in force adding a Git tag and force '
'pushing to the remote repository if the respective options as set (--tag and --push).')
def bump(**kwargs):
args = Args(**kwargs)
data = data_class(args, project, project.load_or_exit(expect=model_type))
do_bump(args, data)
return bump
def do_bump(args: Args, data: VersionBumpData[AbstractProjectModel]) -> None:
# Validate arguments.
if args.push and not args.tag:
sys.exit('error: --push can only be used with --tag')
bump_args = 'version major minor patch post snapshot'.split()
provided_args = {k for k in bump_args if getattr(args, k)}
if len(provided_args) > 1:
sys.exit('error: conflicting options: {}'.format(provided_args))
elif not provided_args:
# TODO(NiklasRosenstein): Bump based on changelog
sys.exit('error: missing version argument or bump option')
# Run checks.
if not args.skip_checks:
res = data.run_checks()
if res != 0:
sys.exit('error: checks failed')
version_refs = list(get_version_refs(data.obj))
if not version_refs:
sys.exit('error: no version refs found')
# Ensure the version is the same accross all refs.
current_version = data.obj.get_version()
is_inconsistent = any(parse_version(x.value) != current_version for x in version_refs)
if is_inconsistent:
if not args.force:
sys.exit('error: inconsistent versions across files need to be fixed first.')
logger.warning('inconsistent versions across files were found.')
# Bump the current version number.
if args.post:
new_version = bump_version(current_version, 'post')
elif args.patch:
new_version = bump_version(current_version, 'patch')
elif args.minor:
new_version = bump_version(current_version, 'minor')
elif args.major:
new_version = bump_version(current_version, 'major')
elif args.snapshot:
new_version = data.get_snapshot_version()
if new_version < current_version:
# The snapshot version number can be considered lower, so we'll allow it.
args.allow_lower = True
else:
new_version = parse_version(args.version)
if not new_version.pep440_compliant:
logger.warning(f'version "{new_version}" is not PEP440 compliant.')
# The new version cannot be lower than the current one, unless forced.
if new_version < current_version and not (args.force or args.allow_lower):
sys.exit(f'version "{new_version}" is lower than current version "{current_version}"')
if str(new_version) == str(current_version) and not args.force:
# NOTE(NiklasRosenstein): Comparing as strings to include pre-release and build number.
logger.warning(f'new version "{new_version}" is equal to current version')
exit(0)
# The substitution logic below does not work if the same file is listed multiple
# times so let's check for now that every file is listed only once.
n_files = set(os.path.normpath(os.path.abspath(ref.filename))
for ref in version_refs)
assert len(n_files) == len(version_refs), "multiple version refs in one file is not currently supported."
# Bump version references.
print(f'bumping {len(version_refs)} version reference(s)')
for ref in version_refs:
print(f' {os.path.relpath(ref.filename)}: {ref.value} → {new_version}')
if not args.dry:
with open(ref.filename) as fp:
contents = fp.read()
contents = contents[:ref.start] + str(new_version) + contents[ref.end:]
with open(ref.filename, 'w') as fp:
fp.write(contents)
changed_files = [x.filename for x in version_refs]
# TODO(NiklasRosenstein): For single-versioned mono repositories, bump inter dependencies.
# TODO(NiklasRosenstein): Release staged changelogs.
if not args.skip_update:
print()
data.update()
if args.tag:
print()
git = Git()
if any(f.mode == 'A' for f in git.porcelain()):
logger.error('cannot tag with non-empty staging area')
exit(1)
tag_name = data.obj.get_tag(new_version)
print(f'tagging {tag_name}')
if not args.dry:
git.add(changed_files)
git.commit('({}) bump version to {}'.format(data.obj.get_name(), new_version), allow_empty=True)
git.tag(tag_name, force=args.force)
if not args.dry and args.push:
git.push('origin', git.get_current_branch_name(), tag_name, force=args.force)

+ 1
- 0
src/shut/commands/commons/checks.py

@ -81,3 +81,4 @@ def get_checks_status(checks: List[Check], warnings_as_errors: bool = False) ->
status = 1
else:
assert False, max_level
return status

+ 1
- 0
src/shut/commands/pkg/__init__.py

@ -30,6 +30,7 @@ def pkg():
"""
from . import bump
from . import checks
from . import new
from . import status

+ 70
- 0
src/shut/commands/pkg/bump.py

@ -0,0 +1,70 @@
# -*- coding: utf8 -*-
# Copyright (c) 2020 Niklas Rosenstein
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
import os
import logging
import sys
from typing import Iterable
import click
from shut.commands.commons.bump import make_bump_command, VersionBumpData, VersionRef
from shut.model import PackageModel, Project
from shut.model.version import get_commit_distance_version, parse_version, Version
from . import pkg
from .checks import check_package
from .update import update_package
logger = logging.getLogger(__name__)
class PackageBumpData(VersionBumpData[PackageModel]):
def loaded(self) -> None:
project = self.project
if project.monorepo and project.monorepo.release.single_version:
if self.args.force:
logger.warning(
'forcing version bump on individual package version that is usually managed '
'by the monorepo.')
return
print('error: cannot bump package version managed by monorepo.', file=sys.stderr)
exit(1)
def run_checks(self) -> int:
return check_package(self.obj, self.args.warnings_as_errors)
def get_snapshot_version(self) -> Version:
project = self.project
if project.monorepo and project.monorepo.release.single_version:
subject = project.monorepo
else:
subject = self.obj
return get_commit_distance_version(
subject.directory,
subject.version,
subject.get_tag(subject.version)) or subject.version
def update(self) -> Version:
update_package(self.obj, dry=self.args.dry)
pkg.command()(make_bump_command(PackageBumpData, PackageModel))

+ 21
- 16
src/shut/commands/pkg/checks.py

@ -19,26 +19,35 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from shut.checks import CheckStatus, get_checks
from shut.commands import project
from shut.commands.commons.checks import print_checks_all, get_checks_status
from shut.commands.pkg import pkg
from shut.model import PackageModel, Project
from nr.stream import Stream
from termcolor import colored
from typing import Iterable, Union
import click
import enum
import logging
import os
import sys
import termcolor
import time
from typing import Iterable, Union
import click
import termcolor
from nr.stream import Stream
from termcolor import colored
from shut.checks import CheckStatus, get_checks
from shut.commands import project
from shut.commands.commons.checks import print_checks_all, get_checks_status
from shut.model import PackageModel, Project
from . import pkg
logger = logging.getLogger(__name__)
def check_package(package: PackageModel, warnings_as_errors: bool = False) -> int:
start_time = time.perf_counter()
checks = sorted(get_checks(project, package), key=lambda c: c.name)
seconds = time.perf_counter() - start_time
print_checks_all(package.data.name, checks, seconds)
return get_checks_status(checks, warnings_as_errors)
@pkg.command()
@click.option('-w', '--warnings-as-errors', is_flag=True)
def checks(warnings_as_errors):
@ -49,9 +58,5 @@ def checks(warnings_as_errors):
on the package configuration and entrypoint definition.
"""
start_time = time.perf_counter()
package = project.load(expect=PackageModel)
checks = sorted(get_checks(project, package), key=lambda c: c.name)
seconds = time.perf_counter() - start_time
print_checks_all(package.data.name, checks, seconds)
sys.exit(get_checks_status(checks, warnings_as_errors))
sys.exit(check_package(package, warnings_as_errors))

+ 0
- 63
src/shut/render/core.py

@ -1,63 +0,0 @@
# -*- coding: utf8 -*-
# Copyright (c) 2020 Niklas Rosenstein
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
class FileToRender(object):
"""
Represents a file that can be rendered to disk on-demand.
# Arguments
filename: The file on disk that should be rendered.
render_func: The function that will be called to render the file's contents. The
function must accept a file-like object as it's first argument. If *convolutional*
is set to `True`, it must also accept an additional file-like object that represents
the current contents of the file, or None if the file did not previously exist on
disk.
args: Additional positional arguments for *render_func*.
convolutional: Whether *render_func* accepts a second file-like object argument.
encoding: The encoding of the file object to pass.
kwargs: Additional keyword arguments for *render_func*.
"""
def __init__(
self,
filename: str,
render_callback: Callable,
*args: Any,
convolutional: bool = False,
encoding: str = None,
**kwargs: Any,
) -> None:
super(FileToRender, self).__init__()
self.filename =
self.name = nr.fs.norm(nr.fs.join(directory or '.', name))
self.encoding = kwargs.pop('encoding', self.encoding)
self._callable = callable
self._args = args
self._kwargs = kwargs
def with_chmod(self, chmod):
self.chmod = chmod
return self
@override
def render(self, current, dst):
return self._callable(current, dst, *self._args, *self._kwargs)

+ 4
- 2
src/shut/update/__init__.py

@ -25,8 +25,10 @@ and package definition.
"""
from .core import (
register_renderer,
get_files,
get_version_refs,
register_renderer,
VersionRef
)
from . import setuptools
from . import generic, setuptools

+ 23
- 1
src/shut/update/core.py

@ -20,18 +20,31 @@
# IN THE SOFTWARE.
import abc
from typing import Generic, T, Type
from typing import Iterable, Generic, T, Type
from databind.core import datamodel
from shut.utils.io.virtual import VirtualFiles
from shut.utils.type_registry import TypeRegistry
@datamodel
class VersionRef:
filename: str
start: int
end: int
value: str
class Renderer(Generic[T], metaclass=abc.ABCMeta):
@abc.abstractmethod
def get_files(self, files: VirtualFiles, obj: T) -> None:
pass
def get_version_refs(self, obj: T) -> Iterable[VersionRef]:
return; yield
registry = TypeRegistry[Type[Renderer]]()
@ -54,3 +67,12 @@ def get_files(obj: T) -> VirtualFiles:
renderer().get_files(files, obj)
return files
def get_version_refs(obj: T) -> Iterable[VersionRef]:
"""
Gets all version refs returned by registered for type *T*.
"""
for renderer in registry.for_type(type(obj)):
yield from renderer().get_version_refs(obj)

src/shut/render/__init__.py → src/shut/update/generic.py

@ -19,4 +19,29 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from .setuptools import render_setuptools_files
import re
from typing import Iterable
from shut.model import AbstractProjectModel
from shut.utils.io.virtual import VirtualFiles
from .core import Renderer, register_renderer, VersionRef
class GenericRenderer(Renderer[AbstractProjectModel]):
# Renderer[AbstractProjectModel] Overrides
def get_files(self, files: VirtualFiles, obj: AbstractProjectModel) -> None:
pass
def get_version_refs(self, obj: AbstractProjectModel) -> Iterable[VersionRef]:
# Return a reference to the version number in the package or monorepo model.
regex = '^\s*version\s*:\s*[\'"]?(.*?)[\'"]?\s*(#.*)?$'
with open(obj.filename) as fp:
match = re.search(regex, fp.read(), re.S | re.M)
if match:
yield VersionRef(obj.filename, match.start(1), match.end(1), match.group(1))
register_renderer(AbstractProjectModel, GenericRenderer)

+ 36
- 10
src/shut/update/setuptools.py

@ -21,8 +21,9 @@
import contextlib
import os
import re
import textwrap
from typing import Dict, List, Optional, TextIO, Tuple
from typing import Dict, Iterable, List, Optional, TextIO, Tuple
import nr.fs
@ -30,7 +31,7 @@ from shut.model import PackageModel
from shut.model.package import PackageData, PythonPackageMetadata, InstallConfiguration
from shut.model.requirements import Requirement
from shut.utils.io.virtual import VirtualFiles
from .core import Renderer, register_renderer
from .core import Renderer, register_renderer, VersionRef
GENERATED_FILE_REMARK = '''
# This file was auto-generated by Shut. DO NOT EDIT
@ -88,14 +89,6 @@ class SetuptoolsRenderer(Renderer[PackageModel]):
'python-major-minor-version': 'sys.version[:3]'
}
def get_files(self, files: VirtualFiles, package: PackageModel) -> None:
files.add_dynamic('setup.py', self._render_setup, package)
files.add_dynamic('MANIFEST.in', self._render_manifest_in, package, inplace=True)
if package.data.typed:
directory = package.get_python_package_metadata().package_directory
files.add_static(os.path.join(directory, 'py.typed'), '')
def _render_setup(
self,
fp: TextIO,
@ -383,5 +376,38 @@ class SetuptoolsRenderer(Renderer[PackageModel]):
for entry in manifest:
fp.write('{}\n'.format(entry))
_ENTRY_VERSION_REGEX = '__version__\s*=\s*[\'"]([^\'"]+)[\'"]'
def _entry_version_ref(self, filename: str) -> Optional[VersionRef]:
if not os.path.isfile(filename):
# This should be captured by the package checks as well.
return None
with open(filename) as fp:
match = re.search(self._ENTRY_VERSION_REGEX, fp.read())
if match:
return VersionRef(filename, match.start(1), match.end(1), match.group(1))
return None
# Renderer[PackageModel] Overrides
def get_files(self, files: VirtualFiles, package: PackageModel) -> None:
files.add_dynamic('setup.py', self._render_setup, package)
files.add_dynamic('MANIFEST.in', self._render_manifest_in, package, inplace=True)
if package.data.typed:
directory = package.get_python_package_metadata().package_directory
files.add_static(os.path.join(directory, 'py.typed'), '')
def get_version_refs(self, package: PackageModel) -> Iterable[VersionRef]:
filename = package.get_python_package_metadata().filename
if not filename or not os.path.isfile(filename):
return; yield
regex = '__version__\s*=\s*[\'"]([^\'"]+)[\'"]'
with open(filename) as fp:
match = re.search(regex, fp.read())
if match:
yield VersionRef(filename, match.start(1), match.end(1), match.group(1))
register_renderer(PackageModel, SetuptoolsRenderer)

Loading…
Cancel
Save