Browse Source

implement some monorepo checks

shut-new-model
Niklas Rosenstein 9 months ago
parent
commit
52c778c191
No known key found for this signature in database GPG Key ID: 6D269B33D25F6C6
12 changed files with 247 additions and 72 deletions
  1. + 2
    - 2
      src/shut/checks/__init__.py
  2. + 12
    - 2
      src/shut/checks/core.py
  3. + 43
    - 0
      src/shut/checks/monorepo.py
  4. + 17
    - 10
      src/shut/checks/package.py
  5. + 4
    - 1
      src/shut/commands/__init__.py
  6. + 72
    - 0
      src/shut/commands/commons/checks.py
  7. + 1
    - 0
      src/shut/commands/mono/__init__.py
  8. + 65
    - 0
      src/shut/commands/mono/checks.py
  9. + 1
    - 10
      src/shut/commands/pkg/__init__.py
  10. + 18
    - 42
      src/shut/commands/pkg/checks.py
  11. + 2
    - 1
      src/shut/commands/pkg/status.py
  12. + 10
    - 4
      src/shut/model/__init__.py

+ 2
- 2
src/shut/checks/__init__.py

@ -19,5 +19,5 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from . import basic, package
from .core import CheckStatus, get_checks
from .core import *
from . import basic, monorepo, package

+ 12
- 2
src/shut/checks/core.py

@ -39,6 +39,7 @@ class CheckStatus(enum.IntEnum):
CheckResult = namedtuple('CheckResult', 'status,message')
Check = namedtuple('Check', 'name,result')
SkipCheck = namedtuple('SkipCheck', '')
def check(name: str) -> Callable[[Callable], Callable]:
@ -69,7 +70,8 @@ class Checker(Generic[T]):
if isinstance(check_value, types.FunctionType) and hasattr(check_value, '__check_name__'):
index = None
for index, result in enumerate(value(project, subject)):
yield Check(value.__check_name__, result)
if not isinstance(result, SkipCheck):
yield Check(value.__check_name__, result)
if index is None:
yield Check(value.__check_name__, CheckResult(CheckStatus.PASSED, None))
@ -84,7 +86,6 @@ def register_checker(checker: Type[Checker[T]], t: Type[T]) -> Type[Checker]:
registry.setdefault(t, []).append(checker)
def get_checks(project: 'Project', obj: T) -> Iterable[Check]:
"""
Returns all checks from the checkers registered for the type of *obj*.
@ -92,3 +93,12 @@ def get_checks(project: 'Project', obj: T) -> Iterable[Check]:
for checker in registry.get(type(obj), []):
yield from checker().get_checks(project, obj)
__all__ = [
'CheckStatus',
'CheckResult',
'Check',
'register_checker',
'get_checks',
]

+ 43
- 0
src/shut/checks/monorepo.py

@ -0,0 +1,43 @@
# -*- coding: utf8 -*-
# Copyright (c) 2020 Niklas Rosenstein
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from .core import Check, CheckStatus, CheckResult, Checker, SkipCheck, check, register_checker
from shut.model import MonorepoModel
class MonorepoChecker(Checker[MonorepoModel]):
@check('invalid-package')
def _check_no_invalid_packages(self, project, monorepo):
for package_name, exc_info in project.invalid_packages:
yield CheckResult(CheckStatus.ERROR, package_name)
@check('inconsistent-single-version')
def _check_consistent_mono_version(self, project, monorepo):
if monorepo.release.single_version and project.packages:
for package in project.packages:
if package.data.version is not None and package.data.version != monorepo.version:
yield CheckResult(CheckStatus.ERROR, package.data.name)
else:
yield SkipCheck()
register_checker(MonorepoChecker, MonorepoModel)

+ 17
- 10
src/shut/checks/package.py

@ -37,7 +37,10 @@ class PackageChecker(Checker[PackageModel]):
@check('license')
def _check_license(self, project: Project, package: PackageModel) -> Iterable[CheckResult]:
if package.data.license and not package.get_license():
if not package.data.license:
yield CheckResult(CheckStatus.WARNING, 'not specified')
elif package.data.license and not package.get_license():
yield CheckResult('license', CheckStatus.WARNING, 'No LICENSE file found.')
monorepo = project.monorepo
@ -52,17 +55,19 @@ class PackageChecker(Checker[PackageModel]):
classifiers = get_classifiers()
unknown_classifiers = [x for x in package.data.classifiers if x not in classifiers]
if unknown_classifiers:
yield CheckResult('classifiers', CheckStatus.WARNING,
yield CheckResult(
CheckStatus.WARNING,
'Unknown classifiers: ' + ', '.join(unknown_classifiers))
@check('package-config')
def _check_metadata_completeness(self, project: Project, package: PackageModel) -> Iterable[CheckResult]:
@check('author')
def _check_author(self, project: Project, package: PackageModel) -> Iterable[CheckResult]:
if not package.data.author:
yield CheckResult('metadata-completeness', CheckStatus.WARNING, 'no $.package.author')
if not package.data.license: #and not package.get_private():
yield CheckResult('metadata-completeness', CheckStatus.WARNING, 'no $.package.license')
yield CheckResult(CheckStatus.WARNING, 'missing')
@check('url')
def _check_author(self, project: Project, package: PackageModel) -> Iterable[CheckResult]:
if not package.data.url:
yield CheckResult('metadata-completeness', CheckStatus.WARNING, 'no $.package.url')
yield CheckResult(CheckStatus.WARNING, 'missing')
@check('consistent-author')
def _check_consistent_author(self, project: Project, package: PackageModel) -> Iterable[CheckResult]:
@ -92,11 +97,13 @@ class PackageChecker(Checker[PackageModel]):
py_typed_file = os.path.join(metadata.package_directory, 'py.typed')
except ValueError:
if package.data.typed:
yield CheckResult('typed', CheckStatus.WARNING,
yield CheckResult(
CheckStatus.WARNING,
'$.package.typed only works with packages, but this is a module')
else:
if os.path.isfile(py_typed_file) and not package.data.typed:
yield CheckResult('typed', CheckStatus.WARNING,
yield CheckResult(
CheckStatus.WARNING,
'file "py.typed" exists but $.typed is not set')

+ 4
- 1
src/shut/commands/__init__.py

@ -23,7 +23,8 @@
This package implements the Shut CLI.
"""
from shore import __version__
from shut import __version__
from shut.model import Project
from nr.proxy import Proxy
import click
@ -31,6 +32,7 @@ import logging
import os
context = Proxy(lambda: click.get_current_context().obj)
project = Proxy(lambda: click.get_current_context().obj['project'])
@click.group()
@ -54,6 +56,7 @@ def shut(cwd, verbose, quiet):
ctx = click.get_current_context()
ctx.ensure_object(dict)
context['quiet'] = quiet
context['project'] = Project()
if quiet:
level = logging.CRITICAL

+ 72
- 0
src/shut/commands/commons/checks.py

@ -0,0 +1,72 @@
# -*- coding: utf8 -*-
# Copyright (c) 2020 Niklas Rosenstein
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from shut.checks import Check, CheckStatus
from typing import List
import termcolor
def print_checks(
checks: List[Check],
emojis: bool = True,
colors: bool = True,
prefix: str = '') -> None:
"""
Prints the list of *checks* to the terminal.
"""
emoji_chars = {
CheckStatus.PASSED: '✔️',
CheckStatus.WARNING: '⚠️',
CheckStatus.ERROR: ''}
color_names = {
CheckStatus.PASSED: 'green',
CheckStatus.WARNING: 'magenta',
CheckStatus.ERROR: 'red'}
if colors:
colored = termcolor.colored
else:
def colored(s, *a, **kw):
return str(s)
for check in checks:
if emojis:
print(prefix, emoji_chars[check.result.status], ' ', check.name, sep='', end='')
else:
print(prefix, colored(check.name, color_names[check.result.status]), sep='', end='')
if check.result.status != CheckStatus.PASSED:
print(':', check.result.message)
else:
print()
def get_checks_status(checks: List[Check], warnings_as_errors: bool = False) -> int:
max_level = max(x.result.status for x in checks)
if max_level == CheckStatus.PASSED:
status = 0
elif max_level == CheckStatus.WARNING:
status = 1 if warnings_as_errors else 0
elif max_level == CheckStatus.ERROR:
status = 1
else:
assert False, max_level

+ 1
- 0
src/shut/commands/mono/__init__.py

@ -35,4 +35,5 @@ def load_monorepo_manifest() -> Monorepo:
return commons.load_manifest(('monorepo.yaml', 'monorepo.yml'), Monorepo)
from . import checks
from . import status

+ 65
- 0
src/shut/commands/mono/checks.py

@ -0,0 +1,65 @@
# -*- coding: utf8 -*-
# Copyright (c) 2020 Niklas Rosenstein
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from shut.checks import CheckStatus, get_checks
from shut.commands import project
from shut.commands.commons.checks import print_checks, get_checks_status
from shut.commands.mono import mono
from shut.model import MonorepoModel, Project
from nr.stream import Stream
from termcolor import colored
from typing import Iterable, Union
import click
import enum
import logging
import os
import sys
import termcolor
import time
logger = logging.getLogger(__name__)
@mono.command()
@click.option('-w', '--warnings-as-errors', is_flag=True)
def checks(warnings_as_errors):
"""
Sanity-check the package configuration and package files. Which checks are performed
depends on the features that are enabled in the package configuration. Usually that
will at least include the "setuptools" feature which will perform basic sanity checks
on the package configuration and entrypoint definition.
"""
start_time = time.perf_counter()
monorepo = project.load(expect=MonorepoModel)
checks = sorted(get_checks(project, monorepo), key=lambda c: c.name)
seconds = time.perf_counter() - start_time
monorepo_name = termcolor.colored(monorepo.name, 'yellow')
print()
print_checks(checks, prefix=' ')
print()
print('run', len(checks), 'checks for repository', monorepo_name, 'in {:.3f}s'.format(seconds))
print()
sys.exit(get_checks_status(checks, warnings_as_errors))

+ 1
- 10
src/shut/commands/pkg/__init__.py

@ -22,22 +22,13 @@
from shut.commands import shut, commons
from shut.model import Project, PackageModel
from nr.proxy import Proxy
import click
project = Proxy(lambda: click.get_current_context().obj['project'])
@shut.group()
@click.pass_context
def pkg(ctx):
def pkg():
"""
Manage the Python package in the current directory.
"""
ctx.ensure_object(dict)
ctx.obj['project'] = Project()
from . import bootstrap
from . import checks

+ 18
- 42
src/shut/commands/pkg/checks.py

@ -20,7 +20,9 @@
# IN THE SOFTWARE.
from shut.checks import CheckStatus, get_checks
from shut.commands.pkg import pkg, project
from shut.commands import project
from shut.commands.commons.checks import print_checks, get_checks_status
from shut.commands.pkg import pkg
from shut.model import PackageModel, Project
from nr.stream import Stream
@ -31,50 +33,12 @@ import enum
import logging
import os
import sys
import termcolor
import time
logger = logging.getLogger(__name__)
def print_package_checks(project: Project, package: PackageModel, warnings_as_errors: bool = False) -> int:
icons = {
CheckStatus.PASSED: '✔️',
CheckStatus.WARNING: '⚠️',
CheckStatus.ERROR: ''}
colors = {
CheckStatus.PASSED: 'green',
CheckStatus.WARNING: 'magenta',
CheckStatus.ERROR: 'red'}
package_name_c = colored(package.data.name, 'yellow')
checks = sorted(get_checks(project, package), key=lambda c: c.name)
print()
for check in checks:
print(' ', icons[check.result.status], check.name, end='')
if check.result.status != CheckStatus.PASSED:
print(':', check.result.message)
else:
print()
print()
print('run', len(checks), 'check(s) for package', package_name_c)
print()
max_level = max(x.result.status for x in checks)
if max_level == CheckStatus.PASSED:
status = 0
elif max_level == CheckStatus.WARNING:
status = 1 if warnings_as_errors else 0
elif max_level == CheckStatus.ERROR:
status = 1
else:
assert False, max_level
logger.debug('exiting with status %s', status)
return status
@pkg.command()
@click.option('-w', '--warnings-as-errors', is_flag=True)
def checks(warnings_as_errors):
@ -85,5 +49,17 @@ def checks(warnings_as_errors):
on the package configuration and entrypoint definition.
"""
start_time = time.perf_counter()
package = project.load(expect=PackageModel)
sys.exit(print_package_checks(project, package, warnings_as_errors))
checks = sorted(get_checks(project, package), key=lambda c: c.name)
seconds = time.perf_counter() - start_time
package_name = termcolor.colored(package.data.name, 'yellow')
print()
print_checks(checks, prefix=' ')
print()
print('run', len(checks), 'checks for package', package_name, 'in {:.3f}s'.format(seconds))
print()
sys.exit(get_checks_status(checks, warnings_as_errors))

+ 2
- 1
src/shut/commands/pkg/status.py

@ -19,8 +19,9 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from shut.commands import project
from shut.commands.commons.status import print_status
from shut.commands.pkg import pkg, project
from shut.commands.pkg import pkg
from shut.model import PackageModel

+ 10
- 4
src/shut/model/__init__.py

@ -19,19 +19,21 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from typing import List, Type, TypeVar, Union
from typing import List, Tuple, Type, TypeVar, Union
from nr.databind.core import ObjectMapper, NodeCollector
from nr.databind.core import ObjectMapper, NodeCollector, SerializationError
from nr.databind.json import JsonModule
from nr.stream import Stream
import yaml
import os
import sys
from .monorepo import MonorepoModel
from .package import PackageModel
T = TypeVar('T')
ExcInfo = Tuple
def get_existing_file(directory: str, choices: List[str]) -> bool:
@ -57,6 +59,7 @@ class Project:
self.subject: Union[MonorepoModel, PackageModel] = None
self.monorepo: MonorepoModel = None
self.packages: List[PackageModel] = []
self.invalid_packages: List[Tuple[str, ExcInfo]] = []
def load(
self,
@ -119,10 +122,13 @@ class Project:
# Load packages in that monorepo.
directory = os.path.dirname(filename)
for item_name in os.path.listdir(directory):
for item_name in os.listdir(directory):
package_fn = get_existing_file(os.path.join(directory, item_name), self.package_filenames)
if package_fn:
self._load_package(package_fn)
try:
self._load_package(package_fn)
except SerializationError as exc:
self.invalid_packages.append((item_name, sys.exc_info()))
return self.monorepo

Loading…
Cancel
Save