Browse Source

move sanity check logic to shut

shut-new-model
Niklas Rosenstein 9 months ago
parent
commit
4c5e320cba
No known key found for this signature in database GPG Key ID: 6D269B33D25F6C6
8 changed files with 237 additions and 34 deletions
  1. + 1
    - 1
      package.yaml
  2. + 1
    - 1
      src/shore/model.py
  3. + 9
    - 9
      src/shut/commands/pkg/__init__.py
  4. + 89
    - 13
      src/shut/commands/pkg/sanity.py
  5. + 4
    - 3
      src/shut/commands/pkg/status.py
  6. + 22
    - 6
      src/shut/model/__init__.py
  7. + 2
    - 0
      src/shut/model/monorepo.py
  8. + 109
    - 1
      src/shut/model/package.py

+ 1
- 1
package.yaml

@ -12,7 +12,7 @@ package:
- click ^7.0
- jinja2 ^2.11.1
- networkx ^2.4
- nr.databind.core ~0.0.20
- nr.databind.core ~0.0.21
- nr.databind.json ~0.0.13
- nr.fs ^1.5.0
- nr.pylang.utils ^0.0.1

+ 1
- 1
src/shore/model.py

@ -262,7 +262,7 @@ class RootRequirements(Requirements):
def __init__(self, python=None, required=None, platforms=None, test=None, extra=None):
super(RootRequirements, self).__init__(python, required, platforms)
self.test = test
self.extra = extra or []
self.extra = extra or {}
def _extract_from_item(self, mapper, node):
item = node.value

+ 9
- 9
src/shut/commands/pkg/__init__.py

@ -19,24 +19,24 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from .. import shut, commons
from shut.commands import shut, commons
from shut.model import Project, PackageModel
from nr.proxy import Proxy
import click
project = Proxy(lambda: click.get_current_context().obj['project'])
@shut.group()
def pkg():
@click.pass_context
def pkg(ctx):
"""
Manage the Python package in the current directory.
"""
def load_package_manifest() -> PackageModel:
project = Project()
project.load('.')
print(project.subject)
assert isinstance(project.subject, PackageModel)
return project.subject
ctx.ensure_object(dict)
ctx.obj['project'] = Project()
from . import bootstrap

+ 89
- 13
src/shut/commands/pkg/sanity.py

@ -19,30 +19,106 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from shore.model import Monorepo
from shore.core.plugins import CheckResult
from shore.util.classifiers import get_classifiers
from shut.commands.pkg import pkg, project
from shut.model import PackageModel, Project
from . import pkg, load_package_manifest
from nr.stream import Stream
from termcolor import colored
from typing import Iterable, Union
import click
import enum
import logging
import os
import sys
logger = logging.getLogger(__name__)
def _run_for_subject(subject, func): # type: (Union[Package, Monorepo], func) -> List[Any]
if isinstance(subject, Monorepo):
subjects = [subject] + sorted(subject.get_packages(), key=lambda x: x.name)
return [func(x) for x in subjects]
class CheckResult:
"""
Represents a sanity check result.
"""
class Level(enum.IntEnum):
INFO = 0
WARNING = 1
ERROR = 2
def __init__(self, on: PackageModel, level: Union[str, Level], message: str):
if isinstance(level, str):
level = self.Level[level]
assert isinstance(level, self.Level)
self.on = on
self.level = level
self.message = message
def __repr__(self):
return 'CheckResult(on={!r}, level={}, message={!r})'.format(
self.on, self.level, self.message)
def sanity_check_package(project: Project, package: PackageModel) -> Iterable[CheckResult]:
for path in package.unknown_keys:
yield CheckResult(package, CheckResult.Level.WARNING, 'unknown key {}'.format(path))
if not package.get_readme():
yield CheckResult(package, 'WARNING', 'No README file found.')
classifiers = get_classifiers()
unknown_classifiers = [x for x in package.data.classifiers if x not in classifiers]
if unknown_classifiers:
yield CheckResult(package, 'WARNING',
'unknown $.classifiers: {}'.format(unknown_classifiers))
if not package.data.author:
yield CheckResult(package, 'WARNING', 'missing $.package.author')
if not package.data.license: #and not package.get_private():
yield CheckResult(package, 'WARNING', 'missing $.license')
if not package.data.url:
yield CheckResult(package, 'WARNING', 'missing $.url')
if package.data.license and project.monorepo and project.monorepo.license \
and project.monorepo.license != package.data.license:
yield CheckResult(package, 'ERROR', '$.license ({!r}) is inconsistent '
'with monorepo license ({!r})'.format(package.license, package.monorepo.license))
if package.data.license:
for name in ('LICENSE', 'LICENSE.txt', 'LICENSE.rst', 'LICENSE.md'):
filename = os.path.join(os.path.dirname(package.filename), name)
if os.path.isfile(filename):
break
else:
yield CheckResult(package, 'WARNING', 'No LICENSE file found.')
metadata = package.get_python_package_metadata()
if package.data.author and metadata.author != str(package.data.author):
yield CheckResult(package, 'ERROR',
'Inconsistent package author (package.yaml: {!r} != {}: {!r})'.format(
str(package.data.author), metadata.filename, metadata.author))
if package.data.version and metadata.version != str(package.data.version):
yield CheckResult(package, 'ERROR',
'Inconsistent package version (package.yaml: {!r} != {}: {!r})'.format(
str(package.data.version), metadata.filename, metadata.version))
try:
py_typed_file = os.path.join(metadata.package_directory, 'py.typed')
except ValueError:
if package.data.typed:
yield CheckResult(package, 'WARNING', '$.package.typed only works with packages, but this is a module')
else:
return [func(subject)]
if os.path.isfile(py_typed_file) and not package.data.typed:
yield CheckResult(package, 'WARNING', 'file "py.typed" exists but $.typed is not set')
def print_package_checks(project: Project, package: PackageModel, warnings_as_errors: bool = False) -> bool:
"""
Formats the checks created with #sanity_check_package().
"""
def run_checks(package, warnings_as_errors): # type: (Package, bool) -> bool
package_name_c = colored(package.name, 'yellow')
checks = list(Stream.concat(x.get_checks(package) for x in package.get_plugins()))
package_name_c = colored(package.data.name, 'yellow')
checks = list(sanity_check_package(project, package))
if not checks:
print('✔ no checks triggered on package {}'.format(package_name_c))
return True
@ -79,7 +155,7 @@ def sanity(warnings_as_errors):
on the package configuration and entrypoint definition.
"""
package = load_package_manifest()
result = run_checks(package, warnings_as_errors)
package = project.load(expect=PackageModel)
result = print_package_checks(project, package, warnings_as_errors)
if not result:
sys.exit(1)

+ 4
- 3
src/shut/commands/pkg/status.py

@ -19,12 +19,13 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from . import pkg, load_package_manifest
from ..commons.status import print_status
from shut.commands.commons.status import print_status
from shut.commands.pkg import pkg, project
from shut.model import PackageModel
@pkg.command(help="""
Shows whether the package was modified since the last release.
""" + print_status.__doc__)
def status():
print_status(load_package_manifest())
print_status(project.load(expect=PackageModel))

+ 22
- 6
src/shut/model/__init__.py

@ -19,10 +19,11 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from typing import List, Type, TypeVar
from typing import List, Type, TypeVar, Union
from nr.databind.core import ObjectMapper
from nr.databind.core import ObjectMapper, NodeCollector
from nr.databind.json import JsonModule
from nr.stream import Stream
import yaml
import os
@ -51,13 +52,17 @@ class Project:
package_filenames = ['package.yml', 'package.yaml']
def __init__(self, mapper: ObjectMapper = None):
self._cache = {}
self._cache: Dict[str, Union[MonorepoModel, PackageModel]] = {}
self.mapper = mapper or ObjectMapper(JsonModule())
self.subject: Union[MonorepoModel, PackageModel] = None
self.monorepo: MonorepoModel = None
self.packages: List[PackageModel] = []
def load(self, directory: str) -> None:
def load(
self,
directory: str = '.',
expect: Type[Union[MonorepoModel, PackageModel]] = None,
) -> Union[MonorepoModel, PackageModel]:
"""
Loads all project information from *directory*. This searches in all parent directories
for a package or monorepo configuration, then loads all resources that belong to the
@ -85,6 +90,12 @@ class Project:
if package_fn:
self.subject = self._load_package(package_fn)
if expect and not isinstance(self.subject, expect):
raise TypeError('expected {!r} at {!r}, got {!r}'.format(
expect.__name__, directory, type(self.subject).__name__))
return self.subject
def _load_object(self, filename: str, type_: Type[T]) -> T:
filename = os.path.normpath(os.path.abspath(filename))
if filename in self._cache:
@ -94,8 +105,13 @@ class Project:
return obj
with open(filename) as fp:
data = yaml.safe_load(fp)
# TODO(NiklasRosenstein): Store unknown keys
obj = self._cache[filename] = self.mapper.deserialize(data, type_, filename=filename)
node_collector = NodeCollector()
obj = self._cache[filename] = self.mapper.deserialize(
data, type_, filename=filename, decorations=[node_collector])
obj.filename = filename
obj.unknown_keys = list(Stream.concat(
(x.locator.append(k) for k in x.unknowns)
for x in node_collector.nodes))
return obj
def _load_monorepo(self, filename: str) -> MonorepoModel:

+ 2
- 0
src/shut/model/monorepo.py

@ -23,10 +23,12 @@ from .author import Author
from .version import Version
from .release import MonorepoReleaseConfiguration
from nr.databind.core import Field, FieldName, Struct
from typing import List
class MonorepoModel(Struct):
filename = Field(str, hidden=True, default=None)
unknown_keys = Field(List[str], hidden=True, default=list)
name = Field(str)
version = Field(Version, default=None)
author = Field(Author, default=None)

+ 109
- 1
src/shut/model/package.py

@ -19,6 +19,9 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from shore.util.ast import load_module_members
from shore.plugins._util import find_readme_file
from .author import Author
from .changelog import ChangelogConfiguration
from .linter import LinterConfiguration
@ -26,7 +29,9 @@ from .release import ReleaseConfiguration
from .requirements import Requirement
from .version import Version
from nr.databind.core import Field, FieldName, Struct
from typing import Dict, List
from typing import Dict, List, Optional
import ast
import os
class PackageData(Struct):
@ -51,6 +56,9 @@ class PackageData(Struct):
keywords = Field(List[str], default=list)
# TODO: Data files
def get_modulename(self) -> str:
return self.modulename or self.name.replace('-', '_')
class InstallConfiguration(Struct):
hooks = Field(dict(
@ -63,8 +71,108 @@ class InstallConfiguration(Struct):
class PackageModel(Struct):
filename = Field(str, hidden=True, default=None)
unknown_keys = Field(List[str], hidden=True, default=list)
data = Field(PackageData, FieldName('package'))
changelog = Field(ChangelogConfiguration, default=Field.DEFAULT_CONSTRUCT)
install = Field(InstallConfiguration, default=Field.DEFAULT_CONSTRUCT)
linter = Field(LinterConfiguration, default=Field.DEFAULT_CONSTRUCT)
release = Field(ReleaseConfiguration, default=Field.DEFAULT_CONSTRUCT)
def get_python_package_metadata(self) -> 'PythonPackageMetadata':
return PythonPackageMetadata(
os.path.join(os.path.dirname(self.filename), self.data.source_directory),
self.data.get_modulename())
def get_readme(self) -> Optional[str]:
"""
Returns the absolute path to the README for this package.
"""
directory = os.path.dirname(__file__)
if self.data.readme:
return os.path.abspath(os.path.join(directory, self.readme))
return find_readme_file(directory)
class PythonPackageMetadata:
def __init__(self, source_directory: str, modulename: str) -> None:
self.source_directory = source_directory
self.modulename = modulename
self._filename = None
self._author = None
self._version = None
@property
def filename(self) -> str:
"""
Returns the file that contains the package metadata in the Python source code. This is
usually the module filename, the package `__init__.py` or `__version__.py`.
"""
if self._filename:
return self._filename
parts = self.modulename.split('.')
prefix = os.sep.join(parts[:-1])
choices = [
parts[-1] + '.py',
os.path.join(parts[-1], '__version__.py'),
os.path.join(parts[-1], '__init__.py'),
]
for filename in choices:
filename = os.path.join(self.source_directory, prefix, filename)
if os.path.isfile(filename):
self._filename = filename
return filename
raise ValueError('Entry file for package "{}" could not be determined'
.format(self.modulename))
@property
def package_directory(self) -> str:
"""
Returns the Python package directory. Raises a #ValueError if this metadata represents
just a single Python module.
"""
dirname, basename = os.path.split(self.filename)
if basename not in ('__init__.py', '__version__.py'):
raise ValueError('this package is in module-only form')
return dirname
@property
def author(self) -> str:
if not self._author:
self._load_metadata()
return self._author
@property
def version(self) -> str:
if not self._version:
self._load_metadata()
return self._version
def _load_metadata(self) -> None:
members = load_module_members(self.filename)
author = None
version = None
if '__version__' in members:
try:
version = ast.literal_eval(members['__version__'])
except ValueError as exc:
version = '<Non-literal expression>'
if '__author__' in members:
try:
author = ast.literal_eval(members['__author__'])
except ValueError as exc:
author = '<Non-literal expression>'
self._author = author
self._version = version

Loading…
Cancel
Save