Browse Source

add "shut pkg publish" command

shut-new-model
Niklas Rosenstein 9 months ago
parent
commit
5c0cbe377f
No known key found for this signature in database GPG Key ID: 6D269B33D25F6C6
9 changed files with 259 additions and 82 deletions
  1. + 0
    - 2
      README.md
  2. + 4
    - 15
      src/shut/builders/setuptools.py
  3. + 18
    - 3
      src/shut/commands/pkg/build.py
  4. + 69
    - 46
      src/shut/commands/pkg/publish.py
  5. + 4
    - 0
      src/shut/model/publish.py
  6. + 6
    - 2
      src/shut/model/target.py
  7. + 21
    - 3
      src/shut/publish/core.py
  8. + 89
    - 11
      src/shut/publish/warehouse.py
  9. + 48
    - 0
      src/shut/utils/io/sp.py

+ 0
- 2
README.md

@ -34,9 +34,7 @@ package:
__Todo__
* [ ] build/publish commands
* [ ] Bump version based on changelog
* [ ] Command to install and save requirements
* [ ] Package data / data files
* [ ] Conda recipe generator and conda-forge helper
* [ ] Automatic check for license headers in files / automatically insert license headers

+ 4
- 15
src/shut/builders/setuptools.py

@ -20,14 +20,13 @@
# IN THE SOFTWARE.
import os
import subprocess
import shutil
import sys
from typing import Iterable, List, Optional
from termcolor import colored
from shut.model import PackageModel
from shut.model.target import TargetId
from shut.utils.io.sp import subprocess_trimmed_call
from . import Builder, BuilderProvider, register_builder_provider
@ -106,18 +105,8 @@ class SetuptoolsBuilder(Builder):
dist_directory = os.path.join(self.package_directory, 'dist')
dist_exists = os.path.exists(dist_directory)
command = [python, 'setup.py', self.build_type] + self.args
proc = subprocess.Popen(
command,
cwd=self.package_directory,
stdout=None if verbose else subprocess.PIPE,
stderr=None if verbose else subprocess.PIPE)
stdout, stderr = proc.communicate()
if stderr:
for line in stderr.decode().splitlines():
if not line:
continue
print(f' {colored(line, "red")}')
res = proc.wait()
res = subprocess_trimmed_call(command, cwd=self.package_directory)
if res != 0:
return False

+ 18
- 3
src/shut/commands/pkg/build.py

@ -19,7 +19,9 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
import os
import sys
from typing import List
import click
from nr.stream import groupby
@ -32,6 +34,19 @@ from . import pkg
from .. import project
def run_builds(builders: List[Builder], build_dir: str, verbose: bool) -> bool:
for builder in builders:
print(colored(f'building {colored(builder.id, "green")}'))
for filename in builder.get_outputs():
print(f' :: {os.path.join(build_dir, filename)}')
print()
success = builder.build(build_dir, verbose)
if not success:
print(f'error: building "{builder.id}" failed', file=sys.stderr)
return False
return True
@pkg.command()
@click.argument('target', type=lambda s: TargetId.parse(s, True), required=False)
@click.option('-l', '--list', 'list_', is_flag=True, help='list available builders')
@ -64,6 +79,6 @@ def build(target, list_, build_dir, verbose):
if not builders:
sys.exit(f'error: no target matches "{target}"')
for builder in builders:
print(colored(f'building {colored(builder.id, "green")}'))
builder.build(build_dir, verbose)
success = run_builds(builders, build_dir, verbose)
if not success:
sys.exit(1)

+ 69
- 46
src/shut/commands/pkg/publish.py

@ -19,69 +19,92 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
import os
import sys
from typing import List
import click
from nr.stream import groupby, Stream
from termcolor import colored
from shut.builders import get_builders
from shut.model import PackageModel
from shut.publish import Publisher
from shut.model.target import TargetId
from shut.publish import Publisher, get_publishers
from shut.publish.warehouse import WarehousePublisher
from . import pkg
from .build import run_builds
from .. import project
def get_publisher(package: PackageModel, target: str) -> Publisher:
if target == 'pypi' and package.data.publish.pypi:
publisher = WarehousePublisher.pypi_from_credentials(
package.data.publish.pypi_credentials)
elif target in package.data.publish.warehouses:
publisher = WarehousePublisher.from_config(package.data.publish.warehouses[target])
else:
raise ValueError(f'unknown publish target {target!r}')
return publisher
def get_publisher_names(package: PackageModel) -> List[str]:
targets = []
if package.data.publish.pypi.enabled:
targets.append('pypi')
targets.extend(package.data.publish.warehouses.keys())
return targets
@pkg.command()
@click.argument('target')
@click.option('--ls', is_flag=True)
def publish(target, ls):
@click.argument('target', type=lambda s: TargetId.parse(s, True), required=False)
@click.option('-t', '--test', is_flag=True, help='publish to the test repository instead')
@click.option('-l', '--list', 'list_', is_flag=True, help='list available publishers')
@click.option('-v', '--verbose', is_flag=True, help='show more output')
@click.option('-b', '--build-dir', default='build', help='build output directory')
@click.option('--skip-build', is_flag=True, help='do not build artifacts that are to be published')
def publish(target, test, list_, verbose, build_dir, skip_build):
"""
Publish the package to PyPI or another target.
"""
if ls and target:
if list_ and target:
sys.exit('error: conflicting options')
if ls:
names = get_publisher_names()
if not names:
print('no publishes configured')
else:
print('available publishers:')
for name in names:
print(f' {name}')
return
package = project.load_or_exit(expect=PackageModel)
publishers = list(get_publishers(package))
try:
publisher = get_publisher(package, target)
except ValueError as exc:
sys.exit(f'error: {exc}')
if isinstance(publisher, WarehousePublisher):
build_targets = get_build_targets('setuptools')
else:
raise RuntimeError
if list_:
for scope, publishers in groupby(publishers, lambda p: p.id.scope):
print(f'{colored(scope, "green")}:')
for publisher in publishers:
print(f' {publisher.id.name} – {publisher.get_description()}')
return
run_build_targets(build_targets)
publisher.publish(build_targets)
if not target:
sys.exit('error: no target specified')
publishers = [p for p in publishers if target.match(p.id)]
if not publishers:
sys.exit(f'error: no target matches "{target}"')
# Prepare the builds that need to be built for the publishers.
all_builders = list(get_builders(package))
builders_for_publisher = {}
for publisher in publishers:
builders = []
for target_id in publisher.get_build_dependencies():
matched_builders = [b for b in all_builders if target_id.match(b.id)]
if not matched_builders:
sys.exit(f'error: publisher "{publisher.id}" depends on build target "{target_id}" '
'which could not be resolved.')
builders.extend(b for b in matched_builders if b not in builders)
builders_for_publisher[publisher.id] = builders
# Build all builders that are needed.
if not skip_build:
built = set()
for publisher in publishers:
print()
builders = builders_for_publisher[publisher.id]
success = run_builds([b for b in builders if b not in built], build_dir, verbose)
if not success:
sys.exit(1)
# Execute the publishers.
for publisher in publishers:
print()
print(f'publishing {colored(publisher.id, "cyan")}')
builders = builders_for_publisher[publisher.id]
files = (Stream
.concat(b.get_outputs() for b in builders)
.map(lambda x: os.path.join(build_dir, x))
.collect()
)
for filename in files:
print(f' :: {filename}')
print()
success = publisher.publish(files, test, verbose)
if not success:
sys.exit(1)

+ 4
- 0
src/shut/model/publish.py

@ -39,6 +39,10 @@ class WarehouseConfiguration(WarehouseCredentials):
test_repository: Optional[str] = None
test_repository_url: Optional[str] = None
def with_creds(self, creds: WarehouseCredentials) -> 'WarehouseConfiguration':
vars(self).update(vars(creds))
return self
@datamodel
class PypiConfiguration:

+ 6
- 2
src/shut/model/target.py

@ -46,8 +46,12 @@ class TargetId:
parts = (parts[0], '*')
return cls(*parts)
def match(self, other_id: 'TargetId') -> bool:
return fnmatch(other_id.scope, self.scope) and fnmatch(other_id.name, self.name)
def match(self, other_id: 'TargetId', allow_match_name: bool = False) -> bool:
if fnmatch(other_id.scope, self.scope) and fnmatch(other_id.name, self.name):
return True
if allow_match_name and self.name == '*':
return self.scope == other_id.name
return False
class Target(metaclass=abc.ABCMeta):

+ 21
- 3
src/shut/publish/core.py

@ -22,6 +22,9 @@
import abc
from typing import Generic, Iterable, List, T, Type
from nr.stream import concat
from shut.builders import Builder
from shut.model import AbstractProjectModel
from shut.model.target import Target, TargetId
from shut.utils.type_registry import TypeRegistry
@ -37,15 +40,30 @@ __all__ = [
class Publisher(Target, metaclass=abc.ABCMeta):
@abc.abstractmethod
def publish(self, verbose: bool) -> bool:
def get_description(self) -> str:
pass
@abc.abstractmethod
def get_build_dependencies(self) -> Iterable[TargetId]:
"""
Return the IDs of build targets that this publisher depends on.
"""
@abc.abstractmethod
def publish(self, files: List[str], test: bool, verbose: bool) -> bool:
"""
Run the publishing logic. The builders resolved from #get_build_dependencies() are
passed to this function. They will already be built when this method is called.
"""
class PublisherProvider(Generic[T], metaclass=abc.ABCMeta):
@abc.abstractmethod
def get_publishers(self) -> List[Publisher]:
pass
def get_publishers(self, obj: T) -> Iterable[Publisher]:
"""
Return the publishers provided by this plugin.
"""
registry = TypeRegistry[PublisherProvider[AbstractProjectModel]]()

+ 89
- 11
src/shut/publish/warehouse.py

@ -19,28 +19,101 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from typing import Iterable
import logging
import sys
from typing import Iterable, List
from urllib.parse import urlparse
from nr.stream import concat
from shut.builders import Builder
from shut.model import PackageModel, Project
from shut.model.publish import WarehouseCredentials, WarehouseConfiguration
from shut.model.target import TargetId
from shut.utils.io.sp import subprocess_trimmed_call
from .core import Publisher, PublisherProvider, register_publisher_provider
logger = logging.getLogger(__name__)
def _resolve_envvars(s):
if s and s.startswith('$'):
value = os.getenv(s[1:])
if not value:
raise RuntimeError('environment variable {} is not set.'.format(s))
return value
return s
class WarehousePublisher(Publisher):
"""
A publisher target that uploads files to a Warehouse (Python Package Index) using the
"twine" command.
"""
def __init__(self, id_: TargetId, config: WarehouseConfiguration) -> None:
def __init__(self, id_: TargetId, config: WarehouseConfiguration, name: str) -> None:
self._id = id_
self.config = config
self.name = name
@classmethod
def from_pypi_credentials(cls, id_: TargetId, creds: WarehouseCredentials) -> 'WarehousePublisher':
return cls(id_, WarehouseCredentials().with_creds(creds))
config = WarehouseConfiguration(
repository='pypi',
test_repository='testpypi',
).with_creds(creds)
return cls(id_, config, 'PyPI')
# Publisher Overrides
def publish(self):
raise NotImplementedError('todo')
def get_description(self) -> str:
return f'Publish the package to {self.name}.'
def get_build_dependencies(self) -> Iterable[TargetId]:
yield TargetId.parse('setuptools:*')
def publish(self, files: List[str], test: bool, verbose: bool):
command = [sys.executable, '-m', 'twine', 'upload', '--non-interactive']
config = self.config
repo = config.test_repository if test else config.repository
repo_url = config.test_repository_url if test else config.repository_url
username = config.test_username if test else config.username
password = config.test_password if test else config.password
username = _resolve_envvars(username)
password = _resolve_envvars(password)
if not repo and not repo_url:
prefix = 'test_' if test else ''
raise RuntimeError('missing {0}repository or {0}repository_url for PypiPublisher'.format(prefix))
if repo:
command += ('--repository', repo)
if repo_url:
command += ('--repository-url', repo_url)
#if config.sign:
# command.append('--sign')
#if config.sign_with:
# command += ('--sign-with', config.sign_with)
#if config.identity:
# command += ('--identity', config.identity)
if username is not None:
command += ('--username', username)
if password is not None:
command += ('--password', password)
#if config.skip_existing:
# command.append('--skip-existing')
#if config.cert:
# command += ('--cert', config.cert)
#if config.client_cert:
# command += ('--client-cert', config.client_cert)
command += files
command += ['--verbose']
logger.debug('invoking twine: %s', command)
res = subprocess_trimmed_call(command)
return res == 0
# Target Overrides
@ -53,14 +126,19 @@ class WarehouseProvider(PublisherProvider[PackageModel]):
# PublisherProvider Overrides
def get_publishers(self) -> Iterable[Publisher]:
if package.data.publish.pypi.enabled:
def get_publishers(self, package: PackageModel) -> Iterable[Publisher]:
if package.publish.pypi.enabled:
yield WarehousePublisher.from_pypi_credentials(
TargetId('warehouse', 'pypi'),
package.data.publish.pypi.credentials)
for name, config in package.data.publish.warehouse.items():
yield WarehouseProvider(TargetId('warehouse', name), config)
package.publish.pypi.credentials)
for name, config in package.publish.warehouses.items():
display_name = name
if config.repository_url:
display_name = urlparse(config.repository_url).netloc
elif config.repository:
display_name = config.repository
yield WarehousePublisher(TargetId('warehouse', name), config, display_name)
register_publisher_provider(PackageModel, WarehouseProvider)

+ 48
- 0
src/shut/utils/io/sp.py

@ -0,0 +1,48 @@
# -*- coding: utf8 -*-
# Copyright (c) 2020 Niklas Rosenstein
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
import subprocess
from termcolor import colored
def subprocess_trimmed_call(*args, verbose: bool = False, **kwargs) -> int:
"""
Executes a subprocess and only shows stderr in red and indented by 2 spaces if there
was any error output. Returns the status code.
If *verbose* is enabled, the stdout and stderr is not redirected.
"""
proc = subprocess.Popen(
*args,
stdout=None if verbose else subprocess.PIPE,
stderr=None if verbose else subprocess.PIPE,
**kwargs)
stdout, stderr = proc.communicate()
if stderr:
for line in stderr.decode().splitlines():
if not line:
continue
print(f' {colored(line, "red")}')
return proc.wait()

Loading…
Cancel
Save