DEPRECATED -- Rewritten and moved to https://github.com/NiklasRosenstein/shut/. 🌊 Shore is a distribution and release management tool for pure Python packages.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

424 lines
15 KiB

  1. # -*- coding: utf8 -*-
  2. # Copyright (c) 2020 Niklas Rosenstein
  3. #
  4. # Permission is hereby granted, free of charge, to any person obtaining a copy
  5. # of this software and associated documentation files (the "Software"), to
  6. # deal in the Software without restriction, including without limitation the
  7. # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  8. # sell copies of the Software, and to permit persons to whom the Software is
  9. # furnished to do so, subject to the following conditions:
  10. #
  11. # The above copyright notice and this permission notice shall be included in
  12. # all copies or substantial portions of the Software.
  13. #
  14. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  17. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  19. # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  20. # IN THE SOFTWARE.
  21. import collections
  22. import contextlib
  23. import os
  24. import re
  25. import textwrap
  26. from typing import Dict, Iterable, List, Optional, TextIO, Tuple
  27. import nr.fs
  28. from shut.model import PackageModel
  29. from shut.model.package import PackageData, PythonPackageMetadata, InstallConfiguration
  30. from shut.model.requirements import Requirement
  31. from shut.utils.io.virtual import VirtualFiles
  32. from .core import Renderer, register_renderer, VersionRef
  33. GENERATED_FILE_REMARK = '''
  34. # This file was auto-generated by Shut. DO NOT EDIT
  35. # For more information about Shut, check out https://pypi.org/project/shut/
  36. '''.strip() + '\n'
  37. _ReadmeStatus = collections.namedtuple('ReadmeStatus', 'path,runtime_path,outside')
  38. def _normpath(x):
  39. return os.path.normpath(x).replace(os.sep, '/')
  40. def _get_readme_content_type(filename: str) -> str:
  41. return {
  42. 'md': 'text/markdown',
  43. 'rst': 'text/x-rst',
  44. }.get(nr.fs.getsuffix(filename), 'text/plain')
  45. def _split_section(data, begin_marker, end_marker):
  46. start = data.find(begin_marker)
  47. end = data.find(end_marker, start)
  48. if start >= 0 and end >= 0:
  49. prefix = data[:start]
  50. middle = data[start+len(begin_marker):end]
  51. suffix = data[end+len(end_marker)+1:]
  52. return prefix, middle, suffix
  53. return (data, '', '')
  54. @contextlib.contextmanager
  55. def _rewrite_section(fp, data, begin_marker, end_marker):
  56. """
  57. Helper to rewrite a section of a file delimited by *begin_marker* and *end_marker*.
  58. """
  59. prefix, suffix = _split_section(data, begin_marker, end_marker)[::2]
  60. fp.write(prefix)
  61. fp.write(begin_marker + '\n')
  62. yield fp
  63. fp.write(end_marker + '\n')
  64. fp.write(suffix)
  65. class SetuptoolsRenderer(Renderer[PackageModel]):
  66. #: Begin an end section for the MANIFEST.in file.
  67. _BEGIN_SECTION = '# This section is auto-generated by Shut. DO NOT EDIT {'
  68. _END_SECTION = '# }'
  69. #: These variables are used to format entrypoints in the setup.py file. It
  70. #: allows the addition of the Python interpreter version to the entrypoint
  71. #: names.
  72. _ENTRTYPOINT_VARS = {
  73. 'python-major-version': 'sys.version[0]',
  74. 'python-major-minor-version': 'sys.version[:3]'
  75. }
  76. def _render_setup(
  77. self,
  78. fp: TextIO,
  79. package: PackageModel,
  80. ) -> None:
  81. metadata = package.get_python_package_metadata()
  82. install = package.install
  83. data = package.data
  84. # Write the header/imports.
  85. fp.write(GENERATED_FILE_REMARK + '\n')
  86. fp.write('from __future__ import print_function\n')
  87. if install.hooks.before_install or install.hooks.after_install:
  88. fp.write('from setuptools.command.install import install as _install_command\n')
  89. if install.hooks.before_develop or install.hooks.after_develop:
  90. fp.write('from setuptools.command.develop import develop as _develop_command\n')
  91. fp.write(textwrap.dedent('''
  92. import io
  93. import os
  94. import setuptools
  95. import sys
  96. ''').lstrip())
  97. # Write hook overrides.
  98. cmdclass = {}
  99. if install.hooks.any():
  100. fp.write('\ninstall_hooks = [\n')
  101. for hook in package.install_hooks:
  102. fp.write(' ' + json.dumps(hook.normalize().to_json(), sort_keys=True) + ',\n')
  103. fp.write(']\n')
  104. fp.write(textwrap.dedent('''
  105. def _run_hooks(event):
  106. import subprocess, shlex, os
  107. def _shebang(fn):
  108. with open(fn) as fp:
  109. line = fp.readline()
  110. if line.startswith('#'):
  111. return shlex.split(line[1:].strip())
  112. return []
  113. for hook in install_hooks:
  114. if not hook['event'] or hook['event'] == event:
  115. command = [x.replace('$SHORE_INSTALL_HOOK_EVENT', event) for x in hook['command']]
  116. if command[0].endswith('.py') or 'python' in _shebang(command[0]):
  117. command.insert(0, sys.executable)
  118. env = os.environ.copy()
  119. env['SHORE_INSTALL_HOOK_EVENT'] = event
  120. res = subprocess.call(command, env=env)
  121. if res != 0:
  122. raise RuntimeError('command {!r} returned exit code {}'.format(command, res))
  123. '''))
  124. if install.hooks.after_install or install.hooks.before_install:
  125. fp.write(textwrap.dedent('''
  126. class install_command(_install_command):
  127. def run(self):
  128. _run_hooks('install')
  129. super(install_command, self).run()
  130. _run_hooks('post-install')
  131. '''))
  132. cmdclass['install'] = 'install_command'
  133. if install.hooks.before_develop or install.hooks.after_develop:
  134. fp.write(textwrap.dedent('''
  135. class develop_command(_develop_command):
  136. def run(self):
  137. _run_hooks('develop')
  138. super(develop_command, self).run()
  139. _run_hooks('post-develop')
  140. '''))
  141. cmdclass['develop'] = 'develop_command'
  142. readme_file, long_description_expr = self._render_readme_code(fp, package)
  143. # Write the install requirements.
  144. fp.write('\n')
  145. self._render_requirements(fp, 'requirements', data.requirements)
  146. if data.test_requirements:
  147. self._render_requirements(fp, 'test_requirements', data.test_requirements)
  148. tests_require = 'test_requirements'
  149. else:
  150. tests_require = '[]'
  151. if data.extra_requirements:
  152. fp.write('extra_requirements = {}\n')
  153. for key, value in data.extra_requirements.items():
  154. self._render_requirements(fp, 'extras_require[{!r}]'.format(key), value)
  155. extras_require = 'extra_requirements'
  156. else:
  157. extras_require = '{}'
  158. exclude_packages = []
  159. for pkg in data.exclude:
  160. exclude_packages.append(pkg)
  161. exclude_packages.append(pkg + '.*')
  162. if metadata.is_single_module:
  163. packages_args = ' py_modules = [{!r}],'.format(data.get_modulename())
  164. else:
  165. packages_args = ' packages = setuptools.find_packages({src_directory!r}, {exclude_packages!r}),'.format(
  166. src_directory=data.source_directory,
  167. exclude_packages=exclude_packages)
  168. # Find the requirement on Python itself.
  169. python_requirement = data.get_python_requirement()
  170. if python_requirement:
  171. python_requires_expr = repr(python_requirement.version.to_setuptools() if python_requirement else None)
  172. else:
  173. python_requires_expr = 'None'
  174. # TODO: data_files/package_data
  175. # TODO: py.typed must be included in package_data (or include_package_data=True)
  176. data_files = '[]'
  177. # MyPy cannot find PEP-561 compatible packages without zip_safe=False.
  178. # See https://mypy.readthedocs.io/en/latest/installed_packages.html#making-pep-561-compatible-packages
  179. zip_safe = not data.typed
  180. # Write the setup function.
  181. fp.write(textwrap.dedent('''
  182. setuptools.setup(
  183. name = {name!r},
  184. version = {version!r},
  185. author = {author_name!r},
  186. author_email = {author_email!r},
  187. description = {description!r},
  188. long_description = {long_description_expr},
  189. long_description_content_type = {long_description_content_type!r},
  190. url = {url!r},
  191. license = {license!r},
  192. {packages_args}
  193. package_dir = {{'': {src_directory!r}}},
  194. include_package_data = {include_package_data!r},
  195. install_requires = requirements,
  196. extras_require = {extras_require},
  197. tests_require = {tests_require},
  198. python_requires = {python_requires_expr},
  199. data_files = {data_files},
  200. entry_points = {entry_points},
  201. cmdclass = {cmdclass},
  202. keywords = {keywords!r},
  203. classifiers = {classifiers!r},
  204. zip_safe = {zip_safe!r},
  205. ''').rstrip().format(
  206. name=data.name,
  207. version=str(data.version),
  208. packages_args=packages_args,
  209. author_name=data.author.name,
  210. author_email=data.author.email,
  211. url=data.url,
  212. license=data.license,
  213. description=data.description.replace('\n\n', '%%%%').replace('\n', ' ').replace('%%%%', '\n').strip(),
  214. long_description_expr=long_description_expr,
  215. long_description_content_type=_get_readme_content_type(readme_file) if readme_file else None,
  216. extras_require=extras_require,
  217. tests_require=tests_require,
  218. python_requires_expr=python_requires_expr,
  219. src_directory=data.source_directory,
  220. include_package_data=True,#package.package_data != [],
  221. data_files=data_files,
  222. entry_points=self._render_entrypoints(data.entrypoints),
  223. cmdclass = '{' + ', '.join('{!r}: {}'.format(k, v) for k, v in cmdclass.items()) + '}',
  224. keywords = data.keywords,
  225. classifiers = data.classifiers,
  226. zip_safe=zip_safe,
  227. ))
  228. if data.is_universal():
  229. fp.write(textwrap.dedent('''
  230. options = {
  231. 'bdist_wheel': {
  232. 'universal': True,
  233. },
  234. },
  235. )
  236. '''))
  237. else:
  238. fp.write('\n)\n')
  239. def _render_entrypoints(self, entrypoints: Dict[str, List[str]]) -> None:
  240. if not entrypoints:
  241. return '{}'
  242. lines = ['{']
  243. for key, value in entrypoints.items():
  244. lines.append(' {!r}: ['.format(key))
  245. for item in value:
  246. item = repr(item)
  247. args = []
  248. for varname, expr in self._ENTRTYPOINT_VARS.items():
  249. varname = '{{' + varname + '}}'
  250. if varname in item:
  251. item = item.replace(varname, '{' + str(len(args)) + '}')
  252. args.append(expr)
  253. if args:
  254. item += '.format(' + ', '.join(args) + ')'
  255. lines.append(' ' + item.strip() + ',')
  256. lines.append(' ],')
  257. lines[-1] = lines[-1][:-1]
  258. lines.append(' }')
  259. return '\n'.join(lines)
  260. @staticmethod
  261. def _format_reqs(reqs: List[Requirement], level: int = 0) -> List[str]:
  262. indent = ' ' * (level + 1)
  263. reqs = [x for x in reqs if x.package != 'python']
  264. if not reqs:
  265. return '[]'
  266. return '[\n' + ''.join(indent + '{!r},\n'.format(x.to_setuptools()) for x in reqs if x.package != 'python') + ']'
  267. def _render_requirements(self, fp: TextIO, target: str, requirements: List[Requirement]):
  268. fp.write('{} = {}\n'.format(target, self._format_reqs(requirements)))
  269. def _get_readme_status(self, package: PackageModel) -> Optional[_ReadmeStatus]:
  270. """
  271. Returns some information on the readme for a package. The readme can be located outside
  272. of the package directory, but that needs to be handled special in various cases.
  273. """
  274. readme = package.get_readme_file()
  275. if not readme:
  276. return None, 'None'
  277. # Make sure the readme is relative (we need it relative either way).
  278. readme = os.path.relpath(readme, package.get_directory())
  279. # If the readme file is _not_ inside the package directory, the setup.py will
  280. # temporarily copy it. The filename at setup time is thus just the readme's
  281. # base filename.
  282. is_inside = nr.fs.issub(readme)
  283. if is_inside:
  284. readme_relative_path = readme
  285. else:
  286. readme_relative_path = os.path.basename(readme)
  287. return _ReadmeStatus(readme, readme_relative_path, not is_inside)
  288. def _render_readme_code(self, fp: TextIO, package: PackageModel) -> Tuple[Optional[str], Optional[str]]:
  289. """
  290. Renders code for the setup.py file, creating a `long_description` variable. If
  291. a readme file is present or explicitly specified in *package*, that readme file
  292. will be read for the setup.
  293. The readme file may be locatated outside of the packages' directory. In this case,
  294. the setup.py file will temporarily copy it into the package root directory during
  295. the setup.
  296. Returns the Python expression to pass into the `long_description` field of the
  297. #setuptools.setup() call.
  298. """
  299. readme = self._get_readme_status(package)
  300. if not readme:
  301. return None, None
  302. fp.write('\nreadme_file = {!r}\n'.format(readme.runtime_path))
  303. if readme.outside:
  304. # Copy the relative README file if it exists.
  305. fp.write(textwrap.dedent('''
  306. source_readme_file = {!r}
  307. if not os.path.isfile(readme_file) and os.path.isfile(source_readme_file):
  308. import shutil; shutil.copyfile(source_readme_file, readme_file)
  309. import atexit; atexit.register(lambda: os.remove(readme_file))
  310. ''').format(readme.path).lstrip())
  311. # Read the contents of the file into the "long_description" variable.
  312. fp.write(textwrap.dedent('''
  313. if os.path.isfile(readme_file):
  314. with io.open(readme_file, encoding='utf8') as fp:
  315. long_description = fp.read()
  316. else:
  317. print("warning: file \\"{}\\" does not exist.".format(readme_file), file=sys.stderr)
  318. long_description = None
  319. ''').lstrip())
  320. return readme.path, 'long_description'
  321. def _render_manifest_in(self, fp: TextIO, current: TextIO, package: PackageModel) -> None:
  322. """
  323. Modifies a `MANIFEST.in` file in place, ensuring that the automatically generatd content
  324. is up to date (or added if it didn't exist before).
  325. """
  326. files = [
  327. package.filename,
  328. package.get_license_file(),
  329. package.get_py_typed_file(),
  330. ]
  331. readme = self._get_readme_status(package)
  332. if readme:
  333. files.append(readme.runtime_path)
  334. manifest = [
  335. os.path.relpath(f, package.get_directory())
  336. for f in files
  337. if f
  338. ]
  339. markers = (self._BEGIN_SECTION, self._END_SECTION)
  340. with _rewrite_section(fp, current.read() if current else '', *markers):
  341. for entry in manifest:
  342. fp.write('include {}\n'.format(entry))
  343. # Renderer[PackageModel] Overrides
  344. def get_files(self, files: VirtualFiles, package: PackageModel) -> None:
  345. files.add_dynamic('setup.py', self._render_setup, package)
  346. files.add_dynamic('MANIFEST.in', self._render_manifest_in, package, inplace=True)
  347. if package.data.typed:
  348. directory = package.get_python_package_metadata().package_directory
  349. files.add_static(os.path.join(directory, 'py.typed'), '')
  350. def get_version_refs(self, package: PackageModel) -> Iterable[VersionRef]:
  351. def _regex_refs(filename: Optional[str], regex: str) -> Iterable[VersionRef]:
  352. if filename and os.path.isfile(filename):
  353. with open(filename) as fp:
  354. text = fp.read()
  355. match = re.search(regex, text, re.M)
  356. if match:
  357. yield VersionRef(filename, match.start(1), match.end(1), match.group(1))
  358. filename = os.path.join(package.get_directory(), 'setup.py')
  359. yield from _regex_refs(filename, r'^\s*version\s*=\s*[\'"]([^\'"]+)[\'"]')
  360. filename = package.get_python_package_metadata().filename
  361. yield from _regex_refs(filename, r'^__version__\s*=\s*[\'"]([^\'"]+)[\'"]')
  362. register_renderer(PackageModel, SetuptoolsRenderer)