From 204b801da2af09c62240512d8207639da0e1f18d Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Jul 18 2019 06:38:42 +0000 Subject: [PATCH 1/8] Add --without tests bcond --- diff --git a/pyproject-rpm-macros.spec b/pyproject-rpm-macros.spec index 7ca1813..09433cb 100644 --- a/pyproject-rpm-macros.spec +++ b/pyproject-rpm-macros.spec @@ -2,6 +2,8 @@ Name: pyproject-rpm-macros Summary: RPM macros for PEP 517 Python packages License: MIT +%bcond_without tests + # Keep the version at zero and increment only release Version: 0 Release: 3%{?dist} @@ -26,12 +28,13 @@ BuildArch: noarch Requires: python3-pip >= 19 Requires: python3-devel -# Test dependencies +%if %{with tests} BuildRequires: python3dist(pytest) BuildRequires: python3dist(pyyaml) BuildRequires: python3dist(packaging) BuildRequires: python3dist(pytoml) BuildRequires: python3dist(pip) +%endif %description @@ -56,8 +59,10 @@ mkdir -p %{buildroot}%{_rpmconfigdir}/redhat install -m 644 macros.pyproject %{buildroot}%{_rpmmacrodir}/ install -m 644 pyproject_buildrequires.py %{buildroot}%{_rpmconfigdir}/redhat/ +%if %{with tests} %check %{__python3} -m pytest -vv +%endif %files From bc156c44600a3d5433cdb1b1ebe2e85f934db706 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Jul 18 2019 06:59:44 +0000 Subject: [PATCH 2/8] Generate run-time requirements for tests --- diff --git a/macros.pyproject b/macros.pyproject index 6679b8e..9ddd652 100644 --- a/macros.pyproject +++ b/macros.pyproject @@ -17,12 +17,14 @@ if [ -d %{buildroot}%{python3_sitearch} ]; then fi } -%pyproject_buildrequires() %{expand:\\\ +%pyproject_buildrequires(r) %{expand:\\\ echo 'python3-devel' echo 'python3dist(packaging)' echo 'python3dist(pip) >= 19' echo 'python3dist(pytoml)' +# setuptools assumes no pre-existing dist-info +rm -rfv *.dist-info/ if [ -f %{__python3} ]; then - %{__python3} -I %{_rpmconfigdir}/redhat/pyproject_buildrequires.py %{?*} + %{__python3} -I %{_rpmconfigdir}/redhat/pyproject_buildrequires.py %{?**} fi } diff --git a/pyproject-rpm-macros.spec b/pyproject-rpm-macros.spec index 09433cb..fd2b805 100644 --- a/pyproject-rpm-macros.spec +++ b/pyproject-rpm-macros.spec @@ -34,6 +34,8 @@ BuildRequires: python3dist(pyyaml) BuildRequires: python3dist(packaging) BuildRequires: python3dist(pytoml) BuildRequires: python3dist(pip) +BuildRequires: python3dist(setuptools) +BuildRequires: python3dist(wheel) %endif diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py index 45155c3..d4e3879 100644 --- a/pyproject_buildrequires.py +++ b/pyproject_buildrequires.py @@ -8,6 +8,7 @@ from io import StringIO import subprocess import pathlib import re +import email.parser print_err = functools.partial(print, file=sys.stderr) @@ -83,7 +84,6 @@ class Requirements: key=lambda s: (s.operator, s.version), ): version = canonicalize_version(specifier.version) - print_err(version) if not VERSION_RE.fullmatch(str(specifier.version)): raise ValueError( f'Unknown character in version: {specifier.version}. ' @@ -154,6 +154,22 @@ def generate_build_requirements(backend, requirements): requirements.extend(new_reqs, source='get_requires_for_build_wheel') +def generate_run_requirements(backend, requirements): + prepare_metadata = getattr(backend, "prepare_metadata_for_build_wheel", None) + if not prepare_metadata: + raise ValueError( + 'build backend cannot provide build metadata ' + + '(incl. runtime requirements) before buld' + ) + with hook_call(): + dir_basename = prepare_metadata('.') + with open(dir_basename + '/METADATA') as f: + message = email.parser.Parser().parse(f, headersonly=True) + for key in 'Requires', 'Requires-Dist': + requires = message.get_all(key, ()) + requirements.extend(requires, source=f'wheel metadata: {key}') + + def python3dist(name, op=None, version=None): if op is None: if version is not None: @@ -163,12 +179,14 @@ def python3dist(name, op=None, version=None): return f'python3dist({name}) {op} {version}' -def generate_requires(freeze_output): +def generate_requires(freeze_output, *, include_runtime=False, toxenv=None): requirements = Requirements(freeze_output) try: backend = get_backend(requirements) generate_build_requirements(backend, requirements) + if include_runtime: + generate_run_requirements(backend, requirements) except EndPass: return @@ -178,11 +196,11 @@ def main(argv): description='Generate BuildRequires for a Python project.' ) parser.add_argument( - '--runtime', action='store_true', + '-r', '--runtime', action='store_true', help='Generate run-time requirements (not implemented)', ) parser.add_argument( - '--toxenv', metavar='TOXENVS', + '-t', '--toxenv', metavar='TOXENVS', help='generate test tequirements from tox environment ' + '(not implemented; implies --runtime)', ) @@ -190,8 +208,7 @@ def main(argv): args = parser.parse_args(argv) if args.toxenv: args.runtime = True - if args.runtime: - print_err('--runtime is not implemented') + print_err('--toxenv is not implemented') exit(1) freeze_output = subprocess.run( @@ -202,7 +219,7 @@ def main(argv): ).stdout try: - generate_requires(freeze_output) + generate_requires(freeze_output, include_runtime=args.runtime) except Exception as e: # Log the traceback explicitly (it's useful debug info) traceback.print_exc() diff --git a/test_pyproject_buildrequires.py b/test_pyproject_buildrequires.py index edc17c5..225d1c3 100644 --- a/test_pyproject_buildrequires.py +++ b/test_pyproject_buildrequires.py @@ -28,10 +28,13 @@ def test_data(case_name, capsys, tmp_path, monkeypatch): try: generate_requires( case['freeze_output'], + include_runtime=case.get('include_runtime', False), ) except SystemExit as e: assert e.code == case['result'] except Exception as e: + if 'except' not in case: + raise assert type(e).__name__ == case['except'] else: assert 0 == case['result'] diff --git a/testcases.yaml b/testcases.yaml index 6b6d5fc..8291074 100644 --- a/testcases.yaml +++ b/testcases.yaml @@ -99,7 +99,7 @@ Build system dependencies in pyproject.toml: python3dist(wheel) result: 0 -Default build system, dependencies in setup.py: +Default build system, build dependencies in setup.py: freeze_output: | setuptools==50 wheel==1 @@ -109,6 +109,7 @@ Default build system, dependencies in setup.py: name='test', version='0.1', setup_requires=['foo', 'bar!=2'], + install_requires=['inst'], ) expected: | python3dist(setuptools) >= 40.8 @@ -117,3 +118,26 @@ Default build system, dependencies in setup.py: python3dist(foo) (python3dist(bar) < 2 or python3dist(bar) > 2.0) result: 0 + +Default build system, run dependencies in setup.py: + freeze_output: | + setuptools==50 + wheel==1 + pyyaml==1 + include_runtime: true + setup.py: | + from setuptools import setup + setup( + name='test', + version='0.1', + setup_requires=['pyyaml'], # nb. setuptools will try to install this + install_requires=['inst > 1', 'inst2 < 3'], + ) + expected: | + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + python3dist(pyyaml) + python3dist(inst) > 1 + python3dist(inst2) < 3 + result: 0 diff --git a/tests/python-entrypoints.spec b/tests/python-entrypoints.spec index ff52bb7..e99a433 100644 --- a/tests/python-entrypoints.spec +++ b/tests/python-entrypoints.spec @@ -27,7 +27,8 @@ Discover and load entry points from installed packages. %generate_buildrequires -%pyproject_buildrequires +rm -rfv *.dist-info/ +%pyproject_buildrequires -r %build diff --git a/tests/python-pytest.spec b/tests/python-pytest.spec index beb9b8e..e427ccf 100644 --- a/tests/python-pytest.spec +++ b/tests/python-pytest.spec @@ -27,7 +27,7 @@ py.test provides simple, yet powerful testing for Python. %generate_buildrequires -%pyproject_buildrequires +%pyproject_buildrequires -r %build From c7e7d1e003cfcc0e75ca16ec52771fe02f9b6773 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Jul 18 2019 07:10:09 +0000 Subject: [PATCH 3/8] Ignore extra requirements (rather than fail) --- diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py index d4e3879..b253ad2 100644 --- a/pyproject_buildrequires.py +++ b/pyproject_buildrequires.py @@ -67,7 +67,9 @@ class Requirements: return name = canonicalize_name(requirement.name) - if requirement.marker is not None and not requirement.marker.evaluate(): + if (requirement.marker is not None + and not requirement.marker.evaluate(environment={'extra': ''}) + ): print_err(f'Ignoring alien requirement:', requirement_str) return diff --git a/testcases.yaml b/testcases.yaml index 8291074..77cc782 100644 --- a/testcases.yaml +++ b/testcases.yaml @@ -141,3 +141,60 @@ Default build system, run dependencies in setup.py: python3dist(inst) > 1 python3dist(inst2) < 3 result: 0 + +Run dependencies with extras: + freeze_output: | + setuptools==50 + wheel==1 + pyyaml==1 + include_runtime: true + setup.py: | + # slightly abriged copy of pytest's setup.py + from setuptools import setup + + INSTALL_REQUIRES = [ + "py>=1.5.0", + "six>=1.10.0", + "setuptools", + "attrs>=17.4.0", + 'more-itertools>=4.0.0,<6.0.0;python_version<="2.7"', + 'more-itertools>=4.0.0;python_version>"2.7"', + "atomicwrites>=1.0", + 'funcsigs>=1.0;python_version<"3.0"', + 'pathlib2>=2.2.0;python_version<"3.6"', + 'colorama;sys_platform=="win32"', + "pluggy>=0.11", + ] + + def main(): + setup( + setup_requires=["setuptools>=40.0"], + # fmt: off + extras_require={ + "testing": [ + "argcomplete", + "hypothesis>=3.56", + "nose", + "requests", + "mock;python_version=='2.7'", + ], + }, + # fmt: on + install_requires=INSTALL_REQUIRES, + ) + + if __name__ == "__main__": + main() + expected: | + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + python3dist(setuptools) >= 40 + python3dist(py) >= 1.5 + python3dist(six) >= 1.10 + python3dist(setuptools) + python3dist(attrs) >= 17.4 + python3dist(atomicwrites) >= 1 + python3dist(pluggy) >= 0.11 + python3dist(more-itertools) >= 4 + result: 0 From 3600e9832dff3927dd62bcfe60593d3f43ba77a4 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Jul 18 2019 07:24:02 +0000 Subject: [PATCH 4/8] Adjust help and error messages --- diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py index b253ad2..a6ebc57 100644 --- a/pyproject_buildrequires.py +++ b/pyproject_buildrequires.py @@ -199,7 +199,7 @@ def main(argv): ) parser.add_argument( '-r', '--runtime', action='store_true', - help='Generate run-time requirements (not implemented)', + help='Generate run-time requirements', ) parser.add_argument( '-t', '--toxenv', metavar='TOXENVS', @@ -210,7 +210,10 @@ def main(argv): args = parser.parse_args(argv) if args.toxenv: args.runtime = True - print_err('--toxenv is not implemented') + print_err('-t (--toxenv) is not implemented') + exit(1) + if args.extras and not args.runtime: + print_err('-x (--extras) are only useful with -r (--runtime)') exit(1) freeze_output = subprocess.run( From d6e6bb7dfbfdafe111e5442abc3c785c856ec1cf Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Jul 18 2019 08:50:13 +0000 Subject: [PATCH 5/8] Allow specifying extras for build dependencies --- diff --git a/macros.pyproject b/macros.pyproject index 9ddd652..6e558b2 100644 --- a/macros.pyproject +++ b/macros.pyproject @@ -17,7 +17,7 @@ if [ -d %{buildroot}%{python3_sitearch} ]; then fi } -%pyproject_buildrequires(r) %{expand:\\\ +%pyproject_buildrequires(rx:) %{expand:\\\ echo 'python3-devel' echo 'python3dist(packaging)' echo 'python3dist(pip) >= 19' diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py index a6ebc57..2a3fa63 100644 --- a/pyproject_buildrequires.py +++ b/pyproject_buildrequires.py @@ -42,7 +42,7 @@ def hook_call(): class Requirements: """Requirement printer""" - def __init__(self, freeze_output): + def __init__(self, freeze_output, extras=''): self.installed_packages = {} for line in freeze_output.splitlines(): line = line.strip() @@ -51,6 +51,8 @@ class Requirements: name, version = line.split('==') self.installed_packages[name.strip()] = Version(version) + self.marker_env = {'extra': extras} + self.missing_requirements = False def add(self, requirement_str, *, source=None): @@ -68,7 +70,7 @@ class Requirements: name = canonicalize_name(requirement.name) if (requirement.marker is not None - and not requirement.marker.evaluate(environment={'extra': ''}) + and not requirement.marker.evaluate(environment=self.marker_env) ): print_err(f'Ignoring alien requirement:', requirement_str) return @@ -181,8 +183,10 @@ def python3dist(name, op=None, version=None): return f'python3dist({name}) {op} {version}' -def generate_requires(freeze_output, *, include_runtime=False, toxenv=None): - requirements = Requirements(freeze_output) +def generate_requires( + freeze_output, *, include_runtime=False, toxenv=None, extras='', +): + requirements = Requirements(freeze_output, extras=extras) try: backend = get_backend(requirements) @@ -206,6 +210,11 @@ def main(argv): help='generate test tequirements from tox environment ' + '(not implemented; implies --runtime)', ) + parser.add_argument( + '-x', '--extras', metavar='EXTRAS', default='', + help='comma separated list of "extras" for runtime requirements ' + + '(e.g. -x testing,feature-x)', + ) args = parser.parse_args(argv) if args.toxenv: @@ -224,7 +233,11 @@ def main(argv): ).stdout try: - generate_requires(freeze_output, include_runtime=args.runtime) + generate_requires( + freeze_output, + include_runtime=args.runtime, + extras=args.extras, + ) except Exception as e: # Log the traceback explicitly (it's useful debug info) traceback.print_exc() diff --git a/test_pyproject_buildrequires.py b/test_pyproject_buildrequires.py index 225d1c3..b6462b4 100644 --- a/test_pyproject_buildrequires.py +++ b/test_pyproject_buildrequires.py @@ -29,6 +29,7 @@ def test_data(case_name, capsys, tmp_path, monkeypatch): generate_requires( case['freeze_output'], include_runtime=case.get('include_runtime', False), + extras=case.get('extras', ''), ) except SystemExit as e: assert e.code == case['result'] diff --git a/testcases.yaml b/testcases.yaml index 77cc782..956e78e 100644 --- a/testcases.yaml +++ b/testcases.yaml @@ -142,13 +142,13 @@ Default build system, run dependencies in setup.py: python3dist(inst2) < 3 result: 0 -Run dependencies with extras: +Run dependencies with extras (not selected): freeze_output: | setuptools==50 wheel==1 pyyaml==1 include_runtime: true - setup.py: | + setup.py: &pytest_setup_py | # slightly abriged copy of pytest's setup.py from setuptools import setup @@ -198,3 +198,29 @@ Run dependencies with extras: python3dist(pluggy) >= 0.11 python3dist(more-itertools) >= 4 result: 0 + +Run dependencies with extras (selected): + freeze_output: | + setuptools==50 + wheel==1 + pyyaml==1 + include_runtime: true + extras: testing + setup.py: *pytest_setup_py + expected: | + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + python3dist(setuptools) >= 40 + python3dist(py) >= 1.5 + python3dist(six) >= 1.10 + python3dist(setuptools) + python3dist(attrs) >= 17.4 + python3dist(atomicwrites) >= 1 + python3dist(pluggy) >= 0.11 + python3dist(more-itertools) >= 4 + python3dist(argcomplete) + python3dist(hypothesis) >= 3.56 + python3dist(nose) + python3dist(requests) + result: 0 diff --git a/tests/python-pytest.spec b/tests/python-pytest.spec index e427ccf..d6ad7ab 100644 --- a/tests/python-pytest.spec +++ b/tests/python-pytest.spec @@ -27,7 +27,7 @@ py.test provides simple, yet powerful testing for Python. %generate_buildrequires -%pyproject_buildrequires -r +%pyproject_buildrequires -r -x testing %build @@ -37,6 +37,12 @@ py.test provides simple, yet powerful testing for Python. %install %pyproject_install +%check +# Only run one test (which uses a test-only dependency, hypothesis). +# (Unfortunately, some other tests still fail.) +export PYTHONPATH=%{buildroot}%{python3_sitelib} +%{__python3} -m pytest -k metafunc + %files -n python3-%{pypi_name} %doc README.rst From aca2f6a0c466f19138e2b62565ef5c35db189cef Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Jul 18 2019 09:00:23 +0000 Subject: [PATCH 6/8] Mark multiple extras as not supported yet --- diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py index 2a3fa63..f5bdcc6 100644 --- a/pyproject_buildrequires.py +++ b/pyproject_buildrequires.py @@ -212,8 +212,10 @@ def main(argv): ) parser.add_argument( '-x', '--extras', metavar='EXTRAS', default='', - help='comma separated list of "extras" for runtime requirements ' - + '(e.g. -x testing,feature-x)', + help='extra for runtime requirements (e.g. -x testing)', + # XXX: a comma-separated list should be possible here + #help='comma separated list of "extras" for runtime requirements ' + # + '(e.g. -x testing,feature-x)', ) args = parser.parse_args(argv) diff --git a/test_pyproject_buildrequires.py b/test_pyproject_buildrequires.py index b6462b4..9a7c983 100644 --- a/test_pyproject_buildrequires.py +++ b/test_pyproject_buildrequires.py @@ -19,6 +19,9 @@ def test_data(case_name, capsys, tmp_path, monkeypatch): cwd.mkdir() monkeypatch.chdir(cwd) + if case.get('xfail'): + pytest.xfail(case.get('xfail')) + if 'pyproject.toml' in case: cwd.joinpath('pyproject.toml').write_text(case['pyproject.toml']) diff --git a/testcases.yaml b/testcases.yaml index 956e78e..2ca9136 100644 --- a/testcases.yaml +++ b/testcases.yaml @@ -224,3 +224,31 @@ Run dependencies with extras (selected): python3dist(nose) python3dist(requests) result: 0 + +Run dependencies with multiple extras: + xfail: requirement.marker.evaluate seems to not support multiple extras + freeze_output: | + setuptools==50 + wheel==1 + pyyaml==1 + include_runtime: true + extras: testing,more-testing, even-more-testing , cool-feature + setup.py: | + from setuptools import setup + setup( + extras_require={ + 'testing': ['dep1'], + 'more-testing': ['dep2'], + 'even-more-testing': ['dep3'], + 'cool-feature': ['dep4'], + }, + ) + expected: | + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + python3dist(dep1) + python3dist(dep2) + python3dist(dep3) + python3dist(dep4) + result: 0 From 03316d81e74239be49463f9e95a9a02e4e565b79 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Jul 18 2019 09:24:18 +0000 Subject: [PATCH 7/8] Document run-time deps --- diff --git a/README.md b/README.md index 3e14c00..148b1d8 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,28 @@ And install the wheel in `%install` with `%pyproject_install`: %pyproject_install +Adding run-time and test-time dependencies +------------------------------------------ + +To run tests in the `%check` section, the package's runtime dependencies +often need to also be included as build requirements. +If the project's build system supports the [`prepare-metadata-for-build-wheel` +hook](https://www.python.org/dev/peps/pep-0517/#prepare-metadata-for-build-wheel), +this can be done using the `-r` flag: + + %generate_buildrequires + %pyproject_buildrequires -r + +For projects that specify test requirements using an [`extra` +provide](https://packaging.python.org/specifications/core-metadata/#provides-extra-multiple-use), +these can be added using the `-x` flag. +For example, if upstream suggests installing test dependencies with +`pip install mypackage[testing]`, the test deps would be generated by: + + %generate_buildrequires + %pyproject_buildrequires -r -x testing + + Limitations ----------- @@ -49,6 +71,7 @@ Extras are currently ignored. Some valid Python version specifiers are not supported. +The `-x` flag does not yet support multiple (comma-separated) extras. [PEP 517]: https://www.python.org/dev/peps/pep-0517/ [PEP 518]: https://www.python.org/dev/peps/pep-0518/ From a1bd01ac866c7ffe04aebc87f556a63a93d100c1 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Jul 18 2019 09:54:04 +0000 Subject: [PATCH 8/8] Clean up python-entrypoints.spec --- diff --git a/tests/python-entrypoints.spec b/tests/python-entrypoints.spec index e99a433..ff52bb7 100644 --- a/tests/python-entrypoints.spec +++ b/tests/python-entrypoints.spec @@ -27,8 +27,7 @@ Discover and load entry points from installed packages. %generate_buildrequires -rm -rfv *.dist-info/ -%pyproject_buildrequires -r +%pyproject_buildrequires %build