From a613e176e37345e765e20881d9dc22eb1f59dee7 Mon Sep 17 00:00:00 2001 From: Miro Hrončok Date: Aug 20 2020 13:30:49 +0000 Subject: Handle Python Extras in %pyproject_buildrequires on Fedora 33+ There is a slight problem when reporting that a dependency with extra is satisfied. In fact, we only check the "base" dependency. This can lead to a problem when a dependency is wrongly assumed as present and the script proceeds to the "next stage" without restarting -- if the next stage tries to use (import) the missing dependency, the script would crash. However, that might be a very unlikely set of events and if such case ever happens, we'll workaround it or fix it. --- diff --git a/macros.pyproject b/macros.pyproject index aa787f6..d9691b5 100644 --- a/macros.pyproject +++ b/macros.pyproject @@ -79,10 +79,16 @@ echo 'python%{python3_pkgversion}dist(toml)' if [ ! -z "%{?python3_version_nodots}" ] && [ %{python3_version_nodots} -lt 38 ]; then echo 'python%{python3_pkgversion}dist(importlib-metadata)' fi +# Check if we can generate dependencies on Python extras +if [ "%{py_dist_name []}" == "[]" ]; then + extras_flag=%{?!_python_no_extras_requires:--generate-extras} +else + extras_flag= +fi # setuptools assumes no pre-existing dist-info rm -rfv *.dist-info/ >&2 if [ -f %{__python3} ]; then - RPM_TOXENV="%{toxenv}" HOSTNAME="rpmbuild" %{__python3} -I %{_rpmconfigdir}/redhat/pyproject_buildrequires.py --python3_pkgversion %{python3_pkgversion} %{?**} + RPM_TOXENV="%{toxenv}" HOSTNAME="rpmbuild" %{__python3} -I %{_rpmconfigdir}/redhat/pyproject_buildrequires.py $extras_flag --python3_pkgversion %{python3_pkgversion} %{?**} fi } diff --git a/pyproject-rpm-macros.spec b/pyproject-rpm-macros.spec index dfe4ac0..6b8eaef 100644 --- a/pyproject-rpm-macros.spec +++ b/pyproject-rpm-macros.spec @@ -6,7 +6,7 @@ License: MIT # Keep the version at zero and increment only release Version: 0 -Release: 24%{?dist} +Release: 25%{?dist} # Macro files Source001: macros.pyproject @@ -88,6 +88,9 @@ export HOSTNAME="rpmbuild" # to speedup tox in network-less mock, see rhbz#1856 %license LICENSE %changelog +* Fri Aug 14 2020 Miro Hrončok - 0-25 +- Handle Python Extras in %%pyproject_buildrequires on Fedora 33+ + * Tue Aug 11 2020 Miro Hrončok - 0-24 - Allow multiple, comma-separated extras in %%pyproject_buildrequires -x diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py index 29a9c1c..88bfd04 100644 --- a/pyproject_buildrequires.py +++ b/pyproject_buildrequires.py @@ -48,7 +48,7 @@ def hook_call(): class Requirements: """Requirement printer""" def __init__(self, get_installed_version, extras='', - python3_pkgversion='3'): + generate_extras=False, python3_pkgversion='3'): self.get_installed_version = get_installed_version if extras: @@ -58,6 +58,7 @@ class Requirements: self.missing_requirements = False + self.generate_extras = generate_extras self.python3_pkgversion = python3_pkgversion def evaluate_all_environamnets(self, requirement): @@ -86,6 +87,7 @@ class Requirements: return try: + # TODO: check if requirements with extras are satisfied installed = self.get_installed_version(requirement.name) except importlib_metadata.PackageNotFoundError: print_err(f'Requirement not satisfied: {requirement_str}') @@ -93,38 +95,46 @@ class Requirements: if installed and installed in requirement.specifier: print_err(f'Requirement satisfied: {requirement_str}') print_err(f' (installed: {requirement.name} {installed})') + if requirement.extras: + print_err(f' (extras are currently not checked)') else: self.missing_requirements = True - together = [] - for specifier in sorted( - requirement.specifier, - key=lambda s: (s.operator, s.version), - ): - version = canonicalize_version(specifier.version) - if not VERSION_RE.fullmatch(str(specifier.version)): - raise ValueError( - f'Unknown character in version: {specifier.version}. ' - + '(This is probably a bug in pyproject-rpm-macros.)', - ) - if specifier.operator == '!=': - lower = python3dist(name, '<', version, - self.python3_pkgversion) - higher = python3dist(name, '>', f'{version}.0', - self.python3_pkgversion) - together.append( - f'({lower} or {higher})' - ) - else: - together.append(python3dist(name, specifier.operator, version, - self.python3_pkgversion)) - if len(together) == 0: - print(python3dist(name, - python3_pkgversion=self.python3_pkgversion)) - elif len(together) == 1: - print(together[0]) + if self.generate_extras: + extra_names = [f'{name}[{extra}]' for extra in sorted(requirement.extras)] else: - print(f"({' and '.join(together)})") + extra_names = [] + + for name in [name] + extra_names: + together = [] + for specifier in sorted( + requirement.specifier, + key=lambda s: (s.operator, s.version), + ): + version = canonicalize_version(specifier.version) + if not VERSION_RE.fullmatch(str(specifier.version)): + raise ValueError( + f'Unknown character in version: {specifier.version}. ' + + '(This is probably a bug in pyproject-rpm-macros.)', + ) + if specifier.operator == '!=': + lower = python3dist(name, '<', version, + self.python3_pkgversion) + higher = python3dist(name, '>', f'{version}.0', + self.python3_pkgversion) + together.append( + f'({lower} or {higher})' + ) + else: + together.append(python3dist(name, specifier.operator, version, + self.python3_pkgversion)) + if len(together) == 0: + print(python3dist(name, + python3_pkgversion=self.python3_pkgversion)) + elif len(together) == 1: + print(together[0]) + else: + print(f"({' and '.join(together)})") def check(self, *, source=None): """End current pass if any unsatisfied dependencies were output""" @@ -259,7 +269,7 @@ def python3dist(name, op=None, version=None, python3_pkgversion="3"): def generate_requires( *, include_runtime=False, toxenv=None, extras='', get_installed_version=importlib_metadata.version, # for dep injection - python3_pkgversion="3", + generate_extras=False, python3_pkgversion="3", ): """Generate the BuildRequires for the project in the current directory @@ -267,6 +277,7 @@ def generate_requires( """ requirements = Requirements( get_installed_version, extras=extras, + generate_extras=generate_extras, python3_pkgversion=python3_pkgversion ) @@ -306,6 +317,10 @@ def main(argv): '(e.g. -x testing,feature-x) (implies --runtime)', ) parser.add_argument( + '--generate-extras', action='store_true', + help='Generate build requirements on Python Extras', + ) + parser.add_argument( '-p', '--python3_pkgversion', metavar='PYTHON3_PKGVERSION', default="3", help=('Python version for pythonXdist()' 'or pythonX.Ydist() requirements'), @@ -329,6 +344,7 @@ def main(argv): include_runtime=args.runtime, toxenv=args.toxenv, extras=args.extras, + generate_extras=args.generate_extras, python3_pkgversion=args.python3_pkgversion, ) except Exception: diff --git a/pyproject_buildrequires_testcases.yaml b/pyproject_buildrequires_testcases.yaml index 104cd20..0bbee1d 100644 --- a/pyproject_buildrequires_testcases.yaml +++ b/pyproject_buildrequires_testcases.yaml @@ -66,7 +66,8 @@ Bad character in version: requires = ["pkg == 0.$.^.*"] except: ValueError -Build system dependencies in pyproject.toml: +Build system dependencies in pyproject.toml with extras: + generate_extras: true installed: setuptools: 50 wheel: 1 @@ -74,27 +75,50 @@ Build system dependencies in pyproject.toml: [build-system] requires = [ "foo", + "bar[baz] > 5", "ne!=1", "ge>=1.2", "le <= 1.2.3", "lt < 1.2.3.4 ", " gt > 1.2.3.4.5", + "multi[extras1,extras2] == 6.0", "combo >2, <5, != 3.0.0", "invalid!!ignored", "py2 ; python_version < '2.7'", "py3 ; python_version > '3.0'", - "pkg [extra-currently-ignored]", ] expected: | python3dist(foo) + python3dist(bar) > 5 + python3dist(bar[baz]) > 5 (python3dist(ne) < 1 or python3dist(ne) > 1.0) python3dist(ge) >= 1.2 python3dist(le) <= 1.2.3 python3dist(lt) < 1.2.3.4 python3dist(gt) > 1.2.3.4.5 + python3dist(multi) == 6 + python3dist(multi[extras1]) == 6 + python3dist(multi[extras2]) == 6 ((python3dist(combo) < 3 or python3dist(combo) > 3.0) and python3dist(combo) < 5 and python3dist(combo) > 2) python3dist(py3) - python3dist(pkg) + python3dist(setuptools) >= 40.8 + python3dist(wheel) + result: 0 + +Build system dependencies in pyproject.toml without extras: + generate_extras: false + installed: + setuptools: 50 + wheel: 1 + pyproject.toml: | + [build-system] + requires = [ + "bar[baz] > 5", + "multi[extras1,extras2] == 6.0", + ] + expected: | + python3dist(bar) > 5 + python3dist(multi) == 6 python3dist(setuptools) >= 40.8 python3dist(wheel) result: 0 diff --git a/test_pyproject_buildrequires.py b/test_pyproject_buildrequires.py index b3a95c9..46b36d2 100644 --- a/test_pyproject_buildrequires.py +++ b/test_pyproject_buildrequires.py @@ -45,6 +45,7 @@ def test_data(case_name, capsys, tmp_path, monkeypatch): include_runtime=case.get('include_runtime', False), extras=case.get('extras', ''), toxenv=case.get('toxenv', None), + generate_extras=case.get('generate_extras', False), ) except SystemExit as e: assert e.code == case['result'] diff --git a/tests/python-httpbin.spec b/tests/python-httpbin.spec new file mode 100644 index 0000000..9467689 --- /dev/null +++ b/tests/python-httpbin.spec @@ -0,0 +1,66 @@ +Name: python-httpbin +Version: 0.7.0 +Release: 0%{?dist} +Summary: HTTP Request & Response Service, written in Python + Flask +License: MIT +URL: https://github.com/Runscope/httpbin +Source0: %{url}/archive/v%{version}/httpbin-%{version}.tar.gz +BuildArch: noarch + +BuildRequires: python3-devel +BuildRequires: pyproject-rpm-macros + +%description +This package buildrequires a package with extra: raven[flask]. + + +%package -n python3-httpbin +Summary: %{summary} + +%if 0%{?fedora} < 33 && 0%{?rhel} < 9 +# Old Fedoras don't understand Python extras yet +# This package needs raven[flask] +# So we add the transitive dependencies manually: +BuildRequires: %{py3_dist blinker flask} +Requires: %{py3_dist blinker flask} +%endif + +%description -n python3-httpbin +%{summary}. + + +%prep +%autosetup -n httpbin-%{version} + +# brotlipy wrapper is not packaged, httpbin works fine with brotli +sed -i s/brotlipy/brotli/ setup.py + +# update test_httpbin.py to reflect new behavior of werkzeug +sed -i /Content-Length/d test_httpbin.py + + +%generate_buildrequires +%pyproject_buildrequires -t + + +%build +%pyproject_wheel + + +%install +%pyproject_install +%pyproject_save_files httpbin + + +%check +%tox + +# Internal check for our macros +# The runtime dependencies contain raven[flask], we assert we got them. +# The %%tox above also dies without it, but this makes it more explicit +%{python3} -c 'import blinker, flask' # transitive deps + + +%files -n python3-httpbin -f %{pyproject_files} +%doc README* +%license LICENSE* diff --git a/tests/tests.yml b/tests/tests.yml index 951d972..ddf64a3 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -31,6 +31,9 @@ - openqa_client: dir: . run: ./mocktest.sh python-openqa_client + - httpbin: + dir: . + run: ./mocktest.sh python-httpbin - ldap: dir: . run: ./mocktest.sh python-ldap