From 9f3eea2ae5ee09efc68bd532ab7caf46c653e486 Mon Sep 17 00:00:00 2001 From: Miro Hrončok Date: Oct 01 2020 14:33:58 +0000 Subject: [PATCH 1/4] Support the extras configuration option of tox in %pyproject_buildrequires -t Fixes https://bugzilla.redhat.com/show_bug.cgi?id=1877977 --- diff --git a/pyproject-rpm-macros.spec b/pyproject-rpm-macros.spec index ed009d5..3c681e0 100644 --- a/pyproject-rpm-macros.spec +++ b/pyproject-rpm-macros.spec @@ -46,7 +46,7 @@ BuildRequires: (python3dist(importlib-metadata) if python3 < 3.8) BuildRequires: python3dist(pip) BuildRequires: python3dist(setuptools) BuildRequires: python3dist(toml) -BuildRequires: python3dist(tox-current-env) >= 0.0.2 +BuildRequires: python3dist(tox-current-env) >= 0.0.3 BuildRequires: python3dist(wheel) %endif @@ -96,6 +96,8 @@ export HOSTNAME="rpmbuild" # to speedup tox in network-less mock, see rhbz#1856 %changelog * Tue Sep 29 2020 Lumír Balhar - 0-30 - Process RECORD files in %%pyproject_install and remove them +- Support the extras configuration option of tox in %%pyproject_buildrequires -t +- Fixes: rhbz#1877977 * Wed Sep 23 2020 Miro Hrončok - 0-29 - Check the requirements after installing "requires_for_build_wheel" diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py index 34ccbd0..f20c378 100644 --- a/pyproject_buildrequires.py +++ b/pyproject_buildrequires.py @@ -52,17 +52,25 @@ class Requirements: def __init__(self, get_installed_version, extras='', generate_extras=False, python3_pkgversion='3'): self.get_installed_version = get_installed_version + self.extras = set() if extras: - self.marker_envs = [{'extra': e.strip()} for e in extras.split(',')] - else: - self.marker_envs = [{'extra': ''}] + self.add_extras(*extras.split(',')) self.missing_requirements = False self.generate_extras = generate_extras self.python3_pkgversion = python3_pkgversion + def add_extras(self, *extras): + self.extras |= set(e.strip() for e in extras) + + @property + def marker_envs(self): + if self.extras: + return [{'extra': e} for e in sorted(self.extras)] + return [{'extra': ''}] + def evaluate_all_environamnets(self, requirement): for marker_env in self.marker_envs: if requirement.marker.evaluate(environment=marker_env): @@ -232,12 +240,14 @@ def parse_tox_requires_lines(lines): def generate_tox_requirements(toxenv, requirements): - requirements.add('tox-current-env >= 0.0.2', source='tox itself') + requirements.add('tox-current-env >= 0.0.3', source='tox itself') requirements.check(source='tox itself') - with tempfile.NamedTemporaryFile('r') as depfile: + with tempfile.NamedTemporaryFile('r') as deps, tempfile.NamedTemporaryFile('r') as extras: r = subprocess.run( - [sys.executable, '-m', 'tox', '--print-deps-to-file', - depfile.name, '-qre', toxenv], + [sys.executable, '-m', 'tox', + '--print-deps-to', deps.name, + '--print-extras-to', extras.name, + '-qre', toxenv], check=False, encoding='utf-8', stdout=subprocess.PIPE, @@ -247,8 +257,9 @@ def generate_tox_requirements(toxenv, requirements): print_err(r.stdout, end='') r.check_returncode() - deplines = depfile.read().splitlines() + deplines = deps.read().splitlines() packages = parse_tox_requires_lines(deplines) + requirements.add_extras(*extras.read().splitlines()) requirements.extend(packages, source=f'tox --print-deps-only: {toxenv}') diff --git a/pyproject_buildrequires_testcases.yaml b/pyproject_buildrequires_testcases.yaml index 77360c6..3f534a3 100644 --- a/pyproject_buildrequires_testcases.yaml +++ b/pyproject_buildrequires_testcases.yaml @@ -290,7 +290,7 @@ Tox dependencies: setuptools: 50 wheel: 1 tox: 3.5.3 - tox-current-env: 0.0.2 + tox-current-env: 0.0.3 toxenv: py3 setup.py: | from setuptools import setup @@ -312,8 +312,52 @@ Tox dependencies: python3dist(setuptools) >= 40.8 python3dist(wheel) python3dist(wheel) - python3dist(tox-current-env) >= 0.0.2 + python3dist(tox-current-env) >= 0.0.3 python3dist(toxdep1) python3dist(toxdep2) python3dist(inst) result: 0 + +Tox extras: + installed: + setuptools: 50 + wheel: 1 + tox: 3.5.3 + tox-current-env: 0.0.3 + toxenv: py3 + setup.py: | + from setuptools import setup + setup( + name='test', + version='0.1', + install_requires=['inst'], + extras_require={ + 'extra1': ['dep11 > 11', 'dep12'], + 'extra2': ['dep21', 'dep22', 'dep23'], + 'nope': ['nopedep'], + } + ) + tox.ini: | + [tox] + envlist = py36,py37,py38 + [testenv] + deps = + toxdep + extras = + extra2 + extra1 + commands = + true + expected: | + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + python3dist(tox-current-env) >= 0.0.3 + python3dist(toxdep) + python3dist(inst) + python3dist(dep11) > 11 + python3dist(dep12) + python3dist(dep21) + python3dist(dep22) + python3dist(dep23) + result: 0 From 2ecbed74410bf24110e8a3d3bb23b633ab28334f Mon Sep 17 00:00:00 2001 From: Miro Hrončok Date: Oct 01 2020 14:35:07 +0000 Subject: [PATCH 2/4] Support multiple -x options for %pyproject_buildrequires Fixes https://bugzilla.redhat.com/show_bug.cgi?id=1877978 --- diff --git a/pyproject-rpm-macros.spec b/pyproject-rpm-macros.spec index 3c681e0..f6617f1 100644 --- a/pyproject-rpm-macros.spec +++ b/pyproject-rpm-macros.spec @@ -97,7 +97,9 @@ export HOSTNAME="rpmbuild" # to speedup tox in network-less mock, see rhbz#1856 * Tue Sep 29 2020 Lumír Balhar - 0-30 - Process RECORD files in %%pyproject_install and remove them - Support the extras configuration option of tox in %%pyproject_buildrequires -t +- Support multiple -x options for %%pyproject_buildrequires - Fixes: rhbz#1877977 +- Fixes: rhbz#1877978 * Wed Sep 23 2020 Miro Hrončok - 0-29 - Check the requirements after installing "requires_for_build_wheel" diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py index f20c378..36139eb 100644 --- a/pyproject_buildrequires.py +++ b/pyproject_buildrequires.py @@ -49,13 +49,14 @@ def hook_call(): class Requirements: """Requirement printer""" - def __init__(self, get_installed_version, extras='', + def __init__(self, get_installed_version, extras=None, generate_extras=False, python3_pkgversion='3'): self.get_installed_version = get_installed_version self.extras = set() if extras: - self.add_extras(*extras.split(',')) + for extra in extras: + self.add_extras(*extra.split(',')) self.missing_requirements = False @@ -276,7 +277,7 @@ def python3dist(name, op=None, version=None, python3_pkgversion="3"): def generate_requires( - *, include_runtime=False, toxenv=None, extras='', + *, include_runtime=False, toxenv=None, extras=None, get_installed_version=importlib_metadata.version, # for dep injection generate_extras=False, python3_pkgversion="3", ): @@ -285,7 +286,7 @@ def generate_requires( This is the main Python entry point. """ requirements = Requirements( - get_installed_version, extras=extras, + get_installed_version, extras=extras or [], generate_extras=generate_extras, python3_pkgversion=python3_pkgversion ) @@ -321,9 +322,9 @@ def main(argv): '(implies --runtime)'), ) parser.add_argument( - '-x', '--extras', metavar='EXTRAS', default='', + '-x', '--extras', metavar='EXTRAS', action='append', help='comma separated list of "extras" for runtime requirements ' - '(e.g. -x testing,feature-x) (implies --runtime)', + '(e.g. -x testing,feature-x) (implies --runtime, can be repeated)', ) parser.add_argument( '--generate-extras', action='store_true', diff --git a/pyproject_buildrequires_testcases.yaml b/pyproject_buildrequires_testcases.yaml index 3f534a3..0f380f4 100644 --- a/pyproject_buildrequires_testcases.yaml +++ b/pyproject_buildrequires_testcases.yaml @@ -238,7 +238,8 @@ Run dependencies with extras (selected): wheel: 1 pyyaml: 1 include_runtime: true - extras: testing + extras: + - testing setup.py: *pytest_setup_py expected: | python3dist(setuptools) >= 40.8 @@ -264,7 +265,9 @@ Run dependencies with multiple extras: wheel: 1 pyyaml: 1 include_runtime: true - extras: testing,more-testing, even-more-testing , cool-feature + extras: + - testing,more-testing + - even-more-testing , cool-feature setup.py: | from setuptools import setup setup( diff --git a/test_pyproject_buildrequires.py b/test_pyproject_buildrequires.py index 46b36d2..cc7ea4e 100644 --- a/test_pyproject_buildrequires.py +++ b/test_pyproject_buildrequires.py @@ -43,7 +43,7 @@ def test_data(case_name, capsys, tmp_path, monkeypatch): generate_requires( get_installed_version=get_installed_version, include_runtime=case.get('include_runtime', False), - extras=case.get('extras', ''), + extras=case.get('extras', []), toxenv=case.get('toxenv', None), generate_extras=case.get('generate_extras', False), ) From 6f6a7636d4340d7233654f2bd9cfbdb9024356fa Mon Sep 17 00:00:00 2001 From: Miro Hrončok Date: Oct 01 2020 14:35:08 +0000 Subject: [PATCH 3/4] Skip empty lines from tox deps An empty line happens when there are not deps: Handling from tox --print-deps-only: WARNING: Skipping invalid requirement: Parse error at "''": Expected W:(abcd...) --- diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py index 36139eb..a34c9e2 100644 --- a/pyproject_buildrequires.py +++ b/pyproject_buildrequires.py @@ -235,7 +235,7 @@ def parse_tox_requires_lines(lines): f'WARNING: Skipping dependency line: {line}\n' + f' tox deps options other than -r are not supported (yet).', ) - else: + elif line: packages.append(line) return packages From b38c0ff5949c24eff42b042d80d34b7c10a68011 Mon Sep 17 00:00:00 2001 From: Miro Hrončok Date: Oct 01 2020 14:35:08 +0000 Subject: [PATCH 4/4] CI: Add dns-lexicon to tests new things This tests: - https://bugzilla.redhat.com/show_bug.cgi?id=1877977 - https://bugzilla.redhat.com/show_bug.cgi?id=1877978 Unfortunately it is impossible to use this spec to test things on Fedora 31/32, so we skip the tests there. We keep python-requests.spec for now to test Fedora 31/32, but it can be removed after Fedora 32 EOL. --- diff --git a/tests/python-dns-lexicon.spec b/tests/python-dns-lexicon.spec new file mode 100644 index 0000000..801a904 --- /dev/null +++ b/tests/python-dns-lexicon.spec @@ -0,0 +1,73 @@ +Name: python-dns-lexicon +Version: 3.4.0 +Release: 0%{?dist} +Summary: Manipulate DNS records on various DNS providers in a standardized/agnostic way +License: MIT +URL: https://github.com/AnalogJ/lexicon +Source0: %{url}/archive/v%{version}/lexicon-%{version}.tar.gz +BuildArch: noarch + +BuildRequires: pyproject-rpm-macros +BuildRequires: python3-devel + +%description +This package has extras specified in tox configuration, +we test that the extras are installed when -e is used. +This package also uses a custom toxenv and creates several extras subpackages. + + +%package -n python3-dns-lexicon +Summary: %{summary} + +%description -n python3-dns-lexicon +... + + +%pyproject_extras_subpackage -n python3-dns-lexicon plesk route53 + + +%prep +%autosetup -n lexicon-%{version} +# The tox configuration lists a [dev] extra, but that installs nothing (is missing). +# The test requirements are only specified via poetry.dev-dependencies. +# Here we amend the data a bit so we can test more things, adding the tests deps to the dev extra: +sed -i \ +'s/\[tool.poetry.extras\]/'\ +'pytest = {version = ">3", optional = true}\n'\ +'vcrpy = {version = ">1", optional = true}\n\n'\ +'[tool.poetry.extras]\n'\ +'dev = ["pytest", "vcrpy"]/' pyproject.toml + + +%generate_buildrequires +%if 0%{?fedora} >= 33 || 0%{?rhel} >= 9 +# We use the "light" toxenv because the default one installs the [full] extra and we don't have all the deps. +# Note that [full] contains [plesk] and [route53] but we specify them manually instead: +%pyproject_buildrequires -e light -x plesk -x route53 +%else +# older Fedora don't have the required runtime dependencies, so we don't test it there +%pyproject_buildrequires +%endif + + +%build +%pyproject_wheel + + +%install +%pyproject_install +%pyproject_save_files lexicon + + +%if 0%{?fedora} >= 33 || 0%{?rhel} >= 9 +%check +# we cannot use %%tox here, because the configured commands call poetry directly :/ +# we use %%pytest instead, running a subset of tests not o waste CI time +%pytest -k "test_route53 or test_plesk" +%endif + + +%files -n python3-dns-lexicon -f %{pyproject_files} +%license LICENSE +%doc README.rst +%{_bindir}/lexicon diff --git a/tests/python-requests.spec b/tests/python-requests.spec index 998668c..d9f9d73 100644 --- a/tests/python-requests.spec +++ b/tests/python-requests.spec @@ -14,6 +14,8 @@ BuildRequires: pyproject-rpm-macros %description This package uses multiple extras in %%pyproject_extras_subpkg and in %%pyproject_buildrequires. +This test is mostly obsoleted by python-dns-lexicon.spec on Fedora 33+, +but we keep it around until Fedora 32 EOL. %package -n python3-requests diff --git a/tests/tests.yml b/tests/tests.yml index 94ad7fb..65ed9fc 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -64,6 +64,9 @@ - printrun: dir: . run: ./mocktest.sh printrun + - dns_lexicon: + dir: . + run: ./mocktest.sh python-dns-lexicon required_packages: - mock - rpmdevtools