From 11da7b793a392f5845fa2d25708a45f97714db2e Mon Sep 17 00:00:00 2001 From: Miro Hrončok Date: Aug 27 2019 12:54:28 +0000 Subject: [PATCH 1/8] Documentation typo --- diff --git a/README.md b/README.md index c473c7e..9b4d416 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ By using the `-e` flag, you can use a different tox environment(s): %check %tox %if %{with integration_test} - %tox -e % {default_toxenv}-integration + %tox -e %{default_toxenv}-integration %endif If you wish to provide custom `tox` flags or arguments, add them after `--`: From f3157b52379136d364070b6a77e4f7f25b48782e Mon Sep 17 00:00:00 2001 From: Miro Hrončok Date: Sep 03 2019 16:16:57 +0000 Subject: [PATCH 2/8] Add test that uses poetry --- diff --git a/tests/python-clikit.spec b/tests/python-clikit.spec new file mode 100644 index 0000000..958893b --- /dev/null +++ b/tests/python-clikit.spec @@ -0,0 +1,46 @@ +%global pypi_name clikit +Name: python-%{pypi_name} +Version: 0.3.1 +Release: 1%{?dist} +Summary: Builds beautiful and testable command line interfaces + +License: MIT +URL: https://github.com/sdispater/clikit +Source0: %{pypi_source} + +BuildArch: noarch +BuildRequires: pyproject-rpm-macros + +%description +%{summary}. + + +%package -n python3-%{pypi_name} +Summary: %{summary} +%{?python_provide:%python_provide python3-%{pypi_name}} + +%description -n python3-%{pypi_name} +%{summary}. + + +%prep +%autosetup -p1 -n %{pypi_name}-%{version} + + +%generate_buildrequires +%pyproject_buildrequires + + +%build +%pyproject_wheel + + +%install +%pyproject_install + + +%files -n python3-%{pypi_name} +%doc README.md +%license LICENSE +%{python3_sitelib}/%{pypi_name}/ +%{python3_sitelib}/%{pypi_name}-%{version}.dist-info/ diff --git a/tests/tests.yml b/tests/tests.yml index cc2fe95..a8955e0 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -22,6 +22,9 @@ - pluggy: dir: . run: ./mocktest.sh python-pluggy + - clikit: + dir: . + run: ./mocktest.sh python-clikit required_packages: - mock - rpmdevtools From 23901d999aa19f7caea60e2ed66adf1ba2fd192f Mon Sep 17 00:00:00 2001 From: Miro Hrončok Date: Sep 12 2019 10:58:17 +0000 Subject: [PATCH 3/8] Fedora CI: Update pluggy to avoid a missing dependency on importlib_metadata Unfortunately, it no longer has custom toxenv --- diff --git a/tests/python-pluggy.spec b/tests/python-pluggy.spec index 449c23d..df5dcd0 100644 --- a/tests/python-pluggy.spec +++ b/tests/python-pluggy.spec @@ -1,6 +1,6 @@ %global pypi_name pluggy Name: python-%{pypi_name} -Version: 0.12.0 +Version: 0.13.0 Release: 1%{?dist} Summary: The plugin manager stripped of pytest specific details @@ -28,7 +28,7 @@ Summary: %{summary} %generate_buildrequires -%pyproject_buildrequires -e %{toxenv}-pytestrelease +%pyproject_buildrequires -t %build From d262d909f56484c8339a19f13b4b946ea3c692a4 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sep 18 2019 14:16:17 +0000 Subject: [PATCH 4/8] Use importlib_metadata rather than pip freeze --- diff --git a/macros.pyproject b/macros.pyproject index 9e55089..1203621 100644 --- a/macros.pyproject +++ b/macros.pyproject @@ -24,7 +24,6 @@ fi %{-e:%{expand:%global toxenv %{-e*}}} echo 'python3-devel' echo 'python3dist(packaging)' -echo 'python3dist(pip) >= 19' echo 'python3dist(pytoml)' # setuptools assumes no pre-existing dist-info rm -rfv *.dist-info/ diff --git a/pyproject-rpm-macros.spec b/pyproject-rpm-macros.spec index 4fb50d7..88fb2f0 100644 --- a/pyproject-rpm-macros.spec +++ b/pyproject-rpm-macros.spec @@ -21,17 +21,28 @@ URL: https://src.fedoraproject.org/rpms/pyproject-rpm-macros BuildArch: noarch -# We keep them here for now to avoid one loop of %%generate_buildrequires -# And to allow the other macros without %%pyproject_buildrequires (e.g. on Fedora 30) -# But those are also always in the output of %%generate_buildrequires -# in order to be removable in the future Requires: python3-pip >= 19 Requires: python3-devel +# We keep these here for now to avoid one loop of %%generate_buildrequires +# But those are also always in the output of %%generate_buildrequires +# in order to be removable in the future +Requires: python3dist(packaging) +Requires: python3dist(pytoml) + +# This is not output from %%generate_buildrequires to work around: +# https://github.com/rpm-software-management/mock/issues/336 +Requires: (python3dist(importlib-metadata) if python3 < 3.8) + %if %{with tests} BuildRequires: python3dist(pytest) BuildRequires: python3dist(pyyaml) BuildRequires: python3dist(packaging) +%if 0%{fedora} < 32 +# The %%if should not be needed, it works around: +# https://github.com/rpm-software-management/mock/issues/336 +BuildRequires: (python3dist(importlib-metadata) if python3 < 3.8) +%endif BuildRequires: python3dist(pytoml) BuildRequires: python3dist(pip) BuildRequires: python3dist(setuptools) diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py index e9eddce..cbe5be9 100644 --- a/pyproject_buildrequires.py +++ b/pyproject_buildrequires.py @@ -26,7 +26,10 @@ try: from packaging.requirements import Requirement, InvalidRequirement from packaging.version import Version from packaging.utils import canonicalize_name, canonicalize_version - import pip + try: + import importlib.metadata as importlib_metadata + except ImportError: + import importlib_metadata except ImportError as e: print_err('Import error:', e) # already echoed by the %pyproject_buildrequires macro @@ -44,14 +47,8 @@ def hook_call(): class Requirements: """Requirement printer""" - def __init__(self, freeze_output, extras=''): - self.installed_packages = {} - for line in freeze_output.splitlines(): - line = line.strip() - if line.startswith('#'): - continue - name, version = line.split('==') - self.installed_packages[name.strip()] = Version(version) + def __init__(self, get_installed_version, extras=''): + self.get_installed_version = get_installed_version self.marker_env = {'extra': extras} @@ -77,7 +74,11 @@ class Requirements: print_err(f'Ignoring alien requirement:', requirement_str) return - installed = self.installed_packages.get(requirement.name) + try: + installed = self.get_installed_version(requirement.name) + except importlib_metadata.PackageNotFoundError: + print_err(f'Requirement not satisfied: {requirement_str}') + installed = None if installed and installed in requirement.specifier: print_err(f'Requirement satisfied: {requirement_str}') print_err(f' (installed: {requirement.name} {installed})') @@ -203,9 +204,14 @@ def python3dist(name, op=None, version=None): def generate_requires( - freeze_output, *, include_runtime=False, toxenv=None, extras='', + *, include_runtime=False, toxenv=None, extras='', + get_installed_version=importlib_metadata.version, # for dep injection ): - requirements = Requirements(freeze_output, extras=extras) + """Generate the BuildRequires for the project in the current directory + + This is the main Python entry point. + """ + requirements = Requirements(get_installed_version, extras=extras) try: backend = get_backend(requirements) @@ -259,16 +265,8 @@ def main(argv): print_err('-x (--extras) are only useful with -r (--runtime)') exit(1) - freeze_output = subprocess.run( - [sys.executable, '-I', '-m', 'pip', 'freeze', '--all'], - encoding='utf-8', - stdout=subprocess.PIPE, - check=True, - ).stdout - try: generate_requires( - freeze_output, include_runtime=args.runtime, toxenv=args.toxenv, extras=args.extras, diff --git a/test_pyproject_buildrequires.py b/test_pyproject_buildrequires.py index 9b08e83..877501f 100644 --- a/test_pyproject_buildrequires.py +++ b/test_pyproject_buildrequires.py @@ -6,6 +6,11 @@ import yaml from pyproject_buildrequires import generate_requires +try: + import importlib.metadata as importlib_metadata +except ImportError: + import importlib_metadata + testcases = {} with Path(__file__).parent.joinpath('testcases.yaml').open() as f: testcases = yaml.safe_load(f) @@ -26,9 +31,17 @@ def test_data(case_name, capsys, tmp_path, monkeypatch): if filename in case: cwd.joinpath(filename).write_text(case[filename]) + def get_installed_version(dist_name): + try: + return str(case['installed'][dist_name]) + except (KeyError, TypeError): + raise importlib_metadata.PackageNotFoundError( + f'info not found for {dist_name}' + ) + try: generate_requires( - case['freeze_output'], + get_installed_version=get_installed_version, include_runtime=case.get('include_runtime', False), extras=case.get('extras', ''), toxenv=case.get('toxenv', None), diff --git a/testcases.yaml b/testcases.yaml index 2c90bd0..05ed361 100644 --- a/testcases.yaml +++ b/testcases.yaml @@ -1,5 +1,5 @@ No pyproject.toml, nothing installed: - freeze_output: | + installed: # empty expected: | python3dist(setuptools) >= 40.8 @@ -7,7 +7,7 @@ No pyproject.toml, nothing installed: result: 0 Nothing installed yet: - freeze_output: | + installed: # empty pyproject.toml: | # empty @@ -17,9 +17,9 @@ Nothing installed yet: result: 0 Insufficient version of setuptools: - freeze_output: | - setuptools==5 - wheel==1 + installed: + setuptools: 5 + wheel: 1 pyproject.toml: | # empty expected: | @@ -28,9 +28,9 @@ Insufficient version of setuptools: result: 0 Empty pyproject.toml, empty setup.py: - freeze_output: | - setuptools==50 - wheel==1 + installed: + setuptools: 50 + wheel: 1 setup.py: | expected: | python3dist(setuptools) >= 40.8 @@ -39,9 +39,9 @@ Empty pyproject.toml, empty setup.py: result: 0 Default build system, empty setup.py: - freeze_output: | - setuptools==50 - wheel==1 + installed: + setuptools: 50 + wheel: 1 pyproject.toml: | # empty setup.py: | @@ -52,24 +52,24 @@ Default build system, empty setup.py: result: 0 Erroring setup.py: - freeze_output: | - setuptools==50 - wheel==1 + installed: + setuptools: 50 + wheel: 1 setup.py: | exit(77) result: 77 Bad character in version: - freeze_output: | + installed: {} pyproject.toml: | [build-system] requires = ["pkg == 0.$.^.*"] except: ValueError Build system dependencies in pyproject.toml: - freeze_output: | - setuptools==50 - wheel==1 + installed: + setuptools: 50 + wheel: 1 pyproject.toml: | [build-system] requires = [ @@ -100,9 +100,9 @@ Build system dependencies in pyproject.toml: result: 0 Default build system, build dependencies in setup.py: - freeze_output: | - setuptools==50 - wheel==1 + installed: + setuptools: 50 + wheel: 1 setup.py: | from setuptools import setup setup( @@ -120,10 +120,10 @@ Default build system, build dependencies in setup.py: result: 0 Default build system, run dependencies in setup.py: - freeze_output: | - setuptools==50 - wheel==1 - pyyaml==1 + installed: + setuptools: 50 + wheel: 1 + pyyaml: 1 include_runtime: true setup.py: | from setuptools import setup @@ -143,10 +143,10 @@ Default build system, run dependencies in setup.py: result: 0 Run dependencies with extras (not selected): - freeze_output: | - setuptools==50 - wheel==1 - pyyaml==1 + installed: + setuptools: 50 + wheel: 1 + pyyaml: 1 include_runtime: true setup.py: &pytest_setup_py | # slightly abriged copy of pytest's setup.py @@ -200,10 +200,10 @@ Run dependencies with extras (not selected): result: 0 Run dependencies with extras (selected): - freeze_output: | - setuptools==50 - wheel==1 - pyyaml==1 + installed: + setuptools: 50 + wheel: 1 + pyyaml: 1 include_runtime: true extras: testing setup.py: *pytest_setup_py @@ -227,10 +227,10 @@ Run dependencies with extras (selected): Run dependencies with multiple extras: xfail: requirement.marker.evaluate seems to not support multiple extras - freeze_output: | - setuptools==50 - wheel==1 - pyyaml==1 + installed: + setuptools: 50 + wheel: 1 + pyyaml: 1 include_runtime: true extras: testing,more-testing, even-more-testing , cool-feature setup.py: | @@ -246,19 +246,18 @@ Run dependencies with multiple extras: expected: | python3dist(setuptools) >= 40.8 python3dist(wheel) - python3dist(wheel) python3dist(dep1) python3dist(dep2) python3dist(dep3) python3dist(dep4) result: 0 -Tox depndencies: - freeze_output: | - setuptools==50 - wheel==1 - tox==3.5.3 - tox-current-env==0.0.2 +Tox dependencies: + installed: + setuptools: 50 + wheel: 1 + tox: 3.5.3 + tox-current-env: 0.0.2 toxenv: py3 setup.py: | from setuptools import setup From 7e01f58f73fef860d26e96ecf17af746c2a61c1c Mon Sep 17 00:00:00 2001 From: Miro Hrončok Date: Sep 18 2019 15:31:26 +0000 Subject: [PATCH 5/8] Fedora CI: Publish the mock logs even when it fails --- diff --git a/tests/mocktest.sh b/tests/mocktest.sh index 5c65abc..5af4d83 100755 --- a/tests/mocktest.sh +++ b/tests/mocktest.sh @@ -23,14 +23,17 @@ spectool -g -R ${1}.spec rpmbuild -bs ${1}.spec # build the SRPM in mock +res=0 mock -r $config --enablerepo=local init -mock -r $config --enablerepo=local ~/rpmbuild/SRPMS/${1}-*.src.rpm +mock -r $config --enablerepo=local ~/rpmbuild/SRPMS/${1}-*.src.rpm || res=$? # move the results to the artifacts directory, so we can examine them artifacts=${TEST_ARTIFACTS:-/tmp/artifacts} pushd /var/lib/mock/fedora-rawhide-x86_64/result -mv *.rpm ${artifacts}/ +mv *.rpm ${artifacts}/ || : for log in *.log; do mv ${log} ${artifacts}/${1}-${log} done popd + +exit $res From 97d785b58d894faa331c86f43ffdb59dd7bfa34f Mon Sep 17 00:00:00 2001 From: Miro Hrončok Date: Sep 19 2019 23:32:33 +0000 Subject: [PATCH 6/8] Don't use --strip-file-prefix with pip, the custom option is gone --- diff --git a/macros.pyproject b/macros.pyproject index 1203621..c826123 100644 --- a/macros.pyproject +++ b/macros.pyproject @@ -5,7 +5,7 @@ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}" \\\ %pyproject_install() %{expand:\\\ -%{__python3} -m pip install --root %{buildroot} --strip-file-prefix %{buildroot} --no-deps --disable-pip-version-check --progress-bar off --verbose --ignore-installed --no-warn-script-location ./*.whl +%{__python3} -m pip install --root %{buildroot} --no-deps --disable-pip-version-check --progress-bar off --verbose --ignore-installed --no-warn-script-location ./*.whl if [ -d %{buildroot}%{_bindir} ]; then pathfix.py -pni "%{__python3} %{py3_shbang_opts}" %{buildroot}%{_bindir}/* fi From 137aa316c4aaa9c7b73e104e43e596e0e69657f4 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sep 20 2019 08:18:12 +0000 Subject: [PATCH 7/8] Bump release --- diff --git a/pyproject-rpm-macros.spec b/pyproject-rpm-macros.spec index 88fb2f0..0e64447 100644 --- a/pyproject-rpm-macros.spec +++ b/pyproject-rpm-macros.spec @@ -6,7 +6,7 @@ License: MIT # Keep the version at zero and increment only release Version: 0 -Release: 5%{?dist} +Release: 6%{?dist} Source0: macros.pyproject Source1: pyproject_buildrequires.py @@ -87,6 +87,9 @@ install -m 644 pyproject_buildrequires.py %{buildroot}%{_rpmconfigdir}/redhat/ %license LICENSE %changelog +* Fri Jul 26 2019 Petr Viktorin - 0-6 +- Use importlib_metadata rather than pip freeze + * Fri Jul 26 2019 Miro Hrončok - 0-5 - Allow to fetch test dependencies from tox - Add %%tox macro to invoke tests From 2a3b101709e2983da68e6ce7a74e70389eea3022 Mon Sep 17 00:00:00 2001 From: Miro Hrončok Date: Sep 23 2019 13:34:26 +0000 Subject: [PATCH 8/8] Fedora CI: Run mock on the current Fedora version (or 31 if less than 31) --- diff --git a/tests/mocktest.sh b/tests/mocktest.sh index 5af4d83..2d1ed8a 100755 --- a/tests/mocktest.sh +++ b/tests/mocktest.sh @@ -1,12 +1,20 @@ #!/usr/bin/bash -eux +. /etc/os-release +fedora=$VERSION_ID -config="/tmp/fedora-rawhide-x86_64-ci.cfg" +# we don't have dynamic BuildRequires on Fedora 30 +# so we at least test that we can build in a Fedora 31 mock +if [ $fedora -lt 31 ]; then + fedora=31 +fi + +config="/tmp/fedora-${fedora}-x86_64-ci.cfg" # create mock config if not present # this makes sure tested version of pyproject-rpm-macros is available # TODO: check if it has precedence if the release was not bumped in tested PR if [ ! -f $config ]; then - original="/etc/mock/fedora-rawhide-x86_64.cfg" + original="/etc/mock/fedora-${fedora}-x86_64.cfg" split=$(sed -n '/\[fedora\]/=' $original | head -n1) head -n$(($split-1)) $original > $config cat /etc/yum.repos.d/test-pyproject-rpm-macros.repo >> $config @@ -29,7 +37,7 @@ mock -r $config --enablerepo=local ~/rpmbuild/SRPMS/${1}-*.src.rpm || res=$? # move the results to the artifacts directory, so we can examine them artifacts=${TEST_ARTIFACTS:-/tmp/artifacts} -pushd /var/lib/mock/fedora-rawhide-x86_64/result +pushd /var/lib/mock/fedora-*-x86_64/result mv *.rpm ${artifacts}/ || : for log in *.log; do mv ${log} ${artifacts}/${1}-${log}