From 8485b55bea9e2019265a0f5900e33e258149ea0d Mon Sep 17 00:00:00 2001 From: Miro Hrončok Date: Mar 13 2021 14:18:26 +0000 Subject: Don't generate runtime dependency on setuptools for console_scripts entrypoints - recent setuptools don't need it, they use importlib.metadata - for this to work, we need to detect setuptools version - the detection is not bulletproof, but it fallbacks to the old behavior --- diff --git a/python-rpm-generators.spec b/python-rpm-generators.spec index 24994bf..47fac27 100644 --- a/python-rpm-generators.spec +++ b/python-rpm-generators.spec @@ -1,7 +1,7 @@ Name: python-rpm-generators Summary: Dependency generators for Python RPMs Version: 12 -Release: 4%{?dist} +Release: 5%{?dist} # Originally all those files were part of RPM, so license is kept here License: GPLv2+ @@ -47,6 +47,10 @@ install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} *.py %{_rpmconfigdir}/pythonbundles.py %changelog +* Sat Mar 13 2021 Miro Hrončok - 12-5 +- Don't generate runtime dependency on setuptools for console_scripts entrypoints, + recent setuptools don't need it + * Thu Mar 11 2021 Tomas Orsava - 12-4 - scripts/pythondistdeps: Treat extras names case-insensitively and always output them in lower case (#1936875) diff --git a/pythondistdeps.py b/pythondistdeps.py index f38f726..1aa2b7c 100755 --- a/pythondistdeps.py +++ b/pythondistdeps.py @@ -16,7 +16,7 @@ import argparse from distutils.sysconfig import get_python_lib from os.path import dirname, sep import re -from sys import argv, stdin, stderr +from sys import argv, stdin, stderr, version_info from warnings import warn from packaging.requirements import Requirement as Requirement_ @@ -44,8 +44,12 @@ packaging.markers._operators["=="] = str_lower_eq try: from importlib.metadata import PathDistribution + from importlib.metadata import PackageNotFoundError + from importlib.metadata import version as version_of except ImportError: from importlib_metadata import PathDistribution + from importlib_metadata import PackageNotFoundError + from importlib_metadata import version as version_of try: from pathlib import Path @@ -252,6 +256,67 @@ def get_marker_env(dist, extra): "extra": extra} +def find_setuptools_version(py_version): + """Figure out setuptools version installed for the given Python X.Y version""" + + # first, do it the sane way, if we run that Python version + # nb: this breaks if we run on PyPy and generate for CPython or vice versa + if '{}.{}'.format(*version_info[:2]) == py_version: + try: + return parse(version_of('setuptools')) + except PackageNotFoundError: + pass + + # next, run the Python version and import setuptools from it + # nb: this also breaks if we run on different implementation + try: + import subprocess + code = 'import setuptools; print(setuptools.__version__)' + output = subprocess.check_output(['python{}'.format(dist.py_version), '-Esc', code], + universal_newlines=True) + return parse(output.strip()) + except Exception: # anything can go wrong here + pass + + warn('Cannot check setuptools version for Python {}; ' + 'assuming an old version for safety.'.format(py_version), + RuntimeWarning) + return parse('0') + + +def console_scripts_deps(dist, metadirname): + if metadirname.endswith('.dist-info'): + # console scripts from .dist-info have no extra deps + return [] + + groups = {ep.group for ep in dist.entry_points} + if not {"console_scripts", "gui_scripts"} & groups: + # a different kind of entrypoint + return [] + + python_version = parse(dist.py_version) + setuptools_version = find_setuptools_version(dist.py_version) + + # https://setuptools.readthedocs.io/en/latest/history.html#v47-3-0 + if setuptools_version >= parse('47.3.0'): + if python_version < parse('3.6'): + # don't use importlib_metadata on very old Pythons, + # use pkg_resources from setuptools + return [Requirement('setuptools')] + if python_version < parse('3.8'): + # technically this can also fallback to pkg_resources, + # but we prefer the lighter requirement + return [Requirement('importlib_metadata')] + return [] + + # https://setuptools.readthedocs.io/en/latest/history.html#v47-2-0 + if setuptools_version >= parse('47.2.0') and python_version >= parse('3.8'): + return [] + + # older versions use pkg_resources from setuptools + return [Requirement('setuptools')] + + if __name__ == "__main__": """To allow this script to be importable (and its classes/functions reused), actions are performed only when run as a main script.""" @@ -438,15 +503,12 @@ if __name__ == "__main__": else: deps = dist.requirements - # console_scripts/gui_scripts entry points need pkg_resources from setuptools - if (dist.entry_points and - (lower.endswith('.egg') or - lower.endswith('.egg-info'))): - groups = {ep.group for ep in dist.entry_points} - if {"console_scripts", "gui_scripts"} & groups: - # stick them first so any more specific requirement - # overrides it - deps.insert(0, Requirement('setuptools')) + # console_scripts/gui_scripts entry points might need extra dependencies + # we insert the requirement to the beginning of the list + # so any more specific requirement on the same package can override it + if dist.entry_points: + deps = console_scripts_deps(dist, lower) + deps + # add requires/recommends based on egg/dist metadata for dep in deps: # Even if we're requiring `foo[bar]`, also require `foo` diff --git a/tests/console_script.sh b/tests/console_script.sh new file mode 100755 index 0000000..45e5d44 --- /dev/null +++ b/tests/console_script.sh @@ -0,0 +1,71 @@ +#!/usr/bin/bash -eux +RPMDIR=$(rpm --eval '%_topdir')/RPMS/noarch +RPMPKG="${RPMDIR}/isort-5.7.0-0.noarch.rpm" + +mkdir -p $(rpm --eval '%_topdir')/SOURCES/ + +spectool -g -R isort.spec + +for py_version in 3.6 3.7 3.8 3.9 3.10; do + rpmbuild -ba --define "python3_test_version ${py_version}" isort.spec + + # sanity check for provides; if this is broken, so is the test + rpm -qp --provides ${RPMPKG} | grep "python${py_version}dist(isort)" + + # only the "main" Python version has setuptools installed, + # everything else does not + if [ "$py_version" = "$(rpm --eval '%python3_version')" ]; then + # all main Python/setuptools versions in Fedora 33+ are recent enough + # not to justify a generated dependency wrt the console_script + rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)" && exit 1 || true + rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)" && exit 1 || true + else + # no setuptools installed, we assume an old version of setuptools was used to prepare the data + # hence the package always requires setuptools + rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)" + rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)" && exit 1 || true + fi + + # the rest is only possible on the CI or in mock/container, where we are root + # never run this script as root on your host OS, it is destructive + # also, it only works once (improvements welcome) + test $EUID -ne 0 && continue + export RPM_BUILD_ROOT=/ + + # install setuptools and build again + python${py_version} -m ensurepip + rpmbuild -ba --define "python3_test_version ${py_version}" isort.spec + + # the ensurepip version of setuptools is recent enough on Fedora 33+ + # WARNING: Once we flip the rpmwheels bcond in python3.6, this assumption will break + rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)" && exit 1 || true + # but older Pythons still need importlib_metadata + if [[ "$py_version" = "3.6" || "$py_version" = "3.7" ]]; then + rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)" + else + rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)" && exit 1 || true + fi + + # install setuptools 47.2 and build again + python${py_version} -m pip uninstall --yes setuptools + python${py_version} -m pip install 'setuptools>=47.2.0,<47.3.0' + rpmbuild -ba --define "python3_test_version ${py_version}" isort.spec + + # this version of setuptools never uses importlib_metadata + rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)" && exit 1 || true + # older Pythons use setuptools, newer Pythons use importlib.metadata + if [[ "$py_version" = "3.6" || "$py_version" = "3.7" ]]; then + rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)" + else + rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)" && exit 1 || true + fi + + # install an even older setuptools version and build again + python${py_version} -m pip uninstall --yes setuptools + python${py_version} -m pip install 'setuptools<47.2.0' + rpmbuild -ba --define "python3_test_version ${py_version}" isort.spec + + # old console_scripts entyrpoint used pkg_resources only + rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)" + rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)" && exit 1 || true +done diff --git a/tests/data/scripts_pythondistdeps/test-data.yaml b/tests/data/scripts_pythondistdeps/test-data.yaml index 35c72a8..6a14801 100644 --- a/tests/data/scripts_pythondistdeps/test-data.yaml +++ b/tests/data/scripts_pythondistdeps/test-data.yaml @@ -131,7 +131,7 @@ python3dist(setuptools) = 41.6 requires: |- python(abi) = 3.7 - python3.7dist(setuptools) + python3.7dist(importlib-metadata) usr/lib/python3.7/site-packages/setuptools-41.6.0.dist-info: provides: |- python3.7dist(setuptools) = 41.6 @@ -233,9 +233,7 @@ provides: |- python3.9dist(setuptools) = 41.6 python3dist(setuptools) = 41.6 - requires: |- - python(abi) = 3.9 - python3.9dist(setuptools) + requires: python(abi) = 3.9 usr/lib/python3.9/site-packages/setuptools-41.6.0.dist-info: provides: |- python3.9dist(setuptools) = 41.6 @@ -360,9 +358,9 @@ python3dist(numpy-stl) = 2.11.2 requires: |- python(abi) = 3.7 + python3.7dist(importlib-metadata) python3.7dist(numpy) python3.7dist(python-utils) >= 1.6.2 - python3.7dist(setuptools) usr/lib64/python3.7/site-packages/scipy-1.2.1.dist-info: provides: |- python3.7dist(scipy) = 1.2.1 @@ -395,7 +393,6 @@ python(abi) = 3.9 python3.9dist(numpy) python3.9dist(python-utils) >= 1.6.2 - python3.9dist(setuptools) usr/lib64/python3.9/site-packages/simplejson-3.16.0-py3.9.egg-info: provides: |- python3.9dist(simplejson) = 3.16 @@ -535,9 +532,7 @@ provides: |- python3.9dist(setuptools) = 41.6 python3dist(setuptools) = 41.6 - requires: |- - python(abi) = 3.9 - python3.9dist(setuptools) + requires: python(abi) = 3.9 usr/lib/python3.9/site-packages/setuptools-41.6.0.dist-info: provides: |- python3.9dist(setuptools) = 41.6 @@ -635,14 +630,12 @@ python2.7dist(zope.interface) >= 4.1 usr/lib/python3.10/site-packages/setuptools-41.6.0-py3.10.egg-info: provides: python3.10dist(setuptools) = 41.6 - requires: |- - python(abi) = 3.10 - python3.10dist(setuptools) + requires: python(abi) = 3.10 usr/lib/python3.7/site-packages/setuptools-41.6.0-py3.7.egg-info: provides: python3.7dist(setuptools) = 41.6 requires: |- python(abi) = 3.7 - python3.7dist(setuptools) + python3.7dist(importlib-metadata) usr/lib/python3.7/site-packages/setuptools-41.6.0.dist-info: provides: python3.7dist(setuptools) = 41.6 requires: python(abi) = 3.7 @@ -697,9 +690,9 @@ provides: python3.7dist(numpy-stl) = 2.11.2 requires: |- python(abi) = 3.7 + python3.7dist(importlib-metadata) python3.7dist(numpy) python3.7dist(python-utils) >= 1.6.2 - python3.7dist(setuptools) usr/lib64/python3.7/site-packages/scipy-1.2.1.dist-info: provides: python3.7dist(scipy) = 1.2.1 requires: |- @@ -728,7 +721,6 @@ python(abi) = 3.9 python3.9dist(numpy) python3.9dist(python-utils) >= 1.6.2 - python3.9dist(setuptools) usr/lib64/python3.9/site-packages/simplejson-3.16.0-py3.9.egg-info: provides: |- python3.9dist(simplejson) = 3.16 @@ -746,9 +738,7 @@ provides: |- python3.10dist(setuptools) = 41.6 python3dist(setuptools) = 41.6 - requires: |- - python(abi) = 3.10 - python3.10dist(setuptools) + requires: python(abi) = 3.10 usr/lib/python3.11/site-packages/pip-20.0.2-py3.11.egg-info: provides: python3.11dist(pip) = 20.0.2 requires: |- @@ -800,9 +790,7 @@ provides: |- python3.10dist(setuptools) = 41.6 python3dist(setuptools) = 41.6 - requires: |- - python(abi) = 3.10 - python3.10dist(setuptools) + requires: python(abi) = 3.10 usr/lib/python3.11/site-packages/pip-20.0.2-py3.11.egg-info: provides: |- python3.11dist(pip) = 20.0.2 diff --git a/tests/isort.spec b/tests/isort.spec new file mode 100644 index 0000000..472a385 --- /dev/null +++ b/tests/isort.spec @@ -0,0 +1,32 @@ +Name: isort +Version: 5.7.0 +Release: 0 +Summary: A Python package with a console_scripts entrypoint +License: MIT +Source0: %{pypi_source} +BuildArch: noarch +BuildRequires: python3-devel +BuildRequires: python3-setuptools +BuildRequires: python%{python3_test_version} + +%description +... + +%prep +%autosetup + +%build +%py3_build + +%install +%py3_install +%if "%{python3_version}" != "%{python3_test_version}" +mv %{buildroot}%{_prefix}/lib/python%{python3_version} \ + %{buildroot}%{_prefix}/lib/python%{python3_test_version} +mv %{buildroot}%{_prefix}/lib/python%{python3_test_version}/site-packages/%{name}-%{version}-py%{python3_version}.egg-info \ + %{buildroot}%{_prefix}/lib/python%{python3_test_version}/site-packages/%{name}-%{version}-py%{python3_test_version}.egg-info +%endif + +%files +%{_bindir}/%{name}* +%{_prefix}/lib/python%{python3_test_version}/site-packages/%{name}* diff --git a/tests/test_scripts_pythondistdeps.py b/tests/test_scripts_pythondistdeps.py index 3636f1c..45e6513 100644 --- a/tests/test_scripts_pythondistdeps.py +++ b/tests/test_scripts_pythondistdeps.py @@ -22,13 +22,15 @@ # Requirements for this script: # - Python >= 3.6 # - pip >= 20.0.1 -# - setuptools +# - setuptools >= 47.3 # - pytest # - pyyaml # - wheel from pathlib import Path +import functools +import os import pytest import shlex import shutil @@ -39,6 +41,7 @@ import yaml PYTHONDISTDEPS_PATH = Path(__file__).parent / '..' / 'pythondistdeps.py' TEST_DATA_PATH = Path(__file__).parent / 'data' / 'scripts_pythondistdeps' +FAKE_PATH = TEST_DATA_PATH / '_path' def run_pythondistdeps(provides_params, requires_params, dist_egg_info_path, expect_failure=False): @@ -47,10 +50,12 @@ def run_pythondistdeps(provides_params, requires_params, dist_egg_info_path, exp info_path = TEST_DATA_PATH / dist_egg_info_path files = '\n'.join(map(str, info_path.iterdir())) + environ = fake_path_pythons() + provides = subprocess.run((sys.executable, PYTHONDISTDEPS_PATH, *shlex.split(provides_params)), - input=files, capture_output=True, check=False, encoding="utf-8") + input=files, capture_output=True, check=False, encoding="utf-8", env=environ) requires = subprocess.run((sys.executable, PYTHONDISTDEPS_PATH, *shlex.split(requires_params)), - input=files, capture_output=True, check=False, encoding="utf-8") + input=files, capture_output=True, check=False, encoding="utf-8", env=environ) if expect_failure: if provides.returncode == 0 or requires.returncode == 0: @@ -67,6 +72,25 @@ def run_pythondistdeps(provides_params, requires_params, dist_egg_info_path, exp return {"provides": provides.stdout.strip(), "requires": requires.stdout.strip()} +@functools.lru_cache(maxsize=1) +def fake_path_pythons(): + """Create fake Pythons, so when the pythondistdeps script tries to detect + setuptools version, it tells them ours.""" + environ = os.environ.copy() + path = environ.get("PATH", "") + path = f"{FAKE_PATH}:{path}" + environ["PATH"] = path + + FAKE_PATH.mkdir(exist_ok=True) + + for ver in "2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10": + exe = FAKE_PATH / f"python{ver}" + exe.unlink(missing_ok=True) + exe.symlink_to(sys.executable) + + return environ + + def load_test_data(): """Reads the test-data.yaml and loads the test data into a dict.""" with TEST_DATA_PATH.joinpath('test-data.yaml').open() as file: diff --git a/tests/tests.yml b/tests/tests.yml index a79a5f1..a5da935 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -34,6 +34,10 @@ dir: ./tests # Use update-test-sources.sh to update the test data run: python3 -m pytest --capture=no -vvv + # WARNING: This test alters the environment, keep it last: + - console_script: + dir: . + run: ./console_script.sh required_packages: - rpm-build - rpmdevtools @@ -43,3 +47,8 @@ - python3-pyyaml - python3-setuptools - python3-wheel + - python3.6 + - python3.7 + - python3.8 + - python3.9 + - python3.10