#37 Don't generate runtime dependency on setuptools for console_scripts entrypoints
Closed 2 years ago by churchyard. Opened 2 years ago by churchyard.
rpms/ churchyard/python-rpm-generators console_scripts_importlib-metadata  into  rawhide

file modified
+5 -1
@@ -1,7 +1,7 @@ 

  Name:           python-rpm-generators

  Summary:        Dependency generators for Python RPMs

  Version:        12

- Release:        4%{?dist}

+ Release:        5%{?dist}

  

  # Originally all those files were part of RPM, so license is kept here

  License:        GPLv2+
@@ -47,6 +47,10 @@ 

  %{_rpmconfigdir}/pythonbundles.py

  

  %changelog

+ * Sat Mar 13 2021 Miro Hrončok <mhroncok@redhat.com> - 12-5

+ - Don't generate runtime dependency on setuptools for console_scripts entrypoints,

+   recent setuptools don't need it

+ 

  * Thu Mar 11 2021 Tomas Orsava <torsava@redhat.com> - 12-4

  - scripts/pythondistdeps: Treat extras names case-insensitively and always

    output them in lower case (#1936875)

file modified
+72 -10
@@ -16,7 +16,7 @@ 

  from distutils.sysconfig import get_python_lib

  from os.path import dirname, sep

  import re

- from sys import argv, stdin, stderr

+ from sys import argv, stdin, stderr, version_info

  from warnings import warn

  

  from packaging.requirements import Requirement as Requirement_
@@ -44,8 +44,12 @@ 

  

  try:

      from importlib.metadata import PathDistribution

+     from importlib.metadata import PackageNotFoundError

+     from importlib.metadata import version as version_of
from importlib.metadata import PackageNotFoundError
from importlib.metadata import version as version_of

Minor nitpick: Why each import on it's own line when other imports use a multi-import style?

  except ImportError:

      from importlib_metadata import PathDistribution

+     from importlib_metadata import PackageNotFoundError

+     from importlib_metadata import version as version_of

  

  try:

      from pathlib import Path
@@ -252,6 +256,67 @@ 

              "extra": extra}

  

  

+ def find_setuptools_version(py_version):

+     """Figure out setuptools version installed for the given Python X.Y version"""

+ 

+     # first, do it the sane way, if we run that Python version

+     # nb: this breaks if we run on PyPy and generate for CPython or vice versa

+     if '{}.{}'.format(*version_info[:2]) == py_version:

+         try:

+             return parse(version_of('setuptools'))

+         except PackageNotFoundError:

+             pass

+ 

+     # next, run the Python version and import setuptools from it

+     # nb: this also breaks if we run on different implementation

+     try:

+         import subprocess

+         code = 'import setuptools; print(setuptools.__version__)'

+         output = subprocess.check_output(['python{}'.format(dist.py_version), '-Esc', code],

As discussed on IRC: The '-E' flag has questionable benefits, especially for upstream.

+                                          universal_newlines=True)

+         return parse(output.strip())

+     except Exception:  # anything can go wrong here

+         pass

+ 

+     warn('Cannot check setuptools version for Python {}; '

+          'assuming an old version for safety.'.format(py_version),

+          RuntimeWarning)

+     return parse('0')

+ 

+ 

+ def console_scripts_deps(dist, metadirname):

+     if metadirname.endswith('.dist-info'):

+         # console scripts from .dist-info have no extra deps

+         return []

+ 

+     groups = {ep.group for ep in dist.entry_points}

+     if not {"console_scripts", "gui_scripts"} & groups:

+         # a different kind of entrypoint

+         return []

+ 

+     python_version = parse(dist.py_version)

+     setuptools_version = find_setuptools_version(dist.py_version)

+ 

+     # https://setuptools.readthedocs.io/en/latest/history.html#v47-3-0

+     if setuptools_version >= parse('47.3.0'):

+         if python_version < parse('3.6'):

+             # don't use importlib_metadata on very old Pythons,

+             # use pkg_resources from setuptools

+             return [Requirement('setuptools')]

+         if python_version < parse('3.8'):

+             # technically this can also fallback to pkg_resources,

+             # but we prefer the lighter requirement

+             return [Requirement('importlib_metadata')]
    if python_version < parse('3.8'):
        ...
        return [Requirement('importlib_metadata')]

Noting for discussion: For new setuptools but old Python 3.6 and 3.7, which is the better default requirement - setuptools or importlib_metadata?

While it's true that this script runs on importlib{.,_}metadata, it also very likely runs on newer version of Python. E.g. in Fedora we run this script on Python 3.9 or soon 3.10. The presence of importlib*metadata there is not a good predictor whether importlib_metadata is available for Python 3.6 and 3.7. And I would wager that in most old Python stacks, setuptools is much more likely to be present than importlib_metadata.

+         return []

+ 

+     # https://setuptools.readthedocs.io/en/latest/history.html#v47-2-0

+     if setuptools_version >= parse('47.2.0') and python_version >= parse('3.8'):

+         return []

+ 

+     # older versions use pkg_resources from setuptools

+     return [Requirement('setuptools')]

+ 

+ 

  if __name__ == "__main__":

      """To allow this script to be importable (and its classes/functions

         reused), actions are performed only when run as a main script."""
@@ -438,15 +503,12 @@ 

                  else:

                      deps = dist.requirements

  

-                 # console_scripts/gui_scripts entry points need pkg_resources from setuptools

-                 if (dist.entry_points and

-                     (lower.endswith('.egg') or

-                      lower.endswith('.egg-info'))):

-                     groups = {ep.group for ep in dist.entry_points}

-                     if {"console_scripts", "gui_scripts"} & groups:

-                         # stick them first so any more specific requirement

-                         # overrides it

-                         deps.insert(0, Requirement('setuptools'))

+                 # console_scripts/gui_scripts entry points might need extra dependencies

+                 # we insert the requirement to the beginning of the list

+                 # so any more specific requirement on the same package can override it

+                 if dist.entry_points:

+                     deps = console_scripts_deps(dist, lower) + deps
                deps = console_scripts_deps(dist, lower) + deps

Minor nitpick, I think the original deps.insert(0, ...) is more clear.

+ 

                  # add requires/recommends based on egg/dist metadata

                  for dep in deps:

                      # Even if we're requiring `foo[bar]`, also require `foo`

@@ -0,0 +1,71 @@ 

+ #!/usr/bin/bash -eux

+ RPMDIR=$(rpm --eval '%_topdir')/RPMS/noarch

+ RPMPKG="${RPMDIR}/isort-5.7.0-0.noarch.rpm"

+ 

+ mkdir -p $(rpm --eval '%_topdir')/SOURCES/

+ 

+ spectool -g -R isort.spec

+ 

+ for py_version in 3.6 3.7 3.8 3.9 3.10; do

+   rpmbuild -ba --define "python3_test_version ${py_version}" isort.spec

+ 

+   # sanity check for provides; if this is broken, so is the test

+   rpm -qp --provides ${RPMPKG} | grep "python${py_version}dist(isort)"

+ 

+   # only the "main" Python version has setuptools installed,

+   # everything else does not

+   if [ "$py_version" = "$(rpm --eval '%python3_version')" ]; then

+     # all main Python/setuptools versions in Fedora 33+ are recent enough

+     # not to justify a generated dependency wrt the console_script

+     rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)" && exit 1 || true

+     rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)"  && exit 1 || true
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)" && exit 1 || true
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)"  && exit 1 || true

Let's print out some error before exit?

+   else

+     # no setuptools installed, we assume an old version of setuptools was used to prepare the data

+     # hence the package always requires setuptools

+     rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)"

+     rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)"  && exit 1 || true
# no setuptools installed, we assume an old version of setuptools was used to prepare the data
# hence the package always requires setuptools
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)"
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)"  && exit 1 || true

So, new version of setuptools would generate importlib-metadata requires on Python 3.6 and 3.7. But since you assume Python 3.6 and 3.7 don't have setuptools in Fedora, you ignore that case?

+   fi

+ 

+   # the rest is only possible on the CI or in mock/container, where we are root

+   # never run this script as root on your host OS, it is destructive

# never run this script as root on your host OS, it is destructive

This really needs to be also on top of this file, people have brainfarts and sometimes run with sudo without checking.

+   # also, it only works once (improvements welcome)

+   test $EUID -ne 0 && continue

+   export RPM_BUILD_ROOT=/

+ 

+   # install setuptools and build again

+   python${py_version} -m ensurepip

+   rpmbuild -ba --define "python3_test_version ${py_version}" isort.spec

+ 

+   # the ensurepip version of setuptools is recent enough on Fedora 33+

+   # WARNING: Once we flip the rpmwheels bcond in python3.6, this assumption will break

# WARNING: Once we flip the rpmwheels bcond in python3.6, this assumption will break

I need more context here.

+   rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)"  && exit 1 || true

+   # but older Pythons still need importlib_metadata

+   if [[ "$py_version" = "3.6" || "$py_version" = "3.7" ]]; then

+     rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)"

+   else

+     rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)" && exit 1 || true

+   fi

+ 

+   # install setuptools 47.2 and build again

+   python${py_version} -m pip uninstall --yes setuptools

+   python${py_version} -m pip install 'setuptools>=47.2.0,<47.3.0'

+   rpmbuild -ba --define "python3_test_version ${py_version}" isort.spec

+ 

+   # this version of setuptools never uses importlib_metadata

+   rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)"  && exit 1 || true

+   # older Pythons use setuptools, newer Pythons use importlib.metadata

+   if [[ "$py_version" = "3.6" || "$py_version" = "3.7" ]]; then

+     rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)"

+   else

+     rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)" && exit 1 || true

+   fi

+ 

+   # install an even older setuptools version and build again

+   python${py_version} -m pip uninstall --yes setuptools

+   python${py_version} -m pip install 'setuptools<47.2.0'

+   rpmbuild -ba --define "python3_test_version ${py_version}" isort.spec

+ 

+   # old console_scripts entyrpoint used pkg_resources only

+   rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)"

+   rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)"  && exit 1 || true

+ done

@@ -131,7 +131,7 @@ 

                  python3dist(setuptools) = 41.6

              requires: |-

                  python(abi) = 3.7

-                 python3.7dist(setuptools)

+                 python3.7dist(importlib-metadata)

          usr/lib/python3.7/site-packages/setuptools-41.6.0.dist-info:

              provides: |-

                  python3.7dist(setuptools) = 41.6
@@ -233,9 +233,7 @@ 

              provides: |-

                  python3.9dist(setuptools) = 41.6

                  python3dist(setuptools) = 41.6

-             requires: |-

-                 python(abi) = 3.9

-                 python3.9dist(setuptools)

+             requires: python(abi) = 3.9

          usr/lib/python3.9/site-packages/setuptools-41.6.0.dist-info:

              provides: |-

                  python3.9dist(setuptools) = 41.6
@@ -360,9 +358,9 @@ 

                  python3dist(numpy-stl) = 2.11.2

              requires: |-

                  python(abi) = 3.7

+                 python3.7dist(importlib-metadata)

                  python3.7dist(numpy)

                  python3.7dist(python-utils) >= 1.6.2

-                 python3.7dist(setuptools)

          usr/lib64/python3.7/site-packages/scipy-1.2.1.dist-info:

              provides: |-

                  python3.7dist(scipy) = 1.2.1
@@ -395,7 +393,6 @@ 

                  python(abi) = 3.9

                  python3.9dist(numpy)

                  python3.9dist(python-utils) >= 1.6.2

-                 python3.9dist(setuptools)

          usr/lib64/python3.9/site-packages/simplejson-3.16.0-py3.9.egg-info:

              provides: |-

                  python3.9dist(simplejson) = 3.16
@@ -535,9 +532,7 @@ 

              provides: |-

                  python3.9dist(setuptools) = 41.6

                  python3dist(setuptools) = 41.6

-             requires: |-

-                 python(abi) = 3.9

-                 python3.9dist(setuptools)

+             requires: python(abi) = 3.9

          usr/lib/python3.9/site-packages/setuptools-41.6.0.dist-info:

              provides: |-

                  python3.9dist(setuptools) = 41.6
@@ -635,14 +630,12 @@ 

                  python2.7dist(zope.interface) >= 4.1

          usr/lib/python3.10/site-packages/setuptools-41.6.0-py3.10.egg-info:

              provides: python3.10dist(setuptools) = 41.6

-             requires: |-

-                 python(abi) = 3.10

-                 python3.10dist(setuptools)

+             requires: python(abi) = 3.10

          usr/lib/python3.7/site-packages/setuptools-41.6.0-py3.7.egg-info:

              provides: python3.7dist(setuptools) = 41.6

              requires: |-

                  python(abi) = 3.7

-                 python3.7dist(setuptools)

+                 python3.7dist(importlib-metadata)

          usr/lib/python3.7/site-packages/setuptools-41.6.0.dist-info:

              provides: python3.7dist(setuptools) = 41.6

              requires: python(abi) = 3.7
@@ -697,9 +690,9 @@ 

              provides: python3.7dist(numpy-stl) = 2.11.2

              requires: |-

                  python(abi) = 3.7

+                 python3.7dist(importlib-metadata)

                  python3.7dist(numpy)

                  python3.7dist(python-utils) >= 1.6.2

-                 python3.7dist(setuptools)

          usr/lib64/python3.7/site-packages/scipy-1.2.1.dist-info:

              provides: python3.7dist(scipy) = 1.2.1

              requires: |-
@@ -728,7 +721,6 @@ 

                  python(abi) = 3.9

                  python3.9dist(numpy)

                  python3.9dist(python-utils) >= 1.6.2

-                 python3.9dist(setuptools)

          usr/lib64/python3.9/site-packages/simplejson-3.16.0-py3.9.egg-info:

              provides: |-

                  python3.9dist(simplejson) = 3.16
@@ -746,9 +738,7 @@ 

              provides: |-

                  python3.10dist(setuptools) = 41.6

                  python3dist(setuptools) = 41.6

-             requires: |-

-                 python(abi) = 3.10

-                 python3.10dist(setuptools)

+             requires: python(abi) = 3.10

          usr/lib/python3.11/site-packages/pip-20.0.2-py3.11.egg-info:

              provides: python3.11dist(pip) = 20.0.2

              requires: |-
@@ -800,9 +790,7 @@ 

              provides: |-

                  python3.10dist(setuptools) = 41.6

                  python3dist(setuptools) = 41.6

-             requires: |-

-                 python(abi) = 3.10

-                 python3.10dist(setuptools)

+             requires: python(abi) = 3.10

          usr/lib/python3.11/site-packages/pip-20.0.2-py3.11.egg-info:

              provides: |-

                  python3.11dist(pip) = 20.0.2

file added
+32
@@ -0,0 +1,32 @@ 

+ Name:           isort

+ Version:        5.7.0

+ Release:        0

+ Summary:        A Python package with a console_scripts entrypoint

+ License:        MIT

+ Source0:        %{pypi_source}

+ BuildArch:      noarch

+ BuildRequires:  python3-devel

+ BuildRequires:  python3-setuptools

+ BuildRequires:  python%{python3_test_version}

+ 

+ %description

+ ...

+ 

+ %prep

+ %autosetup

+ 

+ %build

+ %py3_build

+ 

+ %install

+ %py3_install

+ %if "%{python3_version}" != "%{python3_test_version}"

+ mv %{buildroot}%{_prefix}/lib/python%{python3_version} \

+    %{buildroot}%{_prefix}/lib/python%{python3_test_version}

mv %{buildroot}%{_prefix}/lib/python%{python3_version} \
%{buildroot}%{_prefix}/lib/python%{python3_test_version}

I need a drink :laughing:

+ mv %{buildroot}%{_prefix}/lib/python%{python3_test_version}/site-packages/%{name}-%{version}-py%{python3_version}.egg-info \

+    %{buildroot}%{_prefix}/lib/python%{python3_test_version}/site-packages/%{name}-%{version}-py%{python3_test_version}.egg-info

+ %endif

+ 

+ %files

+ %{_bindir}/%{name}*

+ %{_prefix}/lib/python%{python3_test_version}/site-packages/%{name}*

@@ -22,13 +22,15 @@ 

  # Requirements for this script:

  # - Python >= 3.6

  # - pip >= 20.0.1

- # - setuptools

+ # - setuptools >= 47.3

  # - pytest

  # - pyyaml

  # - wheel

  

  

  from pathlib import Path

+ import functools

+ import os

  import pytest

  import shlex

  import shutil
@@ -39,6 +41,7 @@ 

  

  PYTHONDISTDEPS_PATH = Path(__file__).parent / '..' / 'pythondistdeps.py'

  TEST_DATA_PATH = Path(__file__).parent / 'data' / 'scripts_pythondistdeps'

+ FAKE_PATH = TEST_DATA_PATH / '_path'

  

  

  def run_pythondistdeps(provides_params, requires_params, dist_egg_info_path, expect_failure=False):
@@ -47,10 +50,12 @@ 

      info_path = TEST_DATA_PATH / dist_egg_info_path

      files = '\n'.join(map(str, info_path.iterdir()))

  

+     environ = fake_path_pythons()

+ 

      provides = subprocess.run((sys.executable, PYTHONDISTDEPS_PATH, *shlex.split(provides_params)),

-             input=files, capture_output=True, check=False, encoding="utf-8")

+             input=files, capture_output=True, check=False, encoding="utf-8", env=environ)

      requires = subprocess.run((sys.executable, PYTHONDISTDEPS_PATH, *shlex.split(requires_params)),

-             input=files, capture_output=True, check=False, encoding="utf-8")

+             input=files, capture_output=True, check=False, encoding="utf-8", env=environ)

  

      if expect_failure:

          if provides.returncode == 0 or requires.returncode == 0:
@@ -67,6 +72,25 @@ 

          return {"provides": provides.stdout.strip(), "requires": requires.stdout.strip()}

  

  

+ @functools.lru_cache(maxsize=1)

+ def fake_path_pythons():

+     """Create fake Pythons, so when the pythondistdeps script tries to detect

+     setuptools version, it tells them ours."""

def fake_path_pythons():
"""Create fake Pythons, so when the pythondistdeps script tries to detect
setuptools version, it tells them ours."""

By "ours" you mean it detects version of the python3-setuptools package for the main Python (now 3.9)? But Why?

+     environ = os.environ.copy()

+     path = environ.get("PATH", "")

+     path = f"{FAKE_PATH}:{path}"

+     environ["PATH"] = path

+ 

+     FAKE_PATH.mkdir(exist_ok=True)

+ 

+     for ver in "2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10":

+         exe = FAKE_PATH / f"python{ver}"

+         exe.unlink(missing_ok=True)

+         exe.symlink_to(sys.executable)

+ 

+     return environ

+ 

+ 

  def load_test_data():

      """Reads the test-data.yaml and loads the test data into a dict."""

      with TEST_DATA_PATH.joinpath('test-data.yaml').open() as file:

file modified
+9
@@ -34,6 +34,10 @@ 

          dir: ./tests

          # Use update-test-sources.sh to update the test data

          run: python3 -m pytest --capture=no -vvv

+     # WARNING: This test alters the environment, keep it last:

+     - console_script:

+         dir: .

+         run: ./console_script.sh

      required_packages:

      - rpm-build

      - rpmdevtools
@@ -43,3 +47,8 @@ 

      - python3-pyyaml

      - python3-setuptools

      - python3-wheel

+     - python3.6

+     - python3.7

+     - python3.8

+     - python3.9

+     - python3.10

  • recent setuptools don't need it, they use importlib.metadata
  • for this to work, we need to detect setuptools version
  • the detection is not bulletproof, but it fallbacks to the old behavior

1 new commit added

  • fixup! Don't generate runtime dependency on setuptools for console_scripts entrypoints
2 years ago

rebased onto 8485b55

2 years ago

Build succeeded.

    if python_version < parse('3.8'):
        ...
        return [Requirement('importlib_metadata')]

Noting for discussion: For new setuptools but old Python 3.6 and 3.7, which is the better default requirement - setuptools or importlib_metadata?

While it's true that this script runs on importlib{.,_}metadata, it also very likely runs on newer version of Python. E.g. in Fedora we run this script on Python 3.9 or soon 3.10. The presence of importlib*metadata there is not a good predictor whether importlib_metadata is available for Python 3.6 and 3.7. And I would wager that in most old Python stacks, setuptools is much more likely to be present than importlib_metadata.

As discussed on IRC: The '-E' flag has questionable benefits, especially for upstream.

                deps = console_scripts_deps(dist, lower) + deps

Minor nitpick, I think the original deps.insert(0, ...) is more clear.

from importlib.metadata import PackageNotFoundError
from importlib.metadata import version as version_of

Minor nitpick: Why each import on it's own line when other imports use a multi-import style?

rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)" && exit 1 || true
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)"  && exit 1 || true

Let's print out some error before exit?

# no setuptools installed, we assume an old version of setuptools was used to prepare the data
# hence the package always requires setuptools
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)"
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)"  && exit 1 || true

So, new version of setuptools would generate importlib-metadata requires on Python 3.6 and 3.7. But since you assume Python 3.6 and 3.7 don't have setuptools in Fedora, you ignore that case?

# never run this script as root on your host OS, it is destructive

This really needs to be also on top of this file, people have brainfarts and sometimes run with sudo without checking.

# WARNING: Once we flip the rpmwheels bcond in python3.6, this assumption will break

I need more context here.

mv %{buildroot}%{_prefix}/lib/python%{python3_version} \
%{buildroot}%{_prefix}/lib/python%{python3_test_version}

I need a drink :laughing:

def fake_path_pythons():
"""Create fake Pythons, so when the pythondistdeps script tries to detect
setuptools version, it tells them ours."""

By "ours" you mean it detects version of the python3-setuptools package for the main Python (now 3.9)? But Why?

I've talked to @torsava and we decided to simplify this by using a specific command line option to the script instead. That covers most of the review comments here. I'll send a new PR shortly.

Pull-Request has been closed by churchyard

2 years ago