From 11ab52056d2258404739a62e8ba5abd1025c8a56 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Jan 26 2021 16:37:16 +0000 Subject: Backport --with-wheel-pkg-dir=PATH configure option * Always apply patch 189 * Pass--with-wheel-pkg-dir=/usr/share/python-wheels option to configure if rpmwheels is used. --- diff --git a/ensurepip.patch b/ensurepip.patch new file mode 100644 index 0000000..1cf7c87 --- /dev/null +++ b/ensurepip.patch @@ -0,0 +1,483 @@ +From 75e59a97f5d1fddb0c30ed9747b1b8cb84420a62 Mon Sep 17 00:00:00 2001 +From: Victor Stinner +Date: Wed, 20 Jan 2021 17:07:21 +0100 +Subject: [PATCH 01/30] bpo-42856: Add --with-wheel-pkg-dir=PATH configure + option (GH-24210) + +Add --with-wheel-pkg-dir=PATH option to the ./configure script. If +specified, the ensurepip module looks for setuptools and pip wheel +packages in this directory: if both are present, these wheel packages +are used instead of ensurepip bundled wheel packages. + +Some Linux distribution packaging policies recommend against bundling +dependencies. For example, Fedora installs wheel packages in the +/usr/share/python-wheels/ directory and don't install the +ensurepip._bundled package. + +ensurepip: Remove unused runpy import. +--- + Doc/library/ensurepip.rst | 6 +- + Doc/whatsnew/3.10.rst | 12 ++ + Lib/ensurepip/__init__.py | 114 ++++++++++++++---- + Lib/test/test_ensurepip.py | 69 +++++++++-- + Makefile.pre.in | 2 + + .../2021-01-07-12-51-38.bpo-42856.n3cMHV.rst | 9 ++ + configure | 28 +++++ + configure.ac | 16 +++ + 8 files changed, 220 insertions(+), 36 deletions(-) + create mode 100644 Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst + +diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst +index a5221250c4..fa1b42cf48 100644 +--- a/Doc/library/ensurepip.rst ++++ b/Doc/library/ensurepip.rst +@@ -48,7 +48,7 @@ The simplest possible invocation is:: + + This invocation will install ``pip`` if it is not already installed, + but otherwise does nothing. To ensure the installed version of ``pip`` +-is at least as recent as the one bundled with ``ensurepip``, pass the ++is at least as recent as the one available in ``ensurepip``, pass the + ``--upgrade`` option:: + + python -m ensurepip --upgrade +@@ -86,7 +86,7 @@ Module API + + .. function:: version() + +- Returns a string specifying the bundled version of pip that will be ++ Returns a string specifying the available version of pip that will be + installed when bootstrapping an environment. + + .. function:: bootstrap(root=None, upgrade=False, user=False, \ +@@ -100,7 +100,7 @@ Module API + for the current environment. + + *upgrade* indicates whether or not to upgrade an existing installation +- of an earlier version of ``pip`` to the bundled version. ++ of an earlier version of ``pip`` to the available version. + + *user* indicates whether to use the user scheme rather than installing + globally. +diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst +index 7edc552d82..7fe2b96a00 100644 +--- a/Doc/whatsnew/3.10.rst ++++ b/Doc/whatsnew/3.10.rst +@@ -621,6 +621,18 @@ Build Changes + don't build nor install test modules. + (Contributed by Xavier de Gaye, Thomas Petazzoni and Peixing Xin in :issue:`27640`.) + ++* Add ``--with-wheel-pkg-dir=PATH`` option to the ``./configure`` script. If ++ specified, the :mod:`ensurepip` module looks for ``setuptools`` and ``pip`` ++ wheel packages in this directory: if both are present, these wheel packages ++ are used instead of ensurepip bundled wheel packages. ++ ++ Some Linux distribution packaging policies recommend against bundling ++ dependencies. For example, Fedora installs wheel packages in the ++ ``/usr/share/python-wheels/`` directory and don't install the ++ ``ensurepip._bundled`` package. ++ ++ (Contributed by Victor Stinner in :issue:`42856`.) ++ + + C API Changes + ============= +diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py +index cb2882e336..2276fd7fd8 100644 +--- a/Lib/ensurepip/__init__.py ++++ b/Lib/ensurepip/__init__.py +@@ -1,27 +1,82 @@ ++import collections + import os + import os.path ++import subprocess + import sys +-import runpy ++import sysconfig + import tempfile +-import subprocess + from importlib import resources + +-from . import _bundled +- + + + __all__ = ["version", "bootstrap"] +- +- ++_PACKAGE_NAMES = ('setuptools', 'pip') + _SETUPTOOLS_VERSION = "47.1.0" +- + _PIP_VERSION = "20.2.3" +- + _PROJECTS = [ + ("setuptools", _SETUPTOOLS_VERSION, "py3"), + ("pip", _PIP_VERSION, "py2.py3"), + ] + ++# Packages bundled in ensurepip._bundled have wheel_name set. ++# Packages from WHEEL_PKG_DIR have wheel_path set. ++_Package = collections.namedtuple('Package', ++ ('version', 'wheel_name', 'wheel_path')) ++ ++# Directory of system wheel packages. Some Linux distribution packaging ++# policies recommend against bundling dependencies. For example, Fedora ++# installs wheel packages in the /usr/share/python-wheels/ directory and don't ++# install the ensurepip._bundled package. ++_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR') ++ ++ ++def _find_packages(path): ++ packages = {} ++ try: ++ filenames = os.listdir(path) ++ except OSError: ++ # Ignore: path doesn't exist or permission error ++ filenames = () ++ # Make the code deterministic if a directory contains multiple wheel files ++ # of the same package, but don't attempt to implement correct version ++ # comparison since this case should not happen. ++ filenames = sorted(filenames) ++ for filename in filenames: ++ # filename is like 'pip-20.2.3-py2.py3-none-any.whl' ++ if not filename.endswith(".whl"): ++ continue ++ for name in _PACKAGE_NAMES: ++ prefix = name + '-' ++ if filename.startswith(prefix): ++ break ++ else: ++ continue ++ ++ # Extract '20.2.2' from 'pip-20.2.2-py2.py3-none-any.whl' ++ version = filename.removeprefix(prefix).partition('-')[0] ++ wheel_path = os.path.join(path, filename) ++ packages[name] = _Package(version, None, wheel_path) ++ return packages ++ ++ ++def _get_packages(): ++ global _PACKAGES, _WHEEL_PKG_DIR ++ if _PACKAGES is not None: ++ return _PACKAGES ++ ++ packages = {} ++ for name, version, py_tag in _PROJECTS: ++ wheel_name = f"{name}-{version}-{py_tag}-none-any.whl" ++ packages[name] = _Package(version, wheel_name, None) ++ if _WHEEL_PKG_DIR: ++ dir_packages = _find_packages(_WHEEL_PKG_DIR) ++ # only used the wheel package directory if all packages are found there ++ if all(name in dir_packages for name in _PACKAGE_NAMES): ++ packages = dir_packages ++ _PACKAGES = packages ++ return packages ++_PACKAGES = None ++ + + def _run_pip(args, additional_paths=None): + # Run the bootstraping in a subprocess to avoid leaking any state that happens +@@ -42,7 +97,8 @@ def version(): + """ + Returns a string specifying the bundled version of pip. + """ +- return _PIP_VERSION ++ return _get_packages()['pip'].version ++ + + def _disable_pip_configuration_settings(): + # We deliberately ignore all pip environment variables +@@ -104,16 +160,23 @@ def _bootstrap(*, root=None, upgrade=False, user=False, + # Put our bundled wheels into a temporary directory and construct the + # additional paths that need added to sys.path + additional_paths = [] +- for project, version, py_tag in _PROJECTS: +- wheel_name = "{}-{}-{}-none-any.whl".format(project, version, py_tag) +- whl = resources.read_binary( +- _bundled, +- wheel_name, +- ) +- with open(os.path.join(tmpdir, wheel_name), "wb") as fp: ++ for name, package in _get_packages().items(): ++ if package.wheel_name: ++ # Use bundled wheel package ++ from ensurepip import _bundled ++ wheel_name = package.wheel_name ++ whl = resources.read_binary(_bundled, wheel_name) ++ else: ++ # Use the wheel package directory ++ with open(package.wheel_path, "rb") as fp: ++ whl = fp.read() ++ wheel_name = os.path.basename(package.wheel_path) ++ ++ filename = os.path.join(tmpdir, wheel_name) ++ with open(filename, "wb") as fp: + fp.write(whl) + +- additional_paths.append(os.path.join(tmpdir, wheel_name)) ++ additional_paths.append(filename) + + # Construct the arguments to be passed to the pip command + args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] +@@ -126,7 +189,7 @@ def _bootstrap(*, root=None, upgrade=False, user=False, + if verbosity: + args += ["-" + "v" * verbosity] + +- return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) ++ return _run_pip([*args, *_PACKAGE_NAMES], additional_paths) + + def _uninstall_helper(*, verbosity=0): + """Helper to support a clean default uninstall process on Windows +@@ -139,11 +202,14 @@ def _uninstall_helper(*, verbosity=0): + except ImportError: + return + +- # If the pip version doesn't match the bundled one, leave it alone +- if pip.__version__ != _PIP_VERSION: +- msg = ("ensurepip will only uninstall a matching version " +- "({!r} installed, {!r} bundled)") +- print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr) ++ # If the installed pip version doesn't match the available one, ++ # leave it alone ++ available_version = version() ++ if pip.__version__ != available_version: ++ print(f"ensurepip will only uninstall a matching version " ++ f"({pip.__version__!r} installed, " ++ f"{available_version!r} available)", ++ file=sys.stderr) + return + + _disable_pip_configuration_settings() +@@ -153,7 +219,7 @@ def _uninstall_helper(*, verbosity=0): + if verbosity: + args += ["-" + "v" * verbosity] + +- return _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) ++ return _run_pip([*args, *reversed(_PACKAGE_NAMES)]) + + + def _main(argv=None): +diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py +index 4786d28f39..bfca0cd7fb 100644 +--- a/Lib/test/test_ensurepip.py ++++ b/Lib/test/test_ensurepip.py +@@ -1,19 +1,68 @@ +-import unittest +-import unittest.mock +-import test.support ++import contextlib + import os + import os.path +-import contextlib + import sys ++import tempfile ++import test.support ++import unittest ++import unittest.mock + + import ensurepip + import ensurepip._uninstall + + +-class TestEnsurePipVersion(unittest.TestCase): ++class TestPackages(unittest.TestCase): ++ def touch(self, directory, filename): ++ fullname = os.path.join(directory, filename) ++ open(fullname, "wb").close() ++ ++ def test_version(self): ++ # Test version() ++ with tempfile.TemporaryDirectory() as tmpdir: ++ self.touch(tmpdir, "pip-1.2.3b1-py2.py3-none-any.whl") ++ self.touch(tmpdir, "setuptools-49.1.3-py3-none-any.whl") ++ with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), ++ unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)): ++ self.assertEqual(ensurepip.version(), '1.2.3b1') ++ ++ def test_get_packages_no_dir(self): ++ # Test _get_packages() without a wheel package directory ++ with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), ++ unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None)): ++ packages = ensurepip._get_packages() ++ ++ # when bundled wheel packages are used, we get _PIP_VERSION ++ self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version()) ++ ++ # use bundled wheel packages ++ self.assertIsNotNone(packages['pip'].wheel_name) ++ self.assertIsNotNone(packages['setuptools'].wheel_name) ++ ++ def test_get_packages_with_dir(self): ++ # Test _get_packages() with a wheel package directory ++ setuptools_filename = "setuptools-49.1.3-py3-none-any.whl" ++ pip_filename = "pip-20.2.2-py2.py3-none-any.whl" ++ ++ with tempfile.TemporaryDirectory() as tmpdir: ++ self.touch(tmpdir, setuptools_filename) ++ self.touch(tmpdir, pip_filename) ++ # not used, make sure that it's ignored ++ self.touch(tmpdir, "wheel-0.34.2-py2.py3-none-any.whl") ++ ++ with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), ++ unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)): ++ packages = ensurepip._get_packages() ++ ++ self.assertEqual(packages['setuptools'].version, '49.1.3') ++ self.assertEqual(packages['setuptools'].wheel_path, ++ os.path.join(tmpdir, setuptools_filename)) ++ self.assertEqual(packages['pip'].version, '20.2.2') ++ self.assertEqual(packages['pip'].wheel_path, ++ os.path.join(tmpdir, pip_filename)) ++ ++ # wheel package is ignored ++ self.assertEqual(sorted(packages), ['pip', 'setuptools']) + +- def test_returns_version(self): +- self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version()) + + class EnsurepipMixin: + +@@ -27,6 +76,8 @@ def setUp(self): + real_devnull = os.devnull + os_patch = unittest.mock.patch("ensurepip.os") + patched_os = os_patch.start() ++ # But expose os.listdir() used by _find_packages() ++ patched_os.listdir = os.listdir + self.addCleanup(os_patch.stop) + patched_os.devnull = real_devnull + patched_os.path = os.path +@@ -147,7 +198,7 @@ def test_pip_config_file_disabled(self): + self.assertEqual(self.os_environ["PIP_CONFIG_FILE"], os.devnull) + + @contextlib.contextmanager +-def fake_pip(version=ensurepip._PIP_VERSION): ++def fake_pip(version=ensurepip.version()): + if version is None: + pip = None + else: +@@ -243,7 +294,7 @@ def test_pip_config_file_disabled(self): + + # Basic testing of the main functions and their argument parsing + +-EXPECTED_VERSION_OUTPUT = "pip " + ensurepip._PIP_VERSION ++EXPECTED_VERSION_OUTPUT = "pip " + ensurepip.version() + + class TestBootstrappingMainFunction(EnsurepipMixin, unittest.TestCase): + +diff --git a/Makefile.pre.in b/Makefile.pre.in +index fa0b9d85d3..ca6b5189c7 100644 +--- a/Makefile.pre.in ++++ b/Makefile.pre.in +@@ -146,6 +146,8 @@ CONFINCLUDEDIR= $(exec_prefix)/include + PLATLIBDIR= @PLATLIBDIR@ + SCRIPTDIR= $(prefix)/$(PLATLIBDIR) + ABIFLAGS= @ABIFLAGS@ ++# Variable used by ensurepip ++WHEEL_PKG_DIR= @WHEEL_PKG_DIR@ + + # Detailed destination directories + BINLIBDEST= @BINLIBDEST@ +diff --git a/Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst b/Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst +new file mode 100644 +index 0000000000..6aab7a6e51 +--- /dev/null ++++ b/Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst +@@ -0,0 +1,9 @@ ++Add ``--with-wheel-pkg-dir=PATH`` option to the ``./configure`` script. If ++specified, the :mod:`ensurepip` module looks for ``setuptools`` and ``pip`` ++wheel packages in this directory: if both are present, these wheel packages are ++used instead of ensurepip bundled wheel packages. ++ ++Some Linux distribution packaging policies recommend against bundling ++dependencies. For example, Fedora installs wheel packages in the ++``/usr/share/python-wheels/`` directory and don't install the ++``ensurepip._bundled`` package. +diff --git a/configure b/configure +index 1d81c00c63..37ee3691bb 100755 +--- a/configure ++++ b/configure +@@ -630,6 +630,7 @@ OPENSSL_INCLUDES + ENSUREPIP + SRCDIRS + THREADHEADERS ++WHEEL_PKG_DIR + LIBPL + PY_ENABLE_SHARED + PLATLIBDIR +@@ -847,6 +848,7 @@ with_libm + with_libc + enable_big_digits + with_platlibdir ++with_wheel_pkg_dir + with_computed_gotos + with_ensurepip + with_openssl +@@ -1576,6 +1578,9 @@ Optional Packages: + system-dependent) + --with-platlibdir=DIRNAME + Python library directory name (default is "lib") ++ --with-wheel-pkg-dir=PATH ++ Directory of wheel packages used by ensurepip ++ (default: none) + --with-computed-gotos enable computed gotos in evaluation loop (enabled by + default on supported compilers) + --with-ensurepip[=install|upgrade|no] +@@ -15493,6 +15498,29 @@ else + fi + + ++# Check for --with-wheel-pkg-dir=PATH ++ ++WHEEL_PKG_DIR="" ++{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-wheel-pkg-dir" >&5 ++$as_echo_n "checking for --with-wheel-pkg-dir... " >&6; } ++ ++# Check whether --with-wheel-pkg-dir was given. ++if test "${with_wheel_pkg_dir+set}" = set; then : ++ withval=$with_wheel_pkg_dir; ++if test -n "$withval"; then ++ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 ++$as_echo "yes" >&6; } ++ WHEEL_PKG_DIR="$withval" ++else ++ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 ++$as_echo "no" >&6; } ++fi ++else ++ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 ++$as_echo "no" >&6; } ++fi ++ ++ + # Check whether right shifting a negative integer extends the sign bit + # or fills with zeros (like the Cray J90, according to Tim Peters). + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether right shift extends the sign bit" >&5 +diff --git a/configure.ac b/configure.ac +index 08c462ac9f..99077e9c3a 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -4838,6 +4838,22 @@ else + fi + AC_SUBST(LIBPL) + ++# Check for --with-wheel-pkg-dir=PATH ++AC_SUBST(WHEEL_PKG_DIR) ++WHEEL_PKG_DIR="" ++AC_MSG_CHECKING(for --with-wheel-pkg-dir) ++AC_ARG_WITH(wheel-pkg-dir, ++ AS_HELP_STRING([--with-wheel-pkg-dir=PATH], ++ [Directory of wheel packages used by ensurepip (default: none)]), ++[ ++if test -n "$withval"; then ++ AC_MSG_RESULT(yes) ++ WHEEL_PKG_DIR="$withval" ++else ++ AC_MSG_RESULT(no) ++fi], ++[AC_MSG_RESULT(no)]) ++ + # Check whether right shifting a negative integer extends the sign bit + # or fills with zeros (like the Cray J90, according to Tim Peters). + AC_MSG_CHECKING(whether right shift extends the sign bit) +-- +2.29.2 + diff --git a/python3.10.spec b/python3.10.spec index 0dbb838..fb62dda 100644 --- a/python3.10.spec +++ b/python3.10.spec @@ -17,7 +17,7 @@ URL: https://www.python.org/ %global prerel a4 %global upstream_version %{general_version}%{?prerel} Version: %{general_version}%{?prerel:~%{prerel}} -Release: 2%{?dist} +Release: 3%{?dist} License: Python @@ -269,14 +269,11 @@ Patch1: 00001-rpath.patch # See https://bugzilla.redhat.com/show_bug.cgi?id=556092 Patch111: 00111-no-static-lib.patch -# 00189 # f40d9755abf593ffd64af2e909199958c285084d -# Instead of bundled wheels, use our RPM packaged wheels -# -# We keep them in /usr/share/python-wheels -# -# Downstream only: upstream bundles -# We might eventually pursuit upstream support, but it's low prio -Patch189: 00189-use-rpm-wheels.patch +# 00189 # +# Backport --with-wheel-pkg-dir=PATH configure option +# CPython commit 75e59a97f5d1fddb0c30ed9747b1b8cb84420a62 +Patch189: ensurepip.patch + # The following versions of setuptools/pip are bundled when this patch is not applied. # The versions are written in Lib/ensurepip/__init__.py, this patch removes them. # When the bundled setuptools/pip wheel is updated, the patch no longer applies cleanly. @@ -623,17 +620,13 @@ version once Python %{pybasever} is stable. %gpgverify -k2 -s1 -d0 %autosetup -S git_am -N -n Python-%{upstream_version} -# Apply patches up to 188 -%autopatch -M 188 +# Apply patches +%autopatch %if %{with rpmwheels} -%apply_patch -q %{PATCH189} rm Lib/ensurepip/_bundled/*.whl %endif -# Apply the remaining patches -%autopatch -m 190 - # Remove all exe files to ensure we are not shipping prebuilt binaries # note that those are only used to create Microsoft Windows installers # and that functionality is broken on Linux anyway @@ -730,6 +723,9 @@ BuildPython() { --with-dtrace \ --with-lto \ --with-ssl-default-suites=openssl \ +%if %{with rpmwheels} + --with-wheel-pkg-dir=/usr/share/python-wheels \ +%endif %if %{with valgrind} --with-valgrind \ %endif @@ -1594,6 +1590,9 @@ CheckPython optimized # ====================================================== %changelog +* Tue Jan 26 2021 Victor Stinner - 3.10.0~a4-3 +- Backport --with-wheel-pkg-dir=PATH configure option. + * Fri Jan 15 2021 Charalampos Stratakis - 3.10.0~a4-2 - Compile the debug build with -O0 instead of -Og (rhbz#1818857)