diff --git a/00189-add-rewheel-module.patch b/00189-add-rewheel-module.patch deleted file mode 100644 index d8ba671..0000000 --- a/00189-add-rewheel-module.patch +++ /dev/null @@ -1,245 +0,0 @@ -diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py -index 7ff938b..8c3d062 100644 ---- a/Lib/ensurepip/__init__.py -+++ b/Lib/ensurepip/__init__.py -@@ -1,8 +1,10 @@ - import os - import os.path - import pkgutil -+import shutil - import sys - import tempfile -+from ensurepip import rewheel - - - __all__ = ["version", "bootstrap"] -@@ -24,8 +26,15 @@ def _run_pip(args, additional_paths=None): - sys.path = additional_paths + sys.path - - # Install the bundled software -- import pip._internal -- return pip._internal.main(args) -+ try: -+ # pip 10 -+ from pip._internal import main -+ except ImportError: -+ # pip 9 -+ from pip import main -+ if args[0] in ["install", "list", "wheel"]: -+ args.append('--pre') -+ return main(args) - - - def version(): -@@ -88,20 +97,39 @@ def _bootstrap(*, root=None, upgrade=False, user=False, - # omit pip and easy_install - os.environ["ENSUREPIP_OPTIONS"] = "install" - -+ whls = [] -+ rewheel_dir = None -+ # try to see if we have system-wide versions of _PROJECTS -+ dep_records = rewheel.find_system_records([p[0] for p in _PROJECTS]) -+ # TODO: check if system-wide versions are the newest ones -+ # if --upgrade is used? -+ if all(dep_records): -+ # if we have all _PROJECTS installed system-wide, we'll recreate -+ # wheels from them and install those -+ rewheel_dir = tempfile.TemporaryDirectory() -+ for dr in dep_records: -+ new_whl = rewheel.rewheel_from_record(dr, rewheel_dir.name) -+ whls.append(os.path.join(rewheel_dir.name, new_whl)) -+ else: -+ # if we don't have all the _PROJECTS installed system-wide, -+ # let's just fall back to bundled wheels -+ for project, version in _PROJECTS: -+ whl = os.path.join( -+ os.path.dirname(__file__), -+ "_bundled", -+ "{}-{}-py2.py3-none-any.whl".format(project, version) -+ ) -+ whls.append(whl) -+ - with tempfile.TemporaryDirectory() as tmpdir: - # Put our bundled wheels into a temporary directory and construct the - # additional paths that need added to sys.path - additional_paths = [] -- for project, version in _PROJECTS: -- wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) -- whl = pkgutil.get_data( -- "ensurepip", -- "_bundled/{}".format(wheel_name), -- ) -- with open(os.path.join(tmpdir, wheel_name), "wb") as fp: -- fp.write(whl) -- -- additional_paths.append(os.path.join(tmpdir, wheel_name)) -+ for whl in whls: -+ shutil.copy(whl, tmpdir) -+ additional_paths.append(os.path.join(tmpdir, os.path.basename(whl))) -+ if rewheel_dir: -+ rewheel_dir.cleanup() - - # Construct the arguments to be passed to the pip command - args = ["install", "--no-index", "--find-links", tmpdir] -diff --git a/Lib/ensurepip/rewheel/__init__.py b/Lib/ensurepip/rewheel/__init__.py -new file mode 100644 -index 0000000..753c764 ---- /dev/null -+++ b/Lib/ensurepip/rewheel/__init__.py -@@ -0,0 +1,143 @@ -+import argparse -+import codecs -+import csv -+import email.parser -+import os -+import io -+import re -+import site -+import subprocess -+import sys -+import zipfile -+ -+def run(): -+ parser = argparse.ArgumentParser(description='Recreate wheel of package with given RECORD.') -+ parser.add_argument('record_path', -+ help='Path to RECORD file') -+ parser.add_argument('-o', '--output-dir', -+ help='Dir where to place the wheel, defaults to current working dir.', -+ dest='outdir', -+ default=os.path.curdir) -+ -+ ns = parser.parse_args() -+ retcode = 0 -+ try: -+ print(rewheel_from_record(**vars(ns))) -+ except BaseException as e: -+ print('Failed: {}'.format(e)) -+ retcode = 1 -+ sys.exit(1) -+ -+def find_system_records(projects): -+ """Return list of paths to RECORD files for system-installed projects. -+ -+ If a project is not installed, the resulting list contains None instead -+ of a path to its RECORD -+ """ -+ records = [] -+ # get system site-packages dirs -+ sys_sitepack = site.getsitepackages([sys.base_prefix, sys.base_exec_prefix]) -+ sys_sitepack = [sp for sp in sys_sitepack if os.path.exists(sp)] -+ # try to find all projects in all system site-packages -+ for project in projects: -+ path = None -+ for sp in sys_sitepack: -+ dist_info_re = os.path.join(sp, project) + r'-[^\{0}]+\.dist-info'.format(os.sep) -+ candidates = [os.path.join(sp, p) for p in os.listdir(sp)] -+ # filter out candidate dirs based on the above regexp -+ filtered = [c for c in candidates if re.match(dist_info_re, c)] -+ # if we have 0 or 2 or more dirs, something is wrong... -+ if len(filtered) == 1: -+ path = filtered[0] -+ if path is not None: -+ records.append(os.path.join(path, 'RECORD')) -+ else: -+ records.append(None) -+ return records -+ -+def rewheel_from_record(record_path, outdir): -+ """Recreates a whee of package with given record_path and returns path -+ to the newly created wheel.""" -+ site_dir = os.path.dirname(os.path.dirname(record_path)) -+ record_relpath = record_path[len(site_dir):].strip(os.path.sep) -+ to_write, to_omit = get_records_to_pack(site_dir, record_relpath) -+ new_wheel_name = get_wheel_name(record_path) -+ new_wheel_path = os.path.join(outdir, new_wheel_name + '.whl') -+ -+ new_wheel = zipfile.ZipFile(new_wheel_path, mode='w', compression=zipfile.ZIP_DEFLATED) -+ # we need to write a new record with just the files that we will write, -+ # e.g. not binaries and *.pyc/*.pyo files -+ new_record = io.StringIO() -+ writer = csv.writer(new_record) -+ -+ # handle files that we can write straight away -+ for f, sha_hash, size in to_write: -+ new_wheel.write(os.path.join(site_dir, f), arcname=f) -+ writer.writerow([f, sha_hash,size]) -+ -+ # rewrite the old wheel file with a new computed one -+ writer.writerow([record_relpath, '', '']) -+ new_wheel.writestr(record_relpath, new_record.getvalue()) -+ -+ new_wheel.close() -+ -+ return new_wheel.filename -+ -+def get_wheel_name(record_path): -+ """Return proper name of the wheel, without .whl.""" -+ -+ wheel_info_path = os.path.join(os.path.dirname(record_path), 'WHEEL') -+ with codecs.open(wheel_info_path, encoding='utf-8') as wheel_info_file: -+ wheel_info = email.parser.Parser().parsestr(wheel_info_file.read()) -+ -+ metadata_path = os.path.join(os.path.dirname(record_path), 'METADATA') -+ with codecs.open(metadata_path, encoding='utf-8') as metadata_file: -+ metadata = email.parser.Parser().parsestr(metadata_file.read()) -+ -+ # construct name parts according to wheel spec -+ distribution = metadata.get('Name') -+ version = metadata.get('Version') -+ build_tag = '' # nothing for now -+ lang_tag = [] -+ for t in wheel_info.get_all('Tag'): -+ lang_tag.append(t.split('-')[0]) -+ lang_tag = '.'.join(lang_tag) -+ abi_tag, plat_tag = wheel_info.get('Tag').split('-')[1:3] -+ # leave out build tag, if it is empty -+ to_join = filter(None, [distribution, version, build_tag, lang_tag, abi_tag, plat_tag]) -+ return '-'.join(list(to_join)) -+ -+def get_records_to_pack(site_dir, record_relpath): -+ """Accepts path of sitedir and path of RECORD file relative to it. -+ Returns two lists: -+ - list of files that can be written to new RECORD straight away -+ - list of files that shouldn't be written or need some processing -+ (pyc and pyo files, scripts) -+ """ -+ record_file_path = os.path.join(site_dir, record_relpath) -+ with codecs.open(record_file_path, encoding='utf-8') as record_file: -+ record_contents = record_file.read() -+ # temporary fix for https://github.com/pypa/pip/issues/1376 -+ # we need to ignore files under ".data" directory -+ data_dir = os.path.dirname(record_relpath).strip(os.path.sep) -+ data_dir = data_dir[:-len('dist-info')] + 'data' -+ -+ to_write = [] -+ to_omit = [] -+ for l in record_contents.splitlines(): -+ spl = l.split(',') -+ if len(spl) == 3: -+ # new record will omit (or write differently): -+ # - abs paths, paths with ".." (entry points), -+ # - pyc+pyo files -+ # - the old RECORD file -+ # TODO: is there any better way to recognize an entry point? -+ if os.path.isabs(spl[0]) or spl[0].startswith('..') or \ -+ spl[0].endswith('.pyc') or spl[0].endswith('.pyo') or \ -+ spl[0] == record_relpath or spl[0].startswith(data_dir): -+ to_omit.append(spl) -+ else: -+ to_write.append(spl) -+ else: -+ pass # bad RECORD or empty line -+ return to_write, to_omit -diff --git a/Makefile.pre.in b/Makefile.pre.in -index d07b312..1c6720e 100644 ---- a/Makefile.pre.in -+++ b/Makefile.pre.in -@@ -1301,7 +1301,7 @@ LIBSUBDIRS= tkinter tkinter/test tkinter/test/test_tkinter \ - test/test_asyncio \ - collections concurrent concurrent/futures encodings \ - email email/mime test/test_email test/test_email/data \ -- ensurepip ensurepip/_bundled \ -+ ensurepip ensurepip/_bundled ensurepip/rewheel \ - html json test/test_json http dbm xmlrpc \ - sqlite3 sqlite3/test \ - logging csv wsgiref urllib \ diff --git a/00189-use-rpm-wheels.patch b/00189-use-rpm-wheels.patch new file mode 100644 index 0000000..cc2719b --- /dev/null +++ b/00189-use-rpm-wheels.patch @@ -0,0 +1,51 @@ +diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py +index 4748ba4..fc02255 100644 +--- a/Lib/ensurepip/__init__.py ++++ b/Lib/ensurepip/__init__.py +@@ -1,16 +1,27 @@ ++import distutils.version ++import glob + import os + import os.path +-import pkgutil + import sys + import tempfile + + + __all__ = ["version", "bootstrap"] + ++_WHEEL_DIR = "/usr/share/python-wheels/" + +-_SETUPTOOLS_VERSION = "39.0.1" + +-_PIP_VERSION = "10.0.1" ++def _get_most_recent_wheel_version(pkg): ++ prefix = os.path.join(_WHEEL_DIR, "{}-".format(pkg)) ++ suffix = "-py2.py3-none-any.whl" ++ pattern = "{}*{}".format(prefix, suffix) ++ versions = (p[len(prefix):-len(suffix)] for p in glob.glob(pattern)) ++ return str(max(versions, key=distutils.version.LooseVersion)) ++ ++ ++_SETUPTOOLS_VERSION = _get_most_recent_wheel_version("setuptools") ++ ++_PIP_VERSION = _get_most_recent_wheel_version("pip") + + _PROJECTS = [ + ("setuptools", _SETUPTOOLS_VERSION), +@@ -94,12 +105,9 @@ def _bootstrap(*, root=None, upgrade=False, user=False, + additional_paths = [] + for project, version in _PROJECTS: + wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) +- whl = pkgutil.get_data( +- "ensurepip", +- "_bundled/{}".format(wheel_name), +- ) +- with open(os.path.join(tmpdir, wheel_name), "wb") as fp: +- fp.write(whl) ++ with open(os.path.join(_WHEEL_DIR, wheel_name), "rb") as sfp: ++ with open(os.path.join(tmpdir, wheel_name), "wb") as fp: ++ fp.write(sfp.read()) + + additional_paths.append(os.path.join(tmpdir, wheel_name)) + diff --git a/python3.spec b/python3.spec index 3661213..6e87de8 100644 --- a/python3.spec +++ b/python3.spec @@ -14,7 +14,7 @@ URL: https://www.python.org/ # WARNING When rebasing to a new Python version, # remember to update the python3-docs package as well Version: %{pybasever}.0 -Release: 5%{?dist} +Release: 6%{?dist} License: Python @@ -33,6 +33,9 @@ License: Python # WARNING: This does not change the package name and summary above %bcond_with flatpackage +# Whether to use RPM build wheels from the python-{pip,setuptools}-wheel package +# Uses upstream bundled prebuilt wheels otherwise +%bcond_without rpmwheels # Expensive optimizations (mainly, profile-guided optimizations) %ifarch %{ix86} x86_64 @@ -46,13 +49,6 @@ License: Python # Run the test suite in %%check %bcond_without tests -# Ability to reuse RPM-installed pip using rewheel -%if %{with flatpackage} -%bcond_with rewheel -%else -%bcond_without rewheel -%endif - # Extra build for debugging the interpreter or C-API extensions # (the -debug subpackages) %if %{with flatpackage} @@ -190,9 +186,9 @@ BuildRequires: /usr/bin/dtrace # workaround http://bugs.python.org/issue19804 (test_uuid requires ifconfig) BuildRequires: /usr/sbin/ifconfig -%if %{with rewheel} -BuildRequires: python3-setuptools -BuildRequires: python3-pip +%if %{with rpmwheels} +BuildRequires: python-setuptools-wheel +BuildRequires: python-pip-wheel %endif @@ -283,10 +279,9 @@ Patch170: 00170-gc-assertions.patch Patch178: 00178-dont-duplicate-flags-in-sysconfig.patch # 00189 # -# Add the rewheel module, allowing to recreate wheels from already installed -# ones -# https://github.com/bkabrda/rewheel -Patch189: 00189-add-rewheel-module.patch +# Instead of bundled wheels, use our RPM packaged wheels from +# /usr/share/python-wheels +Patch189: 00189-use-rpm-wheels.patch # 00205 # # LIBPL variable in makefile takes LIBPL from configure.ac @@ -363,11 +358,6 @@ Obsoletes: python%{pyshortver} %global platpyver 3.6.2-20 Obsoletes: platform-python < %{platpyver} -%if %{with rewheel} -Requires: python3-setuptools -Requires: python3-pip -%endif - # This prevents ALL subpackages built from this spec to require # /usr/bin/python3*. Granularity per subpackage is impossible. # It's intended for the libs package not to drag in the interpreter, see @@ -414,6 +404,14 @@ Requires: glibc%{?_isa} >= 2.24.90-26 Requires: gdbm-libs%{?_isa} >= 1:1.13 %endif +%if %{with rpmwheels} +Requires: python-setuptools-wheel +Requires: python-pip-wheel +%else +Provides: bundled(python3-pip) = 10.0.1 +Provides: bundled(python3-setuptools) = 39.0.1 +%endif + # There are files in the standard library that have python shebang. # We've filtered the automatic requirement out so libs are installable without # the main package. This however makes it pulled in by default. @@ -443,12 +441,7 @@ Requires: %{name}-libs%{?_isa} = %{version}-%{release} BuildRequires: python-rpm-macros Requires: python-rpm-macros Requires: python3-rpm-macros - -%if %{with rewheel} -# without rewheel is used to bootstrap setuptools+pip -# python3-rpm-generators needs python3-setuptools, so we cannot have it yet Requires: python3-rpm-generators -%endif # https://bugzilla.redhat.com/show_bug.cgi?id=1217376 # https://bugzilla.redhat.com/show_bug.cgi?id=1496757 @@ -579,9 +572,13 @@ Requires: redhat-rpm-config %global __requires_exclude ^python\\(abi\\) = 3\\..$ %global __provides_exclude ^python\\(abi\\) = 3\\..$ -# We keep those inside on purpose +%if %{with rpmwheels} +Requires: python-setuptools-wheel +Requires: python-pip-wheel +%else Provides: bundled(python3-pip) = 10.0.1 Provides: bundled(python3-setuptools) = 39.0.1 +%endif # The description for the flat package %description @@ -604,11 +601,6 @@ version once Python %{pybasever} is stable. # Remove bundled libraries to ensure that we're using the system copy. rm -r Modules/expat -%if %{with rewheel} -%global pip_version %(pip3 --version | cut -d' ' -f2) -sed -r -i s/'_PIP_VERSION = "[0-9.]+"'/'_PIP_VERSION = "%{pip_version}"'/ Lib/ensurepip/__init__.py -%endif - # # Apply patches: # @@ -625,8 +617,9 @@ sed -r -i s/'_PIP_VERSION = "[0-9.]+"'/'_PIP_VERSION = "%{pip_version}"'/ Lib/en %patch170 -p1 %patch178 -p1 -%if %{with rewheel} +%if %{with rpmwheels} %patch189 -p1 +rm Lib/ensurepip/_bundled/*.whl %endif %patch205 -p1 @@ -1107,20 +1100,13 @@ CheckPython optimized %{pylibdir}/ensurepip/*.py %{pylibdir}/ensurepip/__pycache__/*%{bytecode_suffixes} -%if %{without flatpackage} +%if %{with rpmwheels} %exclude %{pylibdir}/ensurepip/_bundled %else %dir %{pylibdir}/ensurepip/_bundled %{pylibdir}/ensurepip/_bundled/*.whl %endif -%if %{with rewheel} -%dir %{pylibdir}/ensurepip/rewheel/ -%dir %{pylibdir}/ensurepip/rewheel/__pycache__/ -%{pylibdir}/ensurepip/rewheel/*.py -%{pylibdir}/ensurepip/rewheel/__pycache__/*%{bytecode_suffixes} -%endif - %dir %{pylibdir}/test/ %dir %{pylibdir}/test/__pycache__/ %dir %{pylibdir}/test/support/ @@ -1520,6 +1506,9 @@ CheckPython optimized # ====================================================== %changelog +* Wed Aug 15 2018 Miro Hrončok - 3.7.0-6 +- Use RPM built wheels of pip and setuptools in ensurepip instead of our rewheel patch + * Fri Aug 10 2018 Igor Gnatenko - 3.7.0-5 - Fix wrong requirement on gdbm