diff --git a/00189-add-rewheel-module.patch b/00189-add-rewheel-module.patch new file mode 100644 index 0000000..ad80e9f --- /dev/null +++ b/00189-add-rewheel-module.patch @@ -0,0 +1,224 @@ +unchanged: +--- Python-3.4.0rc3/Lib/ensurepip/__init__.py 2014-03-10 07:56:33.000000000 +0100 ++++ Python-3.4.0rc3-rewheel/Lib/ensurepip/__init__.py 2014-03-12 09:57:12.917120853 +0100 +@@ -1,8 +1,10 @@ + import os + import os.path + import pkgutil ++import shutil + import sys + import tempfile ++from ensurepip import rewheel + + + __all__ = ["version", "bootstrap"] +@@ -38,6 +40,8 @@ def _run_pip(args, additional_paths=None + + # Install the bundled software + import pip ++ if args[0] in ["install", "list", "wheel"]: ++ args.append('--pre') + pip.main(args) + + +@@ -87,20 +90,40 @@ def bootstrap(*, root=None, upgrade=Fals + # omit pip and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "install" + +- with tempfile.TemporaryDirectory() as tmpdir: +- # Put our bundled wheels into a temporary directory and construct the +- # additional paths that need added to sys.path +- additional_paths = [] ++ whls = [] ++ rewheel_dir = None ++ # try to see if we have system-wide versions of _PROJECTS ++ dep_records = rewheel.find_system_records([p[0] for p in _PROJECTS]) ++ # TODO: check if system-wide versions are the newest ones ++ # if --upgrade is used? ++ if all(dep_records): ++ # if we have all _PROJECTS installed system-wide, we'll recreate ++ # wheels from them and install those ++ rewheel_dir = tempfile.TemporaryDirectory() ++ for dr in dep_records: ++ new_whl = rewheel.rewheel_from_record(dr, rewheel_dir.name) ++ whls.append(os.path.join(rewheel_dir.name, new_whl)) ++ else: ++ # if we don't have all the _PROJECTS installed system-wide, ++ # let's just fall back to bundled wheels + for project, version in _PROJECTS: +- wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) +- whl = pkgutil.get_data( ++ whl = os.path.join( ++ os.path.dirname(__file__), + "ensurepip", +- "_bundled/{}".format(wheel_name), ++ "bundled", ++ "{}-{}-py2.py3-none-any.whl".format(project, version) + ) +- with open(os.path.join(tmpdir, wheel_name), "wb") as fp: +- fp.write(whl) ++ whls.append(whl) + +- additional_paths.append(os.path.join(tmpdir, wheel_name)) ++ with tempfile.TemporaryDirectory() as tmpdir: ++ # Put our bundled wheels into a temporary directory and construct the ++ # additional paths that need added to sys.path ++ additional_paths = [] ++ for whl in whls: ++ shutil.copy(whl, tmpdir) ++ additional_paths.append(os.path.join(tmpdir, os.path.basename(whl))) ++ if rewheel_dir: ++ rewheel_dir.cleanup() + + # Construct the arguments to be passed to the pip command + args = ["install", "--no-index", "--find-links", tmpdir] +unchanged: +--- Python-3.4.0rc3/Lib/ensurepip/rewheel/__init__.py 1970-01-01 01:00:00.000000000 +0100 ++++ Python-3.4.0rc3-rewheel/Lib/ensurepip/rewheel/__init__.py 2014-03-12 09:55:30.413152104 +0100 +@@ -0,0 +1,133 @@ ++import argparse ++import csv ++import email.parser ++import os ++import io ++import re ++import site ++import subprocess ++import sys ++import zipfile ++ ++def run(): ++ parser = argparse.ArgumentParser(description='Recreate wheel of package with given RECORD.') ++ parser.add_argument('record_path', ++ help='Path to RECORD file') ++ parser.add_argument('-o', '--output-dir', ++ help='Dir where to place the wheel, defaults to current working dir.', ++ dest='outdir', ++ default=os.path.curdir) ++ ++ ns = parser.parse_args() ++ retcode = 0 ++ try: ++ print(rewheel_from_record(**vars(ns))) ++ except BaseException as e: ++ print('Failed: {}'.format(e)) ++ retcode = 1 ++ sys.exit(1) ++ ++def find_system_records(projects): ++ """Return list of paths to RECORD files for system-installed projects. ++ ++ If a project is not installed, the resulting list contains None instead ++ of a path to its RECORD ++ """ ++ records = [] ++ # get system site-packages dirs ++ sys_sitepack = site.getsitepackages([sys.base_prefix, sys.base_exec_prefix]) ++ sys_sitepack = [sp for sp in sys_sitepack if os.path.exists(sp)] ++ # try to find all projects in all system site-packages ++ for project in projects: ++ path = None ++ for sp in sys_sitepack: ++ dist_info_re = os.path.join(sp, project) + '-[^\{0}]+\.dist-info'.format(os.sep) ++ candidates = [os.path.join(sp, p) for p in os.listdir(sp)] ++ # filter out candidate dirs based on the above regexp ++ filtered = [c for c in candidates if re.match(dist_info_re, c)] ++ # if we have 0 or 2 or more dirs, something is wrong... ++ if len(filtered) == 1: ++ path = filtered[0] ++ records.append(os.path.join(path, 'RECORD')) ++ return records ++ ++def rewheel_from_record(record_path, outdir): ++ """Recreates a whee of package with given record_path and returns path ++ to the newly created wheel.""" ++ site_dir = os.path.dirname(os.path.dirname(record_path)) ++ record_relpath = record_path[len(site_dir):].strip(os.path.sep) ++ to_write, to_omit = get_records_to_pack(site_dir, record_relpath) ++ new_wheel_name = get_wheel_name(record_path) ++ new_wheel_path = os.path.join(outdir, new_wheel_name + '.whl') ++ ++ new_wheel = zipfile.ZipFile(new_wheel_path, mode='w', compression=zipfile.ZIP_DEFLATED) ++ # we need to write a new record with just the files that we will write, ++ # e.g. not binaries and *.pyc/*.pyo files ++ new_record = io.StringIO() ++ writer = csv.writer(new_record) ++ ++ # handle files that we can write straight away ++ for f, sha_hash, size in to_write: ++ new_wheel.write(os.path.join(site_dir, f), arcname=f) ++ writer.writerow([f, sha_hash,size]) ++ ++ # rewrite the old wheel file with a new computed one ++ writer.writerow([record_relpath, '', '']) ++ new_wheel.writestr(record_relpath, new_record.getvalue()) ++ ++ new_wheel.close() ++ ++ return new_wheel.filename ++ ++def get_wheel_name(record_path): ++ """Return proper name of the wheel, without .whl.""" ++ wheel_info_path = os.path.join(os.path.dirname(record_path), 'WHEEL') ++ wheel_info = email.parser.Parser().parsestr(open(wheel_info_path).read()) ++ metadata_path = os.path.join(os.path.dirname(record_path), 'METADATA') ++ metadata = email.parser.Parser().parsestr(open(metadata_path).read()) ++ ++ # construct name parts according to wheel spec ++ distribution = metadata.get('Name') ++ version = metadata.get('Version') ++ build_tag = '' # nothing for now ++ lang_tag = [] ++ for t in wheel_info.get_all('Tag'): ++ lang_tag.append(t.split('-')[0]) ++ lang_tag = '.'.join(lang_tag) ++ abi_tag, plat_tag = wheel_info.get('Tag').split('-')[1:3] ++ # leave out build tag, if it is empty ++ to_join = filter(None, [distribution, version, build_tag, lang_tag, abi_tag, plat_tag]) ++ return '-'.join(list(to_join)) ++ ++def get_records_to_pack(site_dir, record_relpath): ++ """Accepts path of sitedir and path of RECORD file relative to it. ++ Returns two lists: ++ - list of files that can be written to new RECORD straight away ++ - list of files that shouldn't be written or need some processing ++ (pyc and pyo files, scripts) ++ """ ++ record_contents = open(os.path.join(site_dir, record_relpath)).read() ++ # temporary fix for https://github.com/pypa/pip/issues/1376 ++ # we need to ignore files under ".data" directory ++ data_dir = os.path.dirname(record_relpath).strip(os.path.sep) ++ data_dir = data_dir[:-len('dist-info')] + 'data' ++ ++ to_write = [] ++ to_omit = [] ++ for l in record_contents.splitlines(): ++ spl = l.split(',') ++ if len(spl) == 3: ++ # new record will omit (or write differently): ++ # - abs paths, paths with ".." (entry points), ++ # - pyc+pyo files ++ # - the old RECORD file ++ # TODO: is there any better way to recognize an entry point? ++ if os.path.isabs(spl[0]) or spl[0].startswith('..') or \ ++ spl[0].endswith('.pyc') or spl[0].endswith('.pyo') or \ ++ spl[0] == record_relpath or spl[0].startswith(data_dir): ++ to_omit.append(spl) ++ else: ++ to_write.append(spl) ++ else: ++ pass # bad RECORD or empty line ++ return to_write, to_omit +only in patch2: +unchanged: +--- Python-3.4.0/Makefile.pre.in 2014-04-01 12:02:48.188136172 +0200 ++++ Python-3.4.0-new/Makefile.pre.in 2014-04-01 12:03:23.770394025 +0200 +@@ -1140,7 +1140,7 @@ LIBSUBDIRS= tkinter tkinter/test tkinter + test/test_asyncio \ + collections concurrent concurrent/futures encodings \ + email email/mime test/test_email test/test_email/data \ +- ensurepip ensurepip/_bundled \ ++ ensurepip ensurepip/_bundled ensurepip/rewheel \ + html json test/test_json http dbm xmlrpc \ + sqlite3 sqlite3/test \ + logging csv wsgiref urllib \ diff --git a/00190-fix-tests-with-sqlite-3.8.4.patch b/00190-fix-tests-with-sqlite-3.8.4.patch new file mode 100644 index 0000000..8a94f5c --- /dev/null +++ b/00190-fix-tests-with-sqlite-3.8.4.patch @@ -0,0 +1,21 @@ + +# HG changeset patch +# User Benjamin Peterson +# Date 1394679139 18000 +# Node ID 4d626a9df062104b61c44c8a5be8b0fd52fae953 +# Parent 6f93ab911d5dafcde364013e21723259fe2c85a8# Parent dbc9e3ed5e9f1bd11240eaa971f6c75d6a7013b5 +merge 3.3 (#20901) + +diff --git a/Lib/sqlite3/test/hooks.py b/Lib/sqlite3/test/hooks.py +--- a/Lib/sqlite3/test/hooks.py ++++ b/Lib/sqlite3/test/hooks.py +@@ -162,7 +162,7 @@ class ProgressTests(unittest.TestCase): + create table bar (a, b) + """) + second_count = len(progress_calls) +- self.assertGreater(first_count, second_count) ++ self.assertGreaterEqual(first_count, second_count) + + def CheckCancelOperation(self): + """ + diff --git a/00193-skip-correct-num-of-pycfile-bytes-in-modulefinder.patch b/00193-skip-correct-num-of-pycfile-bytes-in-modulefinder.patch new file mode 100644 index 0000000..4a82309 --- /dev/null +++ b/00193-skip-correct-num-of-pycfile-bytes-in-modulefinder.patch @@ -0,0 +1,65 @@ + +# HG changeset patch +# User Brett Cannon +# Date 1393602285 18000 +# Node ID 432cb56db05d73f55d211501bf0dfc767768923b +# Parent ade5e4922a54cb84c99ec924ab7c700a014893da +Issue #20778: Fix modulefinder to work with bytecode-only modules. + +Bug filed and initial attempt at a patch by Bohuslav Kabrda. + +diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py +--- a/Lib/modulefinder.py ++++ b/Lib/modulefinder.py +@@ -290,7 +290,7 @@ class ModuleFinder: + if fp.read(4) != imp.get_magic(): + self.msgout(2, "raise ImportError: Bad magic number", pathname) + raise ImportError("Bad magic number in %s" % pathname) +- fp.read(4) ++ fp.read(8) # Skip mtime and size. + co = marshal.load(fp) + else: + co = None +diff --git a/Lib/test/test_modulefinder.py b/Lib/test/test_modulefinder.py +--- a/Lib/test/test_modulefinder.py ++++ b/Lib/test/test_modulefinder.py +@@ -1,5 +1,7 @@ + import os + import errno ++import importlib.machinery ++import py_compile + import shutil + import unittest + import tempfile +@@ -208,6 +210,14 @@ a/module.py + from . import * + """] + ++bytecode_test = [ ++ "a", ++ ["a"], ++ [], ++ [], ++ "" ++] ++ + + def open_file(path): + dirname = os.path.dirname(path) +@@ -288,6 +298,16 @@ class ModuleFinderTest(unittest.TestCase + def test_relative_imports_4(self): + self._do_test(relative_import_test_4) + ++ def test_bytecode(self): ++ base_path = os.path.join(TEST_DIR, 'a') ++ source_path = base_path + importlib.machinery.SOURCE_SUFFIXES[0] ++ bytecode_path = base_path + importlib.machinery.BYTECODE_SUFFIXES[0] ++ with open_file(source_path) as file: ++ file.write('testing_modulefinder = True\n') ++ py_compile.compile(source_path, cfile=bytecode_path) ++ os.remove(source_path) ++ self._do_test(bytecode_test) ++ + + def test_main(): + support.run_unittest(ModuleFinderTest) diff --git a/python3.spec b/python3.spec index 9967e52..e152006 100644 --- a/python3.spec +++ b/python3.spec @@ -2,14 +2,13 @@ # Conditionals and other variables controlling the build # ====================================================== +%global with_rewheel 0 + %global pybasever 3.4 # pybasever without the dot: %global pyshortver 34 -# prereleasetag -%global prerel rc2 - %global pylibdir %{_libdir}/python%{pybasever} %global dynload_dir %{pylibdir}/lib-dynload @@ -129,7 +128,7 @@ Summary: Version 3 of the Python programming language aka Python 3000 Name: python3 Version: %{pybasever}.0 -Release: %{?prerel:0.}1%{?prerel:.%{prerel}}%{?dist} +Release: 1%{?dist} License: Python Group: Development/Languages @@ -187,12 +186,17 @@ BuildRequires: valgrind-devel BuildRequires: xz-devel BuildRequires: zlib-devel +%if 0%{?with_rewheel} +BuildRequires: python3-setuptools +BuildRequires: python3-pip +%endif + # ======================= # Source code and patches # ======================= -Source: http://www.python.org/ftp/python/%{version}/Python-%{version}%{?prerel}.tar.xz +Source: http://www.python.org/ftp/python/%{version}/Python-%{version}.tar.xz # Avoid having various bogus auto-generated Provides lines for the various # python c modules' SONAMEs: @@ -632,6 +636,28 @@ Patch186: 00186-dont-raise-from-py_compile.patch # relying on this will fail (test_filename_changing_on_output_single_dir) Patch188: 00188-fix-lib2to3-tests-when-hashlib-doesnt-compile-properly.patch +# 00189 # +# +# Add the rewheel module, allowing to recreate wheels from already installed +# ones +# https://github.com/bkabrda/rewheel +%if 0%{with_rewheel} +Patch189: 00189-add-rewheel-module.patch +%endif + +# 00190 # +# +# Fix tests with SQLite >= 3.8.4 +# http://bugs.python.org/issue20901 +# http://hg.python.org/cpython/rev/4d626a9df062 +Patch190: 00190-fix-tests-with-sqlite-3.8.4.patch + +# 00193 +# +# Skip correct number of *.pyc file bytes in ModuleFinder.load_module +# rhbz#1060338 +# http://bugs.python.org/issue20778 +Patch193: 00193-skip-correct-num-of-pycfile-bytes-in-modulefinder.patch # (New patches go here ^^^) # @@ -895,6 +921,13 @@ done # 00187: upstream as of Python 3.4.0b1 %patch188 -p1 +%if 0%{with_rewheel} +%patch189 -p1 +%endif + +%patch190 -p1 +%patch193 -p1 + # Currently (2010-01-15), http://docs.python.org/library is for 2.6, and there # are many differences between 2.6 and the Python 3 library. # @@ -1506,6 +1539,11 @@ rm -fr %{buildroot} %{pylibdir}/ensurepip/__pycache__/*%{bytecode_suffixes} %exclude %{pylibdir}/ensurepip/_bundled +%dir %{pylibdir}/ensurepip/rewheel/ +%dir %{pylibdir}/ensurepip/rewheel/__pycache__/ +%{pylibdir}/ensurepip/rewheel/*.py +%{pylibdir}/ensurepip/rewheel/__pycache__/*%{bytecode_suffixes} + %{pylibdir}/html %{pylibdir}/http %{pylibdir}/idlelib @@ -1759,6 +1797,11 @@ rm -fr %{buildroot} # ====================================================== %changelog +* Tue Apr 15 2014 Matej Stuchlik - 3.4.0-1 +- Update to Python 3.4 final +- Add patch adding the rewheel module +- Merge patches from master + * Wed Jan 08 2014 Bohuslav Kabrda - 3.4.0-0.1.b2 - Update to Python 3.4 beta 2. - Refreshed patches: 55 (systemtap), 146 (hashlib-fips), 154 (test_gdb noise)