unchanged: --- Python-3.4.0rc3/Lib/ensurepip/__init__.py 2014-03-10 07:56:33.000000000 +0100 +++ Python-3.4.0rc3-rewheel/Lib/ensurepip/__init__.py 2014-03-12 09:57:12.917120853 +0100 @@ -1,8 +1,10 @@ import os import os.path import pkgutil +import shutil import sys import tempfile +from ensurepip import rewheel __all__ = ["version", "bootstrap"] @@ -38,6 +40,8 @@ def _run_pip(args, additional_paths=None # Install the bundled software import pip + if args[0] in ["install", "list", "wheel"]: + args.append('--pre') pip.main(args) @@ -87,20 +90,39 @@ def bootstrap(*, root=None, upgrade=Fals # omit pip and easy_install os.environ["ENSUREPIP_OPTIONS"] = "install" - with tempfile.TemporaryDirectory() as tmpdir: - # Put our bundled wheels into a temporary directory and construct the - # additional paths that need added to sys.path - additional_paths = [] + whls = [] + rewheel_dir = None + # try to see if we have system-wide versions of _PROJECTS + dep_records = rewheel.find_system_records([p[0] for p in _PROJECTS]) + # TODO: check if system-wide versions are the newest ones + # if --upgrade is used? + if all(dep_records): + # if we have all _PROJECTS installed system-wide, we'll recreate + # wheels from them and install those + rewheel_dir = tempfile.TemporaryDirectory() + for dr in dep_records: + new_whl = rewheel.rewheel_from_record(dr, rewheel_dir.name) + whls.append(os.path.join(rewheel_dir.name, new_whl)) + else: + # if we don't have all the _PROJECTS installed system-wide, + # let's just fall back to bundled wheels for project, version in _PROJECTS: - wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) - whl = pkgutil.get_data( + whl = os.path.join( + os.path.dirname(__file__), - "ensurepip", - "_bundled/{}".format(wheel_name), + "_bundled", + "{}-{}-py2.py3-none-any.whl".format(project, version) ) - with open(os.path.join(tmpdir, wheel_name), "wb") as fp: - fp.write(whl) + whls.append(whl) - additional_paths.append(os.path.join(tmpdir, wheel_name)) + with tempfile.TemporaryDirectory() as tmpdir: + # Put our bundled wheels into a temporary directory and construct the + # additional paths that need added to sys.path + additional_paths = [] + for whl in whls: + shutil.copy(whl, tmpdir) + additional_paths.append(os.path.join(tmpdir, os.path.basename(whl))) + if rewheel_dir: + rewheel_dir.cleanup() # Construct the arguments to be passed to the pip command args = ["install", "--no-index", "--find-links", tmpdir] unchanged: --- Python-3.4.0rc3/Lib/ensurepip/rewheel/__init__.py 1970-01-01 01:00:00.000000000 +0100 +++ Python-3.4.0rc3-rewheel/Lib/ensurepip/rewheel/__init__.py 2014-03-12 09:55:30.413152104 +0100 @@ -0,0 +1,136 @@ +import argparse +import csv +import email.parser +import os +import io +import re +import site +import subprocess +import sys +import zipfile + +def run(): + parser = argparse.ArgumentParser(description='Recreate wheel of package with given RECORD.') + parser.add_argument('record_path', + help='Path to RECORD file') + parser.add_argument('-o', '--output-dir', + help='Dir where to place the wheel, defaults to current working dir.', + dest='outdir', + default=os.path.curdir) + + ns = parser.parse_args() + retcode = 0 + try: + print(rewheel_from_record(**vars(ns))) + except BaseException as e: + print('Failed: {}'.format(e)) + retcode = 1 + sys.exit(1) + +def find_system_records(projects): + """Return list of paths to RECORD files for system-installed projects. + + If a project is not installed, the resulting list contains None instead + of a path to its RECORD + """ + records = [] + # get system site-packages dirs + sys_sitepack = site.getsitepackages([sys.base_prefix, sys.base_exec_prefix]) + sys_sitepack = [sp for sp in sys_sitepack if os.path.exists(sp)] + # try to find all projects in all system site-packages + for project in projects: + path = None + for sp in sys_sitepack: + dist_info_re = os.path.join(sp, project) + '-[^\{0}]+\.dist-info'.format(os.sep) + candidates = [os.path.join(sp, p) for p in os.listdir(sp)] + # filter out candidate dirs based on the above regexp + filtered = [c for c in candidates if re.match(dist_info_re, c)] + # if we have 0 or 2 or more dirs, something is wrong... + if len(filtered) == 1: + path = filtered[0] + if path is not None: + records.append(os.path.join(path, 'RECORD')) + else: + records.append(None) + return records + +def rewheel_from_record(record_path, outdir): + """Recreates a whee of package with given record_path and returns path + to the newly created wheel.""" + site_dir = os.path.dirname(os.path.dirname(record_path)) + record_relpath = record_path[len(site_dir):].strip(os.path.sep) + to_write, to_omit = get_records_to_pack(site_dir, record_relpath) + new_wheel_name = get_wheel_name(record_path) + new_wheel_path = os.path.join(outdir, new_wheel_name + '.whl') + + new_wheel = zipfile.ZipFile(new_wheel_path, mode='w', compression=zipfile.ZIP_DEFLATED) + # we need to write a new record with just the files that we will write, + # e.g. not binaries and *.pyc/*.pyo files + new_record = io.StringIO() + writer = csv.writer(new_record) + + # handle files that we can write straight away + for f, sha_hash, size in to_write: + new_wheel.write(os.path.join(site_dir, f), arcname=f) + writer.writerow([f, sha_hash,size]) + + # rewrite the old wheel file with a new computed one + writer.writerow([record_relpath, '', '']) + new_wheel.writestr(record_relpath, new_record.getvalue()) + + new_wheel.close() + + return new_wheel.filename + +def get_wheel_name(record_path): + """Return proper name of the wheel, without .whl.""" + wheel_info_path = os.path.join(os.path.dirname(record_path), 'WHEEL') + wheel_info = email.parser.Parser().parsestr(open(wheel_info_path).read()) + metadata_path = os.path.join(os.path.dirname(record_path), 'METADATA') + metadata = email.parser.Parser().parsestr(open(metadata_path).read()) + + # construct name parts according to wheel spec + distribution = metadata.get('Name') + version = metadata.get('Version') + build_tag = '' # nothing for now + lang_tag = [] + for t in wheel_info.get_all('Tag'): + lang_tag.append(t.split('-')[0]) + lang_tag = '.'.join(lang_tag) + abi_tag, plat_tag = wheel_info.get('Tag').split('-')[1:3] + # leave out build tag, if it is empty + to_join = filter(None, [distribution, version, build_tag, lang_tag, abi_tag, plat_tag]) + return '-'.join(list(to_join)) + +def get_records_to_pack(site_dir, record_relpath): + """Accepts path of sitedir and path of RECORD file relative to it. + Returns two lists: + - list of files that can be written to new RECORD straight away + - list of files that shouldn't be written or need some processing + (pyc and pyo files, scripts) + """ + record_contents = open(os.path.join(site_dir, record_relpath)).read() + # temporary fix for https://github.com/pypa/pip/issues/1376 + # we need to ignore files under ".data" directory + data_dir = os.path.dirname(record_relpath).strip(os.path.sep) + data_dir = data_dir[:-len('dist-info')] + 'data' + + to_write = [] + to_omit = [] + for l in record_contents.splitlines(): + spl = l.split(',') + if len(spl) == 3: + # new record will omit (or write differently): + # - abs paths, paths with ".." (entry points), + # - pyc+pyo files + # - the old RECORD file + # TODO: is there any better way to recognize an entry point? + if os.path.isabs(spl[0]) or spl[0].startswith('..') or \ + spl[0].endswith('.pyc') or spl[0].endswith('.pyo') or \ + spl[0] == record_relpath or spl[0].startswith(data_dir): + to_omit.append(spl) + else: + to_write.append(spl) + else: + pass # bad RECORD or empty line + return to_write, to_omit only in patch2: unchanged: --- Python-3.4.0/Makefile.pre.in 2014-04-01 12:02:48.188136172 +0200 +++ Python-3.4.0-new/Makefile.pre.in 2014-04-01 12:03:23.770394025 +0200 @@ -1140,7 +1140,7 @@ LIBSUBDIRS= tkinter tkinter/test tkinter test/test_asyncio \ collections concurrent concurrent/futures encodings \ email email/mime test/test_email test/test_email/data \ - ensurepip ensurepip/_bundled \ + ensurepip ensurepip/_bundled ensurepip/rewheel \ html json test/test_json http dbm xmlrpc \ sqlite3 sqlite3/test \ logging csv wsgiref urllib \