churchyard / rpms / python2

Forked from rpms/python2 6 years ago
Clone
f9138c2
diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py
f9138c2
index 14c9adb..e20104e 100644
f9138c2
--- a/Lib/ensurepip/__init__.py
f9138c2
+++ b/Lib/ensurepip/__init__.py
f9138c2
@@ -7,6 +7,7 @@ import pkgutil
f9138c2
 import shutil
f9138c2
 import sys
f9138c2
 import tempfile
f9138c2
+from ensurepip import rewheel
f9138c2
f9138c2
f9138c2
 __all__ = ["version", "bootstrap"]
f9138c2
@@ -43,6 +44,8 @@ def _run_pip(args, additional_paths=None):
f9138c2
f9138c2
     # Install the bundled software
f9138c2
     import pip
f9138c2
+    if args[0] in ["install", "list", "wheel"]:
f9138c2
+        args.append('--pre')
f9138c2
     pip.main(args)
f9138c2
f9138c2
f9138c2
@@ -93,21 +96,40 @@ def bootstrap(root=None, upgrade=False, user=False,
f9138c2
         # omit pip and easy_install
f9138c2
         os.environ["ENSUREPIP_OPTIONS"] = "install"
f9138c2
f9138c2
+    whls = []
f9138c2
+    rewheel_dir = None
f9138c2
+    # try to see if we have system-wide versions of _PROJECTS
f9138c2
+    dep_records = rewheel.find_system_records([p[0] for p in _PROJECTS])
f9138c2
+    # TODO: check if system-wide versions are the newest ones
f9138c2
+    # if --upgrade is used?
f9138c2
+    if all(dep_records):
f9138c2
+        # if we have all _PROJECTS installed system-wide, we'll recreate
f9138c2
+        # wheels from them and install those
f9138c2
+        rewheel_dir = tempfile.mkdtemp()
f9138c2
+        for dr in dep_records:
f9138c2
+            new_whl = rewheel.rewheel_from_record(dr, rewheel_dir)
f9138c2
+            whls.append(os.path.join(rewheel_dir, new_whl))
f9138c2
+    else:
f9138c2
+        # if we don't have all the _PROJECTS installed system-wide,
f9138c2
+        # let's just fall back to bundled wheels
f9138c2
+        for project, version in _PROJECTS:
f9138c2
+            whl = os.path.join(
f9138c2
+                os.path.dirname(__file__),
f9138c2
+                "_bundled",
f9138c2
+                "{}-{}-py2.py3-none-any.whl".format(project, version)
f9138c2
+            )
f9138c2
+            whls.append(whl)
f9138c2
+
f9138c2
     tmpdir = tempfile.mkdtemp()
f9138c2
     try:
f9138c2
         # Put our bundled wheels into a temporary directory and construct the
f9138c2
         # additional paths that need added to sys.path
f9138c2
         additional_paths = []
f9138c2
-        for project, version in _PROJECTS:
f9138c2
-            wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
f9138c2
-            whl = pkgutil.get_data(
f9138c2
-                "ensurepip",
f9138c2
-                "_bundled/{}".format(wheel_name),
f9138c2
-            )
f9138c2
-            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
f9138c2
-                fp.write(whl)
f9138c2
-
f9138c2
-            additional_paths.append(os.path.join(tmpdir, wheel_name))
f9138c2
+        for whl in whls:
f9138c2
+            shutil.copy(whl, tmpdir)
f9138c2
+            additional_paths.append(os.path.join(tmpdir, os.path.basename(whl)))
f9138c2
+        if rewheel_dir:
f9138c2
+            shutil.rmtree(rewheel_dir)
f9138c2
f9138c2
         # Construct the arguments to be passed to the pip command
f9138c2
         args = ["install", "--no-index", "--find-links", tmpdir]
f9138c2
diff --git a/Lib/ensurepip/rewheel/__init__.py b/Lib/ensurepip/rewheel/__init__.py
f9138c2
new file mode 100644
f9138c2
index 0000000..75c2094
f9138c2
--- /dev/null
f9138c2
+++ b/Lib/ensurepip/rewheel/__init__.py
f9138c2
@@ -0,0 +1,158 @@
Matej Stuchlik b1e5a42
+import argparse
Matej Stuchlik b1e5a42
+import codecs
Matej Stuchlik b1e5a42
+import csv
Matej Stuchlik b1e5a42
+import email.parser
Matej Stuchlik b1e5a42
+import os
Matej Stuchlik b1e5a42
+import io
Matej Stuchlik b1e5a42
+import re
Matej Stuchlik b1e5a42
+import site
Matej Stuchlik b1e5a42
+import subprocess
Matej Stuchlik b1e5a42
+import sys
Matej Stuchlik b1e5a42
+import zipfile
Matej Stuchlik b1e5a42
+
Matej Stuchlik b1e5a42
+def run():
Matej Stuchlik b1e5a42
+    parser = argparse.ArgumentParser(description='Recreate wheel of package with given RECORD.')
Matej Stuchlik b1e5a42
+    parser.add_argument('record_path',
Matej Stuchlik b1e5a42
+                        help='Path to RECORD file')
Matej Stuchlik b1e5a42
+    parser.add_argument('-o', '--output-dir',
Matej Stuchlik b1e5a42
+                        help='Dir where to place the wheel, defaults to current working dir.',
Matej Stuchlik b1e5a42
+                        dest='outdir',
Matej Stuchlik b1e5a42
+                        default=os.path.curdir)
Matej Stuchlik b1e5a42
+
Matej Stuchlik b1e5a42
+    ns = parser.parse_args()
Matej Stuchlik b1e5a42
+    retcode = 0
Matej Stuchlik b1e5a42
+    try:
Matej Stuchlik b1e5a42
+        print(rewheel_from_record(**vars(ns)))
Matej Stuchlik b1e5a42
+    except BaseException as e:
Matej Stuchlik b1e5a42
+        print('Failed: {}'.format(e))
Matej Stuchlik b1e5a42
+        retcode = 1
Matej Stuchlik b1e5a42
+    sys.exit(1)
Matej Stuchlik b1e5a42
+
Matej Stuchlik b1e5a42
+def find_system_records(projects):
Matej Stuchlik b1e5a42
+    """Return list of paths to RECORD files for system-installed projects.
Matej Stuchlik b1e5a42
+
Matej Stuchlik b1e5a42
+    If a project is not installed, the resulting list contains None instead
Matej Stuchlik b1e5a42
+    of a path to its RECORD
Matej Stuchlik b1e5a42
+    """
Matej Stuchlik b1e5a42
+    records = []
Matej Stuchlik b1e5a42
+    # get system site-packages dirs
Robert Kuska 7af76a5
+    if hasattr(sys, 'real_prefix'):
Robert Kuska 7af76a5
+        #we are in python2 virtualenv and sys.real_prefix is the original sys.prefix
Robert Kuska 7af76a5
+        _orig_prefixes = site.PREFIXES
Robert Kuska 7af76a5
+        setattr(site, 'PREFIXES', [sys.real_prefix]*2)
Robert Kuska 7af76a5
+        sys_sitepack = site.getsitepackages()
Robert Kuska 7af76a5
+        setattr(site, 'PREFIXES', _orig_prefixes)
Robert Kuska 7af76a5
+    elif hasattr(sys, 'base_prefix'): # python3 venv doesn't inject real_prefix to sys
Robert Kuska 7af76a5
+        # we are on python3 and base(_exec)_prefix is unchanged in venv
Robert Kuska 7af76a5
+        sys_sitepack = site.getsitepackages([sys.base_prefix, sys.base_exec_prefix])
Robert Kuska 7af76a5
+    else:
Robert Kuska 7af76a5
+        # we are in python2 without virtualenv
Robert Kuska 7af76a5
+        sys_sitepack = site.getsitepackages()
Robert Kuska 7af76a5
+
Matej Stuchlik b1e5a42
+    sys_sitepack = [sp for sp in sys_sitepack if os.path.exists(sp)]
Matej Stuchlik b1e5a42
+    # try to find all projects in all system site-packages
Matej Stuchlik b1e5a42
+    for project in projects:
Matej Stuchlik b1e5a42
+        path = None
Matej Stuchlik b1e5a42
+        for sp in sys_sitepack:
Matej Stuchlik b1e5a42
+            dist_info_re = os.path.join(sp, project) + '-[^\{0}]+\.dist-info'.format(os.sep)
Matej Stuchlik b1e5a42
+            candidates = [os.path.join(sp, p) for p in os.listdir(sp)]
Matej Stuchlik b1e5a42
+            # filter out candidate dirs based on the above regexp
Matej Stuchlik b1e5a42
+            filtered = [c for c in candidates if re.match(dist_info_re, c)]
Matej Stuchlik b1e5a42
+            # if we have 0 or 2 or more dirs, something is wrong...
Matej Stuchlik b1e5a42
+            if len(filtered) == 1:
Matej Stuchlik b1e5a42
+                path = filtered[0]
Matej Stuchlik b1e5a42
+        if path is not None:
Matej Stuchlik b1e5a42
+            records.append(os.path.join(path, 'RECORD'))
Matej Stuchlik b1e5a42
+        else:
Matej Stuchlik b1e5a42
+            records.append(None)
Matej Stuchlik b1e5a42
+    return records
Matej Stuchlik b1e5a42
+
Matej Stuchlik b1e5a42
+def rewheel_from_record(record_path, outdir):
Matej Stuchlik b1e5a42
+    """Recreates a whee of package with given record_path and returns path
Matej Stuchlik b1e5a42
+    to the newly created wheel."""
Matej Stuchlik b1e5a42
+    site_dir = os.path.dirname(os.path.dirname(record_path))
Matej Stuchlik b1e5a42
+    record_relpath = record_path[len(site_dir):].strip(os.path.sep)
Matej Stuchlik b1e5a42
+    to_write, to_omit = get_records_to_pack(site_dir, record_relpath)
Matej Stuchlik b1e5a42
+    new_wheel_name = get_wheel_name(record_path)
Matej Stuchlik b1e5a42
+    new_wheel_path = os.path.join(outdir, new_wheel_name + '.whl')
Matej Stuchlik b1e5a42
+
Matej Stuchlik b1e5a42
+    new_wheel = zipfile.ZipFile(new_wheel_path, mode='w', compression=zipfile.ZIP_DEFLATED)
Matej Stuchlik b1e5a42
+    # we need to write a new record with just the files that we will write,
Matej Stuchlik b1e5a42
+    # e.g. not binaries and *.pyc/*.pyo files
Robert Kuska 7af76a5
+    if sys.version_info[0] < 3:
Robert Kuska 7af76a5
+        new_record = io.BytesIO()
Robert Kuska 7af76a5
+    else:
Robert Kuska 7af76a5
+        new_record = io.StringIO()
Matej Stuchlik b1e5a42
+    writer = csv.writer(new_record)
Matej Stuchlik b1e5a42
+
Matej Stuchlik b1e5a42
+    # handle files that we can write straight away
Matej Stuchlik b1e5a42
+    for f, sha_hash, size in to_write:
Matej Stuchlik b1e5a42
+        new_wheel.write(os.path.join(site_dir, f), arcname=f)
Matej Stuchlik b1e5a42
+        writer.writerow([f, sha_hash,size])
Matej Stuchlik b1e5a42
+
Matej Stuchlik b1e5a42
+    # rewrite the old wheel file with a new computed one
Matej Stuchlik b1e5a42
+    writer.writerow([record_relpath, '', ''])
Matej Stuchlik b1e5a42
+    new_wheel.writestr(record_relpath, new_record.getvalue())
Matej Stuchlik b1e5a42
+
Matej Stuchlik b1e5a42
+    new_wheel.close()
Matej Stuchlik b1e5a42
+
Matej Stuchlik b1e5a42
+    return new_wheel.filename
Matej Stuchlik b1e5a42
+
Matej Stuchlik b1e5a42
+def get_wheel_name(record_path):
Matej Stuchlik b1e5a42
+    """Return proper name of the wheel, without .whl."""
Matej Stuchlik b1e5a42
+
Matej Stuchlik b1e5a42
+    wheel_info_path = os.path.join(os.path.dirname(record_path), 'WHEEL')
Matej Stuchlik b1e5a42
+    with codecs.open(wheel_info_path, encoding='utf-8') as wheel_info_file:
Robert Kuska 7af76a5
+        wheel_info = email.parser.Parser().parsestr(wheel_info_file.read().encode('utf-8'))
Matej Stuchlik b1e5a42
+
Matej Stuchlik b1e5a42
+    metadata_path = os.path.join(os.path.dirname(record_path), 'METADATA')
Matej Stuchlik b1e5a42
+    with codecs.open(metadata_path, encoding='utf-8') as metadata_file:
Robert Kuska 7af76a5
+        metadata = email.parser.Parser().parsestr(metadata_file.read().encode('utf-8'))
Matej Stuchlik b1e5a42
+
Matej Stuchlik b1e5a42
+    # construct name parts according to wheel spec
Matej Stuchlik b1e5a42
+    distribution = metadata.get('Name')
Matej Stuchlik b1e5a42
+    version = metadata.get('Version')
Matej Stuchlik b1e5a42
+    build_tag = '' # nothing for now
Matej Stuchlik b1e5a42
+    lang_tag = []
Matej Stuchlik b1e5a42
+    for t in wheel_info.get_all('Tag'):
Matej Stuchlik b1e5a42
+        lang_tag.append(t.split('-')[0])
Matej Stuchlik b1e5a42
+    lang_tag = '.'.join(lang_tag)
Matej Stuchlik b1e5a42
+    abi_tag, plat_tag = wheel_info.get('Tag').split('-')[1:3]
Matej Stuchlik b1e5a42
+    # leave out build tag, if it is empty
Matej Stuchlik b1e5a42
+    to_join = filter(None, [distribution, version, build_tag, lang_tag, abi_tag, plat_tag])
Matej Stuchlik b1e5a42
+    return '-'.join(list(to_join))
Matej Stuchlik b1e5a42
+
Matej Stuchlik b1e5a42
+def get_records_to_pack(site_dir, record_relpath):
Matej Stuchlik b1e5a42
+    """Accepts path of sitedir and path of RECORD file relative to it.
Matej Stuchlik b1e5a42
+    Returns two lists:
Matej Stuchlik b1e5a42
+    - list of files that can be written to new RECORD straight away
Matej Stuchlik b1e5a42
+    - list of files that shouldn't be written or need some processing
Matej Stuchlik b1e5a42
+      (pyc and pyo files, scripts)
Matej Stuchlik b1e5a42
+    """
Matej Stuchlik b1e5a42
+    record_file_path = os.path.join(site_dir, record_relpath)
Matej Stuchlik b1e5a42
+    with codecs.open(record_file_path, encoding='utf-8') as record_file:
Matej Stuchlik b1e5a42
+        record_contents = record_file.read()
Matej Stuchlik b1e5a42
+    # temporary fix for https://github.com/pypa/pip/issues/1376
Matej Stuchlik b1e5a42
+    # we need to ignore files under ".data" directory
Matej Stuchlik b1e5a42
+    data_dir = os.path.dirname(record_relpath).strip(os.path.sep)
Matej Stuchlik b1e5a42
+    data_dir = data_dir[:-len('dist-info')] + 'data'
Matej Stuchlik b1e5a42
+
Matej Stuchlik b1e5a42
+    to_write = []
Matej Stuchlik b1e5a42
+    to_omit = []
Matej Stuchlik b1e5a42
+    for l in record_contents.splitlines():
Matej Stuchlik b1e5a42
+        spl = l.split(',')
Matej Stuchlik b1e5a42
+        if len(spl) == 3:
Matej Stuchlik b1e5a42
+            # new record will omit (or write differently):
Matej Stuchlik b1e5a42
+            # - abs paths, paths with ".." (entry points),
Matej Stuchlik b1e5a42
+            # - pyc+pyo files
Matej Stuchlik b1e5a42
+            # - the old RECORD file
Matej Stuchlik b1e5a42
+            # TODO: is there any better way to recognize an entry point?
Matej Stuchlik b1e5a42
+            if os.path.isabs(spl[0]) or spl[0].startswith('..') or \
Matej Stuchlik b1e5a42
+               spl[0].endswith('.pyc') or spl[0].endswith('.pyo') or \
Matej Stuchlik b1e5a42
+               spl[0] == record_relpath or spl[0].startswith(data_dir):
Matej Stuchlik b1e5a42
+                to_omit.append(spl)
Matej Stuchlik b1e5a42
+            else:
Matej Stuchlik b1e5a42
+                to_write.append(spl)
Matej Stuchlik b1e5a42
+        else:
Matej Stuchlik b1e5a42
+            pass # bad RECORD or empty line
Matej Stuchlik b1e5a42
+    return to_write, to_omit
f9138c2
diff --git a/Makefile.pre.in b/Makefile.pre.in
f9138c2
index ca33158..44bdde5 100644
f9138c2
--- a/Makefile.pre.in
f9138c2
+++ b/Makefile.pre.in
f9138c2
@@ -1066,7 +1066,7 @@ LIBSUBDIRS=	lib-tk lib-tk/test lib-tk/test/test_tkinter \
Robert Kuska 7af76a5
		test/tracedmodules \
Robert Kuska 7af76a5
		encodings compiler hotshot \
Robert Kuska 7af76a5
		email email/mime email/test email/test/data \
Robert Kuska 7af76a5
-		ensurepip ensurepip/_bundled \
Robert Kuska 7af76a5
+		ensurepip ensurepip/_bundled ensurepip/rewheel\
Robert Kuska 7af76a5
		json json/tests \
Robert Kuska 7af76a5
		sqlite3 sqlite3/test \
Robert Kuska 7af76a5
		logging bsddb bsddb/test csv importlib wsgiref \