rpms / pipenv

Clone
Source Code
GIT

#18 devendoring few packages
Merged 4 years ago by churchyard. Opened 4 years ago by pkopkan.
https://github.com/PatrikKopkan/rpm-pipenv master  into  master

Devendored more libraries
Patrik Kopkan • 4 years ago  
d568bc9
Replace big devendoring patch with seds
Patrik Kopkan • 4 years ago  
3f432ac
0002-fix-imports-of-unbundled-pkgs.patch
file removed
-202 
@@ -1,202 +0,0 @@
 

- diff --git a/pipenv/__init__.py b/pipenv/__init__.py
 

- index 4d137e7..1e5230b 100644
 

- --- a/pipenv/__init__.py
 

- +++ b/pipenv/__init__.py
 

- @@ -18,7 +18,7 @@ sys.path.insert(0, PIPENV_VENDOR)
 

-  # Inject patched directory into system path.
 

-  sys.path.insert(0, PIPENV_PATCHED)
 

-  

- -from pipenv.vendor.urllib3.exceptions import DependencyWarning
 

- +from urllib3.exceptions import DependencyWarning
 

-  from pipenv.vendor.vistir.compat import ResourceWarning, fs_str
 

-  warnings.filterwarnings("ignore", category=DependencyWarning)
 

-  warnings.filterwarnings("ignore", category=ResourceWarning)
 

- diff --git a/pipenv/core.py b/pipenv/core.py
 

- index b9abe27..cccdb0e 100644
 

- --- a/pipenv/core.py
 

- +++ b/pipenv/core.py
 

- @@ -1070,7 +1070,7 @@ def do_lock(
 

-  

-      # Support for --keep-outdated…
 

-      if keep_outdated:
 

- -        from pipenv.vendor.packaging.utils import canonicalize_name
 

- +        from packaging.utils import canonicalize_name
 

-          for section_name, section in (
 

-              ("default", project.packages),
 

-              ("develop", project.dev_packages),
 

- @@ -1281,7 +1281,7 @@ def pip_install(
 

-  ):
 

-      from pipenv.patched.notpip._internal import logger as piplogger
 

-      from .utils import Mapping
 

- -    from .vendor.urllib3.util import parse_url
 

- +    from urllib3.util import parse_url
 

-  

-      src = []
 

-      write_to_tmpfile = False
 

- @@ -1640,7 +1640,7 @@ def do_py(system=False):
 

-  def do_outdated(pypi_mirror=None):
 

-      # TODO: Allow --skip-lock here?
 

-      from .vendor.requirementslib.models.requirements import Requirement
 

- -    from .vendor.packaging.utils import canonicalize_name
 

- +    from packaging.utils import canonicalize_name
 

-      from collections import namedtuple
 

-  

-      packages = {}
 

- @@ -2009,7 +2009,7 @@ def do_uninstall(
 

-  ):
 

-      from .environments import PIPENV_USE_SYSTEM
 

-      from .vendor.requirementslib.models.requirements import Requirement
 

- -    from .vendor.packaging.utils import canonicalize_name
 

- +    from packaging.utils import canonicalize_name
 

-  

-      # Automatically use an activated virtualenv.
 

-      if PIPENV_USE_SYSTEM:
 

- diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py
 

- index 62e25d5..00da817 100644
 

- --- a/pipenv/exceptions.py
 

- +++ b/pipenv/exceptions.py
 

- @@ -11,9 +11,9 @@ import six
 

-  from ._compat import fix_utf8
 

-  from .patched import crayons
 

-  from . import environments
 

- -from .vendor.click.utils import echo as click_echo
 

- -from .vendor.click._compat import get_text_stderr
 

- -from .vendor.click.exceptions import (
 

- +from click.utils import echo as click_echo
 

- +from click._compat import get_text_stderr
 

- +from click.exceptions import (
 

-      Abort,
 

-      BadOptionUsage,
 

-      BadParameter,
 

- @@ -23,7 +23,7 @@ from .vendor.click.exceptions import (
 

-      MissingParameter,
 

-      UsageError,
 

-  )
 

- -from .vendor.click.types import Path
 

- +from click.types import Path
 

-  

-  

-  def handle_exception(exc_type, exception, traceback, hook=sys.excepthook):
 

- diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py
 

- index c466ef0..e2782f2 100644
 

- --- a/pipenv/patched/piptools/_compat/pip_compat.py
 

- +++ b/pipenv/patched/piptools/_compat/pip_compat.py
 

- @@ -24,7 +24,7 @@ __all__ = [
 

-      "user_cache_dir"
 

-  ]
 

-  

- -from pipenv.vendor.appdirs import user_cache_dir
 

- +from appdirs import user_cache_dir
 

-  from pip_shims.shims import (
 

-      InstallRequirement,
 

-      parse_requirements,
 

- diff --git a/pipenv/resolver.py b/pipenv/resolver.py
 

- index e87f324..cc42918 100644
 

- --- a/pipenv/resolver.py
 

- +++ b/pipenv/resolver.py
 

- @@ -121,6 +121,6 @@ def main():
 

-  

-  if __name__ == "__main__":
 

-      _patch_path()
 

- -    from pipenv.vendor import colorama
 

- +    import colorama
 

-      colorama.init()
 

-      main()
 

- diff --git a/pipenv/shells.py b/pipenv/shells.py
 

- index 352c01f..78bced8 100644
 

- --- a/pipenv/shells.py
 

- +++ b/pipenv/shells.py
 

- @@ -8,7 +8,7 @@ import sys
 

-  from .environments import PIPENV_SHELL_EXPLICIT, PIPENV_SHELL, PIPENV_EMULATOR
 

-  from .vendor.vistir.compat import get_terminal_size, Path
 

-  from .vendor.vistir.contextmanagers import temp_environ
 

- -from .vendor import shellingham
 

- +import shellingham
 

-  

-  

-  ShellDetectionFailure = shellingham.ShellDetectionFailure
 

- @@ -97,7 +97,7 @@ class Shell(object):
 

-              _handover(self.cmd, self.args + list(args))
 

-  

-      def fork_compat(self, venv, cwd, args):
 

- -        from .vendor import pexpect
 

- +        import pexpect
 

-  

-          # Grab current terminal dimensions to replace the hardcoded default
 

-          # dimensions of pexpect.
 

- diff --git a/pipenv/utils.py b/pipenv/utils.py
 

- index 2dd5615..ba94166 100644
 

- --- a/pipenv/utils.py
 

- +++ b/pipenv/utils.py
 

- @@ -377,7 +377,7 @@ class Resolver(object):
 

-  

-      def resolve(self):
 

-          from pipenv.vendor.pip_shims.shims import DistributionNotFound
 

- -        from pipenv.vendor.requests.exceptions import HTTPError
 

- +        from requests.exceptions import HTTPError
 

-          from pipenv.patched.piptools.exceptions import NoCandidateFound
 

-          from pipenv.patched.piptools.cache import CorruptCacheError
 

-          from .exceptions import CacheError, ResolutionFailure
 

- @@ -504,7 +504,7 @@ def create_spinner(text, nospin=None, spinner_name=None):
 

-  def resolve(cmd, sp):
 

-      from .vendor import delegator
 

-      from .cmdparse import Script
 

- -    from .vendor.pexpect.exceptions import EOF, TIMEOUT
 

- +    from pexpect.exceptions import EOF, TIMEOUT
 

-      from .vendor.vistir.compat import to_native_string
 

-      EOF.__module__ = "pexpect.exceptions"
 

-      from ._compat import decode_output
 

- @@ -696,7 +696,7 @@ def resolve_deps(
 

-      """Given a list of dependencies, return a resolved list of dependencies,
 

-      using pip-tools -- and their hashes, using the warehouse API / pip.
 

-      """
 

- -    from .vendor.requests.exceptions import ConnectionError
 

- +    from requests.exceptions import ConnectionError
 

-      from .vendor.requirementslib.models.requirements import Requirement
 

-  

-      index_lookup = {}
 

- @@ -1076,7 +1076,7 @@ def get_url_name(url):
 

-  

-  def get_canonical_names(packages):
 

-      """Canonicalize a list of packages and return a set of canonical names"""
 

- -    from .vendor.packaging.utils import canonicalize_name
 

- +    from packaging.utils import canonicalize_name
 

-  

-      if not isinstance(packages, Sequence):
 

-          if not isinstance(packages, six.string_types):
 

- @@ -1331,8 +1331,8 @@ def translate_markers(pipfile_entry):
 

-      """
 

-      if not isinstance(pipfile_entry, Mapping):
 

-          raise TypeError("Entry is not a pipfile formatted mapping.")
 

- -    from .vendor.distlib.markers import DEFAULT_CONTEXT as marker_context
 

- -    from .vendor.packaging.markers import Marker
 

- +    from distlib.markers import DEFAULT_CONTEXT as marker_context
 

- +    from packaging.markers import Marker
 

-      from .vendor.vistir.misc import dedup
 

-  

-      allowed_marker_keys = ["markers"] + [k for k in marker_context.keys()]
 

- diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
 

- index 822617a..768ef55 100644
 

- --- a/tests/integration/conftest.py
 

- +++ b/tests/integration/conftest.py
 

- @@ -1,6 +1,8 @@
 

-  import json
 

-  import os
 

- +import requests
 

-  import sys
 

- +import toml
 

-  import warnings
 

-  

-  import pytest
 

- @@ -9,10 +11,9 @@ from pipenv._compat import TemporaryDirectory, Path
 

-  from pipenv.exceptions import VirtualenvActivationException
 

-  from pipenv.utils import temp_environ
 

-  from pipenv.vendor import delegator
 

- -from pipenv.vendor import requests
 

- -from pipenv.vendor import toml
 

-  from pipenv.vendor import tomlkit
 

-  from pytest_pypi.app import prepare_packages as prepare_pypi_packages, prepare_fixtures
 

- +from pytest_pypi.plugin import pypi, pypi_secure
 

-  from vistir.compat import ResourceWarning, fs_str
 

-  from vistir.path import mkdir_p
 

-

0002-import-pytest-pypi.patch
file added
+11 
@@ -0,0 +1,11 @@
 

+ diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
 

+ index 822617ab..4f2c77b1 100644
 

+ --- a/tests/integration/conftest.py
 

+ +++ b/tests/integration/conftest.py
 

+ @@ -13,6 +13,7 @@ from pipenv.vendor import requests
 

+  from pipenv.vendor import toml
 

+  from pipenv.vendor import tomlkit
 

+  from pytest_pypi.app import prepare_packages as prepare_pypi_packages, prepare_fixtures
 

+ +from pytest_pypi.plugin import pypi, pypi_secure
 

+  from vistir.compat import ResourceWarning, fs_str
 

+  from vistir.path import mkdir_p

pipenv.spec
file modified
+50 -30 
@@ -4,7 +4,7 @@
 

  

  Name:           pipenv 

  Version:        2018.11.26
 

- Release:        10%{?dist}
 

+ Release:        11%{?dist}
 

  Summary:        The higher level Python packaging tool
 

  

  # Pipenv source code is MIT, there are bundled packages having different licenses
 
@@ -44,27 +44,17 @@
 

  # pipenv/vendor/cursor is CC-BY-SA
 

  # pipenv/vendor/delegator.py is MIT
 

  # pipenv/vendor/passa is ISC
 

- # pipenv/vendor/pipdeptree.py is MIT
 

- # pipenv/vendor/pipreqs/ is Apache2.0
 

- # pipenv/vendor/pip_shims/ is ISC
 

- # pipenv/vendor/plette/ is ISC
 

- # pipenv/vendor/pythonfinder/ is MIT
 

  # pipenv/vendor/requirementslib/ is (Apache2.0 or BSD)
 

  # pipenv/vendor/resolvelib/ is MIT
 

  # pipenv/vendor/shutilwhich/ is BSD
 

- # pipenv/vendor/tomlkit/ is MIT
 

- # pipenv/vendor/vistir/ is ISC
 

- # pipenv/vendor/yaspin/ is MIT
 

  

  License:        MIT and BSD and ASL 2.0 and LGPLv2+ and Python and ISC and MPLv2.0 and (ASL 2.0 or BSD) and CC-BY-SA
 

  URL:            https://github.com/pypa/pipenv
 

  Source0:        https://github.com/pypa/%{name}/archive/v%{version}/%{name}-%{version}.tar.gz
 

  

- # We unbundle a plenty of packages from vendor directory
 

- # 'from pipenv.vendor' imports must be corrected
 

- # Also adds "pytest_pypi.plugin import pypi, ..." to conftest,
 

+ # Adds "pytest_pypi.plugin import pypi, ..." to conftest,
 

  # as we don't have that plugin installed and it is not autodiscovered
 

- Patch2:         0002-fix-imports-of-unbundled-pkgs.patch
 

+ Patch2:         0002-import-pytest-pypi.patch
 

  

  # A couple of tests fails in the mock environment, add option
 

  # to skip these using special pytest marker
 
@@ -134,6 +124,14 @@
 

  BuildRequires:  python3dist(toml) >= 0.10
 

  BuildRequires:  python3dist(urllib3)
 

  BuildRequires:  python3dist(yarg) >= 0.1.9
 

+ BuildRequires:  python3dist(yaspin)
 

+ BuildRequires:  python3dist(vistir)
 

+ BuildRequires:  python3dist(tomlkit)
 

+ BuildRequires:  python3dist(pipdeptree)
 

+ BuildRequires:  python3dist(pipreqs)
 

+ BuildRequires:  python3dist(pip-shims)
 

+ BuildRequires:  python3dist(plette)
 

+ BuildRequires:  python3dist(pythonfinder)
 

  

  %{?python_provide:%python_provide python3-%{name}}
 

  
@@ -174,25 +172,32 @@
 

  Requires:       python3dist(toml) >= 0.10
 

  Requires:       python3dist(urllib3)
 

  Requires:       python3dist(yarg) >= 0.1.9
 

+ Requires:       python3dist(yaspin)
 

+ Requires:       python3dist(vistir)
 

+ Requires:       python3dist(tomlkit)
 

+ Requires:       python3dist(pipdeptree)
 

+ Requires:       python3dist(pipreqs)
 

+ Requires:       python3dist(pip-shims)
 

+ Requires:       python3dist(plette)
 

+ Requires:       python3dist(pythonfinder)
 

  

  # Following packages bundled under vendor directory are not
 

  # packaged for Fedora yet.
 

  # TODO package for Fedora and unbundle
 

  Provides:       bundled(python3dist(click-didyoumean)) == 0.0.3
 

- Provides:       bundled(python3dist(cursor)) == 1.2
 

  Provides:       bundled(python3dist(delegator.py)) == 0.1.1
 

+ # Needs requirementslib (see below)
 

  Provides:       bundled(python3dist(passa))
 

- Provides:       bundled(python3dist(pipdeptree)) == 0.13
 

- Provides:       bundled(python3dist(pipreqs)) == 0.4.9
 

- Provides:       bundled(python3dist(pip-shims)) == 0.3.2
 

- Provides:       bundled(python3dist(plette)) == 0.2.3.dev0
 

- Provides:       bundled(python3dist(pythonfinder)) == 1.1.10
 

+ # This library uses pip.internals. Some changes in init methods happened there.
 

+ # So version 1.3.3 is useless with pip 19+ and newer versions will break pipenv
 

+ # because pipenv has bundled patched pip.
 

  Provides:       bundled(python3dist(requirementslib)) == 1.3.3
 

+ # Dependency of passa
 

  Provides:       bundled(python3dist(resolvelib)) == 0.2.2
 

+ 

+ # No longer used in upstream master branch, not worth looking into
 

  Provides:       bundled(python3dist(shutilwhich)) == 1.1
 

- Provides:       bundled(python3dist(tomlkit)) == 0.5.2
 

- Provides:       bundled(python3dist(vistir)) == 0.2.5
 

- Provides:       bundled(python3dist(yaspin)) == 0.14
 

+ Provides:       bundled(python3dist(cursor)) == 1.2
 

  

  # The sources contains patched versions of following packages:
 

  Provides:       bundled(python3dist(crayons)) == 0.1.2
 
@@ -251,7 +256,26 @@
 

  

  # Remove packages that are already packaged for Fedora from vendor directory
 

  # pathlib2 and backports are not needed on Python 3.6+
 

- UNBUNDLED="appdirs attr blindspin cached_property cerberus click_completion click colorama distlib docopt first chardet iso8601 jinja2 markupsafe packaging parse pexpect ptyprocess pyparsing dotenv requests certifi idna urllib3 scandir semver shellingham six toml yarg pathlib2 backports"
 

+ UNBUNDLED="appdirs attr blindspin cached_property cerberus click_completion click colorama distlib docopt first chardet iso8601 jinja2 markupsafe packaging parse pexpect ptyprocess pyparsing dotenv requests certifi idna urllib3 scandir semver shellingham six toml yarg pathlib2 backports yaspin vistir pythonfinder plette pipreqs pipdeptree pip_shims tomlkit"
 

+ 

+ # issue: for loop below doesn't handle multiple imports in one line
 

+ # properly. There might be case when library is still not unbundled
 

+ # but is not imported from vendor directory.
 

+ # diff of pyenv.py:
 

+ #  265   │ -from .vendor import attr, delegator
 

+ #  266   │ +import attr, delegator
 

+ # So we unpack such import statements into multiple lines first:
 

+ while matches=$(grep -Elr 'from (\.pipenv)?\.vendor import ([^,]+), (.+)'); do
 

+   sed -Ei 's/from (\.pipenv)?\.vendor import ([^,]+), (.+)/from \1.vendor import \2\nfrom \1.vendor import \3/g' $matches
 

+ done
 

+ 

+ for pkg in ${UNBUNDLED[@]}; do
 

+   find pipenv/* tests/* -not -path '*/\.git*' -type f -exec sed -i -E \
 

+   -e "s/from (pipenv)?\.vendor\.${pkg}(\.\S+)? import/from ${pkg}\2 import/" \
 

+   -e "s/^import (pipenv)?\.vendor\.${pkg}(\.\S+)?/import ${pkg}\2/" \
 

+   -e "s/from (pipenv)?\.vendor import ${pkg}(\.\S+)?/import ${pkg}\2/" \
 

+   -e "s/(pipenv)?\.vendor\.${pkg}(\.\S+)?/${pkg}\2/g" {} \;
 

+ done
 

  

  _vendordir="pipenv/vendor/"
 

  
@@ -362,16 +386,9 @@
 

  %license %{python3_sitelib}/%{name}/vendor/click_didyoumean/LICENSE
 

  %license %{python3_sitelib}/%{name}/vendor/cursor/LICENSE
 

  %license %{python3_sitelib}/%{name}/vendor/passa/LICENSE
 

- %license %{python3_sitelib}/%{name}/vendor/pipreqs/LICENSE
 

- %license %{python3_sitelib}/%{name}/vendor/pip_shims/LICENSE
 

- %license %{python3_sitelib}/%{name}/vendor/plette/LICENSE
 

- %license %{python3_sitelib}/%{name}/vendor/pythonfinder/LICENSE.txt
 

  %license %{python3_sitelib}/%{name}/vendor/requirementslib/LICENSE
 

  %license %{python3_sitelib}/%{name}/vendor/resolvelib/LICENSE
 

  %license %{python3_sitelib}/%{name}/vendor/shutilwhich/LICENSE
 

- %license %{python3_sitelib}/%{name}/vendor/tomlkit/LICENSE
 

- %license %{python3_sitelib}/%{name}/vendor/vistir/LICENSE
 

- %license %{python3_sitelib}/%{name}/vendor/yaspin/LICENSE
 

  

  %doc README.md NOTICES CHANGELOG.rst HISTORY.txt
 

  %{_bindir}/pipenv
 
@@ -384,6 +401,9 @@
 

  %license LICENSE
 

  

  %changelog
 

+ * Fri Oct 11 2019 Patrik Kopkan <pkopkan@redhat.com> - 2018.11.26-11
 

+ - Devendored: yaspin vistir pythonfinder plette pipreqs pipdeptree pip_shims tomlkit
 

+ 

  * Thu Oct 03 2019 Miro Hrončok <mhroncok@redhat.com> - 2018.11.26-10
 

  - Rebuilt for Python 3.8.0rc1 (#1748018)

no initial comment
  • added dynamic import fixing for devendored libraries.
  • devendored: yaspin vistir pythonfinder plette pipreqs pipdeptree pip_shims tomlkit

pipreqs and pipdeptree is on review.

copr build

I don't understand this change

I don't understand this change

Or here.

please, describe this more. Something like: "Careful here, when several modules are imported on one line, but only some are devendored, the following sed will rewrite the import as follows and it will break. As of pip version XYZ this does not happen.

1 new commit added

  • fix typos
4 years ago

@pkopkan Can you please describe it more?

[citest]

7 new commits added

  • fix typos
  • devendored tomlkit
  • cleanup2
  • cleanup
  • test seems passing
  • devendoring
  • getting rid of vendored imports dynamically
4 years ago

[citest]

please do not say stuff like "I patched" in comments, it will not make sense for other people next time they read it. What about "This is patched out in patch X in pipenv VERSION"?

I don't understand this comment.

TODO what?

I see this patch has changed it's purpose quite a bit. Maybe the comment in spec about it shall be adjusted as well?

1 new commit added

  • edit comments
4 years ago

8 new commits added

  • edit comments
  • fix typos
  • devendored tomlkit
  • cleanup2
  • cleanup
  • test seems passing
  • devendoring
  • getting rid of vendored imports dynamically
4 years ago

this isn't the only thing that patch does, is it? also, new versions of pytest use importlib.metadata and importlib.metadata might actually see plugins based on PYTHONPATH, maybe we can try to drop the pytets plugin part of the patch.

I'm going to experiment a bit and will send you a PR into your github fork.

2 new commits added

  • Merge pull request #1 from hroncok/pr18followup
  • Review of PR #18
4 years ago

Just a nitpick, could we keep the same order of these new Requires as the BuildRequires? It's confusing.

Requires: python3dist(vistir) is there twice.

There already is a # No longer used in upstream master branch, not worth looking into section, unify?

The splitting of the imports into different lines is a great idea, kudos!

The splitting of the imports into different lines is a great idea, kudos!

Stylistic note, I'd use ${pkg} in this large sed. For increased readability and disambiguation.

Why are the sed commands ending in /g? AFAIK that's to allow multiple replacements on one line, which shouldn't be needed.

Why not also include (pipenv)? before \.vendor here? Seems like this could backfire later.

Why are we suddenly using $pkg(\.\S+)? here (and on the line below), but on the 2 lines above we have $pkg(\.[a-z,A-Z,\.,_]*)? Seems inconsistent and confusing.

Same here, why $pkg(\.\S+) instead of $pkg(\.[a-z,A-Z,\.,_]*)??

This is the only place where ending the sed command with /g makes sense I believe.

Please don't be discouraged by the amount of small comments. Overall it looks marvellous! I'm honestly surprised with how elegant the solution turned out to be! :)

1 new commit added

  • adress review points
4 years ago

@pkopkan when you address review points, please always comment about it, there is no notification here about new commits.

Anyway, could you please rebase the changes? There is a changelog conflict now.

rebased onto d228b3f4943fe040333ba72de17b20c1fa7b8501
4 years ago

Rebased. Thanks for help.

Do you think that the individual commits here make sense preserving? I am not in favor of blind squashing, but it seems here they are mostly fixup commits. If we actually want more granularity, I suggest:

  1. a commit that changes the sed and the patch
  2. a commit that changes the actual (de)vendored libraries

Or, if the above would be too tedious (it probably would), squashing it might work as well.

rebased onto 3f432ac
4 years ago

Commits look good now! Will wait for the CI and merge.

Pull-Request has been merged by churchyard
4 years ago
Changes Summary 3
-202
file removed
0002-fix-imports-of-unbundled-pkgs.patch
+11
file added
0002-import-pytest-pypi.patch
+50 -30
file changed
pipenv.spec
Powered by Pagure 5.13.3
Documentation • File an Issue • About this Instance • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.