From b1b3a2905e82d535b1ee347d35fa4148bfd3406c Mon Sep 17 00:00:00 2001
From: Frost Ming <mianghong@gmail.com>
Date: Thu, 23 Dec 2021 10:49:17 +0800
Subject: [PATCH 1/4] Fix the index parsing
---
pipenv/core.py | 10 ++++--
pipenv/resolver.py | 2 +-
pipenv/utils.py | 48 ++++++++++---------------
tests/integration/test_install_basic.py | 13 ++++---
tests/unit/test_utils.py | 23 ++++++++++++
5 files changed, 58 insertions(+), 38 deletions(-)
diff --git a/pipenv/core.py b/pipenv/core.py
index 1fb3e4a..1797acf 100644
--- a/pipenv/core.py
+++ b/pipenv/core.py
@@ -212,9 +212,13 @@ def import_requirements(r=None, dev=False):
trusted_hosts = []
# Find and add extra indexes.
for line in contents.split("\n"):
- line_indexes, _trusted_hosts, _ = parse_indexes(line.strip())
- indexes.extend(line_indexes)
- trusted_hosts.extend(_trusted_hosts)
+ index, extra_index, trusted_host, _ = parse_indexes(line.strip(), strict=True)
+ if index:
+ indexes = [index]
+ if extra_index:
+ indexes.append(extra_index)
+ if trusted_host:
+ trusted_hosts.append(trusted_host)
indexes = sorted(set(indexes))
trusted_hosts = sorted(set(trusted_hosts))
reqs = [f for f in parse_requirements(r, session=pip_requests)]
diff --git a/pipenv/resolver.py b/pipenv/resolver.py
index 80de928..1a77860 100644
--- a/pipenv/resolver.py
+++ b/pipenv/resolver.py
@@ -646,7 +646,7 @@ def parse_packages(packages, pre, clear, system, requirements_dir=None):
from pipenv.utils import parse_indexes
parsed_packages = []
for package in packages:
- indexes, trusted_hosts, line = parse_indexes(package)
+ _, _, line = parse_indexes(package)
line = " ".join(line)
pf = dict()
req = Requirement.from_line(line)
diff --git a/pipenv/utils.py b/pipenv/utils.py
index 61e5f68..086b2af 100644
--- a/pipenv/utils.py
+++ b/pipenv/utils.py
@@ -484,10 +484,7 @@ class Resolver(object):
if project is None:
from .project import Project
project = Project()
- url = None
- indexes, trusted_hosts, remainder = parse_indexes(line)
- if indexes:
- url = indexes[0]
+ index, extra_index, trust_host, remainder = parse_indexes(line)
line = " ".join(remainder)
req = None # type: Requirement
try:
@@ -502,10 +499,10 @@ class Resolver(object):
raise ResolutionFailure("Failed to resolve requirement from line: {0!s}".format(line))
else:
raise ResolutionFailure("Failed to resolve requirement from line: {0!s}".format(line))
- if url:
+ if index:
try:
index_lookup[req.normalized_name] = project.get_source(
- url=url, refresh=True).get("name")
+ url=index, refresh=True).get("name")
except TypeError:
pass
try:
@@ -519,12 +516,6 @@ class Resolver(object):
markers_lookup[req.normalized_name] = req.markers.replace('"', "'")
return req, index_lookup, markers_lookup
- @classmethod
- def get_deps_from_line(cls, line):
- # type: (str) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]]
- req, _, _ = cls.parse_line(line)
- return cls.get_deps_from_req(req)
-
@classmethod
def get_deps_from_req(cls, req, resolver=None):
# type: (Requirement, Optional["Resolver"]) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]]
@@ -697,7 +688,7 @@ class Resolver(object):
self._pip_command = self._get_pip_command()
return self._pip_command
- def prepare_pip_args(self, use_pep517=False, build_isolation=True):
+ def prepare_pip_args(self, use_pep517=None, build_isolation=True):
pip_args = []
if self.sources:
pip_args = prepare_pip_source_args(self.sources, pip_args)
@@ -820,7 +811,6 @@ class Resolver(object):
from pipenv.patched.piptools.cache import CorruptCacheError
from .exceptions import CacheError, ResolutionFailure
with temp_environ():
- os.environ["PIP_NO_USE_PEP517"] = str("")
try:
results = self.resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS)
except CorruptCacheError as e:
@@ -2074,24 +2064,22 @@ def looks_like_dir(path):
return any(sep in path for sep in seps)
-def parse_indexes(line):
+def parse_indexes(line, strict=False):
from argparse import ArgumentParser
- parser = ArgumentParser("indexes")
- parser.add_argument(
- "--index", "-i", "--index-url",
- metavar="index_url", action="store", nargs="?",
- )
- parser.add_argument(
- "--extra-index-url", "--extra-index",
- metavar="extra_indexes", action="append",
- )
- parser.add_argument("--trusted-host", metavar="trusted_hosts", action="append")
+ line = line.split("#")[0].strip()
+ parser = ArgumentParser("indexes", exit_on_error=False)
+ parser.add_argument("-i", "--index-url", dest="index")
+ parser.add_argument("--extra-index-url", dest="extra_index")
+ parser.add_argument("--trusted-host", dest="trusted_host")
args, remainder = parser.parse_known_args(line.split())
- index = [] if not args.index else [args.index]
- extra_indexes = [] if not args.extra_index_url else args.extra_index_url
- indexes = index + extra_indexes
- trusted_hosts = args.trusted_host if args.trusted_host else []
- return indexes, trusted_hosts, remainder
+ index = args.index
+ extra_index = args.extra_index
+ trusted_host = args.trusted_host
+ if strict and sum(
+ bool(arg) for arg in (index, extra_index, trusted_host, remainder)
+ ) > 1:
+ raise ValueError("Index arguments must be on their own lines.")
+ return index, extra_index, trusted_host, remainder
@contextmanager
diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py
index f711879..6d70a03 100644
--- a/tests/integration/test_install_basic.py
+++ b/tests/integration/test_install_basic.py
@@ -286,18 +286,23 @@ def test_requirements_to_pipfile(PipenvInstance, pypi):
# Write a requirements file
with open("requirements.txt", "w") as f:
- f.write("-i {}\nrequests[socks]==2.19.1\n".format(pypi.url))
+ f.write(
+ f"-i {pypi.url}\n"
+ "# -i https://private.pypi.org/simple\n"
+ "requests[socks]==2.19.1\n"
+ )
c = p.pipenv("install")
assert c.return_code == 0
print(c.out)
print(c.err)
- print(delegator.run("ls -l").out)
-
# assert stuff in pipfile
assert "requests" in p.pipfile["packages"]
assert "extras" in p.pipfile["packages"]["requests"]
-
+ assert not any(
+ source['url'] == 'https://private.pypi.org/simple'
+ for source in p.pipfile['source']
+ )
# assert stuff in lockfile
assert "requests" in p.lockfile["default"]
assert "chardet" in p.lockfile["default"]
diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
index 24df26a..41d5feb 100644
--- a/tests/unit/test_utils.py
+++ b/tests/unit/test_utils.py
@@ -138,6 +138,29 @@ def test_convert_deps_to_pip_unicode():
assert deps[0] == "django==1.10"
+@pytest.mark.parametrize("line,result", [
+ ("-i https://example.com/simple/", ("https://example.com/simple/", None, None, [])),
+ ("--extra-index-url=https://example.com/simple/", (None, "https://example.com/simple/", None, [])),
+ ("--trusted-host=example.com", (None, None, "example.com", [])),
+ ("# -i https://example.com/simple/", (None, None, None, [])),
+ ("requests", (None, None, None, ["requests"]))
+])
+@pytest.mark.utils
+def test_parse_indexes(line, result):
+ assert pipenv.utils.parse_indexes(line) == result
+
+
+@pytest.mark.parametrize("line", [
+ "-i https://example.com/simple/ --extra-index-url=https://extra.com/simple/",
+ "--extra-index-url https://example.com/simple/ --trusted-host=example.com",
+ "requests -i https://example.com/simple/",
+])
+@pytest.mark.utils
+def test_parse_indexes_individual_lines(line):
+ with pytest.raises(ValueError):
+ pipenv.utils.parse_indexes(line, strict=True)
+
+
class TestUtils:
"""Test utility functions in pipenv"""
--
2.33.1
From 4d3d22394c274879f6e6fd39166d60aa71e4a870 Mon Sep 17 00:00:00 2001
From: Frost Ming <mianghong@gmail.com>
Date: Thu, 23 Dec 2021 11:27:23 +0800
Subject: [PATCH 2/4] remove the useless option
---
news/4899.bugfix.rst | 1 +
pipenv/utils.py | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
create mode 100644 news/4899.bugfix.rst
diff --git a/news/4899.bugfix.rst b/news/4899.bugfix.rst
new file mode 100644
index 0000000..bc61835
--- /dev/null
+++ b/news/4899.bugfix.rst
@@ -0,0 +1 @@
+Fix the index parsing to reject illegal requirements.txt.
diff --git a/pipenv/utils.py b/pipenv/utils.py
index 086b2af..44d7202 100644
--- a/pipenv/utils.py
+++ b/pipenv/utils.py
@@ -2067,7 +2067,7 @@ def looks_like_dir(path):
def parse_indexes(line, strict=False):
from argparse import ArgumentParser
line = line.split("#")[0].strip()
- parser = ArgumentParser("indexes", exit_on_error=False)
+ parser = ArgumentParser("indexes")
parser.add_argument("-i", "--index-url", dest="index")
parser.add_argument("--extra-index-url", dest="extra_index")
parser.add_argument("--trusted-host", dest="trusted_host")
--
2.33.1
From 1775fa1b28a478c11923ab57d27889e805ac1f50 Mon Sep 17 00:00:00 2001
From: Frost Ming <mianghong@gmail.com>
Date: Thu, 23 Dec 2021 12:53:42 +0800
Subject: [PATCH 3/4] fix comment ignorance
---
pipenv/utils.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/pipenv/utils.py b/pipenv/utils.py
index 44d7202..67be2d0 100644
--- a/pipenv/utils.py
+++ b/pipenv/utils.py
@@ -2066,7 +2066,9 @@ def looks_like_dir(path):
def parse_indexes(line, strict=False):
from argparse import ArgumentParser
- line = line.split("#")[0].strip()
+
+ comment_re = re.compile(r"(?:^|\s+)#.*$")
+ line = comment_re.sub("", line)
parser = ArgumentParser("indexes")
parser.add_argument("-i", "--index-url", dest="index")
parser.add_argument("--extra-index-url", dest="extra_index")
--
2.33.1
From 2640cccad9c770255fd529de676b48967faeed90 Mon Sep 17 00:00:00 2001
From: Frost Ming <mianghong@gmail.com>
Date: Thu, 23 Dec 2021 15:20:11 +0800
Subject: [PATCH 4/4] fix pip location
---
pipenv/patched/notpip/_internal/build_env.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pipenv/patched/notpip/_internal/build_env.py b/pipenv/patched/notpip/_internal/build_env.py
index 71fa326..a4db7f1 100644
--- a/pipenv/patched/notpip/_internal/build_env.py
+++ b/pipenv/patched/notpip/_internal/build_env.py
@@ -15,7 +15,7 @@ from sysconfig import get_paths
from pipenv.patched.notpip._vendor.pkg_resources import Requirement, VersionConflict, WorkingSet
-from pipenv.patched.notpip import __file__ as pip_location
+from pip import __file__ as pip_location
from pipenv.patched.notpip._internal.utils.subprocess import call_subprocess
from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory
from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING
--
2.33.1