From b1b3a2905e82d535b1ee347d35fa4148bfd3406c Mon Sep 17 00:00:00 2001

From: Frost Ming <mianghong@gmail.com>

Date: Thu, 23 Dec 2021 10:49:17 +0800

Subject: [PATCH 1/4] Fix the index parsing

---

 pipenv/core.py                          | 10 ++++--

 pipenv/resolver.py                      |  2 +-

 pipenv/utils.py                         | 48 ++++++++++---------------

 tests/integration/test_install_basic.py | 13 ++++---

 tests/unit/test_utils.py                | 23 ++++++++++++

 5 files changed, 58 insertions(+), 38 deletions(-)

diff --git a/pipenv/core.py b/pipenv/core.py

index 1fb3e4a..1797acf 100644

--- a/pipenv/core.py

+++ b/pipenv/core.py

@@ -212,9 +212,13 @@ def import_requirements(r=None, dev=False):

     trusted_hosts = []

     # Find and add extra indexes.

     for line in contents.split("\n"):

-        line_indexes, _trusted_hosts, _ = parse_indexes(line.strip())

-        indexes.extend(line_indexes)

-        trusted_hosts.extend(_trusted_hosts)

+        index, extra_index, trusted_host, _ = parse_indexes(line.strip(), strict=True)

+        if index:

+            indexes = [index]

+        if extra_index:

+            indexes.append(extra_index)

+        if trusted_host:

+            trusted_hosts.append(trusted_host)

     indexes = sorted(set(indexes))

     trusted_hosts = sorted(set(trusted_hosts))

     reqs = [f for f in parse_requirements(r, session=pip_requests)]

diff --git a/pipenv/resolver.py b/pipenv/resolver.py

index 80de928..1a77860 100644

--- a/pipenv/resolver.py

+++ b/pipenv/resolver.py

@@ -646,7 +646,7 @@ def parse_packages(packages, pre, clear, system, requirements_dir=None):

     from pipenv.utils import parse_indexes

     parsed_packages = []

     for package in packages:

-        indexes, trusted_hosts, line = parse_indexes(package)

+        _, _, line = parse_indexes(package)

         line = " ".join(line)

         pf = dict()

         req = Requirement.from_line(line)

diff --git a/pipenv/utils.py b/pipenv/utils.py

index 61e5f68..086b2af 100644

--- a/pipenv/utils.py

+++ b/pipenv/utils.py

@@ -484,10 +484,7 @@ class Resolver(object):

         if project is None:

             from .project import Project

             project = Project()

-        url = None

-        indexes, trusted_hosts, remainder = parse_indexes(line)

-        if indexes:

-            url = indexes[0]

+        index, extra_index, trust_host, remainder = parse_indexes(line)

         line = " ".join(remainder)

         req = None  # type: Requirement

         try:

@@ -502,10 +499,10 @@ class Resolver(object):

                     raise ResolutionFailure("Failed to resolve requirement from line: {0!s}".format(line))

             else:

                 raise ResolutionFailure("Failed to resolve requirement from line: {0!s}".format(line))

-        if url:

+        if index:

             try:

                 index_lookup[req.normalized_name] = project.get_source(

-                    url=url, refresh=True).get("name")

+                    url=index, refresh=True).get("name")

             except TypeError:

                 pass

         try:

@@ -519,12 +516,6 @@ class Resolver(object):

             markers_lookup[req.normalized_name] = req.markers.replace('"', "'")

         return req, index_lookup, markers_lookup

-    @classmethod

-    def get_deps_from_line(cls, line):

-        # type: (str) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]]

-        req, _, _ = cls.parse_line(line)

-        return cls.get_deps_from_req(req)

-

     @classmethod

     def get_deps_from_req(cls, req, resolver=None):

         # type: (Requirement, Optional["Resolver"]) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]]

@@ -697,7 +688,7 @@ class Resolver(object):

             self._pip_command = self._get_pip_command()

         return self._pip_command

-    def prepare_pip_args(self, use_pep517=False, build_isolation=True):

+    def prepare_pip_args(self, use_pep517=None, build_isolation=True):

         pip_args = []

         if self.sources:

             pip_args = prepare_pip_source_args(self.sources, pip_args)

@@ -820,7 +811,6 @@ class Resolver(object):

         from pipenv.patched.piptools.cache import CorruptCacheError

         from .exceptions import CacheError, ResolutionFailure

         with temp_environ():

-            os.environ["PIP_NO_USE_PEP517"] = str("")

             try:

                 results = self.resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS)

             except CorruptCacheError as e:

@@ -2074,24 +2064,22 @@ def looks_like_dir(path):

     return any(sep in path for sep in seps)

-def parse_indexes(line):

+def parse_indexes(line, strict=False):

     from argparse import ArgumentParser

-    parser = ArgumentParser("indexes")

-    parser.add_argument(

-        "--index", "-i", "--index-url",

-        metavar="index_url", action="store", nargs="?",

-    )

-    parser.add_argument(

-        "--extra-index-url", "--extra-index",

-        metavar="extra_indexes", action="append",

-    )

-    parser.add_argument("--trusted-host", metavar="trusted_hosts", action="append")

+    line = line.split("#")[0].strip()

+    parser = ArgumentParser("indexes", exit_on_error=False)

+    parser.add_argument("-i", "--index-url", dest="index")

+    parser.add_argument("--extra-index-url", dest="extra_index")

+    parser.add_argument("--trusted-host", dest="trusted_host")

     args, remainder = parser.parse_known_args(line.split())

-    index = [] if not args.index else [args.index]

-    extra_indexes = [] if not args.extra_index_url else args.extra_index_url

-    indexes = index + extra_indexes

-    trusted_hosts = args.trusted_host if args.trusted_host else []

-    return indexes, trusted_hosts, remainder

+    index = args.index

+    extra_index = args.extra_index

+    trusted_host = args.trusted_host

+    if strict and sum(

+        bool(arg) for arg in (index, extra_index, trusted_host, remainder)

+    ) > 1:

+        raise ValueError("Index arguments must be on their own lines.")

+    return index, extra_index, trusted_host, remainder

 @contextmanager

diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py

index f711879..6d70a03 100644

--- a/tests/integration/test_install_basic.py

+++ b/tests/integration/test_install_basic.py

@@ -286,18 +286,23 @@ def test_requirements_to_pipfile(PipenvInstance, pypi):

         # Write a requirements file

         with open("requirements.txt", "w") as f:

-            f.write("-i {}\nrequests[socks]==2.19.1\n".format(pypi.url))

+            f.write(

+                f"-i {pypi.url}\n"

+                "# -i https://private.pypi.org/simple\n"

+                "requests[socks]==2.19.1\n"

+            )

         c = p.pipenv("install")

         assert c.return_code == 0

         print(c.out)

         print(c.err)

-        print(delegator.run("ls -l").out)

-

         # assert stuff in pipfile

         assert "requests" in p.pipfile["packages"]

         assert "extras" in p.pipfile["packages"]["requests"]

-

+        assert not any(

+            source['url'] == 'https://private.pypi.org/simple'

+            for source in p.pipfile['source']

+        )

         # assert stuff in lockfile

         assert "requests" in p.lockfile["default"]

         assert "chardet" in p.lockfile["default"]

diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py

index 24df26a..41d5feb 100644

--- a/tests/unit/test_utils.py

+++ b/tests/unit/test_utils.py

@@ -138,6 +138,29 @@ def test_convert_deps_to_pip_unicode():

     assert deps[0] == "django==1.10"

+@pytest.mark.parametrize("line,result", [

+    ("-i https://example.com/simple/", ("https://example.com/simple/", None, None, [])),

+    ("--extra-index-url=https://example.com/simple/", (None, "https://example.com/simple/", None, [])),

+    ("--trusted-host=example.com", (None, None, "example.com", [])),

+    ("# -i https://example.com/simple/", (None, None, None, [])),

+    ("requests", (None, None, None, ["requests"]))

+])

+@pytest.mark.utils

+def test_parse_indexes(line, result):

+    assert pipenv.utils.parse_indexes(line) == result

+

+

+@pytest.mark.parametrize("line", [

+    "-i https://example.com/simple/ --extra-index-url=https://extra.com/simple/",

+    "--extra-index-url https://example.com/simple/ --trusted-host=example.com",

+    "requests -i https://example.com/simple/",

+])

+@pytest.mark.utils

+def test_parse_indexes_individual_lines(line):

+    with pytest.raises(ValueError):

+        pipenv.utils.parse_indexes(line, strict=True)

+

+

 class TestUtils:

     """Test utility functions in pipenv"""

-- 

2.33.1

From 4d3d22394c274879f6e6fd39166d60aa71e4a870 Mon Sep 17 00:00:00 2001

From: Frost Ming <mianghong@gmail.com>

Date: Thu, 23 Dec 2021 11:27:23 +0800

Subject: [PATCH 2/4] remove the useless option

---

 news/4899.bugfix.rst | 1 +

 pipenv/utils.py      | 2 +-

 2 files changed, 2 insertions(+), 1 deletion(-)

 create mode 100644 news/4899.bugfix.rst

diff --git a/news/4899.bugfix.rst b/news/4899.bugfix.rst

new file mode 100644

index 0000000..bc61835

--- /dev/null

+++ b/news/4899.bugfix.rst

@@ -0,0 +1 @@

+Fix the index parsing to reject illegal requirements.txt.

diff --git a/pipenv/utils.py b/pipenv/utils.py

index 086b2af..44d7202 100644

--- a/pipenv/utils.py

+++ b/pipenv/utils.py

@@ -2067,7 +2067,7 @@ def looks_like_dir(path):

 def parse_indexes(line, strict=False):

     from argparse import ArgumentParser

     line = line.split("#")[0].strip()

-    parser = ArgumentParser("indexes", exit_on_error=False)

+    parser = ArgumentParser("indexes")

     parser.add_argument("-i", "--index-url", dest="index")

     parser.add_argument("--extra-index-url", dest="extra_index")

     parser.add_argument("--trusted-host", dest="trusted_host")

-- 

2.33.1

From 1775fa1b28a478c11923ab57d27889e805ac1f50 Mon Sep 17 00:00:00 2001

From: Frost Ming <mianghong@gmail.com>

Date: Thu, 23 Dec 2021 12:53:42 +0800

Subject: [PATCH 3/4] fix comment ignorance

---

 pipenv/utils.py | 4 +++-

 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pipenv/utils.py b/pipenv/utils.py

index 44d7202..67be2d0 100644

--- a/pipenv/utils.py

+++ b/pipenv/utils.py

@@ -2066,7 +2066,9 @@ def looks_like_dir(path):

 def parse_indexes(line, strict=False):

     from argparse import ArgumentParser

-    line = line.split("#")[0].strip()

+

+    comment_re = re.compile(r"(?:^|\s+)#.*$")

+    line = comment_re.sub("", line)

     parser = ArgumentParser("indexes")

     parser.add_argument("-i", "--index-url", dest="index")

     parser.add_argument("--extra-index-url", dest="extra_index")

-- 

2.33.1

From 2640cccad9c770255fd529de676b48967faeed90 Mon Sep 17 00:00:00 2001

From: Frost Ming <mianghong@gmail.com>

Date: Thu, 23 Dec 2021 15:20:11 +0800

Subject: [PATCH 4/4] fix pip location

---

 pipenv/patched/notpip/_internal/build_env.py | 2 +-

 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pipenv/patched/notpip/_internal/build_env.py b/pipenv/patched/notpip/_internal/build_env.py

index 71fa326..a4db7f1 100644

--- a/pipenv/patched/notpip/_internal/build_env.py

+++ b/pipenv/patched/notpip/_internal/build_env.py

@@ -15,7 +15,7 @@ from sysconfig import get_paths

 from pipenv.patched.notpip._vendor.pkg_resources import Requirement, VersionConflict, WorkingSet

-from pipenv.patched.notpip import __file__ as pip_location

+from pip import __file__ as pip_location

 from pipenv.patched.notpip._internal.utils.subprocess import call_subprocess

 from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory

 from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING

-- 

2.33.1