#120 F30: Merge with master and 3.7.4
Merged 4 years ago by churchyard. Opened 4 years ago by churchyard.
rpms/ churchyard/python3 3.7.4  into  f30

@@ -1,42 +0,0 @@ 

- diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py

- index 0faf2bb..d0365ec 100644

- --- a/Lib/test/test_urlparse.py

- +++ b/Lib/test/test_urlparse.py

- @@ -1011,6 +1011,12 @@ class UrlParseTestCase(unittest.TestCase):

-          self.assertIn('\u2100', denorm_chars)

-          self.assertIn('\uFF03', denorm_chars)

-  

- +        # bpo-36742: Verify port separators are ignored when they

- +        # existed prior to decomposition

- +        urllib.parse.urlsplit('http://\u30d5\u309a:80')

- +        with self.assertRaises(ValueError):

- +            urllib.parse.urlsplit('http://\u30d5\u309a\ufe1380')

- +

-          for scheme in ["http", "https", "ftp"]:

-              for c in denorm_chars:

-                  url = "{}://netloc{}false.netloc/path".format(scheme, c)

- diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py

- index 8b6c9b1..e2f7b69 100644

- --- a/Lib/urllib/parse.py

- +++ b/Lib/urllib/parse.py

- @@ -402,13 +402,16 @@ def _checknetloc(netloc):

-      # looking for characters like \u2100 that expand to 'a/c'

-      # IDNA uses NFKC equivalence, so normalize for this check

-      import unicodedata

- -    netloc2 = unicodedata.normalize('NFKC', netloc)

- -    if netloc == netloc2:

- +    n = netloc.rpartition('@')[2] # ignore anything to the left of '@'

- +    n = n.replace(':', '')        # ignore characters already included

- +    n = n.replace('#', '')        # but not the surrounding text

- +    n = n.replace('?', '')

- +    netloc2 = unicodedata.normalize('NFKC', n)

- +    if n == netloc2:

-          return

- -    _, _, netloc = netloc.rpartition('@') # anything to the left of '@' is okay

-      for c in '/?#@:':

-          if c in netloc2:

- -            raise ValueError("netloc '" + netloc2 + "' contains invalid " +

- +            raise ValueError("netloc '" + netloc + "' contains invalid " +

-                               "characters under NFKC normalization")

-  

-  def urlsplit(url, scheme='', allow_fragments=True):

@@ -1,150 +0,0 @@ 

- From 7e200e0763f5b71c199aaf98bd5588f291585619 Mon Sep 17 00:00:00 2001

- From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= <miro@hroncok.cz>

- Date: Tue, 7 May 2019 17:28:47 +0200

- Subject: [PATCH] bpo-30458: Disallow control chars in http URLs. (GH-12755)

-  (GH-13154)

- MIME-Version: 1.0

- Content-Type: text/plain; charset=UTF-8

- Content-Transfer-Encoding: 8bit

- 

- Disallow control chars in http URLs in urllib.urlopen.  This addresses a potential security problem for applications that do not sanity check their URLs where http request headers could be injected.

- 

- Disable https related urllib tests on a build without ssl (GH-13032)

- These tests require an SSL enabled build. Skip these tests when python is built without SSL to fix test failures.

- 

- Use http.client.InvalidURL instead of ValueError as the new error case's exception. (GH-13044)

- 

- Backport Co-Authored-By: Miro Hrončok <miro@hroncok.cz>

- ---

-  Lib/http/client.py                            | 15 ++++++

-  Lib/test/test_urllib.py                       | 53 +++++++++++++++++++

-  Lib/test/test_xmlrpc.py                       |  7 ++-

-  .../2019-04-10-08-53-30.bpo-30458.51E-DA.rst  |  1 +

-  4 files changed, 75 insertions(+), 1 deletion(-)

-  create mode 100644 Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst

- 

- diff --git a/Lib/http/client.py b/Lib/http/client.py

- index 1de151c38e..2afd452fe3 100644

- --- a/Lib/http/client.py

- +++ b/Lib/http/client.py

- @@ -140,6 +140,16 @@ _MAXHEADERS = 100

-  _is_legal_header_name = re.compile(rb'[^:\s][^:\r\n]*').fullmatch

-  _is_illegal_header_value = re.compile(rb'\n(?![ \t])|\r(?![ \t\n])').search

-  

- +# These characters are not allowed within HTTP URL paths.

- +#  See https://tools.ietf.org/html/rfc3986#section-3.3 and the

- +#  https://tools.ietf.org/html/rfc3986#appendix-A pchar definition.

- +# Prevents CVE-2019-9740.  Includes control characters such as \r\n.

- +# We don't restrict chars above \x7f as putrequest() limits us to ASCII.

- +_contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]')

- +# Arguably only these _should_ allowed:

- +#  _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$")

- +# We are more lenient for assumed real world compatibility purposes.

- +

-  # We always set the Content-Length header for these methods because some

-  # servers will otherwise respond with a 411

-  _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}

- @@ -1101,6 +1111,11 @@ class HTTPConnection:

-          self._method = method

-          if not url:

-              url = '/'

- +        # Prevent CVE-2019-9740.

- +        match = _contains_disallowed_url_pchar_re.search(url)

- +        if match:

- +            raise InvalidURL(f"URL can't contain control characters. {url!r} "

- +                             f"(found at least {match.group()!r})")

-          request = '%s %s %s' % (method, url, self._http_vsn_str)

-  

-          # Non-ASCII characters should have been eliminated earlier

- diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py

- index 2ac73b58d8..7214492eca 100644

- --- a/Lib/test/test_urllib.py

- +++ b/Lib/test/test_urllib.py

- @@ -329,6 +329,59 @@ class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin, FakeFTPMixin):

-          finally:

-              self.unfakehttp()

-  

- +    @unittest.skipUnless(ssl, "ssl module required")

- +    def test_url_with_control_char_rejected(self):

- +        for char_no in list(range(0, 0x21)) + [0x7f]:

- +            char = chr(char_no)

- +            schemeless_url = f"//localhost:7777/test{char}/"

- +            self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello.")

- +            try:

- +                # We explicitly test urllib.request.urlopen() instead of the top

- +                # level 'def urlopen()' function defined in this... (quite ugly)

- +                # test suite.  They use different url opening codepaths.  Plain

- +                # urlopen uses FancyURLOpener which goes via a codepath that

- +                # calls urllib.parse.quote() on the URL which makes all of the

- +                # above attempts at injection within the url _path_ safe.

- +                escaped_char_repr = repr(char).replace('\\', r'\\')

- +                InvalidURL = http.client.InvalidURL

- +                with self.assertRaisesRegex(

- +                    InvalidURL, f"contain control.*{escaped_char_repr}"):

- +                    urllib.request.urlopen(f"http:{schemeless_url}")

- +                with self.assertRaisesRegex(

- +                    InvalidURL, f"contain control.*{escaped_char_repr}"):

- +                    urllib.request.urlopen(f"https:{schemeless_url}")

- +                # This code path quotes the URL so there is no injection.

- +                resp = urlopen(f"http:{schemeless_url}")

- +                self.assertNotIn(char, resp.geturl())

- +            finally:

- +                self.unfakehttp()

- +

- +    @unittest.skipUnless(ssl, "ssl module required")

- +    def test_url_with_newline_header_injection_rejected(self):

- +        self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello.")

- +        host = "localhost:7777?a=1 HTTP/1.1\r\nX-injected: header\r\nTEST: 123"

- +        schemeless_url = "//" + host + ":8080/test/?test=a"

- +        try:

- +            # We explicitly test urllib.request.urlopen() instead of the top

- +            # level 'def urlopen()' function defined in this... (quite ugly)

- +            # test suite.  They use different url opening codepaths.  Plain

- +            # urlopen uses FancyURLOpener which goes via a codepath that

- +            # calls urllib.parse.quote() on the URL which makes all of the

- +            # above attempts at injection within the url _path_ safe.

- +            InvalidURL = http.client.InvalidURL

- +            with self.assertRaisesRegex(

- +                InvalidURL, r"contain control.*\\r.*(found at least . .)"):

- +                urllib.request.urlopen(f"http:{schemeless_url}")

- +            with self.assertRaisesRegex(InvalidURL, r"contain control.*\\n"):

- +                urllib.request.urlopen(f"https:{schemeless_url}")

- +            # This code path quotes the URL so there is no injection.

- +            resp = urlopen(f"http:{schemeless_url}")

- +            self.assertNotIn(' ', resp.geturl())

- +            self.assertNotIn('\r', resp.geturl())

- +            self.assertNotIn('\n', resp.geturl())

- +        finally:

- +            self.unfakehttp()

- +

-      def test_read_0_9(self):

-          # "0.9" response accepted (but not "simple responses" without

-          # a status line)

- diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py

- index 32263f7f0b..0e002ec4ef 100644

- --- a/Lib/test/test_xmlrpc.py

- +++ b/Lib/test/test_xmlrpc.py

- @@ -945,7 +945,12 @@ class SimpleServerTestCase(BaseServerTestCase):

-      def test_partial_post(self):

-          # Check that a partial POST doesn't make the server loop: issue #14001.

-          conn = http.client.HTTPConnection(ADDR, PORT)

- -        conn.request('POST', '/RPC2 HTTP/1.0\r\nContent-Length: 100\r\n\r\nbye')

- +        conn.send('POST /RPC2 HTTP/1.0\r\n'

- +                  'Content-Length: 100\r\n\r\n'

- +                  'bye HTTP/1.1\r\n'

- +                  f'Host: {ADDR}:{PORT}\r\n'

- +                  'Accept-Encoding: identity\r\n'

- +                  'Content-Length: 0\r\n\r\n'.encode('ascii'))

-          conn.close()

-  

-      def test_context_manager(self):

- diff --git a/Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst b/Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst

- new file mode 100644

- index 0000000000..ed8027fb4d

- --- /dev/null

- +++ b/Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst

- @@ -0,0 +1 @@

- +Address CVE-2019-9740 by disallowing URL paths with embedded whitespace or control characters through into the underlying http client request.  Such potentially malicious header injection URLs now cause an http.client.InvalidURL exception to be raised.

- -- 

- 2.21.0

- 

file modified
+17 -25
@@ -13,11 +13,11 @@ 

  

  #  WARNING  When rebasing to a new Python version,

  #           remember to update the python3-docs package as well

- %global general_version %{pybasever}.3

+ %global general_version %{pybasever}.4

  #global prerel ...

  %global upstream_version %{general_version}%{?prerel}

  Version: %{general_version}%{?prerel:~%{prerel}}

- Release: 3%{?dist}

+ Release: 1%{?dist}

  License: Python

  

  
@@ -126,6 +126,12 @@ 

  # on files that test invalid syntax.

  %undefine py_auto_byte_compile

  

+ # Don't let RPM set SOURCE_DATE_EPOCH based on the latest %%changelog date

+ # It breaks tests with: can't find '__main__' module in .../test_zip.zip

+ # Reported at https://bugs.python.org/issue34022

+ # Tracked at https://bugzilla.redhat.com/show_bug.cgi?id=1724753

+ %global source_date_epoch_from_changelog 0

+ 

  # For multilib support, files that are different between 32- and 64-bit arches

  # need different filenames. Use "64" or "32" according to the word size.

  # Currently, the best way to determine an architecture's word size happens to
@@ -282,18 +288,6 @@ 

  # So we mark the command as unsupported - and the tests are skipped

  Patch316: 00316-mark-bdist_wininst-unsupported.patch

  

- # 00320 #

- # Fix handling of pre-normalization characters in urlsplit()

- # This fixes a regression introduced by the fix for CVE-2019-9636

- # Fixed upstream: https://bugs.python.org/issue36742

- Patch320: 00320-fix-pre-normalization-chars-in-urlsplit.patch

- 

- # 00324 #

- # Disallow control chars in http URLs

- # Security fix for CVE-2019-9740 and CVE-2019-9947

- # Fixed upstream: https://bugs.python.org/issue30458

- Patch324: 00324-disallow-control-chars-in-http-urls.patch

- 

  # (New patches go here ^^^)

  #

  # When adding new patches to "python" and "python3" in Fedora, EL, etc.,
@@ -587,8 +581,6 @@ 

  %patch251 -p1

  %patch274 -p1

  %patch316 -p1

- %patch320 -p1

- %patch324 -p1

  

  

  # Remove files that should be generated by the build
@@ -980,12 +972,6 @@ 

    ConfName=$1

    ConfDir=$(pwd)/build/$ConfName

  

-   # Fedora sets TLSv1 as explicit minimum version.

-   # Python's test suite assumes that the minimum protocol version is set to

-   # a magic marker. We workaround the test problem by setting:

-   export OPENSSL_CONF=/non-existing-file

-   # https://bugzilla.redhat.com/show_bug.cgi?id=1618753

- 

    echo STARTING: CHECKING OF PYTHON FOR CONFIGURATION: $ConfName

  

    # Note that we're running the tests using the version of the code in the
@@ -999,8 +985,6 @@ 

    #   https://bugzilla.redhat.com/show_bug.cgi?id=1196181

    # test_gdb skipped on s390x:

    #   https://bugzilla.redhat.com/show_bug.cgi?id=1678277

-   # test_asyncio skipped:

-   #   https://bugs.python.org/issue35998

    LD_LIBRARY_PATH=$ConfDir $ConfDir/python -m test.regrtest \

      -wW --slowest -j0 \

      -x test_distutils \
@@ -1011,7 +995,6 @@ 

      %ifarch %{mips64}

      -x test_ctypes \

      %endif

-     -x test_asyncio \

  

    echo FINISHED: CHECKING OF PYTHON FOR CONFIGURATION: $ConfName

  
@@ -1508,6 +1491,15 @@ 

  # ======================================================

  

  %changelog

+ * Tue Jul 09 2019 Miro Hrončok <mhroncok@redhat.com> - 3.7.4-1

+ - Update to 3.7.4

+ 

+ * Tue Jul 02 2019 Miro Hrončok <mhroncok@redhat.com> - 3.7.4~rc2-1

+ - Update to 3.7.4rc2

+ 

+ * Tue Jun 25 2019 Miro Hrončok <mhroncok@redhat.com> - 3.7.4~rc1-1

+ - Update to 3.7.4rc1

+ 

  * Tue May 07 2019 Charalampos Stratakis <cstratak@redhat.com> - 3.7.3-3

  - Fix handling of pre-normalization characters in urlsplit

  - Disallow control chars in http URLs (#1695572, #1700684, #1688169, #1706851)

file modified
+1 -1
@@ -1,1 +1,1 @@ 

- SHA512 (Python-3.7.3.tar.xz) = 6d9b7c0f1764e0f655a39430a3af6f7b5e3c9b7166c042e780677a54b17ad4ca6d0d9cba262c82b1b70bba8f7c28883dad4cc0d7cc194fc7d2c1b5f4f08a763a

+ SHA512 (Python-3.7.4.tar.xz) = 71f64668c259f3ed07bd4aa239dbba6cc1b6b0a84d50bbff160664845d7986f757e4d65fca327e62a2d12ba593742ca83a22f7cb6093aff8715ec916c2ba9416

file modified
+3 -3
@@ -13,10 +13,10 @@ 

          run: VERSION=3.7 ./venv.sh

      - selftest:

          dir: python/selftest

-         run: VERSION=3.7 X="-x test_socket -x test_asyncgen -x test_asyncio -x test_compile -x test_concurrent_futures -x test_itertools -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_shutil -x test_time -x test_multiprocessing_spawn -x test_threading -x test_wsgiref" ./parallel.sh

-     - findleaks:

+         run: VERSION=3.7 X="-x test_wsgiref" ./parallel.sh

+     - debugtest:

          dir: python/selftest

-         run: VERSION=3.7 X="-x test_socket -x test_asyncgen -x test_asyncio -x test_compile -x test_concurrent_futures -x test_itertools -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_shutil -x test_time -x test_multiprocessing_spawn -x test_threading -x test_wsgiref" ./findleaks.sh

+         run: VERSION=3.7 PYTHON=python3-debug X="-x test_wsgiref" ./parallel.sh

      required_packages:

      - gcc  # for extension building in venv and selftest

      - gdb  # for test_gdb