#42 Make compatible with python 3.10
Merged 2 years ago by zuul. Opened 2 years ago by mhuin.
rpms/ mhuin/python-gear py310  into  rawhide

@@ -0,0 +1,51 @@ 

+ From 8a76de29e1ceebcde517c6491de190de4e005303 Mon Sep 17 00:00:00 2001

+ From: shangxdy <shang.xiaodong@zte.com.cn>

+ Date: Thu, 4 Jan 2018 10:24:06 +0800

+ Subject: [PATCH 1/9] Modify connection timeout process

+ 

+ When connection to gearman server is timeout, it's necessary to

+ release lock before raising a exception, otherwise the client may

+ be dead locked.

+ 

+ Change-Id: Idc9c9c4bd439b122dc7855ca05d962c4e6687829

+ Signed-off-by: shangxdy <shang.xiaodong@zte.com.cn>

+ Signed-off-by: Matthieu Huin <mhuin@redhat.com>

+ ---

+  gear/__init__.py | 21 ++++++++++++---------

+  1 file changed, 12 insertions(+), 9 deletions(-)

+ 

+ diff --git a/gear/__init__.py b/gear/__init__.py

+ index 0a78fe4..7695490 100644

+ --- a/gear/__init__.py

+ +++ b/gear/__init__.py

+ @@ -1221,15 +1221,18 @@ class BaseClient(BaseClientServer):

+          start_time = time.time()

+          while self.running:

+              self.connections_condition.acquire()

+ -            while self.running and not self.active_connections:

+ -                if timeout is not None:

+ -                    self._checkTimeout(start_time, timeout)

+ -                self.log.debug("Waiting for at least one active connection")

+ -                self.connections_condition.wait(timeout=1)

+ -            if self.active_connections:

+ -                self.log.debug("Active connection found")

+ -                connected = True

+ -            self.connections_condition.release()

+ +            try:

+ +                while self.running and not self.active_connections:

+ +                    if timeout is not None:

+ +                        self._checkTimeout(start_time, timeout)

+ +                    self.log.debug("Waiting for at least one active "

+ +                                   "connection")

+ +                    self.connections_condition.wait(timeout=1)

+ +                if self.active_connections:

+ +                    self.log.debug("Active connection found")

+ +                    connected = True

+ +            finally:

+ +                self.connections_condition.release()

+              if connected:

+                  return

+  

+ -- 

+ 2.31.1

+ 

@@ -0,0 +1,57 @@ 

+ From 9261c8f4a79971e5d5361dd2d0df902bdc44e394 Mon Sep 17 00:00:00 2001

+ From: Tobias Henkel <tobias.henkel@bmw.de>

+ Date: Tue, 24 Mar 2020 17:43:46 +0100

+ Subject: [PATCH 2/9] Move handleDisconnect into BaseClientServer

+ 

+ It's called from there using self.handleDisconnect so define it there

+ as well.

+ 

+ Change-Id: I90fef55a79168e082e69f81a717c08badd4163a9

+ Signed-off-by: Matthieu Huin <mhuin@redhat.com>

+ ---

+  gear/__init__.py | 22 +++++++++++-----------

+  1 file changed, 11 insertions(+), 11 deletions(-)

+ 

+ diff --git a/gear/__init__.py b/gear/__init__.py

+ index f7fc767..891804b 100644

+ --- a/gear/__init__.py

+ +++ b/gear/__init__.py

+ @@ -996,6 +996,17 @@ class BaseClientServer(object):

+          end = time.time()

+          self.reportTimingStats(packet.ptype, end - start)

+  

+ +    def handleDisconnect(self, job):

+ +        """Handle a Gearman server disconnection.

+ +

+ +        If the Gearman server is disconnected, this will be called for any

+ +        jobs currently associated with the server.

+ +

+ +        :arg Job packet: The :py:class:`Job` that was running when the server

+ +            disconnected.

+ +        """

+ +        return job

+ +

+      def reportTimingStats(self, ptype, duration):

+          """Report processing times by packet type

+  

+ @@ -1696,17 +1707,6 @@ class Client(BaseClient):

+          packet.connection.handleOptionRes(packet.getArgument(0))

+          task.setComplete()

+  

+ -    def handleDisconnect(self, job):

+ -        """Handle a Gearman server disconnection.

+ -

+ -        If the Gearman server is disconnected, this will be called for any

+ -        jobs currently associated with the server.

+ -

+ -        :arg Job packet: The :py:class:`Job` that was running when the server

+ -            disconnected.

+ -        """

+ -        return job

+ -

+  

+  class FunctionRecord(object):

+      """Represents a function that should be registered with Gearman.

+ -- 

+ 2.31.1

+ 

0003-Bump-crypto-requirement-to-accomodate-security-stand.patch 0001-Bump-crypto-requirement-to-accomodate-security-stand.patch
file renamed
+6 -4
@@ -1,7 +1,9 @@ 

- From 233eb64672bfa3430daa0a08786145162fd8a8c8 Mon Sep 17 00:00:00 2001

+ From 79e1c3009e491b2cf8275dc52ce861719ff86d25 Mon Sep 17 00:00:00 2001

  From: Fabien Boucher <fboucher@redhat.com>

  Date: Tue, 21 Jul 2020 10:27:57 +0000

- Subject: [PATCH] Bump crypto requirement to accomodate security standards

+ Subject: [PATCH 3/9] Bump crypto requirement to accomodate security standards

+ 

+ Depends-on: https://review.opendev.org/742165

  

  On Fedora rawhide the gear package no longer build.

  https://koschei.fedoraproject.org/package/python-gear?
@@ -18,6 +20,7 @@ 

  tox -epy38

  

  Change-Id: I57cd9c4750f27b7b76e92a0eef03e7de70c13dd5

+ Signed-off-by: Matthieu Huin <mhuin@redhat.com>

  ---

   gear/tests/test_functional.py | 4 ++--

   1 file changed, 2 insertions(+), 2 deletions(-)
@@ -45,6 +48,5 @@ 

               cert.sign(key, 'sha1')

   

  -- 

- 2.28.0.rc1

- 

+ 2.31.1

  

@@ -0,0 +1,50 @@ 

+ From a160d10735c62e046b72e31e8eb7756c5f06e09f Mon Sep 17 00:00:00 2001

+ From: Ahmon Dancy <adancy@wikimedia.org>

+ Date: Mon, 27 Jul 2020 14:46:49 -0700

+ Subject: [PATCH 7/9] wakeConnections: Randomize connections before scanning

+  them

+ 

+ gear/__init__.py:

+  Modified Server.wakeConnections() so that it randomizes the

+  list of active connections before sending out NOOP's to them.

+  This will hopefully spread workload across machines more evenly

+  when there are multiple workers per machine.

+ 

+ Reference: https://phabricator.wikimedia.org/T258630

+ 

+ Change-Id: I05dcb9fa383f3aefc8b5b1bb9dd8b3ff6ff7f37d

+ Signed-off-by: Matthieu Huin <mhuin@redhat.com>

+ ---

+  gear/__init__.py | 9 ++++++++-

+  1 file changed, 8 insertions(+), 1 deletion(-)

+ 

+ diff --git a/gear/__init__.py b/gear/__init__.py

+ index f7fc767..4a0fcb7 100644

+ --- a/gear/__init__.py

+ +++ b/gear/__init__.py

+ @@ -15,6 +15,7 @@

+  import errno

+  import logging

+  import os

+ +import random

+  import select

+  import six

+  import socket

+ @@ -3295,7 +3296,13 @@ class Server(BaseClientServer):

+  

+      def wakeConnections(self, job=None):

+          p = Packet(constants.RES, constants.NOOP, b'')

+ -        for connection in self.active_connections:

+ +

+ +        # Use a randomized copy of active_connections to try

+ +        # to spread workload across the machines that workers are on.

+ +        conns = self.active_connections[:]

+ +        random.shuffle(conns)    # Modifies the list

+ +

+ +        for connection in conns:

+              if connection.state == 'SLEEP':

+                  if ((job and job.name in connection.functions) or

+                          (job is None)):

+ -- 

+ 2.31.1

+ 

@@ -0,0 +1,135 @@ 

+ From 66ba8442dcaba24de0322793876bd241858b49a3 Mon Sep 17 00:00:00 2001

+ From: Guillaume Chauvel <guillaume.chauvel@gmail.com>

+ Date: Wed, 15 Jul 2020 19:34:49 +0200

+ Subject: [PATCH 8/9] Create SSL context using PROTOCOL_TLS, fallback to

+  highest supported version

+ 

+ Zuul test: tests.unit.test_scheduler.TestSchedulerSSL.test_jobs_executed

+ fails on ubuntu focal with the following exception:

+ 

+ Traceback (most recent call last):

+   File "/home/gchauvel/zuul/zuul/.tox/py38/lib/python3.8/site-packages/gear/__init__.py", line 2835, in _doConnectLoop

+     self.connectLoop()

+   File "/home/gchauvel/zuul/zuul/.tox/py38/lib/python3.8/site-packages/gear/__init__.py", line 2865, in connectLoop

+     c = context.wrap_socket(c, server_side=True)

+   File "/usr/lib/python3.8/ssl.py", line 500, in wrap_socket

+     return self.sslsocket_class._create(

+   File "/usr/lib/python3.8/ssl.py", line 1040, in _create

+     self.do_handshake()

+   File "/usr/lib/python3.8/ssl.py", line 1309, in do_handshake

+     self._sslobj.do_handshake()

+ ssl.SSLError: [SSL] internal error (_ssl.c:1108)

+ 

+ This is due to libssl1.1 being compiled with

+ "-DOPENSSL_TLS_SECURITY_LEVEL=2" and gear forcing TLSv1.0

+ 

+ Extracted from ubuntu source package:

+ 

+   The default security level for TLS connections was increased from

+   level 1 to level 2. This moves from the 80 bit security level to the

+   112 bit security level and will require 2048 bit or larger RSA and

+   DHE keys, 224 bit or larger ECC keys, SHA-2, TLSv1.2 or DTLSv1.2.

+ 

+ Allowing to negotiate TLS to the highest available version between

+ server and client solves the issue, provided that TLSv1.2 is useable.

+ The option is supported by in the latest version of all pythons >=3.5

+ [1][2][3]. Unfortunately Xenial doesn't have latest 3.5 and lacks the

+ ssl.PROTOCOL_TLS definition. We provide a fallback to select the highest

+ version of TLS supported in that case.

+ 

+ There is some risk using the fallback beacuse both the client and server

+ need to agree on the version supported in this case. Xenial python 3.5

+ does support TLSv1_2 which means that for all practical purposes TLS

+ v1.2 should be available on all platforms that gear runs avoiding this

+ problem.

+ 

+ Disable TLSv1.3:

+ According to https://bugs.python.org/issue43622#msg389497, an event on

+ ssl socket can happen without data being available at application level.

+ As gear is using a polling loop with multiple file descriptors and ssl

+ socket used as a blocking one, a blocked state could happen.

+ This is highlighted by Zuul SSL test: TestSchedulerSSL, where such

+ blocked state appears consistently.

+ note: gear tests and zuul tests are ok when using TLSv1.2 but the

+ previous behavior could also happen

+ 

+ [1] https://docs.python.org/2.7/library/ssl.html?highlight=protocol_tls#ssl.PROTOCOL_TLS

+ [2] https://docs.python.org/3.5/library/ssl.html?highlight=protocol_tls#ssl.PROTOCOL_TLS

+ [3] https://docs.python.org/3/library/ssl.html?highlight=protocol_tls#ssl.PROTOCOL_TLS

+ 

+ Change-Id: I5efb6c0576987815c5b93f8bc4020cdee2898d04

+ Signed-off-by: Matthieu Huin <mhuin@redhat.com>

+ ---

+  gear/__init__.py | 42 ++++++++++++++++++++++++++++++++++++++++--

+  1 file changed, 40 insertions(+), 2 deletions(-)

+ 

+ diff --git a/gear/__init__.py b/gear/__init__.py

+ index 6287281..6e0f1c8 100644

+ --- a/gear/__init__.py

+ +++ b/gear/__init__.py

+ @@ -91,6 +91,44 @@ def convert_to_bytes(data):

+      return data

+  

+  

+ +def best_tls_version():

+ +    if hasattr(ssl, 'PROTOCOL_TLS'):

+ +        return ssl.PROTOCOL_TLS

+ +    # Note there is some risk in selecting tls 1.2 if available

+ +    # as both the client and server may not support it and need 1.1

+ +    # or 1.0. However, a xenial installation with python 3.5 does

+ +    # support 1.2 which is probably as old a setup as we need to worry

+ +    # about.

+ +    elif hasattr(ssl, 'PROTOCOL_TLSv1_2'):

+ +        return ssl.PROTOCOL_TLSv1_2

+ +    elif hasattr(ssl, 'PROTOCOL_TLSv1_1'):

+ +        return ssl.PROTOCOL_TLSv1_1

+ +    elif hasattr(ssl, 'PROTOCOL_TLSv1'):

+ +        return ssl.PROTOCOL_TLSv1

+ +    else:

+ +        raise ConnectionError('No supported TLS version available.')

+ +

+ +

+ +def create_ssl_context():

+ +    tls_version = best_tls_version()

+ +    context = ssl.SSLContext(tls_version)

+ +

+ +    # Disable TLSv1.3

+ +    # According to https://bugs.python.org/issue43622#msg389497, an event on

+ +    # ssl socket can happen without data being available at application level.

+ +    # As gear is using a polling loop with multiple file descriptors and ssl

+ +    # socket used as a blocking one, a blocked state could happen.

+ +    # This is highlighted by Zuul SSL test: TestSchedulerSSL, where such

+ +    # blocked state appears consistently.

+ +    # note: gear tests and zuul tests are ok for TLSv1.2 but this behavior

+ +    # could also happen

+ +    if (hasattr(ssl, 'PROTOCOL_TLS') and

+ +            tls_version == ssl.PROTOCOL_TLS):

+ +        context.options |= ssl.OP_NO_TLSv1_3

+ +

+ +    return context

+ +

+ +

+  class Task(object):

+      def __init__(self):

+          self._wait_event = threading.Event()

+ @@ -209,7 +247,7 @@ class Connection(object):

+  

+              if self.use_ssl:

+                  self.log.debug("Using SSL")

+ -                context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)

+ +                context = create_ssl_context()

+                  context.verify_mode = ssl.CERT_REQUIRED

+                  context.check_hostname = False

+                  context.load_cert_chain(self.ssl_cert, self.ssl_key)

+ @@ -2862,7 +2900,7 @@ class Server(BaseClientServer):

+                      self.log.debug("Accepting new connection")

+                      c, addr = self.socket.accept()

+                      if self.use_ssl:

+ -                        context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)

+ +                        context = create_ssl_context()

+                          context.verify_mode = ssl.CERT_REQUIRED

+                          context.load_cert_chain(self.ssl_cert, self.ssl_key)

+                          context.load_verify_locations(self.ssl_ca)

+ -- 

+ 2.31.1

+ 

file modified
+11 -3
@@ -3,14 +3,19 @@ 

  

  Name: python-%{srcname}

  Version: 0.15.1

- Release: 7%{?dist}

+ Release: 8%{?dist}

  Summary: Pure Python Async Gear Protocol Library

  

  License: ASL 2.0

  URL: https://opendev.org/opendev/%{srcname}

  Source0: %pypi_source

  

- Patch01: 0001-Bump-crypto-requirement-to-accomodate-security-stand.patch

+ Patch01: 0001-Modify-connection-timeout-process.patch

+ Patch02: 0002-Move-handleDisconnect-into-BaseClientServer.patch

+ Patch03: 0003-Bump-crypto-requirement-to-accomodate-security-stand.patch

+ Patch07: 0007-wakeConnections-Randomize-connections-before-scannin.patch

+ Patch08: 0008-Create-SSL-context-using-PROTOCOL_TLS-fallback-to-hi.patch

+ 

  

  BuildArch: noarch

  
@@ -63,6 +68,9 @@ 

  %{python3_sitelib}/%{srcname}-*egg-info

  

  %changelog

+ * Thu Jun 17 2021 Matthieu Huin <mhuin@redhat.com> - 0.15.1-8

+ - Fix support for Python 3.10

+ 

  * Fri Jun 04 2021 Python Maint <python-maint@redhat.com> - 0.15.1-7

  - Rebuilt for Python 3.10

  
@@ -73,7 +81,7 @@ 

  - Remove BuildRequire on git

  - Let runtime deps to be discovered out from requierements.txt

  

- * Tue Jul 30 2020 Faben Boucher <fboucher@redhat.com> - 0.15.1-4

+ * Thu Jul 30 2020 Faben Boucher <fboucher@redhat.com> - 0.15.1-4

  - Fix FTBFS by providing upstream patch

    0001-Bump-crypto-requirement-to-accomodate-security-stand.patch

  

no initial comment

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

1 new commit added

  • Fix bogus date in changelog
2 years ago

Build succeeded.

Do you really need to carry all those patches ?

Do you really need to carry all those patches ?

Probably not. This is actually pretty much the next version of gear, but upstream doesn't want to release it yet, until zuul 5 is out. I didn't want to delve too much into cherry picking because I know this version works with python 3.10, but if it is really a problem, I can try to pinpoint which patches are absolutely required.

1 new commit added

  • Remove irrelevant patches
2 years ago

Build succeeded.

Metadata Update from @fbo:
- Pull-request tagged with: gateit

2 years ago

Build succeeded (gate pipeline).

Pull-Request has been merged by zuul

2 years ago

Build succeeded (promote pipeline).