diff --git a/master/buildbot/plugins/db.py b/master/buildbot/plugins/db.py
index f08833e9993..15c6ebf8740 100644
--- a/master/buildbot/plugins/db.py
+++ b/master/buildbot/plugins/db.py
@@ -25,6 +25,7 @@
from buildbot.errors import PluginDBError
from buildbot.interfaces import IPlugin
+from buildbot.util.importlib_compat import entry_points_get
# Base namespace for Buildbot specific plugins
_NAMESPACE_BASE = 'buildbot'
@@ -34,8 +35,8 @@ def find_distribution_info(entry_point_name, entry_point_group):
for distribution in distributions():
# each distribution can have many entry points
try:
- for ep in distribution.entry_points:
- if ep.name == entry_point_name and ep.group == entry_point_group:
+ for ep in entry_points_get(distribution.entry_points, entry_point_group):
+ if ep.name == entry_point_name:
return (distribution.metadata['Name'], distribution.metadata['Version'])
except KeyError as exc:
raise PluginDBError("Plugin info was found, but it is invalid.") from exc
@@ -242,7 +243,7 @@ def _load_entry(self, entry):
def _tree(self):
if self._real_tree is None:
self._real_tree = _NSNode()
- entries = entry_points().get(self._group, [])
+ entries = entry_points_get(entry_points(), self._group)
for entry in entries:
self._real_tree.add(entry.name,
_PluginEntry(self._group, entry,
diff --git a/master/buildbot/test/util/www.py b/master/buildbot/test/util/www.py
index df3ab34443c..dc312828842 100644
--- a/master/buildbot/test/util/www.py
+++ b/master/buildbot/test/util/www.py
@@ -29,6 +29,7 @@
from buildbot.test.fake import fakemaster
from buildbot.util import bytes2unicode
from buildbot.util import unicode2bytes
+from buildbot.util.importlib_compat import entry_points_get
from buildbot.www import auth
from buildbot.www import authz
@@ -123,7 +124,7 @@ def getSession(self):
class RequiresWwwMixin:
# mix this into a TestCase to skip if buildbot-www is not installed
- if not [ep for ep in entry_points().get('buildbot.www', []) if ep.name == 'base']:
+ if not [ep for ep in entry_points_get(entry_points(), 'buildbot.www') if ep.name == 'base']:
if 'BUILDBOT_TEST_REQUIRE_WWW' in os.environ:
raise RuntimeError('$BUILDBOT_TEST_REQUIRE_WWW is set but '
'buildbot-www is not installed')
diff --git a/master/buildbot/util/importlib_compat.py b/master/buildbot/util/importlib_compat.py
new file mode 100644
index 00000000000..9c0aa3eecbe
--- /dev/null
+++ b/master/buildbot/util/importlib_compat.py
@@ -0,0 +1,33 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+# This module is for backward compatibility of importlib.
+
+
+def entry_points_get(entry_points, group):
+ """ Since Python 3.12 dictionary access is removed and replaced by new interface.
+ see: https://github.com/python/cpython/issues/97781
+ """
+ if hasattr(entry_points, "select"):
+ return entry_points.select(group=group)
+ else:
+ if isinstance(entry_points, list):
+ filtered_entry_points = []
+ for ep in entry_points:
+ if ep.group == group:
+ filtered_entry_points.append(ep)
+ return filtered_entry_points
+ else:
+ return entry_points.get(group, [])
From 501a2edcc13bdad04cae5f8f01e2bb94009af4b9 Mon Sep 17 00:00:00 2001
From: Pavol Misik <pmisik@gmail.com>
Date: Mon, 20 Nov 2023 23:07:10 +0100
Subject: [PATCH 07/15] test: Disable twisted generated warning "the (type,
exc, tb) signature of throw() is deprecated" on Python 3.12
---
master/buildbot/test/__init__.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/master/buildbot/__init__.py b/master/buildbot/__init__.py
index c7640ce56a1..0681ee8d516 100644
--- a/master/buildbot/__init__.py
+++ b/master/buildbot/__init__.py
@@ -53,7 +53,7 @@ def mTimeVersion(init_file):
for root, _, files in os.walk(cwd):
for f in files:
m = max(os.path.getmtime(os.path.join(root, f)), m)
- d = datetime.datetime.utcfromtimestamp(m)
+ d = datetime.datetime.fromtimestamp(m, datetime.timezone.utc)
return d.strftime("%Y.%m.%d")
@@ -80,7 +80,7 @@ def getVersionFromArchiveId(git_archive_id='$Format:%ct %d$'):
# archived revision is not tagged, use the commit date
tstamp = git_archive_id.strip().split()[0]
- d = datetime.datetime.utcfromtimestamp(int(tstamp))
+ d = datetime.datetime.fromtimestamp(int(tstamp), datetime.timezone.utc)
return d.strftime('%Y.%m.%d')
return None
diff --git a/master/buildbot/changes/gerritchangesource.py b/master/buildbot/changes/gerritchangesource.py
index 3971173603d..308188e035f 100644
--- a/master/buildbot/changes/gerritchangesource.py
+++ b/master/buildbot/changes/gerritchangesource.py
@@ -543,7 +543,7 @@ def reconfigService(self,
@staticmethod
def now():
"""patchable now (datetime is not patchable as builtin)"""
- return datetime.datetime.utcnow()
+ return datetime.datetime.now(datetime.timezone.utc)
@defer.inlineCallbacks
def poll(self):
@@ -553,7 +553,7 @@ def poll(self):
# the last event time to some historical look-back
last_event = self.now() - datetime.timedelta(days=self._first_fetch_lookback)
else:
- last_event = datetime.datetime.utcfromtimestamp(last_event_ts)
+ last_event = datetime.datetime.fromtimestamp(last_event_ts, datetime.timezone.utc)
last_event_formatted = last_event.strftime("%Y-%m-%d %H:%M:%S")
if self.debug:
diff --git a/master/buildbot/changes/mail.py b/master/buildbot/changes/mail.py
index e9907c9aa5b..6f39f3880a5 100644
--- a/master/buildbot/changes/mail.py
+++ b/master/buildbot/changes/mail.py
@@ -116,7 +116,7 @@ def parse(self, m, prefix=None):
else:
when = mktime_tz(dateTuple)
- theTime = datetime.datetime.utcfromtimestamp(float(when))
+ theTime = datetime.datetime.fromtimestamp(float(when), datetime.timezone.utc)
rev = theTime.strftime('%Y-%m-%d %H:%M:%S')
catRE = re.compile(r'^Category:\s*(\S.*)')
diff --git a/master/buildbot/configurators/janitor.py b/master/buildbot/configurators/janitor.py
index adfe8112186..27924edb295 100644
--- a/master/buildbot/configurators/janitor.py
+++ b/master/buildbot/configurators/janitor.py
@@ -32,7 +32,7 @@
def now():
"""patchable now (datetime is not patchable as builtin)"""
- return datetime.datetime.utcnow()
+ return datetime.datetime.now(datetime.timezone.utc)
class LogChunksJanitor(BuildStep):
diff --git a/master/buildbot/schedulers/timed.py b/master/buildbot/schedulers/timed.py
index a36fbb92689..3318d28c1bf 100644
--- a/master/buildbot/schedulers/timed.py
+++ b/master/buildbot/schedulers/timed.py
@@ -286,7 +286,8 @@ def now(self):
def current_utc_offset(self, tm):
return (
- datetime.datetime.fromtimestamp(tm) - datetime.datetime.utcfromtimestamp(tm)
+ datetime.datetime.fromtimestamp(tm).replace(tzinfo=datetime.timezone.utc)
+ - datetime.datetime.fromtimestamp(tm, datetime.timezone.utc)
).total_seconds()
@defer.inlineCallbacks
diff --git a/master/buildbot/test/unit/test_util.py b/master/buildbot/test/unit/test_util.py
index dfd4e1f42a0..8e2eb482a9d 100644
--- a/master/buildbot/test/unit/test_util.py
+++ b/master/buildbot/test/unit/test_util.py
@@ -204,7 +204,7 @@ def test_UTC(self):
datetime.timedelta(0))
self.assertEqual(util.UTC.dst(datetime.datetime.now()),
datetime.timedelta(0))
- self.assertEqual(util.UTC.tzname(datetime.datetime.utcnow()), "UTC")
+ self.assertEqual(util.UTC.tzname(datetime.datetime.now(datetime.timezone.utc)), "UTC")
def test_epoch2datetime(self):
self.assertEqual(util.epoch2datetime(0),
diff --git a/master/buildbot/www/rest.py b/master/buildbot/www/rest.py
index 0685f8bbade..83090779122 100644
--- a/master/buildbot/www/rest.py
+++ b/master/buildbot/www/rest.py
@@ -337,7 +337,7 @@ def writeError(msg, errcode=404, jsonrpccode=None):
# set up caching
if self.cache_seconds:
- now = datetime.datetime.utcnow()
+ now = datetime.datetime.now(datetime.timezone.utc)
expires = now + datetime.timedelta(seconds=self.cache_seconds)
expiresBytes = unicode2bytes(
expires.strftime("%a, %d %b %Y %H:%M:%S GMT"))
diff --git a/master/buildbot/www/service.py b/master/buildbot/www/service.py
index f07c95cd90f..99e2897bc93 100644
--- a/master/buildbot/www/service.py
+++ b/master/buildbot/www/service.py
@@ -130,7 +130,7 @@ def uid(self):
This should actually only be used for cookie generation
"""
- exp = datetime.datetime.utcnow() + self.expDelay
+ exp = datetime.datetime.now(datetime.timezone.utc) + self.expDelay
claims = {
'user_info': self.user_info,
# Note that we use JWT standard 'exp' field to implement session expiration
diff --git a/master/buildbot/process/remotetransfer.py b/master/buildbot/process/remotetransfer.py
index 76a13faadf3..8ac5f888d28 100644
--- a/master/buildbot/process/remotetransfer.py
+++ b/master/buildbot/process/remotetransfer.py
@@ -126,7 +126,10 @@ def remote_unpack(self):
# Unpack archive and clean up after self
with tarfile.open(name=self.tarname, mode=mode) as archive:
- archive.extractall(path=self.destroot)
+ if hasattr(tarfile, 'data_filter'):
+ archive.extractall(path=self.destroot, filter='data')
+ else:
+ archive.extractall(path=self.destroot)
os.remove(self.tarname)
diff --git a/master/buildbot/test/integration/test_upgrade.py b/master/buildbot/test/integration/test_upgrade.py
index 66669b1b340..40867e4fabc 100644
--- a/master/buildbot/test/integration/test_upgrade.py
+++ b/master/buildbot/test/integration/test_upgrade.py
@@ -77,7 +77,10 @@ def setUpUpgradeTest(self):
with tarfile.open(tarball) as tf:
prefixes = set()
for inf in tf:
- tf.extract(inf)
+ if hasattr(tarfile, 'data_filter'):
+ tf.extract(inf, filter='data')
+ else:
+ tf.extract(inf)
prefixes.add(inf.name.split('/', 1)[0])
# (note that tf.extractall isn't available in py2.4)
From c47b372fe1b577ed1aa5c003ee17e25514bcda49 Mon Sep 17 00:00:00 2001
From: Pavol Misik <pmisik@gmail.com>
Date: Tue, 5 Dec 2023 21:47:40 +0100
Subject: [PATCH 11/15] test: Disable datetime.utcnow and
datetime.utcfromtimestamp warnings on Python 3.12
---
master/buildbot/test/__init__.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/master/buildbot/util/private_tempdir.py b/master/buildbot/util/private_tempdir.py
index ee6c4eae3d0..b2dba0b46a6 100644
--- a/master/buildbot/util/private_tempdir.py
+++ b/master/buildbot/util/private_tempdir.py
@@ -16,6 +16,7 @@
import os
import shutil
import stat
+import sys
import tempfile
@@ -46,5 +47,8 @@ def remove_readonly(func, path, _):
"""
os.chmod(path, stat.S_IWRITE)
func(path)
- shutil.rmtree(self.name, onerror=remove_readonly)
+ if sys.version_info >= (3, 12):
+ shutil.rmtree(self.name, onexc=remove_readonly)
+ else:
+ shutil.rmtree(self.name, onerror=remove_readonly)
self._cleanup_needed = False
From c44051639eb2acbb4357c273af46b0e3292bab61 Mon Sep 17 00:00:00 2001
From: Pavol Misik <pmisik@gmail.com>
Date: Wed, 6 Dec 2023 17:13:39 +0100
Subject: [PATCH 13/15] Fix asyncio changes in Python 3.12
Problem observed:
- failing tests buildbot.test.integration.test_worker_proxy
- failing with stack traces like
[ERROR]
Traceback (most recent call last):
File "C:\P\.venv\Lib\site-packages\twisted\internet\defer.py", line 1996, in _inlineCallbacks
result = context.run(
File "C:\P\.venv\Lib\site-packages\twisted\python\failure.py", line 519, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "C:\P\buildbot\master\buildbot\test\integration\interop\test_transfer.py", line 197, in test_no_exist_multiple_file_upload
yield self.setup_config_single_step(step)
File "C:\P\.venv\Lib\site-packages\twisted\internet\defer.py", line 1996, in _inlineCallbacks
result = context.run(
File "C:\P\.venv\Lib\site-packages\twisted\python\failure.py", line 519, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "C:\P\buildbot\master\buildbot\test\integration\interop\test_transfer.py", line 130, in setup_config_single_step
yield self.setup_master(c)
File "C:\P\.venv\Lib\site-packages\twisted\internet\defer.py", line 1996, in _inlineCallbacks
result = context.run(
File "C:\P\.venv\Lib\site-packages\twisted\python\failure.py", line 519, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "C:\P\buildbot\master\buildbot\test\integration\test_worker_proxy.py", line 159, in setup_master
yield super().setup_master(config_dict, startWorker,
File "C:\P\.venv\Lib\site-packages\twisted\internet\defer.py", line 2000, in _inlineCallbacks
result = context.run(gen.send, result)
File "C:\P\buildbot\master\buildbot\test\util\integration.py", line 249, in setup_master
self.w = Worker(
File "C:\P\buildbot\worker\buildbot_worker\pb.py", line 667, in __init__
proxy_endpoint = clientFromString(reactor, proxy_connection_string)
File "C:\P\.venv\Lib\site-packages\twisted\internet\endpoints.py", line 2137, in clientFromString
kwargs = _clientParsers[name](*args, **kwargs)
File "C:\P\.venv\Lib\site-packages\twisted\internet\endpoints.py", line 1861, in _parseClientTCP
kwargs["port"] = int(args[1])
builtins.ValueError: invalid literal for int() with base 10: 'test_worker_proxy_stdout_11208.txt'
But real problem seems to be in start of proxy
Exception Raised: There is no current event loop
Traceback (most recent call last):
File "C:\P\buildbot\master\buildbot\test\integration\test_worker_proxy.py", line 92, in run_proxy
loop = asyncio.get_event_loop()
^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Python312\Lib\asyncio\events.py", line 693, in get_event_loop
warnings.warn('There is no current event loop',
DeprecationWarning: There is no current event loop
Fix is inspired by https://github.com/danigm/jupyter_client/blob/9c1ddf4a50a71181e60be7045d954a3fafa91357/jupyter_client/utils.py#L14-L29
---
.../test/integration/test_worker_proxy.py | 20 ++++++++++++++-----
1 file changed, 15 insertions(+), 5 deletions(-)
diff --git a/master/buildbot/test/integration/test_worker_proxy.py b/master/buildbot/test/integration/test_worker_proxy.py
index 3ac22cd51b2..f62551bdca6 100644
--- a/master/buildbot/test/integration/test_worker_proxy.py
+++ b/master/buildbot/test/integration/test_worker_proxy.py
@@ -18,6 +18,7 @@
import os
import signal
import socket
+import sys
from twisted.internet import defer
@@ -89,12 +90,21 @@ def run_proxy(queue):
try:
try:
- loop = asyncio.get_event_loop()
+ loop = asyncio.get_running_loop()
except RuntimeError:
- # We can get RuntimeError due to current thread being not main thread on Python 3.8.
- # It's not clear why that happens, so work around it.
- loop = asyncio.new_event_loop()
- asyncio.set_event_loop(loop)
+ # https://github.com/python/cpython/issues/83710
+ if sys.version_info <= (3, 10, 8):
+ # Workaround for bugs.python.org/issue39529.
+ try:
+ loop = asyncio.get_event_loop_policy().get_event_loop()
+ except RuntimeError:
+ # We can get RuntimeError due to current thread being not main thread
+ # on Python 3.8. It's not clear why that happens, so work around it.
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ else:
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
coro = asyncio.start_server(handle_client, host="127.0.0.1")
server = loop.run_until_complete(coro)
From 1d0892db19777155f30796bbcfa348dc0317931c Mon Sep 17 00:00:00 2001
From: Pavol Misik <pmisik@gmail.com>
Date: Fri, 8 Dec 2023 22:14:04 +0100
Subject: [PATCH 14/15] test: Disable sqlalchemy generated warning
---
master/buildbot/test/__init__.py | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/master/buildbot/test/__init__.py b/master/buildbot/test/__init__.py
index 2d56617ca2f..0855c555e80 100644
--- a/master/buildbot/test/__init__.py
+++ b/master/buildbot/test/__init__.py
@@ -19,6 +19,7 @@
from unittest import mock
import setuptools # force import setuptools before any other distutils imports
+from sqlalchemy.exc import RemovedIn20Warning
from buildbot import monkeypatches
from buildbot.test.util.warnings import assertProducesWarning # noqa pylint: disable=wrong-import-position
From bcd5b44aae05a8ba6738de3294980c712a1361d9 Mon Sep 17 00:00:00 2001
From: Pavol Misik <pmisik@gmail.com>
Date: Fri, 15 Dec 2023 01:00:31 +0100
Subject: [PATCH 15/15] test_asyncio_gather: Fix test
[ERROR]
Traceback (most recent call last):
File "C:\P\.venv\Lib\site-packages\twisted\internet\defer.py", line 1996, in _inlineCallbacks
result = context.run(
File "C:\P\.venv\Lib\site-packages\twisted\python\failure.py", line 519, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "C:\P\buildbot\master\buildbot\test\unit\test_asyncio.py", line 83, in test_asyncio_gather
yield as_deferred(f1)
File "C:\P\.venv\Lib\site-packages\twisted\internet\defer.py", line 1248, in adapt
extracted: _SelfResultT | Failure = result.result()
File "C:\P\buildbot\master\buildbot\test\unit\test_asyncio.py", line 73, in main_coro
await asyncio.gather(*dl)
File "C:\Python312\Lib\asyncio\tasks.py", line 674, in _wrap_awaitable
return await awaitable
builtins.TypeError: __await__() returned non-iterator of type '_asyncio.Future'
buildbot.test.unit.test_asyncio.TestAsyncioTestLoop.test_asyncio_gather
---
master/buildbot/test/unit/test_asyncio.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/master/buildbot/test/unit/test_asyncio.py b/master/buildbot/test/unit/test_asyncio.py
index e6e18b98d83..355edc4ee73 100644
--- a/master/buildbot/test/unit/test_asyncio.py
+++ b/master/buildbot/test/unit/test_asyncio.py
@@ -40,8 +40,7 @@ async def coro1():
d1.callback(None)
return defer.Deferred.fromFuture(f)
- @defer.inlineCallbacks
- def test_asyncio_gather(self):
+ async def test_asyncio_gather(self):
self.calls = 0
async def coro1():