From fb7d971c91ae74e6fe05d436df34ca32b00b0485 Mon Sep 17 00:00:00 2001 From: Miro Hrončok Date: Jul 12 2019 11:52:30 +0000 Subject: Repeat DynamicBuildRequires if needed (backport) This backports: - https://github.com/rpm-software-management/mock/commit/af02efc5 - https://github.com/rpm-software-management/mock/commit/ba5e4ef2 - https://github.com/rpm-software-management/mock/commit/702bf038 --- diff --git a/0003-Repeat-dynamic-requires-in-needed.patch b/0003-Repeat-dynamic-requires-in-needed.patch new file mode 100644 index 0000000..92117c6 --- /dev/null +++ b/0003-Repeat-dynamic-requires-in-needed.patch @@ -0,0 +1,214 @@ +From af02efc5bdc5bec8c74877b4510eddbbb786909e Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Miroslav=20Such=C3=BD?= +Date: Mon, 24 Jun 2019 13:30:14 +0200 +Subject: [PATCH] repeat dynamic requires in needed [GH#276] + +if rpmbuild fails with exit code 11, run it again, but limit it with dynamic_buildrequires_max_loops + +Resolves: #276 +--- + mock/etc/mock/site-defaults.cfg | 1 + + mock/py/mockbuild/backend.py | 61 ++++++++++++++++++--------------- + mock/py/mockbuild/util.py | 1 + + 3 files changed, 35 insertions(+), 28 deletions(-) + +diff --git a/etc/mock/site-defaults.cfg b/etc/mock/site-defaults.cfg +index a17a58d0..46fd687a 100644 +--- a/etc/mock/site-defaults.cfg ++++ b/etc/mock/site-defaults.cfg +@@ -91,6 +91,7 @@ + + # Dynamic BuildRequires, available since RPM 4.15 + # config_opts['dynamic_buildrequires'] = True ++# config_opts['dynamic_buildrequires_max_loops'] = 10 + + # You can configure Yum, DNF, rpm and rpmbuild executable paths if you need to + # use different versions that the system-wide ones +diff --git a/py/mockbuild/backend.py b/py/mockbuild/backend.py +index f0f73a6d..5f9a986a 100644 +--- a/py/mockbuild/backend.py ++++ b/py/mockbuild/backend.py +@@ -718,35 +718,40 @@ class Commands(object): + return command + + bd_out = self.make_chroot_path(self.buildroot.builddir) ++ max_loops = int(self.config.get('dynamic_buildrequires_max_loops')) ++ success = False + if dynamic_buildrequires and self.config.get('dynamic_buildrequires'): +- # Technically, we should run rpmbuild+installSrpmDeps until +- # * it fails +- # * installSrpmDeps does nothing +- # but we don't have necessary infrastructure in place +- try: +- self.buildroot.doChroot(get_command(['-br']), +- shell=False, logger=self.buildroot.build_log, timeout=timeout, +- uid=self.buildroot.chrootuid, gid=self.buildroot.chrootgid, +- user=self.buildroot.chrootuser, +- nspawn_args=self._get_nspawn_args(), +- unshare_net=self.private_network, +- printOutput=self.config['print_main_output']) +- except Error as e: +- if e.resultcode != 11: +- raise e +- self.buildroot.root_log.info("Dynamic buildrequires detected") +- self.buildroot.root_log.info("Going to install missing buildrequires") +- buildreqs = glob.glob(bd_out + '/SRPMS/*.buildreqs.nosrc.rpm') +- self.installSrpmDeps(*buildreqs) +- for f_buildreqs in buildreqs: +- os.remove(f_buildreqs) +- if not sc: +- # We want to (re-)write src.rpm with dynamic BuildRequires, +- # but with short-circuit it doesn't matter +- mode = ['-ba'] +- # rpmbuild -br already does %prep, so we don't need waste time +- # on re-doing it +- mode += ['--noprep'] ++ while not success or max_loops > 0: ++ # run rpmbuild+installSrpmDeps until ++ # * it fails ++ # * installSrpmDeps does nothing ++ # * or we run out of dynamic_buildrequires_max_loops tries ++ try: ++ self.buildroot.doChroot(get_command(['-br']), ++ shell=False, logger=self.buildroot.build_log, timeout=timeout, ++ uid=self.buildroot.chrootuid, gid=self.buildroot.chrootgid, ++ user=self.buildroot.chrootuser, ++ nspawn_args=self._get_nspawn_args(), ++ unshare_net=self.private_network, ++ printOutput=self.config['print_main_output']) ++ success = True ++ except Error as e: ++ if e.resultcode != 11: ++ raise e ++ max_loops -= 1 ++ self.buildroot.root_log.info("Dynamic buildrequires detected") ++ self.buildroot.root_log.info("Going to install missing buildrequires") ++ buildreqs = glob.glob(bd_out + '/SRPMS/*.buildreqs.nosrc.rpm') ++ self.installSrpmDeps(*buildreqs) ++ for f_buildreqs in buildreqs: ++ os.remove(f_buildreqs) ++ if not sc: ++ # We want to (re-)write src.rpm with dynamic BuildRequires, ++ # but with short-circuit it doesn't matter ++ mode = ['-ba'] ++ # rpmbuild -br already does %prep, so we don't need waste time ++ # on re-doing it ++ mode += ['--noprep'] + self.buildroot.doChroot(get_command(mode), + shell=False, logger=self.buildroot.build_log, timeout=timeout, + uid=self.buildroot.chrootuid, gid=self.buildroot.chrootgid, +diff --git a/py/mockbuild/util.py b/py/mockbuild/util.py +index 008f1bfb..7d1f0c89 100644 +--- a/py/mockbuild/util.py ++++ b/py/mockbuild/util.py +@@ -1096,6 +1096,7 @@ def setup_default_config_opts(unprivUid, version, pkgpythondir): + config_opts['package_manager'] = 'yum' + + config_opts['dynamic_buildrequires'] = True ++ config_opts['dynamic_buildrequires_max_loops'] = 10 + + # configurable commands executables + config_opts['yum_command'] = '/usr/bin/yum' +-- +2.21.0 + +From ba5e4ef2d16d389383207430019f292457eb7f02 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= +Date: Tue, 2 Jul 2019 18:24:57 +0200 +Subject: [PATCH] DynamicBuildrequires: Detect when no new packages were + installed + +Prior to this change, the DynamicBuildrequires loop never ended. + +This commit fixes it in 2 ways: + + 1. It now actually stops when max_loops (10) happened. + 2. It also stops when the package manager hasn't installed anything. + +Both is consistent with the comment present in the previously broken code. + +The detection of 2. is based on packages returned by `rpm -qa` before +and after the installation. If they haven't changed, the package +manager hasn't installed anything. +--- + mock/py/mockbuild/backend.py | 6 +++++- + mock/py/mockbuild/buildroot.py | 6 ++++++ + 2 files changed, 11 insertions(+), 1 deletion(-) + +diff --git a/py/mockbuild/backend.py b/py/mockbuild/backend.py +index 5f9a986a..1c86504e 100644 +--- a/py/mockbuild/backend.py ++++ b/py/mockbuild/backend.py +@@ -721,11 +721,12 @@ class Commands(object): + max_loops = int(self.config.get('dynamic_buildrequires_max_loops')) + success = False + if dynamic_buildrequires and self.config.get('dynamic_buildrequires'): +- while not success or max_loops > 0: ++ while not success and max_loops > 0: + # run rpmbuild+installSrpmDeps until + # * it fails + # * installSrpmDeps does nothing + # * or we run out of dynamic_buildrequires_max_loops tries ++ packages_before = self.buildroot.all_chroot_packages() + try: + self.buildroot.doChroot(get_command(['-br']), + shell=False, logger=self.buildroot.build_log, timeout=timeout, +@@ -743,6 +744,9 @@ class Commands(object): + self.buildroot.root_log.info("Going to install missing buildrequires") + buildreqs = glob.glob(bd_out + '/SRPMS/*.buildreqs.nosrc.rpm') + self.installSrpmDeps(*buildreqs) ++ packages_after = self.buildroot.all_chroot_packages() ++ if packages_after == packages_before: ++ success = True + for f_buildreqs in buildreqs: + os.remove(f_buildreqs) + if not sc: +diff --git a/py/mockbuild/buildroot.py b/py/mockbuild/buildroot.py +index 0f4104db..b55e54b0 100644 +--- a/py/mockbuild/buildroot.py ++++ b/py/mockbuild/buildroot.py +@@ -207,6 +207,12 @@ class Buildroot(object): + self.uid_manager.restorePrivs() + return result + ++ def all_chroot_packages(self): ++ """package set, result of rpm -qa in the buildroot""" ++ out, _ = self.doChroot([self.config['rpm_command'], "-qa"], ++ returnOutput=True, printOutput=False, shell=False) ++ return set(out.splitlines()) ++ + @traceLog() + def _copy_config(self, filename): + etcdir = self.make_chroot_path('etc') +-- +2.21.0 + +From 702bf03858563203258740e671e425e693dfc51b Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= +Date: Wed, 3 Jul 2019 00:30:08 +0200 +Subject: [PATCH] Fixup: Use rpm -qa --root instead of running rpm -qa in + chroot + +--- + mock/py/mockbuild/buildroot.py | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/py/mockbuild/buildroot.py b/py/mockbuild/buildroot.py +index b55e54b0..9dfcd05a 100644 +--- a/py/mockbuild/buildroot.py ++++ b/py/mockbuild/buildroot.py +@@ -209,8 +209,9 @@ class Buildroot(object): + + def all_chroot_packages(self): + """package set, result of rpm -qa in the buildroot""" +- out, _ = self.doChroot([self.config['rpm_command'], "-qa"], +- returnOutput=True, printOutput=False, shell=False) ++ out = util.do([self.config['rpm_command'], "-qa", ++ "--root", self.make_chroot_path()], ++ returnOutput=True, printOutput=False, shell=False) + return set(out.splitlines()) + + @traceLog() +-- +2.21.0 + diff --git a/mock.spec b/mock.spec index 643006b..2662cf5 100644 --- a/mock.spec +++ b/mock.spec @@ -15,7 +15,7 @@ Summary: Builds packages inside chroots Name: mock Version: 1.4.16 -Release: 2%{?dist} +Release: 3%{?dist} License: GPLv2+ # Source is created by # git clone https://github.com/rpm-software-management/mock.git @@ -27,6 +27,11 @@ Source: %{name}-%{version}.tar.gz Patch0001: 0001-Enable-dynamic-BuildRequires-by-default.patch # https://github.com/rpm-software-management/mock/commit/9c804837c4229bbeda1e9773a94bde5596acadbe Patch0002: 0002-Fix-compatibility-with-pre-4.15-RPM-versions-with-Dy.patch +# https://github.com/rpm-software-management/mock/commit/af02efc5 +# https://github.com/rpm-software-management/mock/commit/ba5e4ef2 +# https://github.com/rpm-software-management/mock/commit/702bf038 +Patch0003: 0003-Repeat-dynamic-requires-in-needed.patch + URL: https://github.com/rpm-software-management/mock/ BuildArch: noarch Requires: tar @@ -236,6 +241,9 @@ pylint-3 py/mockbuild/ py/*.py py/mockbuild/plugins/* || : %endif %changelog +* Fri Jul 12 2019 Miro Hrončok - 1.4.16-3 +- Repeat DynamicBuildRequires if needed (backport) + * Mon Jun 17 2019 Igor Gnatenko - 1.4.16-2 - Backport pre-4.15 DynamicBuildRequires fix - Enable support for DynamicBuildRequires by default