From d678a6a69f94f0659344207dcb8cdd4114a8fa97 Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Aug 23 2019 10:52:16 +0000 Subject: Cap number of threads on 32bit platforms (#1729382) Also drop %_lto_cflags macro (reverted upstream) before people start adapting its use. --- diff --git a/0001-Drop-_lto_cflags-macro-afterall.patch b/0001-Drop-_lto_cflags-macro-afterall.patch new file mode 100644 index 0000000..4555f32 --- /dev/null +++ b/0001-Drop-_lto_cflags-macro-afterall.patch @@ -0,0 +1,50 @@ +From 7faf8eda1358f8a877b9b3d6e1197b814e80b50b Mon Sep 17 00:00:00 2001 +Message-Id: <7faf8eda1358f8a877b9b3d6e1197b814e80b50b.1566557361.git.pmatilai@redhat.com> +From: Panu Matilainen +Date: Tue, 13 Aug 2019 16:26:31 +0300 +Subject: [PATCH] Drop %_lto_cflags macro afterall + +This was only added in commit 2bb7b0cf066c97a9d92eb0bf59618896000cb29d, +but turns out that this kind of usage is bad for build reproducability +because the system-specific CPU count gets recorded RPMTAG_OPTFLAGS +and the resulting binaries too (depending on gcc flags). +In addition, gcc upstream has decided to make -flto default to +autodetected parallelism. Since -flto can be overridden with +by simply appending -fno-lto for the packages that need to disable it, +there's no practical need for us to provide such a macro for disabling +either. +--- + macros.in | 2 +- + platform.in | 3 --- + 2 files changed, 1 insertion(+), 4 deletions(-) + +diff --git a/macros.in b/macros.in +index 633d5ca6e..b7da4ea2f 100644 +--- a/macros.in ++++ b/macros.in +@@ -1025,7 +1025,7 @@ package or when debugging this package.\ + %build_fflags %{optflags} %{?_fmoddir:-I%{_fmoddir}} + + # Link editor flags. This is usually called LDFLAGS in makefiles. +-#%build_ldflags -Wl,-z,relro %{?_lto_cflags} ++#%build_ldflags -Wl,-z,relro + + # Expands to shell code to seot the compiler/linker environment + # variables CFLAGS, CXXFLAGS, FFLAGS, FCFLAGS, LDFLAGS if they have +diff --git a/platform.in b/platform.in +index e1efc42b0..db6d2382f 100644 +--- a/platform.in ++++ b/platform.in +@@ -59,9 +59,6 @@ + + %_smp_mflags -j%{_smp_build_ncpus} + +-# Enable LTO optimization with a maximal parallelism +-%_lto_cflags -flto=%{_smp_build_ncpus} +- + #============================================================================== + # ---- Build policy macros. + # +-- +2.21.0 + diff --git a/0001-Fix-build-code-thread-cap-logic-for-unlimited-CPUs.patch b/0001-Fix-build-code-thread-cap-logic-for-unlimited-CPUs.patch new file mode 100644 index 0000000..e77329e --- /dev/null +++ b/0001-Fix-build-code-thread-cap-logic-for-unlimited-CPUs.patch @@ -0,0 +1,32 @@ +From 3ba0136000c986c692baab7b112aba163173d2d3 Mon Sep 17 00:00:00 2001 +Message-Id: <3ba0136000c986c692baab7b112aba163173d2d3.1566556488.git.pmatilai@redhat.com> +From: Panu Matilainen +Date: Fri, 23 Aug 2019 11:09:40 +0300 +Subject: [PATCH 1/2] Fix build code thread cap logic for unlimited CPUs + +If there's no clear cap set from rpm configuration, just let OMP do +its own thing (ie use all available CPUs) instead of artificially +limiting to 1. +--- + build/parseSpec.c | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/build/parseSpec.c b/build/parseSpec.c +index 055bdf012..61858aabd 100644 +--- a/build/parseSpec.c ++++ b/build/parseSpec.c +@@ -1036,9 +1036,8 @@ static rpmSpec parseSpec(const char *specFile, rpmSpecFlags flags, + #ifdef ENABLE_OPENMP + /* Set number of OMP threads centrally */ + int ncpus = rpmExpandNumeric("%{?_smp_build_ncpus}"); +- if (ncpus <= 0) +- ncpus = 1; +- omp_set_num_threads(ncpus); ++ if (ncpus > 0) { ++ omp_set_num_threads(ncpus); + #endif + + if (spec->clean == NULL) { +-- +2.21.0 + diff --git a/0002-Cap-number-of-threads-on-32bit-platforms-add-a-tunab.patch b/0002-Cap-number-of-threads-on-32bit-platforms-add-a-tunab.patch new file mode 100644 index 0000000..f8835bb --- /dev/null +++ b/0002-Cap-number-of-threads-on-32bit-platforms-add-a-tunab.patch @@ -0,0 +1,62 @@ +From bcaa30872087a1002648653358ab9514f829de9a Mon Sep 17 00:00:00 2001 +Message-Id: +In-Reply-To: <3ba0136000c986c692baab7b112aba163173d2d3.1566556488.git.pmatilai@redhat.com> +References: <3ba0136000c986c692baab7b112aba163173d2d3.1566556488.git.pmatilai@redhat.com> +From: Panu Matilainen +Date: Fri, 23 Aug 2019 11:17:21 +0300 +Subject: [PATCH 2/2] Cap number of threads on 32bit platforms, add a tunable + (RhBug:1729382) + +On 32bit plaforms, address space is easily exhausted with multiple +threads, causing arbitrary builds failure regressions (RhBug:1729382). +Simply cap the number of threads to maximum of four on any 32bit +platform to play it safe. It's still three more than we were able to +use on older releases... + +In addition, introduce a separate tunable for tweaking the maximum +number of threads just in case, defaulting to max CPUs available. +--- + build/parseSpec.c | 10 +++++++++- + platform.in | 3 +++ + 2 files changed, 12 insertions(+), 1 deletion(-) + +diff --git a/build/parseSpec.c b/build/parseSpec.c +index 61858aabd..d987bdc7b 100644 +--- a/build/parseSpec.c ++++ b/build/parseSpec.c +@@ -1035,9 +1035,17 @@ static rpmSpec parseSpec(const char *specFile, rpmSpecFlags flags, + + #ifdef ENABLE_OPENMP + /* Set number of OMP threads centrally */ +- int ncpus = rpmExpandNumeric("%{?_smp_build_ncpus}"); ++ int ncpus = rpmExpandNumeric("%{?_smp_nthreads_max}"); ++#if __WORDSIZE == 32 ++ /* On 32bit platforms, address space shortage is an issue. Play safe. */ ++ if (ncpus <= 0 || ncpus > 4) { ++ ncpus = 4; ++ rpmlog(RPMLOG_DEBUG, ++ "limiting number of threads to %d due to platform\n", ncpus); ++ } + if (ncpus > 0) { + omp_set_num_threads(ncpus); ++#endif + #endif + + if (spec->clean == NULL) { +diff --git a/platform.in b/platform.in +index db6d2382f..c70f44193 100644 +--- a/platform.in ++++ b/platform.in +@@ -59,6 +59,9 @@ + + %_smp_mflags -j%{_smp_build_ncpus} + ++# Maximum number of CPU's to use for threads ++%_smp_nthreads_max %{_smp_build_ncpus} ++ + #============================================================================== + # ---- Build policy macros. + # +-- +2.21.0 + diff --git a/rpm.spec b/rpm.spec index 4a3ee44..8301f57 100644 --- a/rpm.spec +++ b/rpm.spec @@ -21,7 +21,7 @@ %global rpmver 4.15.0 %global snapver beta -%global rel 3 +%global rel 4 %global srcver %{version}%{?snapver:-%{snapver}} %global srcdir %{?snapver:testing}%{!?snapver:%{name}-%(echo %{version} | cut -d'.' -f1-2).x} @@ -57,8 +57,12 @@ Patch101: 0001-Do-not-set-RPMTAG_BUILDTIME-to-SOURCE_DATE_EPOCH-whe.patch Patch110: 0001-Support-running-rpmfcExec-without-any-piped-input-ou.patch Patch111: 0002-Restore-strict-order-of-build-scriptlet-stdout-stder.patch Patch112: 0003-Drop-the-no-longer-needed-rpmfcExec-output-duplicati.patch +Patch113: 0001-Drop-_lto_cflags-macro-afterall.patch # These are not yet upstream +Patch200: 0001-Fix-build-code-thread-cap-logic-for-unlimited-CPUs.patch +Patch201: 0002-Cap-number-of-threads-on-32bit-platforms-add-a-tunab.patch + Patch906: rpm-4.7.1-geode-i686.patch # Probably to be upstreamed in slightly different form Patch907: rpm-4.15.x-ldflags.patch @@ -543,6 +547,10 @@ make check || (cat tests/rpmtests.log; exit 0) %doc doc/librpm/html/* %changelog +* Fri Aug 23 2019 Panu Matilainen - 4.15.0-0.beta.4 +- Cap number of threads on 32bit platforms (#1729382) +- Drop %%_lto_cflags macro (reverted upstream) + * Fri Aug 23 2019 Panu Matilainen - 4.15.0-0.beta.3 - Restore strict order of build scriptlet stdout/stderr output