diff --git a/brp-fix-pyc-reproducibility b/brp-fix-pyc-reproducibility new file mode 100644 index 0000000..4118d97 --- /dev/null +++ b/brp-fix-pyc-reproducibility @@ -0,0 +1,18 @@ +#!/bin/bash -e + +# If using normal root, avoid changing anything. +if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then + exit 0 +fi + +# Defined as %py_reproducible_pyc_path macro and passed here as +# the first command-line argument +path_to_fix=$1 + +# First, check that the parser is available: +if [ ! -x /usr/bin/marshalparser ]; then + echo "ERROR: If %py_reproducible_pyc_path is defined, you have to also BuildRequire: /usr/bin/marshalparser !" + exit 1 +fi + +find "$path_to_fix" -type f -name "*.pyc" | xargs /usr/bin/marshalparser --fix --overwrite diff --git a/brp-python-bytecompile b/brp-python-bytecompile new file mode 100644 index 0000000..71f4d2d --- /dev/null +++ b/brp-python-bytecompile @@ -0,0 +1,141 @@ +#!/bin/bash +errors_terminate=$2 + +# Usage of %_python_bytecompile_extra is not allowed anymore +# See: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3 +# Therefore $1 ($default_python) is not needed and is invoked with "" by default. +# $default_python stays in the arguments for backward compatibility and $extra for the following check: +extra=$3 +if [ 0$extra -eq 1 ]; then + echo -e "%_python_bytecompile_extra is discontinued, use %py_byte_compile instead.\nSee: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3" >/dev/stderr + exit 1 +fi + +# If using normal root, avoid changing anything. +if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then + exit 0 +fi + +# Figure out how deep we need to descend. We could pick an insanely high +# number and hope it's enough, but somewhere, somebody's sure to run into it. +depth=`(find "$RPM_BUILD_ROOT" -type f -name "*.py" -print0 ; echo /) | \ + xargs -0 -n 1 dirname | sed 's,[^/],,g' | sort -u | tail -n 1 | wc -c` +if [ -z "$depth" -o "$depth" -le "1" ]; then + exit 0 +fi + +# This function now implements Python byte-compilation in three different ways: +# Python >= 3.4 and < 3.9 uses a new module compileall2 - https://github.com/fedora-python/compileall2 +# Python < 3.4 (inc. Python 2) uses compileall module from stdlib with some hacks +# When we drop support for Python 2, we'd be able to use all compileall2 features like: +# - -s and -p options to manipulate with a path baked into pyc files instead of $real_libdir +# - -o 0 -o 1 to produce multiple files in one run - each with a different optimization level - instead of $options +# - removed useless $depth - both compileall and compileall2 are limited by sys.getrecursionlimit() +# These changes will make this script much simpler +# In Python >= 3.9, compileall2 was merged back to standard library (compileall) so we can use it directly again. +function python_bytecompile() +{ + local options=$1 + local python_binary=$2 + local exclude=$3 + local python_libdir="$4" + local depth=$5 # Not used for Python >= 3.4 + local real_libdir=$6 # Not used for Python >= 3.4 + + python_version=$($python_binary -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") + + # + # Python 3.9 and higher + # + if [ "$python_version" -ge 39 ]; then + + [ ! -z $exclude ] && exclude="-x '$exclude'" + # -q disables verbose output + # -f forces the process to overwrite existing compiled files + # -x excludes paths defined by regex + # -e excludes symbolic links pointing outside the build root + # -x and -e together implements the same functionality as the Filter class below + # -s strips $RPM_BUILD_ROOT from the path + # -p prepends the leading slash to the path to make it absolute + $python_binary -B $options -m compileall -q -f $exclude -s "$RPM_BUILD_ROOT" -p / -e "$RPM_BUILD_ROOT" "$python_libdir" + + # + # Python 3.4 and higher + # + elif [ "$python_version" -ge 34 ]; then + + [ ! -z $exclude ] && exclude="-x '$exclude'" + # /usr/lib/rpm/redhat/ contains compileall2 Python module + # -q disables verbose output + # -f forces the process to overwrite existing compiled files + # -x excludes paths defined by regex + # -e excludes symbolic links pointing outside the build root + # -x and -e together implements the same functionality as the Filter class below + # -s strips $RPM_BUILD_ROOT from the path + # -p prepends the leading slash to the path to make it absolute + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B $options -m compileall2 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / -e "$RPM_BUILD_ROOT" "$python_libdir" + else +# +# Python 3.3 and lower (incl. Python 2) +# + +cat << EOF | $python_binary $options +import compileall, sys, os, re + +python_libdir = "$python_libdir" +depth = $depth +real_libdir = "$real_libdir" +build_root = "$RPM_BUILD_ROOT" +exclude = r"$exclude" + +class Filter: + def search(self, path): + ret = not os.path.realpath(path).startswith(build_root) + if exclude: + ret = ret or re.search(exclude, path) + return ret + +sys.exit(not compileall.compile_dir(python_libdir, depth, real_libdir, force=1, rx=Filter(), quiet=1)) +EOF + +fi +} + +# .pyc/.pyo files embed a "magic" value, identifying the ABI version of Python +# bytecode that they are for. +# +# The files below RPM_BUILD_ROOT could be targeting multiple versions of +# python (e.g. a single build that emits several subpackages e.g. a +# python26-foo subpackage, a python31-foo subpackage etc) +# +# Support this by assuming that below each /usr/lib/python$VERSION/, all +# .pyc/.pyo files are to be compiled for /usr/bin/python$VERSION. +# +# For example, below /usr/lib/python2.6/, we're targeting /usr/bin/python2.6 +# and below /usr/lib/python3.1/, we're targeting /usr/bin/python3.1 + +# Disable Python hash seed randomization +# This should help with byte-compilation reproducibility: https://bugzilla.redhat.com/show_bug.cgi?id=1686078 +export PYTHONHASHSEED=0 + +shopt -s nullglob +find "$RPM_BUILD_ROOT" -type d -print0|grep -z -E "/(usr|app)/lib(64)?/python[0-9]\.[0-9]+$" | while read -d "" python_libdir; +do + python_binary=$(basename "$python_libdir") + real_libdir=${python_libdir/$RPM_BUILD_ROOT/} + echo "Bytecompiling .py files below $python_libdir using $python_binary" + + # Generate normal (.pyc) byte-compiled files. + python_bytecompile "" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir" + if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + # One or more of the files had a syntax error + exit 1 + fi + + # Generate optimized (.pyo) byte-compiled files. + python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir" + if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + # One or more of the files had a syntax error + exit 1 + fi +done diff --git a/macros.python-srpm b/macros.python-srpm index 5a5799e..f115a85 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -50,9 +50,32 @@ %python3_pkgversion 3 -# BRP scripts, they need to be included in redhat-rpm-macros, in %%__os_install_post +### BRP scripts (and related macros) + +## Automatically compile python files +%py_auto_byte_compile 1 +## Should python bytecompilation errors terminate a build? +%_python_bytecompile_errors_terminate_build 1 +## Should python bytecompilation compile outside python specific directories? +## This always causes errors when enabled, see https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3 +%_python_bytecompile_extra 0 + +## The individual BRP scripts +%__brp_python_bytecompile %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" +%__brp_fix_pyc_reproducibility %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility %__brp_python_hardlink %{_rpmconfigdir}/redhat/brp-python-hardlink +## This macro is included in redhat-rpm-config's %%__os_install_post +# Note that the order matters: +# 1. brp-python-bytecompile can create (or replace) pyc files +# 2. brp-fix-pyc-reproducibility can modify the pyc files from above +# 3. brp-python-hardlink de-duplicates identical pyc files +%__os_install_post_python \ + %{?py_auto_byte_compile:%{?__brp_python_bytecompile}} \ + %{?py_reproducible_pyc_path:%{?__brp_fix_pyc_reproducibility} "%{py_reproducible_pyc_path}"} \ + %{?__brp_python_hardlink} \ +%{nil} + # === Macros for Build/Requires tags using Python dist tags === # - https://fedoraproject.org/wiki/Changes/Automatic_Provides_for_Python_RPM_Packages diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 46e2d1b..a007695 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -17,14 +17,22 @@ Source201: python.lua Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py # BRP scripts +# This one is from redhat-rpm-config < 190 +# A new upstream is forming in https://github.com/rpm-software-management/python-rpm-packaging/blob/main/scripts/brp-python-bytecompile +# But our version is riddled with Fedora-isms +# We might eventually move to upstream source + Fedora patches, but we are not there yet +Source401: brp-python-bytecompile # This one is from https://github.com/rpm-software-management/python-rpm-packaging/blob/main/scripts/brp-python-hardlink # But we don't use a link in case it changes in upstream, there are no "versions" there yet # This was removed from RPM 4.17+ so we maintain it here instead -Source401: brp-python-hardlink +Source402: brp-python-hardlink +# This one is from redhat-rpm-config < 190 +# It has no upstream yet +Source403: brp-fix-pyc-reproducibility # macros and lua: MIT # compileall2.py: PSFv2 -# brp-python-hardlink: GPLv2+ +# brp scripts: GPLv2+ License: MIT and Python and GPLv2+ # The package version MUST be always the same as %%{__default_python3_version}. @@ -39,7 +47,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 5%{?dist} +Release: 6%{?dist} BuildArch: noarch @@ -60,7 +68,8 @@ python?-devel packages require it. So install a python-devel package instead. Summary: RPM macros for building Python source packages # For directory structure and flags macros -Requires: redhat-rpm-config +# Versions before 190 contained some brp scripts moved into python-srpm-macros +Requires: redhat-rpm-config >= 190 # We bundle our own software here :/ Provides: bundled(python3dist(compileall2)) = %{compileall2_version} @@ -105,7 +114,9 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ # It also ensures that: # - our BRPs can execute # - if our BRPs affect this package, we don't need to build it twice +%global __brp_python_bytecompile %{buildroot}%{__brp_python_bytecompile} %global __brp_python_hardlink %{buildroot}%{__brp_python_hardlink} +%global __brp_fix_pyc_reproducibility %{buildroot}%{__brp_fix_pyc_reproducibility} %check @@ -120,7 +131,9 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm %{_rpmconfigdir}/redhat/compileall2.py +%{_rpmconfigdir}/redhat/brp-python-bytecompile %{_rpmconfigdir}/redhat/brp-python-hardlink +%{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility %{_rpmluadir}/fedora/srpm/python.lua %files -n python3-rpm-macros @@ -128,6 +141,9 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Wed Jul 07 2021 Miro Hrončok - 3.10-6 +- Move Python related BuildRoot Policy scripts from redhat-rpm-config to python-srpm-macros + * Wed Jul 07 2021 Miro Hrončok - 3.10-5 - Introduce %%py3_check_import