#64 [WIP] Move brp-python-bytecompile and use compileall2 module in it
Merged 2 months ago by ignatenkobrain. Opened 2 months ago by lbalhar.
rpms/ lbalhar/redhat-rpm-config compileall2  into  master

file added
+142

@@ -0,0 +1,142 @@ 

+ #!/bin/bash

+ errors_terminate=$2

+ extra=$3

+ 

+ # If using normal root, avoid changing anything.

+ if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then

+ 	exit 0

+ fi

+ 

+ # Figure out how deep we need to descend.  We could pick an insanely high

+ # number and hope it's enough, but somewhere, somebody's sure to run into it.

+ depth=`(find "$RPM_BUILD_ROOT" -type f -name "*.py" -print0 ; echo /) | \

+        xargs -0 -n 1 dirname | sed 's,[^/],,g' | sort -u | tail -n 1 | wc -c`

+ if [ -z "$depth" -o "$depth" -le "1" ]; then

+ 	exit 0

+ fi

+ 

+ # This function now implements Python byte-compilation in two different ways:

+ # Python >= 3.4 uses a new module compileall2 - https://github.com/fedora-python/compileall2

+ # Python < 3.4 (inc. Python 2) uses compileall module from stdlib with some hacks

+ # When we drop support for Python 2, we'd be able to use all compileall2 features like:

+ # - -s and -p options to manipulate with a path baked into pyc files instead of $real_libdir

+ # - -o 0 -o 1 to produce multiple files in one run - each with a different optimization level - instead of $options

+ # - removed useless $depth - both compileall and compileall2 are limited by sys.getrecursionlimit()

+ # These changes will make this script much simpler

+ function python_bytecompile()

+ {

+     local options=$1

+     local python_binary=$2

+     local exclude=$3

+     local python_libdir=$4

+     local depth=$5

+     local real_libdir=$6

+ 

+ 	python_version=$($python_binary -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")

+ 

+ 	#

+ 	# Python 3.4 and higher

+ 	#

+ 	if [ "$python_version" -ge 34 ]; then

+ 

+ 		[ ! -z $exclude ] && exclude="-x '$exclude'"

+ 		# /usr/lib/rpm/redhat/ contains compileall2 Python module

+ 		# -q disables verbose output

+ 		# -f forces the process to overwrite existing compiled files

+ 		# -x excludes paths defined by regex

+ 		# -e excludes symbolic links pointing outside the build root

+ 		# -x and -e together implements the same functionality as the Filter class below

+ 		PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary $options -m compileall2 -q -f $exclude -d $real_libdir -e $RPM_BUILD_ROOT $python_libdir

+ 	else

+ #

+ # Python 3.3 and lower (incl. Python 2)

+ #

+ 

+ cat << EOF | $python_binary $options

+ import compileall, sys, os, re

+ 

+ python_libdir = "$python_libdir"

+ depth = $depth

+ real_libdir = "$real_libdir"

+ build_root = "$RPM_BUILD_ROOT"

+ exclude = r"$exclude"

+ 

+ class Filter:

+     def search(self, path):

+         ret = not os.path.realpath(path).startswith(build_root)

+         if exclude:

+             ret = ret or re.search(exclude, path)

+         return ret

+ 

+ sys.exit(not compileall.compile_dir(python_libdir, depth, real_libdir, force=1, rx=Filter(), quiet=1))

+ EOF

+ 

+ fi

+ }

+ 

+ # .pyc/.pyo files embed a "magic" value, identifying the ABI version of Python

+ # bytecode that they are for.

+ #

+ # The files below RPM_BUILD_ROOT could be targeting multiple versions of

+ # python (e.g. a single build that emits several subpackages e.g. a

+ # python26-foo subpackage, a python31-foo subpackage etc)

+ #

+ # Support this by assuming that below each /usr/lib/python$VERSION/, all

+ # .pyc/.pyo files are to be compiled for /usr/bin/python$VERSION.

+ # 

+ # For example, below /usr/lib/python2.6/, we're targeting /usr/bin/python2.6

+ # and below /usr/lib/python3.1/, we're targeting /usr/bin/python3.1

+ 

+ shopt -s nullglob

+ for python_libdir in `find "$RPM_BUILD_ROOT" -type d|grep -E "/usr/lib(64)?/python[0-9]\.[0-9]$"`;

+ do

+ 	python_binary=/usr/bin/$(basename $python_libdir)

+ 	real_libdir=${python_libdir/$RPM_BUILD_ROOT/}

+ 	echo "Bytecompiling .py files below $python_libdir using $python_binary"

+ 

+ 	# Generate normal (.pyc) byte-compiled files.

+ 	python_bytecompile "" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir"

+ 	if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then

+ 		# One or more of the files had a syntax error

+ 		exit 1

+ 	fi

+ 

+ 	# Generate optimized (.pyo) byte-compiled files.

+ 	python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir"

+ 	if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then

+ 		# One or more of the files had a syntax error

+ 		exit 1

+ 	fi

+ done

+ 

+ 

+ # Handle other locations in the filesystem using the default python implementation

+ # if extra is set to 0, don't do this

+ if [ 0$extra -eq 0 ]; then

+ 	exit 0

+ fi

+ 

+ # If we don't have a default python interpreter, we cannot proceed

+ default_python=${1:-/usr/bin/python}

+ if [ ! -x "$default_python" ]; then

+ 	exit 0

+ fi

+ 

+ # Figure out if there are files to be bytecompiled with the default_python at all

+ # this prevents unnecessary default_python invocation

+ find "$RPM_BUILD_ROOT" -type f -name "*.py" | grep -Ev "/bin/|/sbin/|/usr/lib(64)?/python[0-9]\.[0-9]|/usr/share/doc" || exit 0

+ 

+ # Generate normal (.pyc) byte-compiled files.

+ python_bytecompile "" $default_python "/bin/|/sbin/|/usr/lib(64)?/python[0-9]\.[0-9]|/usr/share/doc" "$RPM_BUILD_ROOT" "$depth" "/"

+ if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then

+ 	# One or more of the files had a syntax error

+ 	exit 1

+ fi

+ 

+ # Generate optimized (.pyo) byte-compiled files.

+ python_bytecompile "-O" $default_python "/bin/|/sbin/|/usr/lib(64)?/python[0-9]\.[0-9]|/usr/share/doc" "$RPM_BUILD_ROOT" "$depth" "/"

+ if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then

+ 	# One or more of the files had a syntax error

+ 	exit 1

+ fi

+ exit 0

file modified
+1 -1

@@ -146,7 +146,7 @@ 

  %__brp_strip /usr/lib/rpm/brp-strip %{__strip}

  %__brp_strip_comment_note /usr/lib/rpm/brp-strip-comment-note %{__strip} %{__objdump}

  %__brp_strip_static_archive /usr/lib/rpm/brp-strip-static-archive %{__strip}

- %__brp_python_bytecompile /usr/lib/rpm/brp-python-bytecompile "%{__python}" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}"

+ %__brp_python_bytecompile /usr/lib/rpm/redhat/brp-python-bytecompile "%{__python}" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}"

  %__brp_python_hardlink /usr/lib/rpm/brp-python-hardlink

  # __brp_mangle_shebangs_exclude - shebangs to exclude

  # __brp_mangle_shebangs_exclude_file - file from which to get shebangs to exclude

file modified
+13 -2

@@ -6,7 +6,7 @@ 

  

  Summary: Red Hat specific rpm configuration files

  Name: redhat-rpm-config

- Version: 137

+ Version: 139

  Release: 1%{?dist}

  # No version specified.

  License: GPL+

@@ -48,6 +48,10 @@ 

  # and an echo when the mangling happens

  Source201: brp-mangle-shebangs

  

+ # this comes from rpm itself

+ # however, now we can do Fedora changes within

+ Source202: brp-python-bytecompile

+ 

  # Dependency generator scripts (deprecated)

  Source300: find-provides

  Source301: find-provides.ksyms

@@ -95,7 +99,8 @@ 

  Requires: ocaml-srpm-macros

  Requires: openblas-srpm-macros

  Requires: perl-srpm-macros

- Requires: python-srpm-macros

+ # ↓ Provides compileall2 Python module

+ Requires: python-srpm-macros >= 46

  Requires: rust-srpm-macros

  Requires: qt5-srpm-macros

  

@@ -202,6 +207,12 @@ 

  %{_rpmconfigdir}/macros.d/macros.kmp

  

  %changelog

+ * Wed Jul 17 2019 Lumír Balhar <lbalhar@redhat.com> - 139-1

+ - Use compileall2 Python module for byte-compilation in brp-python-bytecompile

+ 

+ * Tue Jul 09 2019 Miro Hrončok <mhroncok@redhat.com> - 138-1

+ - Move brp-python-bytecompile from rpm, so we can easily adapt it

+ 

  * Mon Jul 08 2019 Nicolas Mailhot <nim@fedoraproject.org> - 137-1

  - listfiles: make it robust against all kinds of “interesting” inputs

  - wordwrap: make list indenting smarter, to produce something with enough

The first commit moves brp-python-bytecompile script to this package so it can be easily modified. Authored by @churchyard

The second commit adapts the script to use the compileall2 module for byte-compilation with Python 3.4. I hope that you can find all important information in the comments in the script. It might look a little bit messy now but compileall2 makes the script much better when we drop support for Python 2. Also by then the compileall2 module will be well tested here and by %py_byte_compile macro where compileall2 is already implemented.

I did some basic tests with one package in COPR and my plan is to rebuild a few tens of Python packages and check the results. If you find something wrong here, let me know sooner than later before I dive deep into testing this change.

It's ready for review. I'll provide more information about testing packages in COPR.

Only one package fails to build in COPR (qemu, release candidate, tests failed) and it's not caused by this change.

The pull-request cannot be merged due to conflicts

rebased onto 5361740

2 months ago

It would be nice to have some description why it is good to move to compileall2 :)

Generally the main motivator is https://bugzilla.redhat.com/show_bug.cgi?id=1536445

A short summary:

  • there are multiple custom hacked bash/Python scripts in our RPM machinery that bytecompile Python - most of them proven to be somewhat buggy over time
  • we'd like to have one way to do this - preferably by a tool supported and provided by Python upstream
  • compileall2 is currently downstream only as well, but our plan is to move our improvements to upstream's compileall module and we use Fedora as an incubator for this

Ack, let's merge this and see how much breaks during mass rebuild :)

Pull-Request has been merged by ignatenkobrain

2 months ago

Please, don't merge Pull Requests marked as [WIP].