#25 New module compileall2 for Python byte compilation
Merged 2 months ago by churchyard. Opened 2 months ago by lbalhar.
rpms/ lbalhar/python-rpm-macros compileall2  into  master

file added
+464

@@ -0,0 +1,464 @@ 

+ """Module/script to byte-compile all .py files to .pyc files.

+ 

+ When called as a script with arguments, this compiles the directories

+ given as arguments recursively; the -l option prevents it from

+ recursing into directories.

+ 

+ Without arguments, if compiles all modules on sys.path, without

+ recursing into subdirectories.  (Even though it should do so for

+ packages -- for now, you'll have to deal with packages separately.)

+ 

+ See module py_compile for details of the actual byte-compilation.

add something like "See https://www.python.org/psf/license/ for licensing information" or actually include the license text in the file (but that's quite long).

+ """

+ import os

+ import sys

+ import importlib.util

+ import py_compile

+ import struct

+ 

+ from functools import partial

+ from pathlib import Path

+ 

+ # Python 3.7 and higher

+ PY37 = sys.version_info[0:2] >= (3, 7)

+ # Python 3.6 and higher

+ PY36 = sys.version_info[0:2] >= (3, 6)

+ # Python 3.5 and higher

+ PY35 = sys.version_info[0:2] >= (3, 5)

+ 

+ # Python 3.7 and above has a different structure and length

+ # of pyc files header. Also, multiple ways how to invalidate pyc file was

+ # introduced in Python 3.7. These cases are covered by variables here or by PY37

+ # variable itself.

+ if PY37:

+     pyc_struct_format = '<4sll'

+     pyc_header_lenght = 12

+     pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER, 0)

+ else:

+     pyc_struct_format = '<4sl'

+     pyc_header_lenght = 8

+     pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER)

+ 

+ RECURSION_LIMIT = sys.getrecursionlimit()

+ 

+ __all__ = ["compile_dir","compile_file","compile_path"]

+ 

+ def optimization_kwarg(opt):

+     """Returns opt as a dictionary {optimization: opt} for use as **kwarg

+     for Python >= 3.5 and empty dictionary for Python 3.4"""

+     if PY35:

+         return dict(optimization=opt)

+     else:

+         # `debug_override` is a way how to enable optimized byte-compiled files

+         # (.pyo) in Python <= 3.4

+         if opt:

+             return dict(debug_override=False)

+         else:

+             return dict()

+ 

+ def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0):

+     if PY36 and quiet < 2 and isinstance(dir, os.PathLike):

+         dir = os.fspath(dir)

+     else:

+         dir = str(dir)

+     if not quiet:

+         print('Listing {!r}...'.format(dir))

+     try:

+         names = os.listdir(dir)

+     except OSError:

+         if quiet < 2:

+             print("Can't list {!r}".format(dir))

+         names = []

+     names.sort()

+     for name in names:

+         if name == '__pycache__':

+             continue

+         fullname = os.path.join(dir, name)

+         if not os.path.isdir(fullname):

+             yield fullname

+         elif (maxlevels > 0 and name != os.curdir and name != os.pardir and

+               os.path.isdir(fullname) and not os.path.islink(fullname)):

+             yield from _walk_dir(fullname, maxlevels=maxlevels - 1,

+                                  quiet=quiet)

+ 

+ def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False,

+                 rx=None, quiet=0, legacy=False, optimize=-1, workers=1,

+                 invalidation_mode=None, stripdir=None,

+                 prependdir=None, limit_sl_dest=None):

+     """Byte-compile all modules in the given directory tree.

+ 

+     Arguments (only dir is required):

+ 

+     dir:       the directory to byte-compile

+     maxlevels: maximum recursion level (default `sys.getrecursionlimit()`)

+     ddir:      the directory that will be prepended to the path to the

+                file as it is compiled into each byte-code file.

+     force:     if True, force compilation, even if timestamps are up-to-date

+     quiet:     full output with False or 0, errors only with 1,

+                no output with 2

+     legacy:    if True, produce legacy pyc paths instead of PEP 3147 paths

+     optimize:  int or list of optimization levels or -1 for level of

+                the interpreter. Multiple levels leads to multiple compiled

+                files each with one optimization level.

+     workers:   maximum number of parallel workers

+     invalidation_mode: how the up-to-dateness of the pyc will be checked

+     stripdir:  part of path to left-strip from source file path

+     prependdir: path to prepend to beggining of original file path, applied

+                after stripdir

+     limit_sl_dest: ignore symlinks if they are pointing outside of

+                    the defined path

+     """

+     ProcessPoolExecutor = None

+     if workers is not None:

+         if workers < 0:

+             raise ValueError('workers must be greater or equal to 0')

+         elif workers != 1:

+             try:

+                 # Only import when needed, as low resource platforms may

+                 # fail to import it

+                 from concurrent.futures import ProcessPoolExecutor

+             except ImportError:

+                 workers = 1

+     files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)

+     success = True

+     if workers is not None and workers != 1 and ProcessPoolExecutor is not None:

+         workers = workers or None

+         with ProcessPoolExecutor(max_workers=workers) as executor:

+             results = executor.map(partial(compile_file,

+                                            ddir=ddir, force=force,

+                                            rx=rx, quiet=quiet,

+                                            legacy=legacy,

+                                            optimize=optimize,

+                                            invalidation_mode=invalidation_mode,

+                                            stripdir=stripdir,

+                                            prependdir=prependdir,

+                                            limit_sl_dest=limit_sl_dest),

+                                    files)

+             success = min(results, default=True)

+     else:

+         for file in files:

+             if not compile_file(file, ddir, force, rx, quiet,

+                                 legacy, optimize, invalidation_mode,

+                                 stripdir=stripdir, prependdir=prependdir,

+                                 limit_sl_dest=limit_sl_dest):

+                 success = False

+     return success

+ 

+ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,

+                  legacy=False, optimize=-1,

+                  invalidation_mode=None, stripdir=None, prependdir=None,

+                  limit_sl_dest=None):

+     """Byte-compile one file.

+ 

+     Arguments (only fullname is required):

+ 

+     fullname:  the file to byte-compile

+     ddir:      if given, the directory name compiled in to the

+                byte-code file.

+     force:     if True, force compilation, even if timestamps are up-to-date

+     quiet:     full output with False or 0, errors only with 1,

+                no output with 2

+     legacy:    if True, produce legacy pyc paths instead of PEP 3147 paths

+     optimize:  int or list of optimization levels or -1 for level of

+                the interpreter. Multiple levels leads to multiple compiled

+                files each with one optimization level.

+     invalidation_mode: how the up-to-dateness of the pyc will be checked

+     stripdir:  part of path to left-strip from source file path

+     prependdir: path to prepend to beggining of original file path, applied

+                after stripdir

+     limit_sl_dest: ignore symlinks if they are pointing outside of

+                    the defined path.

+     """

+     success = True

+     if PY36 and quiet < 2 and isinstance(fullname, os.PathLike):

+         fullname = os.fspath(fullname)

+     else:

+         fullname = str(fullname)

+     name = os.path.basename(fullname)

+ 

+     dfile = None

+ 

+     if ddir is not None:

+         if not PY36:

+             ddir = str(ddir)

+         dfile = os.path.join(ddir, name)

+ 

+     if stripdir is not None:

+         fullname_parts = fullname.split(os.path.sep)

+         stripdir_parts = stripdir.split(os.path.sep)

+         ddir_parts = list(fullname_parts)

+ 

+         for spart, opart in zip(stripdir_parts, fullname_parts):

+             if spart == opart:

+                 ddir_parts.remove(spart)

+ 

+         dfile = os.path.join(*ddir_parts)

+ 

+     if prependdir is not None:

+         if dfile is None:

+             dfile = os.path.join(prependdir, fullname)

+         else:

+             dfile = os.path.join(prependdir, dfile)

+ 

+     if isinstance(optimize, int):

+         optimize = [optimize]

+ 

+     if rx is not None:

+         mo = rx.search(fullname)

+         if mo:

+             return success

+ 

+     if limit_sl_dest is not None and os.path.islink(fullname):

+         if Path(limit_sl_dest).resolve() not in Path(fullname).resolve().parents:

+             return success

+ 

+     opt_cfiles = {}

+ 

+     if os.path.isfile(fullname):

+         for opt_level in optimize:

+             if legacy:

+                 opt_cfiles[opt_level] = fullname + 'c'

+             else:

+                 if opt_level >= 0:

+                     opt = opt_level if opt_level >= 1 else ''

+                     opt_kwarg = optimization_kwarg(opt)

+                     cfile = (importlib.util.cache_from_source(

+                              fullname, **opt_kwarg))

+                     opt_cfiles[opt_level] = cfile

+                 else:

+                     cfile = importlib.util.cache_from_source(fullname)

+                     opt_cfiles[opt_level] = cfile

+ 

+         head, tail = name[:-3], name[-3:]

+         if tail == '.py':

+             if not force:

+                 try:

+                     mtime = int(os.stat(fullname).st_mtime)

+                     expect = struct.pack(*(pyc_header_format + (mtime,)))

+                     for cfile in opt_cfiles.values():

+                         with open(cfile, 'rb') as chandle:

+                             actual = chandle.read(pyc_header_lenght)

+                         if expect != actual:

+                             break

+                     else:

+                         return success

+                 except OSError:

+                     pass

+             if not quiet:

+                 print('Compiling {!r}...'.format(fullname))

+             try:

+                 for opt_level, cfile in opt_cfiles.items():

+                     if PY37:

+                         ok = py_compile.compile(fullname, cfile, dfile, True,

+                                                 optimize=opt_level,

+                                                 invalidation_mode=invalidation_mode)

+                     else:

+                         ok = py_compile.compile(fullname, cfile, dfile, True,

+                                                 optimize=opt_level)

+             except py_compile.PyCompileError as err:

+                 success = False

+                 if quiet >= 2:

+                     return success

+                 elif quiet:

+                     print('*** Error compiling {!r}...'.format(fullname))

+                 else:

+                     print('*** ', end='')

+                 # escape non-printable characters in msg

+                 msg = err.msg.encode(sys.stdout.encoding,

+                                      errors='backslashreplace')

+                 msg = msg.decode(sys.stdout.encoding)

+                 print(msg)

+             except (SyntaxError, UnicodeError, OSError) as e:

+                 success = False

+                 if quiet >= 2:

+                     return success

+                 elif quiet:

+                     print('*** Error compiling {!r}...'.format(fullname))

+                 else:

+                     print('*** ', end='')

+                 print(e.__class__.__name__ + ':', e)

+             else:

+                 if ok == 0:

+                     success = False

+     return success

+ 

+ def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0,

+                  legacy=False, optimize=-1,

+                  invalidation_mode=None):

+     """Byte-compile all module on sys.path.

+ 

+     Arguments (all optional):

+ 

+     skip_curdir: if true, skip current directory (default True)

+     maxlevels:   max recursion level (default 0)

+     force: as for compile_dir() (default False)

+     quiet: as for compile_dir() (default 0)

+     legacy: as for compile_dir() (default False)

+     optimize: as for compile_dir() (default -1)

+     invalidation_mode: as for compiler_dir()

+     """

+     success = True

+     for dir in sys.path:

+         if (not dir or dir == os.curdir) and skip_curdir:

+             if quiet < 2:

+                 print('Skipping current directory')

+         else:

+             success = success and compile_dir(

+                 dir,

+                 maxlevels,

+                 None,

+                 force,

+                 quiet=quiet,

+                 legacy=legacy,

+                 optimize=optimize,

+                 invalidation_mode=invalidation_mode,

+             )

+     return success

+ 

+ 

+ def main():

+     """Script main program."""

+     import argparse

+ 

+     parser = argparse.ArgumentParser(

+         description='Utilities to support installing Python libraries.')

+     parser.add_argument('-l', action='store_const', const=0,

+                         default=RECURSION_LIMIT, dest='maxlevels',

+                         help="don't recurse into subdirectories")

+     parser.add_argument('-r', type=int, dest='recursion',

+                         help=('control the maximum recursion level. '

+                               'if `-l` and `-r` options are specified, '

+                               'then `-r` takes precedence.'))

+     parser.add_argument('-f', action='store_true', dest='force',

+                         help='force rebuild even if timestamps are up to date')

+     parser.add_argument('-q', action='count', dest='quiet', default=0,

+                         help='output only error messages; -qq will suppress '

+                              'the error messages as well.')

+     parser.add_argument('-b', action='store_true', dest='legacy',

+                         help='use legacy (pre-PEP3147) compiled file locations')

+     parser.add_argument('-d', metavar='DESTDIR',  dest='ddir', default=None,

+                         help=('directory to prepend to file paths for use in '

+                               'compile-time tracebacks and in runtime '

+                               'tracebacks in cases where the source file is '

+                               'unavailable'))

+     parser.add_argument('-s', metavar='STRIPDIR',  dest='stripdir',

+                         default=None,

+                         help=('part of path to left-strip from path '

+                               'to source file - for example buildroot. '

+                               'if `-d` and `-s` options are specified, '

+                               'then `-d` takes precedence.'))

+     parser.add_argument('-p', metavar='PREPENDDIR',  dest='prependdir',

+                         default=None,

+                         help=('path to add as prefix to path '

+                               'to source file - for example / to make '

+                               'it absolute when some part is removed '

+                               'by `-s` option'

+                               'if `-d` and `-a` options are specified, '

+                               'then `-d` takes precedence.'))

+     parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,

+                         help=('skip files matching the regular expression; '

+                               'the regexp is searched for in the full path '

+                               'of each file considered for compilation'))

+     parser.add_argument('-i', metavar='FILE', dest='flist',

+                         help=('add all the files and directories listed in '

+                               'FILE to the list considered for compilation; '

+                               'if "-", names are read from stdin'))

+     parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*',

+                         help=('zero or more file and directory names '

+                               'to compile; if no arguments given, defaults '

+                               'to the equivalent of -l sys.path'))

+     parser.add_argument('-j', '--workers', default=1,

+                         type=int, help='Run compileall concurrently')

+     parser.add_argument('-o', action='append', type=int, dest='opt_levels',

+                         help=('Optimization levels to run compilation with.'

+                               'Default is -1 which uses optimization level of'

+                               'Python interpreter itself (specified by -O).'))

+     parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest',

+                         help='Ignore symlinks pointing outsite of the DIR')

+ 

+     if PY37:

+         invalidation_modes = [mode.name.lower().replace('_', '-')

+                               for mode in py_compile.PycInvalidationMode]

+         parser.add_argument('--invalidation-mode',

+                             choices=sorted(invalidation_modes),

+                             help=('set .pyc invalidation mode; defaults to '

+                                   '"checked-hash" if the SOURCE_DATE_EPOCH '

+                                   'environment variable is set, and '

+                                   '"timestamp" otherwise.'))

+ 

+     args = parser.parse_args()

+     compile_dests = args.compile_dest

+ 

+     if args.rx:

+         import re

+         args.rx = re.compile(args.rx)

+ 

+     if args.limit_sl_dest == "":

+         args.limit_sl_dest = None

+ 

+     if args.recursion is not None:

+         maxlevels = args.recursion

+     else:

+         maxlevels = args.maxlevels

+ 

+     if args.opt_levels is None:

+         args.opt_levels = [-1]

+ 

+     # if flist is provided then load it

+     if args.flist:

+         try:

+             with (sys.stdin if args.flist=='-' else open(args.flist)) as f:

+                 for line in f:

+                     compile_dests.append(line.strip())

+         except OSError:

+             if args.quiet < 2:

+                 print("Error reading file list {}".format(args.flist))

+             return False

+ 

+     if args.workers is not None:

+         args.workers = args.workers or None

+ 

+     if PY37 and args.invalidation_mode:

+         ivl_mode = args.invalidation_mode.replace('-', '_').upper()

+         invalidation_mode = py_compile.PycInvalidationMode[ivl_mode]

+     else:

+         invalidation_mode = None

+ 

+     success = True

+     try:

+         if compile_dests:

+             for dest in compile_dests:

+                 if os.path.isfile(dest):

+                     if not compile_file(dest, args.ddir, args.force, args.rx,

+                                         args.quiet, args.legacy,

+                                         invalidation_mode=invalidation_mode,

+                                         stripdir=args.stripdir,

+                                         prependdir=args.prependdir,

+                                         optimize=args.opt_levels,

+                                         limit_sl_dest=args.limit_sl_dest):

+                         success = False

+                 else:

+                     if not compile_dir(dest, maxlevels, args.ddir,

+                                        args.force, args.rx, args.quiet,

+                                        args.legacy, workers=args.workers,

+                                        invalidation_mode=invalidation_mode,

+                                        stripdir=args.stripdir,

+                                        prependdir=args.prependdir,

+                                        optimize=args.opt_levels,

+                                        limit_sl_dest=args.limit_sl_dest):

+                         success = False

+             return success

+         else:

+             return compile_path(legacy=args.legacy, force=args.force,

+                                 quiet=args.quiet,

+                                 invalidation_mode=invalidation_mode)

+     except KeyboardInterrupt:

+         if args.quiet < 2:

+             print("\n[interrupted]")

+         return False

+     return True

+ 

+ 

+ if __name__ == '__main__':

+     exit_status = int(not main())

+     sys.exit(exit_status)

file modified
+23 -11

@@ -1,7 +1,7 @@ 

  # Note that the path could itself be a python file, or a directory

  

- # Python's compile_all module only works on directories, and requires a max

- # recursion depth

+ # Note that the py_byte_compile macro should work for all Python versions

+ # Which unfortunately makes the definition more complicated than it should be

  

  # Usage:

  #    %py_byte_compile <interpereter> <path>

@@ -13,12 +13,24 @@ 

  #    (%{py_byte_compile <interpereter> <path>}) || :

  

  %py_byte_compile()\

- python_binary="%1"\

- buildroot_path="%2"\

- bytecode_compilation_path=".${buildroot_path/#$RPM_BUILD_ROOT}"\

- failure=0\

- pushd $RPM_BUILD_ROOT\

- find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -m py_compile || failure=1\

- find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -m py_compile || failure=1\

- popd\

- test $failure -eq 0

+ py2_byte_compile () {\

+     python_binary="%1"\

+     bytecode_compilation_path="%2"\

+     failure=0\

+     find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -s -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("'"$RPM_BUILD_ROOT"'")[2], doraise=True) for f in sys.argv[1:]]' || failure=1\

+     find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -s -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("'"$RPM_BUILD_ROOT"'")[2], doraise=True) for f in sys.argv[1:]]' || failure=1\

+     test $failure -eq 0\

I wonder why the implementation for Python 2 was changed. Care to elaborate on that?

+ }\

+ \

+ py3_byte_compile () {\

+     python_binary="%1"\

+     bytecode_compilation_path="%2"\

+     PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / $bytecode_compilation_path \

+ }\

+ \

+ # Path to intepreter should not contain any arguments \

+ [[ "%1" =~ " -" ]] && echo "ERROR py_byte_compile: Path to interpreter should not contain any arguments" >&2 && exit 1 \

+ # Get version without a dot (36 instead of 3.6), bash doesn't compare floats well \

+ python_version=$(%1 -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") \

+ # compileall2 Python module is not compatible with Python < 3.4 \

+ if [ "$python_version" -ge 34 ]; then py3_byte_compile "%1" "%2"; else py2_byte_compile "%1" "%2"; fi

file modified
+15 -7

@@ -3,16 +3,18 @@ 

  Release:        46%{?dist}

  Summary:        The unversioned Python RPM macros

  

- License:        MIT

+ # macros: MIT, compileall2.py: PSFv2

+ License:        MIT and Python

  Source0:        macros.python

  Source1:        macros.python-srpm

  Source2:        macros.python2

  Source3:        macros.python3

  Source4:        macros.pybytecompile

+ Source5:        https://github.com/fedora-python/compileall2/raw/v0.5.0/compileall2.py

  

  BuildArch:      noarch

- # For %%python3_pkgversion used in %%python_provide

- Requires:       python-srpm-macros

+ # For %%python3_pkgversion used in %%python_provide and compileall2.py

+ Requires:       python-srpm-macros >= 3-46

  Obsoletes:      python-macros < 3

  Provides:       python-macros = %{version}-%{release}

  

@@ -25,6 +27,7 @@ 

  

  %package -n python-srpm-macros

  Summary:        RPM macros for building Python source packages

+ Requires:       redhat-rpm-config

  

  %description -n python-srpm-macros

  RPM macros for building Python source packages.

@@ -41,8 +44,6 @@ 

  %package -n python3-rpm-macros

  Summary:        RPM macros for building Python 3 packages

  Requires:       python-srpm-macros >= 3-38

- # Would need to be different for each release - worth it?

- #Conflicts:      python3-devel < 3.5.1-3

  

  %description -n python3-rpm-macros

  RPM macros for building Python 3 packages.

@@ -53,10 +54,13 @@ 

  %build

  

  %install

- mkdir -p %{buildroot}/%{rpmmacrodir}

+ mkdir -p %{buildroot}%{rpmmacrodir}

  install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \

-   %{buildroot}/%{rpmmacrodir}/

+   %{buildroot}%{rpmmacrodir}/

  

+ mkdir -p %{buildroot}%{_rpmconfigdir}/redhat

+ install -m 644 %{SOURCE5} \

+   %{buildroot}%{_rpmconfigdir}/redhat/

  

  %files

  %{rpmmacrodir}/macros.python

@@ -64,6 +68,7 @@ 

  

  %files -n python-srpm-macros

  %{rpmmacrodir}/macros.python-srpm

+ %{_rpmconfigdir}/redhat/compileall2.py

  

  %files -n python2-rpm-macros

  %{rpmmacrodir}/macros.python2

@@ -76,6 +81,9 @@ 

  * Fri Jul 12 2019 Miro Hrončok <mhroncok@redhat.com> - 3-46

  - %%python_provide: Switch python2 and python3 behavior

  - https://fedoraproject.org/wiki/Changes/Python_means_Python3

+ - Use compileall2 module for byte-compilation with Python >= 3.4

+ - Do not allow passing arguments to Python during byte-compilation

+ - Use `-s` argument for Python during byte-compilation

  

  * Tue Jul 09 2019 Miro Hrončok <mhroncok@redhat.com> - 3-45

  - %%python_provide: Don't try to obsolete %%_isa provides

This change is introducing a new module compileall2 (enhanced clone of Python's compileall module which hopefully finds a way back to Python upstream) for Python byte-compilation with %py_byte_compile macro and Python >= 3.4. Compileall2 implements a lot of features we need for byte-compilation here and in brp-python-bytecompile. It'll make all byte-compilation phases much simpler when we drop support of Python 2 at it will be well tested by that time.

There is also a new check which prevents from passing a path to Python interpreter with any arguments. That's important because for example -I (isolated mode) causes that Python is not able to load compileall2 module. There is currently only one package which uses it and we agreed with the maintainer that it can be removed. In the meantime, you can take a look at clufter buld in COPR repo and see that the check works and produce a reasonable error message. However, it might be useful to isolate Python interpreter used to byte-compilation a little bit so I added -s argument to all Python invocations for byte-compilation.

I've compared some packages in COPR and it seems that the new way produces the same results as I can find in Koji.

Copr repository with all packages which uses %py_byte_compile macro: https://copr.fedorainfracloud.org/coprs/lbalhar/compileall2/
(there are a few failures but none of them caused by this change)
clufter build failure: https://copr-be.cloud.fedoraproject.org/results/lbalhar/compileall2/fedora-rawhide-x86_64/00967728-clufter/build.log.gz
compileall2: https://github.com/fedora-python/compileall2

Don't forget to update the license information. compileall, by its nature, is licensed under "Python".

errors should go to stderr

2 new commits added

  • fixup! Do not allow passing arguments to Python during byte-compilation
  • fixup! Use a new module compileall2 for Python byte-compilation
2 months ago

@churchyard Both issues fixed in fixup commits. I am testing the error output.

Fix works:

BUILDSTDERR: ERROR py_byte_compile: Path to interpreter should not contain any arguments

See https://copr-be.cloud.fedoraproject.org/results/lbalhar/compileall2/fedora-rawhide-x86_64/00968406-clufter/build.log.gz

this needs an explanation.

add something like "See https://www.python.org/psf/license/ for licensing information" or actually include the license text in the file (but that's quite long).

1 new commit added

  • fixup! fixup! Use a new module compileall2 for Python byte-compilation
2 months ago

License breakdown added as a comment to specfile and info about licensing will be part of the module file in the next release. Upstream issue: https://github.com/fedora-python/compileall2/issues/10

I wonder why the implementation for Python 2 was changed. Care to elaborate on that?

I wonder why the implementation for Python 2 was changed. Care to elaborate on that?

There is only one difference between this version and the version proposed by you I am aware of - -s argument for Python interpreter which is described in the PR description.

Reviewing your own work after 3 months be like: What idiot wrote this? :D

Maybe the Python 2 bit now solves the leaking buildroot better? I seriously have no idea. This is however the code you've tested, right?

Reviewing your own work after 3 months be like: What idiot wrote this? :D
Maybe the Python 2 bit now solves the leaking buildroot better? I seriously have no idea. This is however the code you've tested, right?

Yes. I'd say that it's well tested but I don't want to give you a loaded gun when something will break :smile:

Still, byte-compilation seems to strip buildroot properly for Python 3:

# cd /usr/lib/python3.7/site-packages/Pmw/Pmw_2_0_0/demos/__pycache__/
# python3 TextDialog.cpython-37.pyc 
Traceback (most recent call last):
  File "/usr/lib/python3.7/site-packages/Pmw/Pmw_2_0_0/demos/TextDialog.py", line 68, in <module>
    root = tkinter.Tk()
  File "/usr/lib64/python3.7/tkinter/__init__.py", line 2023, in __init__
    self.tk = _tkinter.create(screenName, baseName, className, interactive, wantobjects, useTk, sync, use)
_tkinter.TclError: no display name and no $DISPLAY environment variable

There are actually very few packages which byte-compiles with Python 2 and even less where it has some effect so I did a test with python-pmw where I removed all *.py[co] files before byte-compilation to check it's behavior and be sure that the files are byte-compiled by the macro.

# cd /usr/lib/python2.7/site-packages/Pmw/Pmw_1_3_3/demos/
# python2 TextDialog.pyc 
Traceback (most recent call last):
  File "/usr/lib/python2.7/site-packages/Pmw/Pmw_1_3_3/demos/TextDialog.py", line 68, in <module>
    root = Tkinter.Tk()
  File "/usr/lib64/python2.7/lib-tk/Tkinter.py", line 1825, in __init__
    self.tk = _tkinter.create(screenName, baseName, className, interactive, wantobjects, useTk, sync, use)
_tkinter.TclError: no display name and no $DISPLAY environment variable

Everything seems to be okay.

If this was done in rpmbuild, the files were overridden by brp-python-bytecompile (unless explicitly disabled).

rebased onto 76681ad

2 months ago

Prior to this change:

%py_byte_compile %{__python2} $RPM_BUILD_ROOT%{_datadir}/pybliographer/

...

<mock-chroot> sh-5.0# python2 /usr/share/pybliographer/pybliocheck.pyc
Traceback (most recent call last):
  File "./usr/share/pybliographer/pybliocheck.py", line 31, in <module>
    print _("usage: pybliocheck <file | directory>...").encode (charset)
NameError: name '_' is not defined

Prior to this change:
...

After:

# python2 /usr/share/pybliographer/pybliocheck.py
Traceback (most recent call last):
  File "/usr/share/pybliographer/pybliocheck.py", line 31, in <module>
    print _("usage: pybliocheck <file | directory>...").encode (charset)
NameError: name '_' is not defined

Pull-Request has been merged by churchyard

2 months ago