#6 Refactor and add tests
Merged 2 months ago by churchyard. Opened 2 months ago by pviktori.
rpms/ pviktori/pyproject-rpm-macros refactor  into  master

file modified
+1

@@ -0,0 +1,1 @@ 

+ __pycache__/

file modified
+4

@@ -45,6 +45,10 @@ 

  The PEPs don't (yet) define a way to specify test dependencies and test runners.

  That means you still need to handle test dependencies and `%check` on your own.

  

+ Extras are currently ignored.

+ 

+ Some valid Python version specifiers are not supported.

+ 

  

  [PEP 517]: https://www.python.org/dev/peps/pep-0517/

  [PEP 518]: https://www.python.org/dev/peps/pep-0518/

file modified
+1 -1

@@ -23,6 +23,6 @@ 

  echo 'python3dist(pip) >= 19'

  echo 'python3dist(pytoml)'

  if [ -f %{__python3} ]; then

-   %{__python3} -I %{_rpmconfigdir}/redhat/pyproject_buildrequires.py

+   %{__python3} -I %{_rpmconfigdir}/redhat/pyproject_buildrequires.py %{?*}

  fi

  }

file modified
+15

@@ -12,6 +12,9 @@ 

  Source8:        README.md

  Source9:        LICENSE

  

+ Source10:       test_pyproject_buildrequires.py

+ Source11:       testcases.yaml

+ 

  URL:            https://src.fedoraproject.org/rpms/pyproject-rpm-macros

  

  BuildArch:      noarch

@@ -23,6 +26,14 @@ 

  Requires: python3-pip >= 19

  Requires: python3-devel

  

+ # Test dependencies

+ BuildRequires: python3dist(pytest)

+ BuildRequires: python3dist(pyyaml)

+ BuildRequires: python3dist(packaging)

+ BuildRequires: python3dist(pytoml)

+ BuildRequires: python3dist(pip)

+ 

+ 

  %description

  This is a provisional implementation of pyproject RPM macros for Fedora 30+.

  These macros are useful for packaging Python projects that use the PEP 517

@@ -45,6 +56,10 @@ 

  install -m 644 macros.pyproject %{buildroot}%{_rpmmacrodir}/

  install -m 644 pyproject_buildrequires.py %{buildroot}%{_rpmconfigdir}/redhat/

  

+ %check

+ %{__python3} -m pytest -vv

+ 

+ 

  %files

  %{_rpmmacrodir}/macros.pyproject

  %{_rpmconfigdir}/redhat/pyproject_buildrequires.py

file modified
+187 -63

@@ -1,89 +1,213 @@ 

  import sys

+ import importlib

+ import argparse

+ import functools

+ import traceback

+ import contextlib

+ from io import StringIO

+ import subprocess

+ import pathlib

+ import re

+ 

+ print_err = functools.partial(print, file=sys.stderr)

+ 

+ # Some valid Python version specifiers are not supported.

+ # Whitelist characters we can handle.

+ VERSION_RE = re.compile('[a-zA-Z0-9.-]+')

+ 

+ class EndPass(Exception):

+     """End current pass of generating requirements"""

  

  try:

      import pytoml

      from packaging.requirements import Requirement, InvalidRequirement

+     from packaging.version import Version

      from packaging.utils import canonicalize_name, canonicalize_version

- except ImportError:

+     import pip

+ except ImportError as e:

+     print_err('Import error:', e)

      # already echoed by the %pyproject_buildrequires macro

      sys.exit(0)

  

  

- try:

-     with open("pyproject.toml") as f:

-         pyproject_data = pytoml.load(f)

- except FileNotFoundError:

-     pyproject_data = {}

- except Exception as e:

-     sys.exit(e)

- else:

-     import importlib

+ @contextlib.contextmanager

+ def hook_call():

+     captured_out = StringIO()

+     with contextlib.redirect_stdout(captured_out):

+         yield

+     for line in captured_out.getvalue().splitlines():

+         print_err('HOOK STDOUT:', line)

+ 

+ 

+ class Requirements:

+     """Requirement printer"""

+     def __init__(self, freeze_output):

+         self.installed_packages = {}

+         for line in freeze_output.splitlines():

+             line = line.strip()

+             if line.startswith('#'):

+                 continue

+             name, version = line.split('==')

+             self.installed_packages[name.strip()] = Version(version)

+ 

+         self.missing_requirements = False

+ 

+     def add(self, requirement_str, *, source=None):

+         """Output a Python-style requirement string as RPM dep"""

+         print_err(f'Handling {requirement_str} from {source}')

  

-     try:

-         backend = importlib.import_module(

-             pyproject_data["build-system"]["build-backend"]

-         )

-     except KeyError:

          try:

-             import setuptools.build_meta

-         except ImportError:

-             print("python3dist(setuptools) >= 40.8")

-             print("python3dist(wheel)")

-             sys.exit(0)

+             requirement = Requirement(requirement_str)

+         except InvalidRequirement as e:

+             print_err(

+                 f'"WARNING: Skipping invalid requirement: {requirement_str}\n'

+                 + f'    {e}',

+             )

+             return

  

-         backend = setuptools.build_meta

-     except ImportError:

-         backend = None

+         name = canonicalize_name(requirement.name)

+         if requirement.marker is not None and not requirement.marker.evaluate():

+             print_err(f'Ignoring alien requirement:', requirement_str)

+             return

  

+         installed = self.installed_packages.get(requirement.name)

+         if installed and installed in requirement.specifier:

+             print_err(f'Requirement satisfied: {requirement_str}')

+             print_err(f'   (installed: {requirement.name} {installed})')

+         else:

+             self.missing_requirements = True

+ 

+         together = []

+         for specifier in sorted(

+             requirement.specifier,

+             key=lambda s: (s.operator, s.version),

+         ):

+             version = canonicalize_version(specifier.version)

+             print_err(version)

+             if not VERSION_RE.fullmatch(str(specifier.version)):

+                 raise ValueError(

+                     f'Unknown character in version: {specifier.version}. '

+                     + '(This is probably a bug in pyproject-rpm-macros.)',

+                 )

+             if specifier.operator == "!=":

+                 lower = python3dist(name, '<', version)

+                 higher = python3dist(name, '>', f'{version}.0')

+                 together.append(

+                     f"({lower} or {higher})"

+                 )

+             else:

+                 together.append(python3dist(name, specifier.operator, version))

+         if len(together) == 0:

+             print(python3dist(name))

+         elif len(together) == 1:

+             print(together[0])

+         else:

+             print(f"({' and '.join(together)})")

  

- requirements = set()

- rpm_requirements = set()

+     def check(self, *, source=None):

+         """End current pass if any unsatisfied dependencies were output"""

+         if self.missing_requirements:

+             print_err(f'Exiting dependency generation pass: {source}')

+             raise EndPass(source)

  

+     def extend(self, requirement_strs, *, source=None):

+         """add() several requirements"""

+         for req_str in requirement_strs:

+             self.add(req_str, source=source)

  

- def add_requirement(requirement):

+ def get_backend(requirements):

      try:

-         requirements.add(Requirement(requirement))

-     except InvalidRequirement as e:

-         print(

-             f"WARNING: Skipping invalid requirement: {requirement}\n         {e}",

-             file=sys.stderr,

-         )

+         f = open('pyproject.toml')

+     except FileNotFoundError:

+         pyproject_data = {}

+     else:

+         with f:

+             pyproject_data = pytoml.load(f)

  

+     buildsystem_data = pyproject_data.get("build-system", {})

+     requirements.extend(

+         buildsystem_data.get("requires", ()),

+         source='build-system.requires',

+     )

  

- if "requires" in pyproject_data.get("build-system", {}):

-     try:

-         for requirement in pyproject_data["build-system"]["requires"]:

-             add_requirement(requirement)

-     except Exception as e:

-         sys.exit(e)

+     backend_name = buildsystem_data.get('build-backend')

+     if not backend_name:

+         requirements.add("setuptools >= 40.8", source='default build backend')

+         requirements.add("wheel", source='default build backend')

  

+         backend_name = 'setuptools.build_meta'

+ 

+     requirements.check(source='build backend')

+ 

+     backend_path = buildsystem_data.get('backend-path')

+     if backend_path:

+         sys.path.insert(0, backend_path)

+ 

+     return importlib.import_module(backend_name)

+ 

+ 

+ def generate_build_requirements(backend, requirements):

+     get_requires = getattr(backend, "get_requires_for_build_wheel", None)

+     if get_requires:

+         with hook_call():

+             new_reqs = get_requires()

+         requirements.extend(new_reqs, source='get_requires_for_build_wheel')

+ 

+ 

+ def python3dist(name, op=None, version=None):

+     if op is None:

+         if version is not None:

+             raise AssertionError('op and version go together')

+         return f'python3dist({name})'

+     else:

+         return f'python3dist({name}) {op} {version}'

+ 

+ 

+ def generate_requires(freeze_output):

+     requirements = Requirements(freeze_output)

+ 

+     try:

+         backend = get_backend(requirements)

+         generate_build_requirements(backend, requirements)

+     except EndPass:

+         return

+ 

+ 

+ def main(argv):

+     parser = argparse.ArgumentParser(

+         description='Generate BuildRequires for a Python project.'

+     )

+     parser.add_argument(

+         '--runtime', action='store_true',

+         help='Generate run-time requirements (not implemented)',

+     )

+     parser.add_argument(

+         '--toxenv', metavar='TOXENVS',

+         help='generate test tequirements from tox environment '

+             + '(not implemented; implies --runtime)',

+     )

+ 

+     args = parser.parse_args(argv)

+     if args.toxenv:

+         args.runtime = True

+     if args.runtime:

+         print_err('--runtime is not implemented')

+         exit(1)

+ 

+     freeze_output = subprocess.run(

+         [sys.executable, '-I', '-m', 'pip', 'freeze', '--all'],

+         encoding='utf-8',

+         stdout=subprocess.PIPE,

+         check=True,

+     ).stdout

  

- if hasattr(backend, "get_requires_for_build_wheel"):

      try:

-         for requirement in backend.get_requires_for_build_wheel():

-             add_requirement(requirement)

+         generate_requires(freeze_output)

      except Exception as e:

-         sys.exit(e)

- 

- for requirement in requirements:

-     name = canonicalize_name(requirement.name)

-     if requirement.marker is not None and not requirement.marker.evaluate():

-         continue

-     together = []

-     for specifier in requirement.specifier:

-         version = canonicalize_version(specifier.version)

-         if specifier.operator == "!=":

-             together.append(

-                 f"(python3dist({name}) < {version} or python3dist({name}) >= {version}.0)"

-             )

-         else:

-             together.append(f"python3dist({name}) {specifier.operator} {version}")

-     if len(together) == 0:

-         rpm_requirements.add(f"python3dist({name})")

-     if len(together) == 1:

-         rpm_requirements.add(together[0])

-     elif len(together) > 1:

-         rpm_requirements.add(f"({' and '.join(together)})")

+         # Log the traceback explicitly (it's useful debug info)

+         traceback.print_exc()

+         exit(1)

  

  

- print(*sorted(rpm_requirements), sep="\n")

+ if __name__ == '__main__':

+     main(sys.argv[1:])

@@ -0,0 +1,40 @@ 

+ from pathlib import Path

+ import io

+ 

+ import pytest

+ import yaml

+ 

+ from pyproject_buildrequires import generate_requires

+ 

+ testcases = {}

+ with Path(__file__).parent.joinpath('testcases.yaml').open() as f:

+     testcases = yaml.safe_load(f)

+ 

+ 

+ @pytest.mark.parametrize('case_name', testcases)

+ def test_data(case_name, capsys, tmp_path, monkeypatch):

+     case = testcases[case_name]

+ 

+     cwd = tmp_path.joinpath('cwd')

+     cwd.mkdir()

+     monkeypatch.chdir(cwd)

+ 

+     if 'pyproject.toml' in case:

+         cwd.joinpath('pyproject.toml').write_text(case['pyproject.toml'])

+ 

+     if 'setup.py' in case:

+         cwd.joinpath('setup.py').write_text(case['setup.py'])

+ 

+     try:

+         generate_requires(

+             case['freeze_output'],

+         )

+     except SystemExit as e:

+         assert e.code == case['result']

+     except Exception as e:

+         assert type(e).__name__ == case['except']

+     else:

+         assert 0 == case['result']

+ 

+         captured = capsys.readouterr()

+         assert captured.out == case['expected']

file added
+119

@@ -0,0 +1,119 @@ 

+ No pyproject.toml, nothing installed:

+   freeze_output: |

+     # empty

+   expected: |

+     python3dist(setuptools) >= 40.8

+     python3dist(wheel)

+   result: 0

+ 

+ Nothing installed yet:

+   freeze_output: |

+     # empty

+   pyproject.toml: |

+     # empty

+   expected: |

+     python3dist(setuptools) >= 40.8

+     python3dist(wheel)

+   result: 0

+ 

+ Insufficient version of setuptools:

+   freeze_output: |

+     setuptools==5

+     wheel==1

+   pyproject.toml: |

+     # empty

+   expected: |

+     python3dist(setuptools) >= 40.8

+     python3dist(wheel)

+   result: 0

+ 

+ Empty pyproject.toml, empty setup.py:

+   freeze_output: |

+     setuptools==50

+     wheel==1

+   setup.py: |

+   expected: |

+     python3dist(setuptools) >= 40.8

+     python3dist(wheel)

+     python3dist(wheel)

+   result: 0

+ 

+ Default build system, empty setup.py:

+   freeze_output: |

+     setuptools==50

+     wheel==1

+   pyproject.toml: |

+     # empty

+   setup.py: |

+   expected: |

+     python3dist(setuptools) >= 40.8

+     python3dist(wheel)

+     python3dist(wheel)

+   result: 0

+ 

+ Erroring setup.py:

+   freeze_output: |

+     setuptools==50

+     wheel==1

+   setup.py: |

+     exit(77)

+   result: 77

+ 

+ Bad character in version:

+   freeze_output: |

+   pyproject.toml: |

+     [build-system]

+     requires = ["pkg == 0.$.^.*"]

+   except: ValueError

+ 

+ Build system dependencies in pyproject.toml:

+   freeze_output: |

+     setuptools==50

+     wheel==1

+   pyproject.toml: |

+     [build-system]

+     requires = [

+         "foo",

+         "ne!=1",

+         "ge>=1.2",

+         "le <= 1.2.3",

+         "lt < 1.2.3.4      ",

+         "    gt > 1.2.3.4.5",

+         "combo >2, <5, != 3.0.0",

+         "invalid!!ignored",

+         "py2 ; python_version < '2.7'",

+         "py3 ; python_version > '3.0'",

+         "pkg [extra-currently-ignored]",

+     ]

+   expected: |

+     python3dist(foo)

+     (python3dist(ne) < 1 or python3dist(ne) > 1.0)

+     python3dist(ge) >= 1.2

+     python3dist(le) <= 1.2.3

+     python3dist(lt) < 1.2.3.4

+     python3dist(gt) > 1.2.3.4.5

+     ((python3dist(combo) < 3 or python3dist(combo) > 3.0) and python3dist(combo) < 5 and python3dist(combo) > 2)

+     python3dist(py3)

+     python3dist(pkg)

+     python3dist(setuptools) >= 40.8

+     python3dist(wheel)

+   result: 0

+ 

+ Default build system, dependencies in setup.py:

+   freeze_output: |

+     setuptools==50

+     wheel==1

+   setup.py: |

+     from setuptools import setup

+     setup(

+         name='test',

+         version='0.1',

+         setup_requires=['foo', 'bar!=2'],

+     )

+   expected: |

+     python3dist(setuptools) >= 40.8

+     python3dist(wheel)

+     python3dist(wheel)

+     python3dist(foo)

+     (python3dist(bar) < 2 or python3dist(bar) > 2.0)

+   result: 0