From d6ad9a778aac86a83eecbc460085f30d2e045181 Mon Sep 17 00:00:00 2001 From: Tomas Hrnciar Date: Jul 08 2021 11:08:04 +0000 Subject: Generate BuildRequires from file %pyproject_buildrequires macro now accepts multiple file names to load additional dependencies from them. New option -N was added to disable automatical generation of requirements in case package does not use build system. Option -N cannot be used in combination with options -r, -e, -t, -x. Co-authored-by: Miro Hrončok --- diff --git a/README.md b/README.md index 1a33dfe..c36bbba 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,15 @@ because runtime dependencies are always required for testing. [tox]: https://tox.readthedocs.io/ [tox-current-env]: https://github.com/fedora-python/tox-current-env/ +Additionaly to generated requirements you can supply multiple file names to `%pyproject_buildrequires` macro. +Dependencies will be loaded from them: + + %pyproject_buildrequires -r requirements/tests.in requirements/docs.in requirements/dev.in + +For packages not using build system you can use `-N` to entirely skip automatical +generation of requirements and install requirements only from manually specified files. +`-N` option cannot be used in combination with other options mentioned above +(`-r`, `-e`, `-t`, `-x`). Running tox based tests ----------------------- diff --git a/macros.pyproject b/macros.pyproject index a5c39a3..4b531be 100644 --- a/macros.pyproject +++ b/macros.pyproject @@ -84,18 +84,24 @@ fi %toxenv %{default_toxenv} -%pyproject_buildrequires(rxte:) %{expand:\\\ +%pyproject_buildrequires(rxtNe:) %{expand:\\\ +%{-N: +%{-r:%{error:The -N and -r options are mutually exclusive}} +%{-x:%{error:The -N and -x options are mutually exclusive}} +%{-e:%{error:The -N and -e options are mutually exclusive}} +%{-t:%{error:The -N and -t options are mutually exclusive}} +} %{-e:%{expand:%global toxenv %(%{__python3} -s %{_rpmconfigdir}/redhat/pyproject_construct_toxenv.py %{?**})}} echo 'python%{python3_pkgversion}-devel' echo 'python%{python3_pkgversion}dist(pip) >= 19' echo 'python%{python3_pkgversion}dist(packaging)' -if [ -f pyproject.toml ]; then +%{!-N:if [ -f pyproject.toml ]; then echo 'python%{python3_pkgversion}dist(toml)' else # Note: If the default requirements change, also change them in the script! echo 'python%{python3_pkgversion}dist(setuptools) >= 40.8' echo 'python%{python3_pkgversion}dist(wheel)' -fi +fi} # Check if we can generate dependencies on Python extras if [ "%{py_dist_name []}" == "[]" ]; then extras_flag=%{?!_python_no_extras_requires:--generate-extras} diff --git a/pyproject-rpm-macros.spec b/pyproject-rpm-macros.spec index 4a3a68e..0bf588e 100644 --- a/pyproject-rpm-macros.spec +++ b/pyproject-rpm-macros.spec @@ -6,7 +6,7 @@ License: MIT # Keep the version at zero and increment only release Version: 0 -Release: 42%{?dist} +Release: 43%{?dist} # Macro files Source001: macros.pyproject @@ -104,6 +104,10 @@ export HOSTNAME="rpmbuild" # to speedup tox in network-less mock, see rhbz#1856 %license LICENSE %changelog +* Thu Jul 01 2021 Tomas Hrnciar - 0-43 +- Generate BuildRequires from file +- Fixes: rhbz#1936448 + * Tue Jun 29 2021 Miro Hrončok - 0-42 - Don't accidentally treat "~= X.0" requirement as "~= X" - Fixes rhzb#1977060 diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py index f5b6330..0232126 100644 --- a/pyproject_buildrequires.py +++ b/pyproject_buildrequires.py @@ -11,6 +11,7 @@ import subprocess import re import tempfile import email.parser +import pathlib print_err = functools.partial(print, file=sys.stderr) @@ -228,14 +229,23 @@ def generate_run_requirements(backend, requirements): requirements.extend(requires, source=f'wheel metadata: {key}') -def parse_tox_requires_lines(lines): +def parse_requirements_lines(lines, path=None): packages = [] for line in lines: + line, _, comment = line.partition('#') + if comment.startswith('egg='): + # not a real comment + # e.g. git+https://github.com/monty/spam.git@master#egg=spam&... + egg, *_ = comment.strip().partition(' ') + egg, *_ = egg.strip().partition('&') + line = egg[4:] line = line.strip() if line.startswith('-r'): - path = line[2:] - with open(path) as f: - packages.extend(parse_tox_requires_lines(f.read().splitlines())) + recursed_path = line[2:].strip() + if path: + recursed_path = path.parent / recursed_path + with open(recursed_path) as f: + packages.extend(parse_requirements_lines(f.read().splitlines(), recursed_path)) elif line.startswith('-'): print_err( f'WARNING: Skipping dependency line: {line}\n' @@ -284,7 +294,7 @@ def generate_tox_requirements(toxenv, requirements): r.check_returncode() deplines = deps.read().splitlines() - packages = parse_tox_requires_lines(deplines) + packages = parse_requirements_lines(deplines) requirements.add_extras(*extras.read().splitlines()) requirements.extend(packages, source=f'tox --print-deps-only: {toxenv}') @@ -304,7 +314,7 @@ def python3dist(name, op=None, version=None, python3_pkgversion="3"): def generate_requires( *, include_runtime=False, toxenv=None, extras=None, get_installed_version=importlib.metadata.version, # for dep injection - generate_extras=False, python3_pkgversion="3", + generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True ): """Generate the BuildRequires for the project in the current directory @@ -317,8 +327,18 @@ def generate_requires( ) try: - backend = get_backend(requirements) - generate_build_requirements(backend, requirements) + if (include_runtime or toxenv) and not use_build_system: + raise ValueError('-N option cannot be used in combination with -r, -e, -t, -x options') + if requirement_files: + for req_file in requirement_files: + lines = req_file.read().splitlines() + packages = parse_requirements_lines(lines, pathlib.Path(req_file.name)) + requirements.extend(packages, + source=f'requirements file {req_file.name}') + requirements.check(source='all requirement files') + if use_build_system: + backend = get_backend(requirements) + generate_build_requirements(backend, requirements) if toxenv: include_runtime = True generate_tox_requirements(toxenv, requirements) @@ -360,6 +380,14 @@ def main(argv): default="3", help=('Python version for pythonXdist()' 'or pythonX.Ydist() requirements'), ) + parser.add_argument( + '-N', '--no-use-build-system', dest='use_build_system', + action='store_false', help='Use -N to indicate that project does not use any build system', + ) + parser.add_argument( + 'requirement_files', nargs='*', type=argparse.FileType('r'), + help=('Add buildrequires from file'), + ) args = parser.parse_args(argv) @@ -382,6 +410,8 @@ def main(argv): extras=args.extras, generate_extras=args.generate_extras, python3_pkgversion=args.python3_pkgversion, + requirement_files=args.requirement_files, + use_build_system=args.use_build_system, ) except Exception: # Log the traceback explicitly (it's useful debug info) diff --git a/pyproject_buildrequires_testcases.yaml b/pyproject_buildrequires_testcases.yaml index c762240..7bc83d9 100644 --- a/pyproject_buildrequires_testcases.yaml +++ b/pyproject_buildrequires_testcases.yaml @@ -446,3 +446,156 @@ Tox provision satisfied: python3dist(toxdep2) python3dist(inst) result: 0 + +Default build system, unmet deps in requirements file: + installed: + setuptools: 50 + wheel: 1 + setup.py: | + from setuptools import setup + setup( + name='test', + version='0.1', + ) + requirements.txt: | + lxml + ncclient + cryptography + paramiko + SQLAlchemy + requirement_files: + - requirements.txt + expected: | + python3dist(lxml) + python3dist(ncclient) + python3dist(cryptography) + python3dist(paramiko) + python3dist(sqlalchemy) + result: 0 + +Default build system, met deps in requirements file: + installed: + setuptools: 50 + wheel: 1 + lxml: 3.9 + ncclient: 1 + cryptography: 2 + paramiko: 1 + SQLAlchemy: 1.0.90 + setup.py: | + from setuptools import setup + setup( + name='test', + version='0.1', + ) + requirements.txt: | + lxml!=3.7.0,>=2.3 # OF-Config + ncclient # OF-Config + cryptography!=1.5.2 # Required by paramiko + paramiko # NETCONF, BGP speaker (SSH console) + SQLAlchemy>=1.0.10,<1.1.0 # Zebra protocol service + requirement_files: + - requirements.txt + expected: | + ((python3dist(lxml) < 3.7 or python3dist(lxml) > 3.7) with python3dist(lxml) >= 2.3) + python3dist(ncclient) + (python3dist(cryptography) < 1.5.2 or python3dist(cryptography) > 1.5.2) + python3dist(paramiko) + (python3dist(sqlalchemy) < 1.1 with python3dist(sqlalchemy) >= 1.0.10) + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + result: 0 + +With pyproject.toml, requirements file and with -N option: + use_build_system: false + installed: + setuptools: 50 + wheel: 1 + toml: 1 + lxml: 3.9 + ncclient: 1 + cryptography: 2 + paramiko: 1 + SQLAlchemy: 1.0.90 + pyproject.toml: | + [build-system] + requires = [ + "foo", + ] + build-backend = "foo.build" + requirements.txt: | + lxml + ncclient + cryptography + paramiko + SQLAlchemy + git+https://github.com/monty/spam.git@master#egg=spam + requirement_files: + - requirements.txt + expected: | + python3dist(lxml) + python3dist(ncclient) + python3dist(cryptography) + python3dist(paramiko) + python3dist(sqlalchemy) + python3dist(spam) + result: 0 + +With pyproject.toml, requirements file and without -N option: + use_build_system: true + installed: + setuptools: 50 + wheel: 1 + toml: 1 + lxml: 3.9 + ncclient: 1 + cryptography: 2 + paramiko: 1 + SQLAlchemy: 1.0.90 + argcomplete: 1 + hypothesis: 1 + pyproject.toml: | + [build-system] + requires = [ + "foo", + ] + build-backend = "foo.build" + requirements.txt: | + lxml + ncclient + cryptography + paramiko + SQLAlchemy + requirements1.in: | + argcomplete + hypothesis + requirement_files: + - requirements.txt + - requirements1.in + expected: | + python3dist(lxml) + python3dist(ncclient) + python3dist(cryptography) + python3dist(paramiko) + python3dist(sqlalchemy) + python3dist(argcomplete) + python3dist(hypothesis) + python3dist(foo) + result: 0 + +Value error if -N and -r arguments are present: + installed: + # empty + include_runtime: true + use_build_system: false + except: ValueError + +Value error if -N and -e arguments are present: + installed: + # empty + toxenv: + - py3 + use_build_system: false + except: ValueError + diff --git a/test_pyproject_buildrequires.py b/test_pyproject_buildrequires.py index 6f70361..5170c29 100644 --- a/test_pyproject_buildrequires.py +++ b/test_pyproject_buildrequires.py @@ -24,8 +24,9 @@ def test_data(case_name, capsys, tmp_path, monkeypatch): if case.get('xfail'): pytest.xfail(case.get('xfail')) - for filename in 'pyproject.toml', 'setup.py', 'tox.ini': - if filename in case: + for filename in case: + file_types = ('.toml', '.py', '.in', '.ini', '.txt') + if filename.endswith(file_types): cwd.joinpath(filename).write_text(case[filename]) def get_installed_version(dist_name): @@ -35,7 +36,8 @@ def test_data(case_name, capsys, tmp_path, monkeypatch): raise importlib.metadata.PackageNotFoundError( f'info not found for {dist_name}' ) - + requirement_files = case.get('requirement_files', []) + requirement_files = [open(f) for f in requirement_files] try: generate_requires( get_installed_version=get_installed_version, @@ -43,6 +45,8 @@ def test_data(case_name, capsys, tmp_path, monkeypatch): extras=case.get('extras', []), toxenv=case.get('toxenv', None), generate_extras=case.get('generate_extras', False), + requirement_files=requirement_files, + use_build_system=case.get('use_build_system', True), ) except SystemExit as e: assert e.code == case['result'] @@ -55,3 +59,6 @@ def test_data(case_name, capsys, tmp_path, monkeypatch): captured = capsys.readouterr() assert captured.out == case['expected'] + finally: + for req in requirement_files: + req.close() diff --git a/tests/fake-requirements.spec b/tests/fake-requirements.spec new file mode 100644 index 0000000..ee0f8bc --- /dev/null +++ b/tests/fake-requirements.spec @@ -0,0 +1,28 @@ +Name: fake-requirements +Version: 0 +Release: 0%{?dist} + +Summary: ... +License: MIT + +BuildRequires: pyproject-rpm-macros + + +%description +Fake spec file to test %%pyproject_buildrequires -N works as expected + +%prep +cat > requirements.txt <=4.1 # comment to increase test complexity +toml>=0.10.0 +EOF + +%generate_buildrequires +%pyproject_buildrequires requirements.txt -N + + +%check +pip show toml click +! pip show setuptools +! pip show wheel + diff --git a/tests/python-markupsafe.spec b/tests/python-markupsafe.spec new file mode 100644 index 0000000..b67d59d --- /dev/null +++ b/tests/python-markupsafe.spec @@ -0,0 +1,55 @@ +Name: python-markupsafe +Version: 2.0.1 +Release: 0%{?dist} +Summary: Implements a XML/HTML/XHTML Markup safe string for Python +License: BSD +URL: https://github.com/pallets/markupsafe +Source0: %{url}/archive/%{version}/MarkupSafe-%{version}.tar.gz + +BuildRequires: gcc +BuildRequires: make +BuildRequires: python3-devel +BuildRequires: pyproject-rpm-macros + +%description +This package installs test- and docs-requirements from files +and uses them to run tests and build documentation. + + +%package -n python3-markupsafe +Summary: %{summary} + +%description -n python3-markupsafe +... + +%prep +%autosetup -n markupsafe-%{version} + +# we don't have pip-tools packaged in Fedora yet +sed -i /pip-tools/d requirements/dev.in + + +%generate_buildrequires +# requirements/dev.in recursively includes tests.in and docs.in +# we also list tests.in manually to verify we can pass multiple arguments, +# but it should be redundant if this was a real package +%pyproject_buildrequires -r requirements/dev.in requirements/tests.in + + +%build +%pyproject_wheel +%make_build -C docs html SPHINXOPTS='-n %{?_smp_mflags}' + + +%install +%pyproject_install +%pyproject_save_files markupsafe + + +%check +%pytest + + +%files -n python3-markupsafe -f %{pyproject_files} +%license LICENSE.rst +%doc CHANGES.rst README.rst diff --git a/tests/tests.yml b/tests/tests.yml index f2a6f51..c1a0704 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -73,6 +73,12 @@ - setuptools: dir: . run: ./mocktest.sh python-setuptools + - markupsafe: + dir: . + run: ./mocktest.sh python-markupsafe + - fake_requirements: + dir: . + run: ./mocktest.sh fake-requirements required_packages: - mock - rpmdevtools