#19 pythondistdeps: Implement provides/requires for extras packages
Merged a month ago by churchyard. Opened 2 months ago by torsava.
rpms/ torsava/python-rpm-generators generators-extras  into  master

file modified
+5 -1

@@ -1,7 +1,7 @@ 

  Name:           python-rpm-generators

  Summary:        Dependency generators for Python RPMs

  Version:        11

- Release:        8%{?dist}

+ Release:        9%{?dist}

  

  # Originally all those files were part of RPM, so license is kept here

  License:        GPLv2+

@@ -47,6 +47,10 @@ 

  %{_rpmconfigdir}/pythonbundles.py

  

  %changelog

+ * Fri Jul 10 2020 Tomas Orsava <torsava@redhat.com> - 11-9

+ - pythondistdeps: Implement provides/requires for extras packages

+ - Enable --require-extras-subpackages

+ 

  * Fri Jun 26 2020 Miro Hrončok <mhroncok@redhat.com> - 11-8

  - Fix python(abi) requires generator, it picked files from almost good directories

  - Add a script to generate Python bundled provides

file modified
+2 -2

@@ -1,3 +1,3 @@ 

- %__pythondist_provides	%{_rpmconfigdir}/pythondistdeps.py --provides --normalized-names-format pep503 --normalized-names-provide-both --majorver-provides-versions 2.7,%{__default_python3_version}

- %__pythondist_requires	%{_rpmconfigdir}/pythondistdeps.py --requires --normalized-names-format pep503

+ %__pythondist_provides	%{_rpmconfigdir}/pythondistdeps.py --provides --normalized-names-format pep503 --package-name %{name} --normalized-names-provide-both --majorver-provides-versions 2.7,%{__default_python3_version}

+ %__pythondist_requires	%{_rpmconfigdir}/pythondistdeps.py --requires --normalized-names-format pep503 --package-name %{name} %{?!_python_no_extras_requires:--require-extras-subpackages}

  %__pythondist_path		^/usr/lib(64)?/python[[:digit:]]\\.[[:digit:]]+/site-packages/[^/]+\\.(dist-info|egg-info|egg-link)$

file modified
+59 -27

@@ -18,7 +18,7 @@ 

  from __future__ import print_function

  import argparse

  from os.path import basename, dirname, isdir, sep

- from sys import argv, stdin, version

+ from sys import argv, stdin, stderr, version

  from distutils.sysconfig import get_python_lib

  from warnings import warn

  

@@ -65,11 +65,13 @@ 

  

  def convert_compatible(name, operator, version_id):

      if version_id.endswith('.*'):

-         print('Invalid requirement: {} {} {}'.format(name, operator, version_id))

+         print("*** INVALID_REQUIREMENT_ERROR___SEE_STDERR ***")

+         print('Invalid requirement: {} {} {}'.format(name, operator, version_id), file=stderr)

          exit(65)  # os.EX_DATAERR

      version = RpmVersion(version_id)

      if len(version.version) == 1:

-         print('Invalid requirement: {} {} {}'.format(name, operator, version_id))

+         print("*** INVALID_REQUIREMENT_ERROR___SEE_STDERR ***")

+         print('Invalid requirement: {} {} {}'.format(name, operator, version_id), file=stderr)

          exit(65)  # os.EX_DATAERR

      upper_version = RpmVersion(version_id)

      upper_version.version.pop()

@@ -88,7 +90,8 @@ 

  

  def convert_arbitrary_equal(name, operator, version_id):

      if version_id.endswith('.*'):

-         print('Invalid requirement: {} {} {}'.format(name, operator, version_id))

+         print("*** INVALID_REQUIREMENT_ERROR___SEE_STDERR ***")

+         print('Invalid requirement: {} {} {}'.format(name, operator, version_id), file=stderr)

          exit(65)  # os.EX_DATAERR

      version = RpmVersion(version_id)

      return '{} = {}'.format(name, version)

@@ -157,7 +160,7 @@ 

      group.add_argument('-R', '--requires', action='store_true', help='Print Requires')

      group.add_argument('-r', '--recommends', action='store_true', help='Print Recommends')

      group.add_argument('-C', '--conflicts', action='store_true', help='Print Conflicts')

-     group.add_argument('-E', '--extras', action='store_true', help='Print Extras')

+     group.add_argument('-E', '--extras', action='store_true', help='[Unused] Generate spec file snippets for extras subpackages')

      group_majorver = parser.add_mutually_exclusive_group()

      group_majorver.add_argument('-M', '--majorver-provides', action='store_true', help='Print extra Provides with Python major version only')

      group_majorver.add_argument('--majorver-provides-versions', action='append',

@@ -171,7 +174,10 @@ 

                          help='Provide both `pep503` and `legacy-dots` format of normalized names (useful for a transition period)')

      parser.add_argument('-L', '--legacy-provides', action='store_true', help='Print extra legacy pythonegg Provides')

      parser.add_argument('-l', '--legacy', action='store_true', help='Print legacy pythonegg Provides/Requires instead')

-     parser.add_argument('files', nargs=argparse.REMAINDER)

+     parser.add_argument('--require-extras-subpackages', action='store_true',

+                         help="If there is a dependency on a package with extras functionality, require the extras subpackage")

+     parser.add_argument('--package-name', action='store', help="Name of the RPM package that's being inspected. Required for extras requires/provides to work.")

+     parser.add_argument('files', nargs=argparse.REMAINDER, help="Files from the RPM package that are to be inspected, can also be supplied on stdin")

      args = parser.parse_args()

  

      py_abi = args.requires

@@ -197,6 +203,12 @@ 

      # At least one type of normalization must be provided

      assert normalized_names_provide_pep503 or normalized_names_provide_legacy

  

+     # Is this script being run for an extras subpackage?

+     extras_subpackage = None

+     if args.package_name:

+         package_name_parts = args.package_name.partition('+')

+         extras_subpackage = package_name_parts[2] or None

+ 

      for f in (args.files or stdin.readlines()):

          f = f.strip()

          lower = f.lower()

@@ -267,11 +279,19 @@ 

              # See https://bugzilla.redhat.com/show_bug.cgi?id=1791530

              normalized_name = normalize_name(dist.project_name)

  

+             # If we're processing an extras subpackage, check that the extras exists

+             if extras_subpackage and extras_subpackage not in dist.extras:

+                 print("*** PYTHON_EXTRAS_NOT_FOUND_ERROR___SEE_STDERR ***")

+                 print(f"\nError: The package name contains an extras name `{extras_subpackage}` that was not found in the metadata.\n"

+                       "Check if the extras were removed from the project. If so, consider removing the subpackage and obsoleting it from another.\n", file=stderr)

+                 exit(65)  # os.EX_DATAERR

+ 

              if args.majorver_provides or args.majorver_provides_versions or \

                      args.majorver_only or args.legacy_provides or args.legacy:

                  # Get the Python major version

                  pyver_major = dist.py_version.split('.')[0]

              if args.provides:

+                 extras_suffix = f"[{extras_subpackage}]" if extras_subpackage else ""

                  # If egg/dist metadata says package name is python, we provide python(abi)

                  if dist.key == 'python':

                      name = 'python(abi)'

@@ -280,21 +300,21 @@ 

                      py_deps[name].append(('==', dist.py_version))

                  if not args.legacy or not args.majorver_only:

                      if normalized_names_provide_legacy:

-                         name = 'python{}dist({})'.format(dist.py_version, dist.key)

+                         name = 'python{}dist({}{})'.format(dist.py_version, dist.key, extras_suffix)

                          if name not in py_deps:

                              py_deps[name] = []

                      if normalized_names_provide_pep503:

-                         name_ = 'python{}dist({})'.format(dist.py_version, normalized_name)

+                         name_ = 'python{}dist({}{})'.format(dist.py_version, normalized_name, extras_suffix)

                          if name_ not in py_deps:

                              py_deps[name_] = []

                  if args.majorver_provides or args.majorver_only or \

                          (args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions):

                      if normalized_names_provide_legacy:

-                         pymajor_name = 'python{}dist({})'.format(pyver_major, dist.key)

+                         pymajor_name = 'python{}dist({}{})'.format(pyver_major, dist.key, extras_suffix)

                          if pymajor_name not in py_deps:

                              py_deps[pymajor_name] = []

                      if normalized_names_provide_pep503:

-                         pymajor_name_ = 'python{}dist({})'.format(pyver_major, normalized_name)

+                         pymajor_name_ = 'python{}dist({}{})'.format(pyver_major, normalized_name, extras_suffix)

                          if pymajor_name_ not in py_deps:

                              py_deps[pymajor_name_] = []

                  if args.legacy or args.legacy_provides:

@@ -341,6 +361,9 @@ 

                              if dep in deps:

                                  depsextras.remove(dep)

                      deps = depsextras

+                 elif extras_subpackage:

+                     # Extras requires also contain the base requires included

+                     deps = [d for d in dist.requires(extras=[extras_subpackage]) if d not in dist.requires()]

                  # console_scripts/gui_scripts entry points need pkg_resources from setuptools

                  if ((dist.get_entry_map('console_scripts') or

                      dist.get_entry_map('gui_scripts')) and

@@ -350,25 +373,34 @@ 

                      deps.insert(0, Requirement.parse('setuptools'))

                  # add requires/recommends based on egg/dist metadata

                  for dep in deps:

-                     if normalized_names_require_pep503:

-                         dep_normalized_name = normalize_name(dep.project_name)

-                     else:

-                         dep_normalized_name = dep.key

- 

-                     if args.legacy:

-                         name = 'pythonegg({})({})'.format(pyver_major, dep.key)

-                     else:

-                         if args.majorver_only:

-                             name = 'python{}dist({})'.format(pyver_major, dep_normalized_name)

+                     # Even if we're requiring `foo[bar]`, also require `foo`

+                     # to be safe, and to make it discoverable through

+                     # `repoquery --whatrequires`

+                     extras_suffixes = [""]

+                     if args.require_extras_subpackages and dep.extras:

+                         # A dependency can have more than one extras,

+                         # i.e. foo[bar,baz], so let's go through all of them

+                         extras_suffixes += [f"[{e}]" for e in dep.extras]

+                     for extras_suffix in extras_suffixes:

+                         if normalized_names_require_pep503:

+                             dep_normalized_name = normalize_name(dep.project_name)

                          else:

-                             name = 'python{}dist({})'.format(dist.py_version, dep_normalized_name)

-                     for spec in dep.specs:

-                         if name not in py_deps:

+                             dep_normalized_name = dep.key

+ 

+                         if args.legacy:

+                             name = 'pythonegg({})({})'.format(pyver_major, dep.key)

+                         else:

+                             if args.majorver_only:

+                                 name = 'python{}dist({}{})'.format(pyver_major, dep_normalized_name, extras_suffix)

+                             else:

+                                 name = 'python{}dist({}{})'.format(dist.py_version, dep_normalized_name, extras_suffix)

+                         for spec in dep.specs:

+                             if name not in py_deps:

+                                 py_deps[name] = []

+                             if spec not in py_deps[name]:

+                                 py_deps[name].append(spec)

+                         if not dep.specs:

                              py_deps[name] = []

-                         if spec not in py_deps[name]:

-                             py_deps[name].append(spec)

-                     if not dep.specs:

-                         py_deps[name] = []

              # Unused, for automatic sub-package generation based on 'extras' from egg/dist metadata

              # TODO: implement in rpm later, or...?

              if args.extras:

@@ -497,6 +497,20 @@ 

                  python3.9dist(simplejson) = 3.16

                  python3dist(simplejson) = 3.16

              requires: python(abi) = 3.9

+     --provides --majorver-provides-versions 3.9:

+         usr/lib/python2.7/site-packages/zope.interface-4.6.0.dist-info:

+             provides: python2.7dist(zope.interface) = 4.6

+             requires: |-

+                 python(abi) = 2.7

+                 python2.7dist(setuptools)

+         usr/lib64/python3.7/site-packages/lxml-4.4.0.dist-info:

+             provides: python3.7dist(lxml) = 4.4

+             requires: python(abi) = 3.7

+         usr/lib64/python3.9/site-packages/simplejson-3.16.0-py3.9.egg-info:

+             provides: |-

+                 python3.9dist(simplejson) = 3.16

+                 python3dist(simplejson) = 3.16

+             requires: python(abi) = 3.9

      --provides --majorver-provides-versions 3.9 --majorver-provides-versions 2.7:

          usr/lib/python2.7/site-packages/attrs-19.1.0-py2.7.egg-info:

              provides: |-

@@ -844,21 +858,6 @@ 

                  python3.9dist(simplejson) = 3.16

                  python3dist(simplejson) = 3.16

              requires: python(abi) = 3.9

-     --provides --majorver-provides-versions 3.9:

-         usr/lib/python2.7/site-packages/zope.interface-4.6.0.dist-info:

-             provides: |-

-                 python2.7dist(zope.interface) = 4.6

-             requires: |-

-                 python(abi) = 2.7

-                 python2.7dist(setuptools)

-         usr/lib64/python3.7/site-packages/lxml-4.4.0.dist-info:

-             provides: python3.7dist(lxml) = 4.4

-             requires: python(abi) = 3.7

-         usr/lib64/python3.9/site-packages/simplejson-3.16.0-py3.9.egg-info:

-             provides: |-

-                 python3.9dist(simplejson) = 3.16

-                 python3dist(simplejson) = 3.16

-             requires: python(abi) = 3.9

  --requires --normalized-names-format legacy-dots:

      --provides --majorver-provides --normalized-names-format legacy-dots:

          usr/lib/python2.7/site-packages/zope.component-4.3.0-py2.7.egg-info:

@@ -1215,3 +1214,65 @@ 

                  python3dist(backports-range) = 3.7.2

                  python3dist(backports.range) = 3.7.2

              requires: python(abi) = 3.7

+ --requires --normalized-names-format pep503 --package-name python3-zope-component+testing:

+     --provides --majorver-provides --normalized-names-format pep503 --package-name python3-zope-component+testing:

+         usr/lib/python3.9/site-packages/zope.component-4.3.0-py3.9.egg-info:

+             provides: |-

+                 python3.9dist(zope-component[testing]) = 4.3

+                 python3dist(zope-component[testing]) = 4.3

+             requires: |-

+                 python(abi) = 3.9

+                 python3.9dist(coverage)

+                 python3.9dist(nose)

+                 python3.9dist(zope-component)

+                 python3.9dist(zope-testing)

+ --requires --normalized-names-format pep503 --package-name python3-zope-schema+docs:

+     --provides --majorver-provides --normalized-names-format pep503 --package-name python3-zope-schema+docs:

+         usr/lib/python3.9/site-packages/zope.schema-4.4.2-py3.9.egg-info:

+             provides: |-

+                 python3.9dist(zope-schema[docs]) = 4.4.2

+                 python3dist(zope-schema[docs]) = 4.4.2

+             requires: |-

+                 python(abi) = 3.9

+                 python3.9dist(sphinx)

+ --requires --normalized-names-format pep503 --package-name python3-zope-schema+testing:

+     --provides --majorver-provides --normalized-names-format pep503 --package-name python3-zope-schema+testing:

+         usr/lib/python3.9/site-packages/zope.schema-4.4.2-py3.9.egg-info:

+             provides: |-

+                 python3.9dist(zope-schema[testing]) = 4.4.2

+                 python3dist(zope-schema[testing]) = 4.4.2

+             requires: |-

+                 python(abi) = 3.9

+                 python3.9dist(coverage)

+                 python3.9dist(nose)

+                 python3.9dist(zope-testing)

+ --requires --normalized-names-format pep503 --require-extras-subpackages --package-name python3-zope-component+missing:

+     --provides --majorver-provides --normalized-names-format pep503 --package-name python3-zope-component+missing:

+         usr/lib/python3.9/site-packages/zope.component-4.3.0-py3.9.egg-info:

+             stderr:

+                 provides: |-

+                     Error: The package name contains an extras name `missing` that was not found in the metadata.

+                     Check if the extras were removed from the project. If so, consider removing the subpackage and obsoleting it from another.

+                 requires: |-

+                     Error: The package name contains an extras name `missing` that was not found in the metadata.

+                     Check if the extras were removed from the project. If so, consider removing the subpackage and obsoleting it from another.

+             stdout:

+                 provides: '*** PYTHON_EXTRAS_NOT_FOUND_ERROR___SEE_STDERR ***'

+                 requires: '*** PYTHON_EXTRAS_NOT_FOUND_ERROR___SEE_STDERR ***'

+ --requires --normalized-names-format pep503 --require-extras-subpackages --package-name python3-zope-component+testing:

+     --provides --majorver-provides --normalized-names-format pep503 --package-name python3-zope-component+testing:

+         usr/lib/python3.9/site-packages/zope.component-4.3.0-py3.9.egg-info:

+             provides: |-

+                 python3.9dist(zope-component[testing]) = 4.3

+                 python3dist(zope-component[testing]) = 4.3

+             requires: |-

+                 python(abi) = 3.9

+                 python3.9dist(coverage)

+                 python3.9dist(nose)

+                 python3.9dist(zope-component)

+                 python3.9dist(zope-component[hook])

+                 python3.9dist(zope-component[persistentregistry])

+                 python3.9dist(zope-component[security])

+                 python3.9dist(zope-component[zcml])

+                 python3.9dist(zope-testing)

+ 

@@ -1,5 +1,5 @@ 

  # Run tests using pytest, e.g. from the root directory

- #   $ python3 -m pytest --ignore tests/testing/ -vvv

+ #   $ python3 -m pytest --ignore tests/testing/ -s -vvv

  #

  # If there are any breakags, the best way to see differences is using a diff:

  #   $ diff tests/data/scripts_pythondistdeps/test-data.yaml <(python3 tests/test_scripts_pythondistdeps.py)

@@ -41,18 +41,30 @@ 

  TEST_DATA_PATH = Path(__file__).parent / 'data' / 'scripts_pythondistdeps'

  

  

- def run_pythondistdeps(provides_params, requires_params, dist_egg_info_path):

+ def run_pythondistdeps(provides_params, requires_params, dist_egg_info_path, expect_failure=False):

      """Runs pythondistdeps.py on `dits_egg_info_path` with given

      provides and requires parameters and returns a dict with generated provides and requires"""

      info_path = TEST_DATA_PATH / dist_egg_info_path

      files = '\n'.join(map(str, info_path.iterdir()))

  

-     provides = subprocess.check_output((sys.executable, PYTHONDISTDEPS_PATH, *shlex.split(provides_params)),

-             input=files, encoding="utf-8")

-     requires = subprocess.check_output((sys.executable, PYTHONDISTDEPS_PATH, *shlex.split(requires_params)),

-             input=files, encoding="utf-8")

+     provides = subprocess.run((sys.executable, PYTHONDISTDEPS_PATH, *shlex.split(provides_params)),

+             input=files, capture_output=True, check=False, encoding="utf-8")

+     requires = subprocess.run((sys.executable, PYTHONDISTDEPS_PATH, *shlex.split(requires_params)),

+             input=files, capture_output=True, check=False, encoding="utf-8")

  

-     return {"provides": provides.strip(), "requires": requires.strip()}

+     if expect_failure:

+         if provides.returncode == 0 or requires.returncode == 0:

+             raise RuntimeError(f"pythondistdeps.py did not exit with a non-zero code as expected.\n"

+                                f"Used parameters: ({provides_params}, {requires_params}, {dist_egg_info_path})")

+         stdout = {"provides": provides.stdout.strip(), "requires": requires.stdout.strip()}

+         stderr = {"provides": provides.stderr.strip(), "requires": requires.stderr.strip()}

+         return {"stderr": stderr, "stdout": stdout}

+ 

+     else:

+         if provides.returncode != 0 or requires.returncode != 0:

+             raise RuntimeError(f"pythondistdeps.py unexpectedly exited with a non-zero code.\n"

+                                f"Used parameters: ({provides_params}, {requires_params}, {dist_egg_info_path})")

+         return {"provides": provides.stdout.strip(), "requires": requires.stdout.strip()}

  

  

  def load_test_data():

@@ -212,7 +224,8 @@ 

  def test_pythondistdeps(provides_params, requires_params, dist_egg_info_path, expected):

      """Runs pythondistdeps with the given parameters and dist-info/egg-info

      path, compares the results with the expected results"""

-     assert expected == run_pythondistdeps(provides_params, requires_params, dist_egg_info_path)

+     expect_failure = "stderr" in expected

+     assert expected == run_pythondistdeps(provides_params, requires_params, dist_egg_info_path, expect_failure)

  

  

  if __name__ == "__main__":

@@ -235,8 +248,10 @@ 

      for provides_params, requires_params, dist_egg_info_path, expected in generate_test_cases(test_data):

          # Print a dot to stderr for each test run to keep user informed about progress

          print(".", end="", flush=True, file=sys.stderr)

+ 

+         expect_failure = "stderr" in test_data[requires_params][provides_params][dist_egg_info_path]

          test_data[requires_params][provides_params][dist_egg_info_path] = \

-             run_pythondistdeps(provides_params, requires_params, dist_egg_info_path)

+             run_pythondistdeps(provides_params, requires_params, dist_egg_info_path, expect_failure)

  

      print(yaml.dump(test_data, indent=4))

  

PR with changes from https://github.com/torsava/rpm/pull/2

The option --require-extras-subpackages has not been enabled yet so we can first build the extras packages before requiring them.

Build succeeded.

Thanks. If you are available, please do bump the release, create a @python group copr and build this there. If not, I'll get back to it tomorrow.

1 new commit added

  • Bump release
2 months ago

Build failed.

Hm, zuul failed on the pythonabi test:

+ rpm -qp --requires /root/rpmbuild/RPMS/noarch/python-misplaced-library-0-0.noarch.rpm
+ grep '^python(abi) = 3.9$'
python(abi) = 3.9
+ exit 1

But nothing has changed except for the release bump. Maybe it's a fluke? [test]

Build failed.

Did you delete my comments? You can do that? :smile:

Yes, I wanted to have the conversation easier to follow and the comments were not needed :)

I can do almost anything :)

Yes, I wanted to have the conversation easier to follow and the comments were not needed :)
I can do almost anything :)

https://giphy.com/gifs/KxsZcxieHS30fxKO4M/html5 :smile:

In order to test this, we need to enable --require-extras-subpackages, no?

I was also thinking we should do:

%{?!_python_no_extras_requires:--require-extras-subpackages}

So the maintainers have an easy opt out.

rebased onto f2d51b3

2 months ago

In order to test this, we need to enable --require-extras-subpackages, no?
I was also thinking we should do:
%{?!_python_no_extras_requires:--require-extras-subpackages}

So the maintainers have an easy opt out.

Sounds good, done.

Build succeeded.

Building all Python packages there. Interesting queries:

$ repoquery --repo=python-extras --whatrequires 'python3.9dist(*\[*\])'
$ repoquery --repo=python-extras --whatrequires 'python2.7dist(*\[*\])'

The repoqueries were wrong. Before I figure out how to use globs properly in them, here is a command that gives proper results:

repoquery --refresh --repo=python-extras --requires -a | grep -E 'python(3\.9|2\.7)dist\(\S+\[\S+\]\)'

The [ and ] symbols needed to be escaped in the original queries. I've edited the comment.

1 new commit added

  • scripts/pythondistdeps: Rework error messages
a month ago

rebased onto 3b1100b

a month ago

5 new commits added

  • scripts/pythondistdeps: Rework error messages
  • Enable --require-extras-subpackages and bump release
  • scripts/pythondistdeps: Add tests for: Implement provides/requires for extras packages
  • scripts/pythondistdeps: Implement provides/requires for extras packages
  • scripts/pythondistdeps: Add parameter --package-name
a month ago

1 new commit added

  • scripts/pythondistdeps: Add tests for: Rework error messages
a month ago

1 new commit added

  • scripts/pythondistdeps: Tests: small tweaks
a month ago

Build succeeded.

Pull-Request has been merged by churchyard

a month ago

Build succeeded.