From 6481c71ec7dba2b64c395ca4f3452064f73a95de Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Thu, 2 Apr 2020 13:26:19 +0200 Subject: [PATCH 1/9] scripts/pythondistdeps: Also provide pythonXdist() with PEP 503 normalized names Upstreaming from Fedora (BZ#1791530) That is, we add new provides that replace dots with a dash. Package that used to provide python3dist(zope.component) and python3.8dist(zope.component) now also provides python3dist(zope-component) and python3.8dist(zope-component). Package that used to provide python3dist(a.-.-.-.a) now provides python3dist(a-a) as well. This is consistent with pip behavior, `pip install zope-component` installs zope.component. Historically, we have always used dist.key (safe_name) from setuptools, but that is a non-standardized convention -- whether or not it replaces dots with dashes is not even documented. We say we use "canonical name" or "normalized name" everywhere, yet we didn't. We really need to follow the standard (PEP 503): https://www.python.org/dev/peps/pep-0503/#normalized-names The proper function here would be packaging.utils.canonicalize_name https://packaging.pypa.io/en/latest/utils/#packaging.utils.canonicalize_name -- we reimplement it here to avoid an external dependency. This is the first required step needed if we want to change our requirements later. If we decide we don't, for whatever reason, this doesn't break anything. --- scripts/pythondistdeps.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/scripts/pythondistdeps.py b/scripts/pythondistdeps.py index 4032857021..f1c96baf78 100755 --- a/scripts/pythondistdeps.py +++ b/scripts/pythondistdeps.py @@ -133,6 +133,12 @@ def convert(name, operator, version_id): return OPERATORS[operator](name, operator, version_id) +def normalize_name(name): + """https://www.python.org/dev/peps/pep-0503/#normalized-names""" + import re + return re.sub(r'[-_.]+', '-', name).lower() + + parser = argparse.ArgumentParser(prog=argv[0]) group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-P', '--provides', action='store_true', help='Print Provides') @@ -199,6 +205,13 @@ def convert(name, operator, version_id): else: warn("Version for {!r} has not been found".format(dist), RuntimeWarning) continue + # This is the PEP 503 normalized name. + # It does also convert dots to dashes, unlike dist.key. + # In the current code, we only add additional provides with this. + # Later, we can start requiring them. + # See https://bugzilla.redhat.com/show_bug.cgi?id=1791530 + normalized_name = normalize_name(dist.project_name) + if args.majorver_provides or args.majorver_only or args.legacy_provides or args.legacy: # Get the Python major version pyver_major = dist.py_version.split('.')[0] @@ -213,10 +226,16 @@ def convert(name, operator, version_id): name = 'python{}dist({})'.format(dist.py_version, dist.key) if name not in py_deps: py_deps[name] = [] + name_ = 'python{}dist({})'.format(dist.py_version, normalized_name) + if name_ not in py_deps: + py_deps[name_] = [] if args.majorver_provides or args.majorver_only: pymajor_name = 'python{}dist({})'.format(pyver_major, dist.key) if pymajor_name not in py_deps: py_deps[pymajor_name] = [] + pymajor_name_ = 'python{}dist({})'.format(pyver_major, normalized_name) + if pymajor_name_ not in py_deps: + py_deps[pymajor_name_] = [] if args.legacy or args.legacy_provides: legacy_name = 'pythonegg({})({})'.format(pyver_major, dist.key) if legacy_name not in py_deps: @@ -227,8 +246,12 @@ def convert(name, operator, version_id): if spec not in py_deps[name]: if not args.legacy: py_deps[name].append(spec) + if name != name_: + py_deps[name_].append(spec) if args.majorver_provides: py_deps[pymajor_name].append(spec) + if pymajor_name != pymajor_name_: + py_deps[pymajor_name_].append(spec) if args.legacy or args.legacy_provides: py_deps[legacy_name].append(spec) if args.requires or (args.recommends and dist.extras): From 80330b3f0306435584009134a00d76c10c54e763 Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Thu, 2 Apr 2020 13:28:09 +0200 Subject: [PATCH 2/9] scripts/pythondistdeps: "Fix" support of environment markers Upstreaming from Fedora --- scripts/pythondistdeps.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/pythondistdeps.py b/scripts/pythondistdeps.py index f1c96baf78..cff25f62d5 100755 --- a/scripts/pythondistdeps.py +++ b/scripts/pythondistdeps.py @@ -205,6 +205,20 @@ def normalize_name(name): else: warn("Version for {!r} has not been found".format(dist), RuntimeWarning) continue + + # pkg_resources use platform.python_version to evaluate if a + # dependency is relevant based on environment markers [1], + # e.g. requirement `argparse;python_version<"2.7"` + # + # Since we're running this script on one Python version while + # possibly evaluating packages for different versions, we mock the + # platform.python_version function. Discussed upstream [2]. + # + # [1] https://www.python.org/dev/peps/pep-0508/#environment-markers + # [2] https://github.com/pypa/setuptools/pull/1275 + import platform + platform.python_version = lambda: dist.py_version + # This is the PEP 503 normalized name. # It does also convert dots to dashes, unlike dist.key. # In the current code, we only add additional provides with this. From 10f69ba1b7e57a2b89f4dbac9793cc7cc87033fe Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Wed, 8 Apr 2020 18:12:09 +0200 Subject: [PATCH 3/9] scripts/pythondistdeps: Notes from an attempted rewrite to importlib.metadata Notes from an attempted rewrite from pkg_resources to importlib.metadata in 2020: 1. While pkg_resources can open a metadata on a specified path (Distribution.from_location()), importlib provides access only to "installed package metadata", i.e. the the dist-info or egg-info directory must be "discoverable", i.e. on the sys.path. - Thankfully only the dist/egg-info directory must exist, the corresponding Python module does not have to be present. - The problems this causes: (a) You have to manipulate the sys.path to add the specific location of the site-packages directory inside the buildroot (b) If you have package "foo" in this newly added directory on sys.path and there is some problem and its dist/egg-info metadata are not found, importlib.metadata continues searching the sys.path and may discover a package with the same name (possibly same version) outside the buildroot. To get around this, you can manipulate the sys.path to remove all other "site-packages" directories. But you have to leave the standard library there, because importlib may import other modules (in my testing: base64, quopri, random, socket, calendar, uu) (c) I have not tested how well it works if you're ispecting metadata of different Python versions than the one you run the script with (especially Python 2 vs Python 3). This might also cause problems with dependency specifiers (i.e. python_version != "3.4") 2. Handling of dependencies (requires) is problematic in importlib.metadata - pkg_resources provides a way to separately list standard requires and a requires for each "extras" category. importlib does not provide this, it only spits out a list of strings, each string in the format: - 'packaging>=14', - 'towncrier>=18.5.0; extra == "docs"', or - 'psutil<6,>=5.6.1; (python_version != "3.4") and extra == "testing" you can either parse these with a regex (fragile) or use the external `packaging` Python module. `packaging`, however, also doesn't have a great support for figuring out extra dependencies, it provides the marker api: - you can use Marker api to evaluate the condition, but not to parse. For parsing you can access the private api Marker._markers: - marker._markers=[[(, , \ )], 'and', (, , \ )] which beyond the problem of being private is also not very useful for parsing due to its structure. - pkg_resources also provides version parsing, which importlib does not and `packaging` needs to be used - importlib is part of the standard library, but packaging and its 2 runtime dependencies (pyparsing and six) are not, and therefore we would go from 1 dependency to 3 3. A few minor issues, more in the next section about equivalents. importlib.metadata.distribution equivalents of pkg_resources.Distribution attributes: - pkg_resources: dist.py_version importlib: # not implemented (but can be guessed from the /usr/lib/pythonXX.YY/ path) - pkg_resources: dist.project_name importlib: dist.metadata['name'] - pkg_resources: dist.key importlib: # not implemented - pkg_resources: dist.version importlib: dist.version - pkg_resources: dist.requires() importlib: dist.requires # but returns strings with almost no parsing done, and also lists extras - pkg_resources: dist.requires(extras=dist.extras) importlib: # not implemented, has to be parsed from dist.requires - pkg_resources: dist.get_entry_map('console_scripts') importlib: [ep for ep in importlib.metadata.entry_points()['console_scripts'] if ep.name == pkg][0] # I have not found a better way to get the console_scripts - pkg_resources: dist.get_entry_map('gui_scripts') importlib: # Presumably same as console_scripts, but untested --- scripts/pythondistdeps.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/pythondistdeps.py b/scripts/pythondistdeps.py index cff25f62d5..b601380d56 100755 --- a/scripts/pythondistdeps.py +++ b/scripts/pythondistdeps.py @@ -11,6 +11,10 @@ # RPM python dependency generator, using .egg-info/.egg-link/.dist-info data # +# Please know: +# - Notes from an attempted rewrite from pkg_resources to importlib.metadata in +# 2020 can be found in the message of the commit that added this line. + from __future__ import print_function import argparse from os.path import basename, dirname, isdir, sep @@ -185,6 +189,9 @@ def normalize_name(name): lower.endswith('.egg-info') or \ lower.endswith('.dist-info'): # This import is very slow, so only do it if needed + # - Notes from an attempted rewrite from pkg_resources to + # importlib.metadata in 2020 can be found in the message of + # the commit that added this line. from pkg_resources import Distribution, FileMetadata, PathMetadata, Requirement, parse_version dist_name = basename(f) if isdir(f): From d959a2ddda3b977ab0cb3445e58ae2ccb48617ca Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Tue, 7 Apr 2020 01:16:34 +0200 Subject: [PATCH 4/9] scripts/pythondistdeps: Sort generated provides/requires So that they can be compared with expected results in tests --- scripts/pythondistdeps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/pythondistdeps.py b/scripts/pythondistdeps.py index b601380d56..336edf55e9 100755 --- a/scripts/pythondistdeps.py +++ b/scripts/pythondistdeps.py @@ -367,7 +367,8 @@ def normalize_name(name): if len(spec_list) == 1: print(spec_list[0]) else: - print('({})'.format(' with '.join(spec_list))) + # Sort spec_list so that the results can be tested easily + print('({})'.format(' with '.join(sorted(spec_list)))) else: # Print out unversioned provides, requires, recommends, conflicts print(name) From 87ec370ed37c6c666d19b51208d2aa54f5d4a879 Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Wed, 27 Nov 2019 18:16:43 +0100 Subject: [PATCH 5/9] scripts/pythondistdeps: Add option to generate major-version provides only for specified Python versions --- scripts/pythondistdeps.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/scripts/pythondistdeps.py b/scripts/pythondistdeps.py index 336edf55e9..d714dcd15c 100755 --- a/scripts/pythondistdeps.py +++ b/scripts/pythondistdeps.py @@ -150,7 +150,11 @@ def normalize_name(name): group.add_argument('-r', '--recommends', action='store_true', help='Print Recommends') group.add_argument('-C', '--conflicts', action='store_true', help='Print Conflicts') group.add_argument('-E', '--extras', action='store_true', help='Print Extras') -parser.add_argument('-M', '--majorver-provides', action='store_true', help='Print extra Provides with Python major version only') +group_majorver = parser.add_mutually_exclusive_group() +group_majorver.add_argument('-M', '--majorver-provides', action='store_true', help='Print extra Provides with Python major version only') +group_majorver.add_argument('--majorver-provides-versions', action='append', + help='Print extra Provides with Python major version only for listed ' + 'Python VERSIONS (appended or comma separated without spaces, e.g. 2.7,3.9)') parser.add_argument('-m', '--majorver-only', action='store_true', help='Print Provides/Requires with Python major version only') parser.add_argument('-L', '--legacy-provides', action='store_true', help='Print extra legacy pythonegg Provides') parser.add_argument('-l', '--legacy', action='store_true', help='Print legacy pythonegg Provides/Requires instead') @@ -160,6 +164,11 @@ def normalize_name(name): py_abi = args.requires py_deps = {} +if args.majorver_provides_versions: + # Go through the arguments (can be specified multiple times), + # and parse individual versions (can be comma-separated) + args.majorver_provides_versions = [v for vstring in args.majorver_provides_versions + for v in vstring.split(",")] for f in (args.files or stdin.readlines()): f = f.strip() @@ -233,7 +242,8 @@ def normalize_name(name): # See https://bugzilla.redhat.com/show_bug.cgi?id=1791530 normalized_name = normalize_name(dist.project_name) - if args.majorver_provides or args.majorver_only or args.legacy_provides or args.legacy: + if args.majorver_provides or args.majorver_provides_versions or \ + args.majorver_only or args.legacy_provides or args.legacy: # Get the Python major version pyver_major = dist.py_version.split('.')[0] if args.provides: @@ -250,7 +260,9 @@ def normalize_name(name): name_ = 'python{}dist({})'.format(dist.py_version, normalized_name) if name_ not in py_deps: py_deps[name_] = [] - if args.majorver_provides or args.majorver_only: + if args.majorver_provides or args.majorver_only or \ + (args.majorver_provides_versions and + dist.py_version in args.majorver_provides_versions): pymajor_name = 'python{}dist({})'.format(pyver_major, dist.key) if pymajor_name not in py_deps: py_deps[pymajor_name] = [] @@ -269,7 +281,9 @@ def normalize_name(name): py_deps[name].append(spec) if name != name_: py_deps[name_].append(spec) - if args.majorver_provides: + if args.majorver_provides or \ + (args.majorver_provides_versions and + dist.py_version in args.majorver_provides_versions): py_deps[pymajor_name].append(spec) if pymajor_name != pymajor_name_: py_deps[pymajor_name_].append(spec) From 610f48b7a9f11465698267e710c32e0014bd3932 Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Tue, 7 Apr 2020 01:10:42 +0200 Subject: [PATCH 6/9] scripts/pythondistdeps: Implement --normalized-name-* options --normalized-names-format FORMAT FORMAT of normalized names can be `pep503` [default] or `legacy-dots` (dots allowed) --normalized-names-provide-both Provede both `pep503` and `legacy-dots` format of normalized names (useful for a transition period) --- scripts/pythondistdeps.py | 86 ++++++++++++++++++++++++++------------- 1 file changed, 58 insertions(+), 28 deletions(-) diff --git a/scripts/pythondistdeps.py b/scripts/pythondistdeps.py index d714dcd15c..f786ab7263 100755 --- a/scripts/pythondistdeps.py +++ b/scripts/pythondistdeps.py @@ -156,11 +156,17 @@ def normalize_name(name): help='Print extra Provides with Python major version only for listed ' 'Python VERSIONS (appended or comma separated without spaces, e.g. 2.7,3.9)') parser.add_argument('-m', '--majorver-only', action='store_true', help='Print Provides/Requires with Python major version only') +parser.add_argument('-n', '--normalized-names-format', action='store', + default="legacy-dots", choices=["pep503", "legacy-dots"], + help='Format of normalized names according to pep503 or legacy format that allows dots [default]') +parser.add_argument('--normalized-names-provide-both', action='store_true', + help='Provide both `pep503` and `legacy-dots` format of normalized names (useful for a transition period)') parser.add_argument('-L', '--legacy-provides', action='store_true', help='Print extra legacy pythonegg Provides') parser.add_argument('-l', '--legacy', action='store_true', help='Print legacy pythonegg Provides/Requires instead') parser.add_argument('files', nargs=argparse.REMAINDER) args = parser.parse_args() + py_abi = args.requires py_deps = {} @@ -170,6 +176,20 @@ def normalize_name(name): args.majorver_provides_versions = [v for vstring in args.majorver_provides_versions for v in vstring.split(",")] +# If normalized_names_require_pep503 is True we require the pep503 +# normalized name, if it is False we provide the legacy normalized name +normalized_names_require_pep503 = args.normalized_names_format == "pep503" + +# If normalized_names_provide_pep503/legacy is True we provide the +# pep503/legacy normalized name, if it is False we don't +normalized_names_provide_pep503 = \ + args.normalized_names_format == "pep503" or args.normalized_names_provide_both +normalized_names_provide_legacy = \ + args.normalized_names_format == "legacy-dots" or args.normalized_names_provide_both + +# At least one type of normalization must be provided +assert normalized_names_provide_pep503 or normalized_names_provide_legacy + for f in (args.files or stdin.readlines()): f = f.strip() lower = f.lower() @@ -237,8 +257,6 @@ def normalize_name(name): # This is the PEP 503 normalized name. # It does also convert dots to dashes, unlike dist.key. - # In the current code, we only add additional provides with this. - # Later, we can start requiring them. # See https://bugzilla.redhat.com/show_bug.cgi?id=1791530 normalized_name = normalize_name(dist.project_name) @@ -254,21 +272,24 @@ def normalize_name(name): py_deps[name] = [] py_deps[name].append(('==', dist.py_version)) if not args.legacy or not args.majorver_only: - name = 'python{}dist({})'.format(dist.py_version, dist.key) - if name not in py_deps: - py_deps[name] = [] - name_ = 'python{}dist({})'.format(dist.py_version, normalized_name) - if name_ not in py_deps: - py_deps[name_] = [] + if normalized_names_provide_legacy: + name = 'python{}dist({})'.format(dist.py_version, dist.key) + if name not in py_deps: + py_deps[name] = [] + if normalized_names_provide_pep503: + name_ = 'python{}dist({})'.format(dist.py_version, normalized_name) + if name_ not in py_deps: + py_deps[name_] = [] if args.majorver_provides or args.majorver_only or \ - (args.majorver_provides_versions and - dist.py_version in args.majorver_provides_versions): - pymajor_name = 'python{}dist({})'.format(pyver_major, dist.key) - if pymajor_name not in py_deps: - py_deps[pymajor_name] = [] - pymajor_name_ = 'python{}dist({})'.format(pyver_major, normalized_name) - if pymajor_name_ not in py_deps: - py_deps[pymajor_name_] = [] + (args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions): + if normalized_names_provide_legacy: + pymajor_name = 'python{}dist({})'.format(pyver_major, dist.key) + if pymajor_name not in py_deps: + py_deps[pymajor_name] = [] + if normalized_names_provide_pep503: + pymajor_name_ = 'python{}dist({})'.format(pyver_major, normalized_name) + if pymajor_name_ not in py_deps: + py_deps[pymajor_name_] = [] if args.legacy or args.legacy_provides: legacy_name = 'pythonegg({})({})'.format(pyver_major, dist.key) if legacy_name not in py_deps: @@ -276,18 +297,21 @@ def normalize_name(name): if dist.version: version = dist.version spec = ('==', version) - if spec not in py_deps[name]: - if not args.legacy: + + if normalized_names_provide_legacy: + if spec not in py_deps[name]: py_deps[name].append(spec) - if name != name_: - py_deps[name_].append(spec) - if args.majorver_provides or \ - (args.majorver_provides_versions and - dist.py_version in args.majorver_provides_versions): - py_deps[pymajor_name].append(spec) - if pymajor_name != pymajor_name_: + if args.majorver_provides or \ + (args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions): + py_deps[pymajor_name].append(spec) + if normalized_names_provide_pep503: + if spec not in py_deps[name_]: + py_deps[name_].append(spec) + if args.majorver_provides or \ + (args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions): py_deps[pymajor_name_].append(spec) - if args.legacy or args.legacy_provides: + if args.legacy or args.legacy_provides: + if spec not in py_deps[legacy_name]: py_deps[legacy_name].append(spec) if args.requires or (args.recommends and dist.extras): name = 'python(abi)' @@ -319,13 +343,18 @@ def normalize_name(name): deps.insert(0, Requirement.parse('setuptools')) # add requires/recommends based on egg/dist metadata for dep in deps: + if normalized_names_require_pep503: + dep_normalized_name = normalize_name(dep.project_name) + else: + dep_normalized_name = dep.key + if args.legacy: name = 'pythonegg({})({})'.format(pyver_major, dep.key) else: if args.majorver_only: - name = 'python{}dist({})'.format(pyver_major, dep.key) + name = 'python{}dist({})'.format(pyver_major, dep_normalized_name) else: - name = 'python{}dist({})'.format(dist.py_version, dep.key) + name = 'python{}dist({})'.format(dist.py_version, dep_normalized_name) for spec in dep.specs: if name not in py_deps: py_deps[name] = [] @@ -370,6 +399,7 @@ def normalize_name(name): spec = ('==', spec[1]) if spec not in py_deps[name]: py_deps[name].append(spec) + names = list(py_deps.keys()) names.sort() for name in names: From df2f04fb517233adc12c189e300e7234d81e4904 Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Mon, 20 Apr 2020 16:37:29 +0200 Subject: [PATCH 7/9] scripts/pythondistdeps: Do anything only when called as a main script Note that the code is completely unchanged except for the indentation under the new if __name__ == "__main__": Note that this change is necessary, but not sufficient to use the RpmVersion class. The init of the RpmVersion class will fail when called from an outside script, because the `parse_version()` function is lazily imported from the code outside the class. However, adding the import of parse_version() to RpmVersion class is not done right now, because while we would import it from `pkg_resources`, other scripts might want to rely instead of the lightweight `packaging` module for the import. Thus I'm leaving this conondrum to be addressed in the future. --- scripts/pythondistdeps.py | 519 +++++++++++++++++++------------------- 1 file changed, 261 insertions(+), 258 deletions(-) diff --git a/scripts/pythondistdeps.py b/scripts/pythondistdeps.py index f786ab7263..b66c98b8fb 100755 --- a/scripts/pythondistdeps.py +++ b/scripts/pythondistdeps.py @@ -143,276 +143,279 @@ def normalize_name(name): return re.sub(r'[-_.]+', '-', name).lower() -parser = argparse.ArgumentParser(prog=argv[0]) -group = parser.add_mutually_exclusive_group(required=True) -group.add_argument('-P', '--provides', action='store_true', help='Print Provides') -group.add_argument('-R', '--requires', action='store_true', help='Print Requires') -group.add_argument('-r', '--recommends', action='store_true', help='Print Recommends') -group.add_argument('-C', '--conflicts', action='store_true', help='Print Conflicts') -group.add_argument('-E', '--extras', action='store_true', help='Print Extras') -group_majorver = parser.add_mutually_exclusive_group() -group_majorver.add_argument('-M', '--majorver-provides', action='store_true', help='Print extra Provides with Python major version only') -group_majorver.add_argument('--majorver-provides-versions', action='append', - help='Print extra Provides with Python major version only for listed ' - 'Python VERSIONS (appended or comma separated without spaces, e.g. 2.7,3.9)') -parser.add_argument('-m', '--majorver-only', action='store_true', help='Print Provides/Requires with Python major version only') -parser.add_argument('-n', '--normalized-names-format', action='store', - default="legacy-dots", choices=["pep503", "legacy-dots"], - help='Format of normalized names according to pep503 or legacy format that allows dots [default]') -parser.add_argument('--normalized-names-provide-both', action='store_true', - help='Provide both `pep503` and `legacy-dots` format of normalized names (useful for a transition period)') -parser.add_argument('-L', '--legacy-provides', action='store_true', help='Print extra legacy pythonegg Provides') -parser.add_argument('-l', '--legacy', action='store_true', help='Print legacy pythonegg Provides/Requires instead') -parser.add_argument('files', nargs=argparse.REMAINDER) -args = parser.parse_args() - - -py_abi = args.requires -py_deps = {} - -if args.majorver_provides_versions: - # Go through the arguments (can be specified multiple times), - # and parse individual versions (can be comma-separated) - args.majorver_provides_versions = [v for vstring in args.majorver_provides_versions - for v in vstring.split(",")] - -# If normalized_names_require_pep503 is True we require the pep503 -# normalized name, if it is False we provide the legacy normalized name -normalized_names_require_pep503 = args.normalized_names_format == "pep503" - -# If normalized_names_provide_pep503/legacy is True we provide the -# pep503/legacy normalized name, if it is False we don't -normalized_names_provide_pep503 = \ - args.normalized_names_format == "pep503" or args.normalized_names_provide_both -normalized_names_provide_legacy = \ - args.normalized_names_format == "legacy-dots" or args.normalized_names_provide_both - -# At least one type of normalization must be provided -assert normalized_names_provide_pep503 or normalized_names_provide_legacy - -for f in (args.files or stdin.readlines()): - f = f.strip() - lower = f.lower() - name = 'python(abi)' - # add dependency based on path, versioned if within versioned python directory - if py_abi and (lower.endswith('.py') or lower.endswith('.pyc') or lower.endswith('.pyo')): - if name not in py_deps: - py_deps[name] = [] - purelib = get_python_lib(standard_lib=0, plat_specific=0).split(version[:3])[0] - platlib = get_python_lib(standard_lib=0, plat_specific=1).split(version[:3])[0] - for lib in (purelib, platlib): - if lib in f: - spec = ('==', f.split(lib)[1].split(sep)[0]) - if spec not in py_deps[name]: - py_deps[name].append(spec) - - # XXX: hack to workaround RPM internal dependency generator not passing directories - lower_dir = dirname(lower) - if lower_dir.endswith('.egg') or \ - lower_dir.endswith('.egg-info') or \ - lower_dir.endswith('.dist-info'): - lower = lower_dir - f = dirname(f) - # Determine provide, requires, conflicts & recommends based on egg/dist metadata - if lower.endswith('.egg') or \ - lower.endswith('.egg-info') or \ - lower.endswith('.dist-info'): - # This import is very slow, so only do it if needed - # - Notes from an attempted rewrite from pkg_resources to - # importlib.metadata in 2020 can be found in the message of - # the commit that added this line. - from pkg_resources import Distribution, FileMetadata, PathMetadata, Requirement, parse_version - dist_name = basename(f) - if isdir(f): - path_item = dirname(f) - metadata = PathMetadata(path_item, f) - else: - path_item = f - metadata = FileMetadata(f) - dist = Distribution.from_location(path_item, dist_name, metadata) - # Check if py_version is defined in the metadata file/directory name - if not dist.py_version: - # Try to parse the Python version from the path the metadata - # resides at (e.g. /usr/lib/pythonX.Y/site-packages/...) - import re - res = re.search(r"/python(?P\d+\.\d+)/", path_item) - if res: - dist.py_version = res.group('pyver') - else: - warn("Version for {!r} has not been found".format(dist), RuntimeWarning) - continue - - # pkg_resources use platform.python_version to evaluate if a - # dependency is relevant based on environment markers [1], - # e.g. requirement `argparse;python_version<"2.7"` - # - # Since we're running this script on one Python version while - # possibly evaluating packages for different versions, we mock the - # platform.python_version function. Discussed upstream [2]. - # - # [1] https://www.python.org/dev/peps/pep-0508/#environment-markers - # [2] https://github.com/pypa/setuptools/pull/1275 - import platform - platform.python_version = lambda: dist.py_version - - # This is the PEP 503 normalized name. - # It does also convert dots to dashes, unlike dist.key. - # See https://bugzilla.redhat.com/show_bug.cgi?id=1791530 - normalized_name = normalize_name(dist.project_name) - - if args.majorver_provides or args.majorver_provides_versions or \ - args.majorver_only or args.legacy_provides or args.legacy: - # Get the Python major version - pyver_major = dist.py_version.split('.')[0] - if args.provides: - # If egg/dist metadata says package name is python, we provide python(abi) - if dist.key == 'python': - name = 'python(abi)' - if name not in py_deps: - py_deps[name] = [] - py_deps[name].append(('==', dist.py_version)) - if not args.legacy or not args.majorver_only: - if normalized_names_provide_legacy: - name = 'python{}dist({})'.format(dist.py_version, dist.key) - if name not in py_deps: - py_deps[name] = [] - if normalized_names_provide_pep503: - name_ = 'python{}dist({})'.format(dist.py_version, normalized_name) - if name_ not in py_deps: - py_deps[name_] = [] - if args.majorver_provides or args.majorver_only or \ - (args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions): - if normalized_names_provide_legacy: - pymajor_name = 'python{}dist({})'.format(pyver_major, dist.key) - if pymajor_name not in py_deps: - py_deps[pymajor_name] = [] - if normalized_names_provide_pep503: - pymajor_name_ = 'python{}dist({})'.format(pyver_major, normalized_name) - if pymajor_name_ not in py_deps: - py_deps[pymajor_name_] = [] - if args.legacy or args.legacy_provides: - legacy_name = 'pythonegg({})({})'.format(pyver_major, dist.key) - if legacy_name not in py_deps: - py_deps[legacy_name] = [] - if dist.version: - version = dist.version - spec = ('==', version) - - if normalized_names_provide_legacy: +if __name__ == "__main__": + """To allow this script to be importable (and its classes/functions + reused), actions are performed only when run as a main script.""" + + parser = argparse.ArgumentParser(prog=argv[0]) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('-P', '--provides', action='store_true', help='Print Provides') + group.add_argument('-R', '--requires', action='store_true', help='Print Requires') + group.add_argument('-r', '--recommends', action='store_true', help='Print Recommends') + group.add_argument('-C', '--conflicts', action='store_true', help='Print Conflicts') + group.add_argument('-E', '--extras', action='store_true', help='Print Extras') + group_majorver = parser.add_mutually_exclusive_group() + group_majorver.add_argument('-M', '--majorver-provides', action='store_true', help='Print extra Provides with Python major version only') + group_majorver.add_argument('--majorver-provides-versions', action='append', + help='Print extra Provides with Python major version only for listed ' + 'Python VERSIONS (appended or comma separated without spaces, e.g. 2.7,3.9)') + parser.add_argument('-m', '--majorver-only', action='store_true', help='Print Provides/Requires with Python major version only') + parser.add_argument('-n', '--normalized-names-format', action='store', + default="legacy-dots", choices=["pep503", "legacy-dots"], + help='Format of normalized names according to pep503 or legacy format that allows dots [default]') + parser.add_argument('--normalized-names-provide-both', action='store_true', + help='Provide both `pep503` and `legacy-dots` format of normalized names (useful for a transition period)') + parser.add_argument('-L', '--legacy-provides', action='store_true', help='Print extra legacy pythonegg Provides') + parser.add_argument('-l', '--legacy', action='store_true', help='Print legacy pythonegg Provides/Requires instead') + parser.add_argument('files', nargs=argparse.REMAINDER) + args = parser.parse_args() + + py_abi = args.requires + py_deps = {} + + if args.majorver_provides_versions: + # Go through the arguments (can be specified multiple times), + # and parse individual versions (can be comma-separated) + args.majorver_provides_versions = [v for vstring in args.majorver_provides_versions + for v in vstring.split(",")] + + # If normalized_names_require_pep503 is True we require the pep503 + # normalized name, if it is False we provide the legacy normalized name + normalized_names_require_pep503 = args.normalized_names_format == "pep503" + + # If normalized_names_provide_pep503/legacy is True we provide the + # pep503/legacy normalized name, if it is False we don't + normalized_names_provide_pep503 = \ + args.normalized_names_format == "pep503" or args.normalized_names_provide_both + normalized_names_provide_legacy = \ + args.normalized_names_format == "legacy-dots" or args.normalized_names_provide_both + + # At least one type of normalization must be provided + assert normalized_names_provide_pep503 or normalized_names_provide_legacy + + for f in (args.files or stdin.readlines()): + f = f.strip() + lower = f.lower() + name = 'python(abi)' + # add dependency based on path, versioned if within versioned python directory + if py_abi and (lower.endswith('.py') or lower.endswith('.pyc') or lower.endswith('.pyo')): + if name not in py_deps: + py_deps[name] = [] + purelib = get_python_lib(standard_lib=0, plat_specific=0).split(version[:3])[0] + platlib = get_python_lib(standard_lib=0, plat_specific=1).split(version[:3])[0] + for lib in (purelib, platlib): + if lib in f: + spec = ('==', f.split(lib)[1].split(sep)[0]) if spec not in py_deps[name]: py_deps[name].append(spec) - if args.majorver_provides or \ - (args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions): - py_deps[pymajor_name].append(spec) - if normalized_names_provide_pep503: - if spec not in py_deps[name_]: - py_deps[name_].append(spec) - if args.majorver_provides or \ - (args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions): - py_deps[pymajor_name_].append(spec) - if args.legacy or args.legacy_provides: - if spec not in py_deps[legacy_name]: - py_deps[legacy_name].append(spec) - if args.requires or (args.recommends and dist.extras): - name = 'python(abi)' - # If egg/dist metadata says package name is python, we don't add dependency on python(abi) - if dist.key == 'python': - py_abi = False - if name in py_deps: - py_deps.pop(name) - elif py_abi and dist.py_version: - if name not in py_deps: - py_deps[name] = [] - spec = ('==', dist.py_version) - if spec not in py_deps[name]: - py_deps[name].append(spec) - deps = dist.requires() - if args.recommends: - depsextras = dist.requires(extras=dist.extras) - if not args.requires: - for dep in reversed(depsextras): - if dep in deps: - depsextras.remove(dep) - deps = depsextras - # console_scripts/gui_scripts entry points need pkg_resources from setuptools - if ((dist.get_entry_map('console_scripts') or - dist.get_entry_map('gui_scripts')) and - (lower.endswith('.egg') or - lower.endswith('.egg-info'))): - # stick them first so any more specific requirement overrides it - deps.insert(0, Requirement.parse('setuptools')) - # add requires/recommends based on egg/dist metadata - for dep in deps: - if normalized_names_require_pep503: - dep_normalized_name = normalize_name(dep.project_name) - else: - dep_normalized_name = dep.key - if args.legacy: - name = 'pythonegg({})({})'.format(pyver_major, dep.key) + # XXX: hack to workaround RPM internal dependency generator not passing directories + lower_dir = dirname(lower) + if lower_dir.endswith('.egg') or \ + lower_dir.endswith('.egg-info') or \ + lower_dir.endswith('.dist-info'): + lower = lower_dir + f = dirname(f) + # Determine provide, requires, conflicts & recommends based on egg/dist metadata + if lower.endswith('.egg') or \ + lower.endswith('.egg-info') or \ + lower.endswith('.dist-info'): + # This import is very slow, so only do it if needed + # - Notes from an attempted rewrite from pkg_resources to + # importlib.metadata in 2020 can be found in the message of + # the commit that added this line. + from pkg_resources import Distribution, FileMetadata, PathMetadata, Requirement, parse_version + dist_name = basename(f) + if isdir(f): + path_item = dirname(f) + metadata = PathMetadata(path_item, f) + else: + path_item = f + metadata = FileMetadata(f) + dist = Distribution.from_location(path_item, dist_name, metadata) + # Check if py_version is defined in the metadata file/directory name + if not dist.py_version: + # Try to parse the Python version from the path the metadata + # resides at (e.g. /usr/lib/pythonX.Y/site-packages/...) + import re + res = re.search(r"/python(?P\d+\.\d+)/", path_item) + if res: + dist.py_version = res.group('pyver') else: - if args.majorver_only: - name = 'python{}dist({})'.format(pyver_major, dep_normalized_name) - else: - name = 'python{}dist({})'.format(dist.py_version, dep_normalized_name) - for spec in dep.specs: + warn("Version for {!r} has not been found".format(dist), RuntimeWarning) + continue + + # pkg_resources use platform.python_version to evaluate if a + # dependency is relevant based on environment markers [1], + # e.g. requirement `argparse;python_version<"2.7"` + # + # Since we're running this script on one Python version while + # possibly evaluating packages for different versions, we mock the + # platform.python_version function. Discussed upstream [2]. + # + # [1] https://www.python.org/dev/peps/pep-0508/#environment-markers + # [2] https://github.com/pypa/setuptools/pull/1275 + import platform + platform.python_version = lambda: dist.py_version + + # This is the PEP 503 normalized name. + # It does also convert dots to dashes, unlike dist.key. + # See https://bugzilla.redhat.com/show_bug.cgi?id=1791530 + normalized_name = normalize_name(dist.project_name) + + if args.majorver_provides or args.majorver_provides_versions or \ + args.majorver_only or args.legacy_provides or args.legacy: + # Get the Python major version + pyver_major = dist.py_version.split('.')[0] + if args.provides: + # If egg/dist metadata says package name is python, we provide python(abi) + if dist.key == 'python': + name = 'python(abi)' if name not in py_deps: py_deps[name] = [] + py_deps[name].append(('==', dist.py_version)) + if not args.legacy or not args.majorver_only: + if normalized_names_provide_legacy: + name = 'python{}dist({})'.format(dist.py_version, dist.key) + if name not in py_deps: + py_deps[name] = [] + if normalized_names_provide_pep503: + name_ = 'python{}dist({})'.format(dist.py_version, normalized_name) + if name_ not in py_deps: + py_deps[name_] = [] + if args.majorver_provides or args.majorver_only or \ + (args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions): + if normalized_names_provide_legacy: + pymajor_name = 'python{}dist({})'.format(pyver_major, dist.key) + if pymajor_name not in py_deps: + py_deps[pymajor_name] = [] + if normalized_names_provide_pep503: + pymajor_name_ = 'python{}dist({})'.format(pyver_major, normalized_name) + if pymajor_name_ not in py_deps: + py_deps[pymajor_name_] = [] + if args.legacy or args.legacy_provides: + legacy_name = 'pythonegg({})({})'.format(pyver_major, dist.key) + if legacy_name not in py_deps: + py_deps[legacy_name] = [] + if dist.version: + version = dist.version + spec = ('==', version) + + if normalized_names_provide_legacy: + if spec not in py_deps[name]: + py_deps[name].append(spec) + if args.majorver_provides or \ + (args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions): + py_deps[pymajor_name].append(spec) + if normalized_names_provide_pep503: + if spec not in py_deps[name_]: + py_deps[name_].append(spec) + if args.majorver_provides or \ + (args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions): + py_deps[pymajor_name_].append(spec) + if args.legacy or args.legacy_provides: + if spec not in py_deps[legacy_name]: + py_deps[legacy_name].append(spec) + if args.requires or (args.recommends and dist.extras): + name = 'python(abi)' + # If egg/dist metadata says package name is python, we don't add dependency on python(abi) + if dist.key == 'python': + py_abi = False + if name in py_deps: + py_deps.pop(name) + elif py_abi and dist.py_version: + if name not in py_deps: + py_deps[name] = [] + spec = ('==', dist.py_version) if spec not in py_deps[name]: py_deps[name].append(spec) - if not dep.specs: - py_deps[name] = [] - # Unused, for automatic sub-package generation based on 'extras' from egg/dist metadata - # TODO: implement in rpm later, or...? - if args.extras: - deps = dist.requires() - extras = dist.extras - print(extras) - for extra in extras: - print('%%package\textras-{}'.format(extra)) - print('Summary:\t{} extra for {} python package'.format(extra, dist.key)) - print('Group:\t\tDevelopment/Python') - depsextras = dist.requires(extras=[extra]) - for dep in reversed(depsextras): - if dep in deps: - depsextras.remove(dep) - deps = depsextras + deps = dist.requires() + if args.recommends: + depsextras = dist.requires(extras=dist.extras) + if not args.requires: + for dep in reversed(depsextras): + if dep in deps: + depsextras.remove(dep) + deps = depsextras + # console_scripts/gui_scripts entry points need pkg_resources from setuptools + if ((dist.get_entry_map('console_scripts') or + dist.get_entry_map('gui_scripts')) and + (lower.endswith('.egg') or + lower.endswith('.egg-info'))): + # stick them first so any more specific requirement overrides it + deps.insert(0, Requirement.parse('setuptools')) + # add requires/recommends based on egg/dist metadata for dep in deps: - for spec in dep.specs: - if spec[0] == '!=': - print('Conflicts:\t{} {} {}'.format(dep.key, '==', spec[1])) + if normalized_names_require_pep503: + dep_normalized_name = normalize_name(dep.project_name) + else: + dep_normalized_name = dep.key + + if args.legacy: + name = 'pythonegg({})({})'.format(pyver_major, dep.key) + else: + if args.majorver_only: + name = 'python{}dist({})'.format(pyver_major, dep_normalized_name) else: - print('Requires:\t{} {} {}'.format(dep.key, spec[0], spec[1])) - print('%%description\t{}'.format(extra)) - print('{} extra for {} python package'.format(extra, dist.key)) - print('%%files\t\textras-{}\n'.format(extra)) - if args.conflicts: - # Should we really add conflicts for extras? - # Creating a meta package per extra with recommends on, which has - # the requires/conflicts in stead might be a better solution... - for dep in dist.requires(extras=dist.extras): - name = dep.key - for spec in dep.specs: - if spec[0] == '!=': + name = 'python{}dist({})'.format(dist.py_version, dep_normalized_name) + for spec in dep.specs: if name not in py_deps: py_deps[name] = [] - spec = ('==', spec[1]) if spec not in py_deps[name]: py_deps[name].append(spec) - -names = list(py_deps.keys()) -names.sort() -for name in names: - if py_deps[name]: - # Print out versioned provides, requires, recommends, conflicts - spec_list = [] - for spec in py_deps[name]: - spec_list.append(convert(name, spec[0], spec[1])) - if len(spec_list) == 1: - print(spec_list[0]) + if not dep.specs: + py_deps[name] = [] + # Unused, for automatic sub-package generation based on 'extras' from egg/dist metadata + # TODO: implement in rpm later, or...? + if args.extras: + deps = dist.requires() + extras = dist.extras + print(extras) + for extra in extras: + print('%%package\textras-{}'.format(extra)) + print('Summary:\t{} extra for {} python package'.format(extra, dist.key)) + print('Group:\t\tDevelopment/Python') + depsextras = dist.requires(extras=[extra]) + for dep in reversed(depsextras): + if dep in deps: + depsextras.remove(dep) + deps = depsextras + for dep in deps: + for spec in dep.specs: + if spec[0] == '!=': + print('Conflicts:\t{} {} {}'.format(dep.key, '==', spec[1])) + else: + print('Requires:\t{} {} {}'.format(dep.key, spec[0], spec[1])) + print('%%description\t{}'.format(extra)) + print('{} extra for {} python package'.format(extra, dist.key)) + print('%%files\t\textras-{}\n'.format(extra)) + if args.conflicts: + # Should we really add conflicts for extras? + # Creating a meta package per extra with recommends on, which has + # the requires/conflicts in stead might be a better solution... + for dep in dist.requires(extras=dist.extras): + name = dep.key + for spec in dep.specs: + if spec[0] == '!=': + if name not in py_deps: + py_deps[name] = [] + spec = ('==', spec[1]) + if spec not in py_deps[name]: + py_deps[name].append(spec) + + names = list(py_deps.keys()) + names.sort() + for name in names: + if py_deps[name]: + # Print out versioned provides, requires, recommends, conflicts + spec_list = [] + for spec in py_deps[name]: + spec_list.append(convert(name, spec[0], spec[1])) + if len(spec_list) == 1: + print(spec_list[0]) + else: + # Sort spec_list so that the results can be tested easily + print('({})'.format(' with '.join(sorted(spec_list)))) else: - # Sort spec_list so that the results can be tested easily - print('({})'.format(' with '.join(sorted(spec_list)))) - else: - # Print out unversioned provides, requires, recommends, conflicts - print(name) + # Print out unversioned provides, requires, recommends, conflicts + print(name) From 41af75a56b1f6dbfb29ded2e6ba13113271a9066 Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Mon, 20 Apr 2020 16:39:28 +0200 Subject: [PATCH 8/9] scripts/pythondistdeps: Version handling exception with better information --- scripts/pythondistdeps.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/pythondistdeps.py b/scripts/pythondistdeps.py index b66c98b8fb..897c5aae56 100755 --- a/scripts/pythondistdeps.py +++ b/scripts/pythondistdeps.py @@ -134,7 +134,11 @@ def convert_ordered(name, operator, version_id): def convert(name, operator, version_id): - return OPERATORS[operator](name, operator, version_id) + try: + return OPERATORS[operator](name, operator, version_id) + except Exception as exc: + raise RuntimeError("Cannot process Python package version `{}` for name `{}`". + format(version_id, name)) from exc def normalize_name(name): From 592a6d5980010c63eb76c31ddd8954fba9cbaa92 Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Thu, 23 Apr 2020 20:41:13 +0200 Subject: [PATCH 9/9] scripts/pythondistdeps: Modify handling of dev versions --- scripts/pythondistdeps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pythondistdeps.py b/scripts/pythondistdeps.py index 897c5aae56..102fcb0e80 100755 --- a/scripts/pythondistdeps.py +++ b/scripts/pythondistdeps.py @@ -55,7 +55,7 @@ def __str__(self): if self.pre: rpm_suffix = '~{}'.format(''.join(str(x) for x in self.pre)) elif self.dev: - rpm_suffix = '~{}'.format(''.join(str(x) for x in self.dev)) + rpm_suffix = '~~{}'.format(''.join(str(x) for x in self.dev)) elif self.post: rpm_suffix = '^post{}'.format(self.post[1]) else: