Synchronise master with upstream

global-requirements.txt

@@ -24,8 +24,11 @@ bcrypt==4.0.1  # Apache-2.0
 beautifulsoup4  # MIT
 betamax  # Apache-2.0
 boto  # MIT
-boto3  # Apache-2.0
-botocore  # Apache-2.0
+# Capped until https://bugs.launchpad.net/glance/+bug/2121144 is resolved
+boto3<1.36  # Apache-2.0
+botocore<1.36  # Apache-2.0
+# indirect from boto3/botocore
+s3transfer<0.11  # Apache-2.0
 cachetools  # MIT License
 cassandra-driver!=3.6.0  # Apache-2.0
 cffi  # MIT
@@ -132,7 +135,7 @@ PyMySQL  # MIT License
 pyOpenSSL  # Apache-2.0
 pyparsing  # MIT
 pyroute2!=0.5.4,!=0.5.5,!=0.7.1,!=0.9.1,!=0.9.2,!=0.9.3,!=0.9.4;sys_platform!='win32'  # Apache-2.0 (+ dual licensed GPL2)
-pysaml2!=4.0.3,!=4.0.4,!=4.0.5,!=4.0.5rc1,!=4.1.0,!=4.2.0,!=4.3.0,!=4.4.0,!=4.6.0  # Apache-2.0
+pysaml2!=4.0.3,!=4.0.4,!=4.0.5rc1,!=4.0.5,!=4.1.0,!=4.2.0,!=4.3.0,!=4.4.0,!=4.6.0  # Apache-2.0
 pysnmp-lextudio  # BSD
 pystache  # MIT
 # Only required for sasl/binary protocol

openstack_requirements/check.py

@@ -18,7 +18,6 @@
 import re
 
 from packaging import markers
-from packaging import specifiers
 
 from openstack_requirements import project
 from openstack_requirements import requirement
@@ -303,130 +302,3 @@ def validate(
             )
 
     return failed
-
-
-def _find_constraint(req, constraints):
-    """Return the constraint matching the markers for req.
-
-    Given a requirement, find the constraint with matching markers.
-    If none match, find a constraint without any markers at all.
-    Otherwise return None.
-    """
-    if req.markers:
-        req_markers = markers.Marker(req.markers)
-        for constraint_setting, _ in constraints:
-            if constraint_setting.markers == req.markers:
-                return constraint_setting
-            if not constraint_setting.markers:
-                # There is no point in performing the complex
-                # comparison for a constraint that has no markers, so
-                # we skip it here. If we find no closer match then the
-                # loop at the end of the function will look for a
-                # constraint without a marker and use that.
-                continue
-            # NOTE(dhellmann): This is a very naive attempt to check
-            # marker compatibility that relies on internal
-            # implementation details of the packaging library.  The
-            # best way to ensure the constraint and requirements match
-            # is to use the same marker string in the corresponding
-            # lines.
-            c_markers = markers.Marker(constraint_setting.markers)
-            env = {
-                str(var): str(val)
-                for var, op, val in c_markers._markers  # WARNING: internals
-            }
-            if req_markers.evaluate(env):
-                return constraint_setting
-    # Try looking for a constraint without any markers.
-    for constraint_setting, _ in constraints:
-        if not constraint_setting.markers:
-            return constraint_setting
-    return None
-
-
-def validate_lower_constraints(req_list, constraints, denylist):
-    """Return True if there is an error.
-
-    :param reqs: RequirementsList for the head of the branch
-    :param constraints: Parsed lower-constraints.txt or None
-
-    """
-    if constraints is None:
-        return False
-
-    parsed_constraints = requirement.parse(constraints)
-
-    failed = False
-
-    for fname, freqs in req_list.reqs_by_file.items():
-
-        if fname == 'doc/requirements.txt':
-            # Skip things that are not needed for unit or functional
-            # tests.
-            continue
-
-        print("Validating lower constraints of {}".format(fname))
-
-        for name, reqs in freqs.items():
-
-            if name in denylist:
-                continue
-
-            if name not in parsed_constraints:
-                print('ERROR: Package {!r} is used in {} '
-                      'but not in lower-constraints.txt'.format(
-                          name, fname))
-                failed = True
-                continue
-
-            for req in reqs:
-                spec = specifiers.SpecifierSet(req.specifiers)
-                # FIXME(dhellmann): This will only find constraints
-                # where the markers match the requirements list
-                # exactly, so we can't do things like use different
-                # constrained versions for different versions of
-                # python 3 if the requirement range is expressed as
-                # python_version>3.0. We can support different
-                # versions if there is a different requirement
-                # specification for each version of python. I don't
-                # really know how smart we want this to be, because
-                # I'm not sure we want to support extremely
-                # complicated dependency sets.
-                constraint_setting = _find_constraint(
-                    req,
-                    parsed_constraints[name],
-                )
-                if not constraint_setting:
-                    print('ERROR: Unable to find constraint for {} '
-                          'matching {!r} or without any markers.'.format(
-                              name, req.markers))
-                    failed = True
-                    continue
-
-                version = constraint_setting.specifiers.lstrip('=')
-
-                if not spec.contains(version):
-                    print('ERROR: Package {!r} is constrained to {} '
-                          'which is incompatible with the settings {} '
-                          'from {}.'.format(
-                              name, version, req, fname))
-                    failed = True
-
-                min = [
-                    s
-                    for s in req.specifiers.split(',')
-                    if '>' in s
-                ]
-                if not min:
-                    # No minimum specified. Ignore this and let some
-                    # other validation trap the error.
-                    continue
-
-                expected = min[0].lstrip('>=')
-                if version != expected:
-                    print('ERROR: Package {!r} is constrained to {} '
-                          'which does not match '
-                          'the minimum version specifier {} in {}'.format(
-                              name, version, expected, fname))
-                    failed = True
-    return failed

openstack_requirements/cmds/generate.py

@@ -27,13 +27,13 @@
 
 
 SECURITY_WARNING = [
-    "# WARNING: OpenStack makes no security guarantees about third-party",
-    "# dependencies listed here, and does not keep track of any",
-    "# vulnerabilities they contain. Versions of these dependencies are",
-    "# frozen at each coordinated release in order to stabilize upstream",
-    "# testing, and can contain known vulnerabilities. Consumers are",
-    "# *STRONGLY* encouraged to rely on curated distributions of OpenStack",
-    "# or manage security patching of dependencies themselves.",
+    "# WARNING: OpenStack makes no security guarantees about third-party\n",
+    "# dependencies listed here, and does not keep track of any\n",
+    "# vulnerabilities they contain. Versions of these dependencies are\n",
+    "# frozen at each coordinated release in order to stabilize upstream\n",
+    "# testing, and can contain known vulnerabilities. Consumers are\n",
+    "# *STRONGLY* encouraged to rely on curated distributions of OpenStack\n",
+    "# or manage security patching of dependencies themselves.\n",
     ]
 
 

openstack_requirements/project.py

@@ -71,8 +71,4 @@ def read(root):
         target_files.append('test-requirements-py%s.txt' % py_version)
     for target_file in target_files:
         _safe_read(result, target_file, output=requirements)
-    # Read lower-constraints.txt and ensure the key is always present
-    # in case the file is missing.
-    result['lower-constraints.txt'] = None
-    _safe_read(result, 'lower-constraints.txt')
     return result

openstack_requirements/requirement.py

@@ -15,8 +15,8 @@
 # This module has no IO at all, and none should be added.
 
 import collections
-import distutils.version
 import packaging.specifiers
+import packaging.version
 import pkg_resources
 import re
 
@@ -26,7 +26,7 @@ def key_specifier(a):
               '===': 1, '==': 1, '~=': 1, '!=': 1,
               '<': 2, '<=': 2}
     a = a._spec
-    return (weight[a[0]], distutils.version.LooseVersion(a[1]))
+    return (weight[a[0]], packaging.version.parse(a[1]))
 
 
 class Requirement(collections.namedtuple('Requirement',
@@ -179,6 +179,11 @@ def _pass_through(req_line, permit_urls=False):
 def to_reqs(content, permit_urls=False):
     for content_line in content.splitlines(True):
         req_line = content_line.strip()
+
+        # skip comments, blank lines
+        if req_line.startswith('#') or not req_line:
+            continue
+
         if _pass_through(req_line, permit_urls=permit_urls):
             yield None, content_line
         else: