diff --git a/global-requirements.txt b/global-requirements.txt index 5c7080c6b..0619639df 100644 --- a/global-requirements.txt +++ b/global-requirements.txt @@ -24,8 +24,11 @@ bcrypt==4.0.1 # Apache-2.0 beautifulsoup4 # MIT betamax # Apache-2.0 boto # MIT -boto3 # Apache-2.0 -botocore # Apache-2.0 +# Capped until https://bugs.launchpad.net/glance/+bug/2121144 is resolved +boto3<1.36 # Apache-2.0 +botocore<1.36 # Apache-2.0 +# indirect from boto3/botocore +s3transfer<0.11 # Apache-2.0 cachetools # MIT License cassandra-driver!=3.6.0 # Apache-2.0 cffi # MIT @@ -132,7 +135,7 @@ PyMySQL # MIT License pyOpenSSL # Apache-2.0 pyparsing # MIT pyroute2!=0.5.4,!=0.5.5,!=0.7.1,!=0.9.1,!=0.9.2,!=0.9.3,!=0.9.4;sys_platform!='win32' # Apache-2.0 (+ dual licensed GPL2) -pysaml2!=4.0.3,!=4.0.4,!=4.0.5,!=4.0.5rc1,!=4.1.0,!=4.2.0,!=4.3.0,!=4.4.0,!=4.6.0 # Apache-2.0 +pysaml2!=4.0.3,!=4.0.4,!=4.0.5rc1,!=4.0.5,!=4.1.0,!=4.2.0,!=4.3.0,!=4.4.0,!=4.6.0 # Apache-2.0 pysnmp-lextudio # BSD pystache # MIT # Only required for sasl/binary protocol diff --git a/openstack_requirements/check.py b/openstack_requirements/check.py index 872301a36..e14547485 100644 --- a/openstack_requirements/check.py +++ b/openstack_requirements/check.py @@ -18,7 +18,6 @@ import re from packaging import markers -from packaging import specifiers from openstack_requirements import project from openstack_requirements import requirement @@ -303,130 +302,3 @@ def validate( ) return failed - - -def _find_constraint(req, constraints): - """Return the constraint matching the markers for req. - - Given a requirement, find the constraint with matching markers. - If none match, find a constraint without any markers at all. - Otherwise return None. - """ - if req.markers: - req_markers = markers.Marker(req.markers) - for constraint_setting, _ in constraints: - if constraint_setting.markers == req.markers: - return constraint_setting - if not constraint_setting.markers: - # There is no point in performing the complex - # comparison for a constraint that has no markers, so - # we skip it here. If we find no closer match then the - # loop at the end of the function will look for a - # constraint without a marker and use that. - continue - # NOTE(dhellmann): This is a very naive attempt to check - # marker compatibility that relies on internal - # implementation details of the packaging library. The - # best way to ensure the constraint and requirements match - # is to use the same marker string in the corresponding - # lines. - c_markers = markers.Marker(constraint_setting.markers) - env = { - str(var): str(val) - for var, op, val in c_markers._markers # WARNING: internals - } - if req_markers.evaluate(env): - return constraint_setting - # Try looking for a constraint without any markers. - for constraint_setting, _ in constraints: - if not constraint_setting.markers: - return constraint_setting - return None - - -def validate_lower_constraints(req_list, constraints, denylist): - """Return True if there is an error. - - :param reqs: RequirementsList for the head of the branch - :param constraints: Parsed lower-constraints.txt or None - - """ - if constraints is None: - return False - - parsed_constraints = requirement.parse(constraints) - - failed = False - - for fname, freqs in req_list.reqs_by_file.items(): - - if fname == 'doc/requirements.txt': - # Skip things that are not needed for unit or functional - # tests. - continue - - print("Validating lower constraints of {}".format(fname)) - - for name, reqs in freqs.items(): - - if name in denylist: - continue - - if name not in parsed_constraints: - print('ERROR: Package {!r} is used in {} ' - 'but not in lower-constraints.txt'.format( - name, fname)) - failed = True - continue - - for req in reqs: - spec = specifiers.SpecifierSet(req.specifiers) - # FIXME(dhellmann): This will only find constraints - # where the markers match the requirements list - # exactly, so we can't do things like use different - # constrained versions for different versions of - # python 3 if the requirement range is expressed as - # python_version>3.0. We can support different - # versions if there is a different requirement - # specification for each version of python. I don't - # really know how smart we want this to be, because - # I'm not sure we want to support extremely - # complicated dependency sets. - constraint_setting = _find_constraint( - req, - parsed_constraints[name], - ) - if not constraint_setting: - print('ERROR: Unable to find constraint for {} ' - 'matching {!r} or without any markers.'.format( - name, req.markers)) - failed = True - continue - - version = constraint_setting.specifiers.lstrip('=') - - if not spec.contains(version): - print('ERROR: Package {!r} is constrained to {} ' - 'which is incompatible with the settings {} ' - 'from {}.'.format( - name, version, req, fname)) - failed = True - - min = [ - s - for s in req.specifiers.split(',') - if '>' in s - ] - if not min: - # No minimum specified. Ignore this and let some - # other validation trap the error. - continue - - expected = min[0].lstrip('>=') - if version != expected: - print('ERROR: Package {!r} is constrained to {} ' - 'which does not match ' - 'the minimum version specifier {} in {}'.format( - name, version, expected, fname)) - failed = True - return failed diff --git a/openstack_requirements/cmds/check_conflicts.py b/openstack_requirements/cmds/check_conflicts.py deleted file mode 100644 index fbea72b6f..000000000 --- a/openstack_requirements/cmds/check_conflicts.py +++ /dev/null @@ -1,75 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Apply validation rules to the various requirements lists. - -""" - -import argparse -import sys -import traceback - -import pkg_resources - -from openstack_requirements.utils import read_requirements_file - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - 'upper_constraints', - default='upper-constraints.txt', - help='path to the upper-constraints.txt file') - parser.add_argument( - 'uc_xfails', - default='upper-constraints-xfails.txt', - help='Path to the upper-constraints-xfails.txt file', - ) - args = parser.parse_args() - - error_count = 0 - - print('\nChecking %s' % args.upper_constraints) - upper_constraints = read_requirements_file(args.upper_constraints) - xfails = read_requirements_file(args.uc_xfails) - for name, spec_list in upper_constraints.items(): - try: - if name: - pyver = "python_version=='%s.%s'" % (sys.version_info[0], - sys.version_info[1]) - for req, original_line in spec_list: - if req.markers in ["", pyver]: - pkg_resources.require(name) - except pkg_resources.ContextualVersionConflict as e: - if e.dist.key in xfails: - xfail_requirement = xfails[e.dist.key][0][0] - xfail_denylists = set(xfail_requirement.markers.split(',')) - conflict = e.dist.as_requirement() - conflict_specifiers = ''.join(conflict.specs[0]) - conflict_name = conflict.name.lower() - - if (e.required_by.issubset(xfail_denylists) and - xfail_requirement.package == conflict_name and - conflict_specifiers == xfail_requirement.specifiers): - - print('XFAIL while checking conflicts ' - 'for %s: %s conflicts with %s' % - (name, e.dist, str(e.req))) - continue - - print('Checking conflicts for %s:\n' - 'ContextualVersionConflict: %s' % (name, str(e))) - - traceback.print_exc(file=sys.stdout) - error_count += 1 - - return 1 if error_count else 0 diff --git a/openstack_requirements/cmds/generate.py b/openstack_requirements/cmds/generate.py index 43090832b..0ad2e6280 100644 --- a/openstack_requirements/cmds/generate.py +++ b/openstack_requirements/cmds/generate.py @@ -27,13 +27,13 @@ SECURITY_WARNING = [ - "# WARNING: OpenStack makes no security guarantees about third-party", - "# dependencies listed here, and does not keep track of any", - "# vulnerabilities they contain. Versions of these dependencies are", - "# frozen at each coordinated release in order to stabilize upstream", - "# testing, and can contain known vulnerabilities. Consumers are", - "# *STRONGLY* encouraged to rely on curated distributions of OpenStack", - "# or manage security patching of dependencies themselves.", + "# WARNING: OpenStack makes no security guarantees about third-party\n", + "# dependencies listed here, and does not keep track of any\n", + "# vulnerabilities they contain. Versions of these dependencies are\n", + "# frozen at each coordinated release in order to stabilize upstream\n", + "# testing, and can contain known vulnerabilities. Consumers are\n", + "# *STRONGLY* encouraged to rely on curated distributions of OpenStack\n", + "# or manage security patching of dependencies themselves.\n", ] diff --git a/openstack_requirements/project.py b/openstack_requirements/project.py index 8fdaec34b..02ec716b9 100644 --- a/openstack_requirements/project.py +++ b/openstack_requirements/project.py @@ -71,8 +71,4 @@ def read(root): target_files.append('test-requirements-py%s.txt' % py_version) for target_file in target_files: _safe_read(result, target_file, output=requirements) - # Read lower-constraints.txt and ensure the key is always present - # in case the file is missing. - result['lower-constraints.txt'] = None - _safe_read(result, 'lower-constraints.txt') return result diff --git a/openstack_requirements/requirement.py b/openstack_requirements/requirement.py index dd6aaac36..56ba2c0d6 100644 --- a/openstack_requirements/requirement.py +++ b/openstack_requirements/requirement.py @@ -15,8 +15,8 @@ # This module has no IO at all, and none should be added. import collections -import distutils.version import packaging.specifiers +import packaging.version import pkg_resources import re @@ -26,7 +26,7 @@ def key_specifier(a): '===': 1, '==': 1, '~=': 1, '!=': 1, '<': 2, '<=': 2} a = a._spec - return (weight[a[0]], distutils.version.LooseVersion(a[1])) + return (weight[a[0]], packaging.version.parse(a[1])) class Requirement(collections.namedtuple('Requirement', @@ -179,6 +179,11 @@ def _pass_through(req_line, permit_urls=False): def to_reqs(content, permit_urls=False): for content_line in content.splitlines(True): req_line = content_line.strip() + + # skip comments, blank lines + if req_line.startswith('#') or not req_line: + continue + if _pass_through(req_line, permit_urls=permit_urls): yield None, content_line else: diff --git a/openstack_requirements/tests/test_check.py b/openstack_requirements/tests/test_check.py index 43ce81dc8..8983e5f62 100644 --- a/openstack_requirements/tests/test_check.py +++ b/openstack_requirements/tests/test_check.py @@ -541,256 +541,6 @@ def test_new_item_matches_py3_allowed_no_py2(self): ) -class TestValidateLowerConstraints(testtools.TestCase): - - def setUp(self): - super(TestValidateLowerConstraints, self).setUp() - self._stdout_fixture = fixtures.StringStream('stdout') - self.stdout = self.useFixture(self._stdout_fixture).stream - self.useFixture(fixtures.MonkeyPatch('sys.stdout', self.stdout)) - - def test_no_constraints_file(self): - constraints_content = None - project_data = { - 'requirements': {'requirements.txt': 'name>=1.2,!=1.4'}, - 'lower-constraints.txt': constraints_content, - } - head_reqs = check.RequirementsList('testproj', project_data) - head_reqs.process(False) - self.assertFalse( - check.validate_lower_constraints( - req_list=head_reqs, - constraints=project_data['lower-constraints.txt'], - denylist=requirement.parse(''), - ) - ) - - def test_no_min(self): - constraints_content = textwrap.dedent(""" - name==1.2 - """) - project_data = { - 'requirements': {'requirements.txt': 'name!=1.4'}, - 'lower-constraints.txt': constraints_content, - } - head_reqs = check.RequirementsList('testproj', project_data) - head_reqs.process(False) - self.assertFalse( - check.validate_lower_constraints( - req_list=head_reqs, - constraints=project_data['lower-constraints.txt'], - denylist=requirement.parse(''), - ) - ) - - def test_matches(self): - constraints_content = textwrap.dedent(""" - name==1.2 - """) - project_data = { - 'requirements': {'requirements.txt': 'name>=1.2,!=1.4'}, - 'lower-constraints.txt': constraints_content, - } - head_reqs = check.RequirementsList('testproj', project_data) - head_reqs.process(False) - self.assertFalse( - check.validate_lower_constraints( - req_list=head_reqs, - constraints=project_data['lower-constraints.txt'], - denylist=requirement.parse(''), - ) - ) - - def test_not_constrained(self): - constraints_content = textwrap.dedent(""" - """) - project_data = { - 'requirements': {'requirements.txt': 'name>=1.2,!=1.4'}, - 'lower-constraints.txt': constraints_content, - } - head_reqs = check.RequirementsList('testproj', project_data) - head_reqs.process(False) - self.assertTrue( - check.validate_lower_constraints( - req_list=head_reqs, - constraints=project_data['lower-constraints.txt'], - denylist=requirement.parse(''), - ) - ) - - def test_mismatch_denylisted(self): - constraints_content = textwrap.dedent(""" - name==1.2 - """) - project_data = { - 'requirements': {'requirements.txt': 'name>=1.3,!=1.4'}, - 'lower-constraints.txt': constraints_content, - } - head_reqs = check.RequirementsList('testproj', project_data) - head_reqs.process(False) - self.assertFalse( - check.validate_lower_constraints( - req_list=head_reqs, - constraints=project_data['lower-constraints.txt'], - denylist=requirement.parse('name'), - ) - ) - - def test_lower_bound_lower(self): - constraints_content = textwrap.dedent(""" - name==1.2 - """) - project_data = { - 'requirements': {'requirements.txt': 'name>=1.1,!=1.4'}, - 'lower-constraints.txt': constraints_content, - } - head_reqs = check.RequirementsList('testproj', project_data) - head_reqs.process(False) - self.assertTrue( - check.validate_lower_constraints( - req_list=head_reqs, - constraints=project_data['lower-constraints.txt'], - denylist=requirement.parse(''), - ) - ) - - def test_lower_bound_higher(self): - constraints_content = textwrap.dedent(""" - name==1.2 - """) - project_data = { - 'requirements': {'requirements.txt': 'name>=1.3,!=1.4'}, - 'lower-constraints.txt': constraints_content, - } - head_reqs = check.RequirementsList('testproj', project_data) - head_reqs.process(False) - self.assertTrue( - check.validate_lower_constraints( - req_list=head_reqs, - constraints=project_data['lower-constraints.txt'], - denylist=requirement.parse(''), - ) - ) - - def test_constrained_version_excluded(self): - constraints_content = textwrap.dedent(""" - name==1.2 - """) - project_data = { - 'requirements': {'requirements.txt': 'name>=1.1,!=1.2'}, - 'lower-constraints.txt': constraints_content, - } - head_reqs = check.RequirementsList('testproj', project_data) - head_reqs.process(False) - self.assertTrue( - check.validate_lower_constraints( - req_list=head_reqs, - constraints=project_data['lower-constraints.txt'], - denylist=requirement.parse(''), - ) - ) - - def test_constraints_with_markers(self): - constraints_content = textwrap.dedent(""" - name==1.1;python_version=='2.7' - name==2.0;python_version=='3.5' - name==2.0;python_version=='3.6' - """) - project_data = { - 'requirements': { - 'requirements.txt': textwrap.dedent(""" - name>=1.1,!=1.2;python_version=='2.7' - name>=2.0;python_version=='3.5' - name>=2.0;python_version=='3.6' - """), - }, - 'lower-constraints.txt': constraints_content, - } - head_reqs = check.RequirementsList('testproj', project_data) - head_reqs.process(False) - self.assertFalse( - check.validate_lower_constraints( - req_list=head_reqs, - constraints=project_data['lower-constraints.txt'], - denylist=requirement.parse(''), - ) - ) - - def test_constraints_with_markers_missing_one_req(self): - constraints_content = textwrap.dedent(""" - name==1.1;python_version=='2.7' - name==2.0;python_version=='3.5' - name==2.0;python_version=='3.6' - """) - project_data = { - 'requirements': { - 'requirements.txt': textwrap.dedent(""" - name>=1.1,!=1.2;python_version=='2.7' - name>=2.0;python_version=='3.5' - """), - }, - 'lower-constraints.txt': constraints_content, - } - head_reqs = check.RequirementsList('testproj', project_data) - head_reqs.process(False) - self.assertFalse( - check.validate_lower_constraints( - req_list=head_reqs, - constraints=project_data['lower-constraints.txt'], - denylist=requirement.parse(''), - ) - ) - - def test_constraints_with_markers_missing_one_marker(self): - constraints_content = textwrap.dedent(""" - name==1.1;python_version=='2.7' - name==2.0;python_version=='3.5' - """) - project_data = { - 'requirements': { - 'requirements.txt': textwrap.dedent(""" - name>=1.1,!=1.2;python_version=='2.7' - name>=2.0;python_version=='3.5' - name>=2.0;python_version=='3.6' - """), - }, - 'lower-constraints.txt': constraints_content, - } - head_reqs = check.RequirementsList('testproj', project_data) - head_reqs.process(False) - self.assertTrue( - check.validate_lower_constraints( - req_list=head_reqs, - constraints=project_data['lower-constraints.txt'], - denylist=requirement.parse(''), - ) - ) - - def test_complex_marker_evaluation(self): - constraints_content = textwrap.dedent(""" - name===0.8.0;python_version=='2.7' - name===1.0.0;python_version>='3.0' - """) - project_data = { - 'requirements': { - 'requirements.txt': textwrap.dedent(""" - name>=0.8.0;python_version<'3.0' # BSD - name>=1.0.0;python_version>='3.0' # BSD - """), - }, - 'lower-constraints.txt': constraints_content, - } - head_reqs = check.RequirementsList('testproj', project_data) - head_reqs.process(False) - self.assertFalse( - check.validate_lower_constraints( - req_list=head_reqs, - constraints=project_data['lower-constraints.txt'], - denylist=requirement.parse(''), - ) - ) - - class TestBackportPythonMarkers(testtools.TestCase): def setUp(self): diff --git a/openstack_requirements/tests/test_project.py b/openstack_requirements/tests/test_project.py index fb046d0e1..6394a1699 100644 --- a/openstack_requirements/tests/test_project.py +++ b/openstack_requirements/tests/test_project.py @@ -42,8 +42,8 @@ def test_no_setup_py(self): root = self.useFixture(fixtures.TempDir()).path proj = project.read(root) self.expectThat( - proj, matchers.Equals({'root': root, 'requirements': {}, - 'lower-constraints.txt': None})) + proj, matchers.Equals({'root': root, 'requirements': {}}) + ) class TestProjectExtras(testtools.TestCase): diff --git a/playbooks/files/project-requirements-change.py b/playbooks/files/project-requirements-change.py index 57362666c..551053a85 100755 --- a/playbooks/files/project-requirements-change.py +++ b/playbooks/files/project-requirements-change.py @@ -145,15 +145,6 @@ def main(): allow_3_only=python_3_branch, ) - failed = ( - check.validate_lower_constraints( - head_reqs, - head_proj['lower-constraints.txt'], - denylist, - ) - or failed - ) - # report the results if failed or head_reqs.failed: print("*** Incompatible requirement found!") diff --git a/setup.cfg b/setup.cfg index af56b935d..0235e9647 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,7 +29,6 @@ packages = console_scripts = edit-constraints = openstack_requirements.cmds.edit_constraint:main generate-constraints = openstack_requirements.cmds.generate:main - check-conflicts = openstack_requirements.cmds.check_conflicts:main validate-constraints = openstack_requirements.cmds.validate:main validate-projects = openstack_requirements.cmds.validate_projects:main normalize-requirements = openstack_requirements.cmds.normalize_requirements:main diff --git a/tox.ini b/tox.ini index d0a0fc3c9..39badc67a 100644 --- a/tox.ini +++ b/tox.ini @@ -13,23 +13,35 @@ commands = [testenv:py310-check-uc] basepython = python3.10 -deps = -r{toxinidir}/upper-constraints.txt -commands = check-conflicts {toxinidir}/upper-constraints.txt {toxinidir}/upper-constraints-xfails.txt +skip_install = true +# For now we need something due to an issue in the tox_install_sibling_packages +# AnseibleModule +deps = setuptools +commands = python -m pip install --dry-run -r{toxinidir}/upper-constraints.txt [testenv:py311-check-uc] basepython = python3.11 -deps = -r{toxinidir}/upper-constraints.txt -commands = check-conflicts {toxinidir}/upper-constraints.txt {toxinidir}/upper-constraints-xfails.txt +skip_install = true +# For now we need something due to an issue in the tox_install_sibling_packages +# AnseibleModule +deps = setuptools +commands = {[testenv:py310-check-uc]commands} [testenv:py312-check-uc] basepython = python3.12 -deps = -r{toxinidir}/upper-constraints.txt -commands = check-conflicts {toxinidir}/upper-constraints.txt {toxinidir}/upper-constraints-xfails.txt +skip_install = true +# For now we need something due to an issue in the tox_install_sibling_packages +# AnseibleModule +deps = setuptools +commands = {[testenv:py310-check-uc]commands} [testenv:py313-check-uc] basepython = python3.13 -deps = -r{toxinidir}/upper-constraints.txt -commands = check-conflicts {toxinidir}/upper-constraints.txt {toxinidir}/upper-constraints-xfails.txt +skip_install = true +# For now we need something due to an issue in the tox_install_sibling_packages +# AnseibleModule +deps = setuptools +commands = {[testenv:py310-check-uc]commands} [testenv:venv] commands = {posargs} diff --git a/upper-constraints.txt b/upper-constraints.txt index 03c348fb3..8e8df98a8 100644 --- a/upper-constraints.txt +++ b/upper-constraints.txt @@ -1,3 +1,10 @@ +# WARNING: OpenStack makes no security guarantees about third-party +# dependencies listed here, and does not keep track of any +# vulnerabilities they contain. Versions of these dependencies are +# frozen at each coordinated release in order to stabilize upstream +# testing, and can contain known vulnerabilities. Consumers are +# *STRONGLY* encouraged to rely on curated distributions of OpenStack +# or manage security patching of dependencies themselves. voluptuous===0.15.2 chardet===5.2.0 enum-compat===0.0.3 @@ -308,7 +315,7 @@ retrying===1.4.1 XStatic-Dagre===0.6.4.1 platformdirs===4.3.8 pydotplus===2.0.2 -boto3===1.40.1 +boto3===1.35.99 jeepney===0.9.0 stestr===4.2.0 pillow===11.3.0 @@ -321,7 +328,7 @@ aiomysql===0.2.0 types-simplejson===3.20.0.20250326 sphinxcontrib-httpdomain===1.8.1 metalsmith===2.5.0 -s3transfer===0.13.1 +s3transfer===0.10.0 text-unidecode===1.3 sphinxcontrib-svg2pdfconverter===1.3.0 oslo.vmware===4.7.0 @@ -438,7 +445,7 @@ tomli===2.2.1;python_version=='3.9' oslo.upgradecheck===2.6.0 sherlock===0.4.1 stevedore===5.5.0 -botocore===1.40.1 +botocore===1.35.99 xmltodict===0.14.2 pyasn1===0.6.0 oslo.rootwrap===7.7.0 @@ -472,7 +479,7 @@ sentinels===1.0.0 kombu===5.5.4 distro===1.9.0 zstd===1.5.7.2 -yaql===3.1.0 +yaql===3.2.0 durationpy===0.10 requestsexceptions===1.4.0 testresources===2.0.2