From 9f233a8c859a0e61952f71055f0d5225b681bb18 Mon Sep 17 00:00:00 2001 From: Gabriel Curio Date: Mon, 11 Dec 2017 17:57:57 -0600 Subject: [PATCH] Use pyparsing to control extra normalization. --- packaging/markers.py | 119 +++---------------- packaging/utils.py | 30 +++++ tests/test_markers.py | 264 +----------------------------------------- 3 files changed, 52 insertions(+), 361 deletions(-) diff --git a/packaging/markers.py b/packaging/markers.py index 6743bbe9..4061c532 100644 --- a/packaging/markers.py +++ b/packaging/markers.py @@ -14,7 +14,7 @@ from ._compat import string_types from .specifiers import Specifier, InvalidSpecifier -from .utils import canonicalize_name +from .utils import safe_extra __all__ = [ @@ -92,7 +92,7 @@ def serialize(self): L("platform.version") | # PEP-345 L("platform.machine") | # PEP-345 L("platform.python_implementation") | # PEP-345 - L("python_implementation") | # undocumented setuptools legacy + L("python_implementation") | # undocumented setuptools legacy L("extra") ) ALIASES = { @@ -144,9 +144,11 @@ def serialize(self): MARKER = stringStart + MARKER_EXPR + stringEnd +MARKER_EXTRA_VALUE = QuotedString("'") | QuotedString('"') +MARKER_EXTRA_VALUE.setParseAction(lambda s, l, t: Value(safe_extra(t[0]))) MARKER_EXTRA_ITEM = Group( - MARKER_EXTRA_VARIABLE + MARKER_EXTRA_OP + MARKER_VALUE + MARKER_EXTRA_VARIABLE + MARKER_EXTRA_OP + MARKER_EXTRA_VALUE ) MARKER_EXTRA_ITEM.setParseAction(lambda s, l, t: tuple(t[0])) MARKER_EXTRA_EXPR = Forward() @@ -293,17 +295,19 @@ def default_environment(): class Marker(object): def __init__(self, marker): - self._marker_string = marker - extra_markers = MarkerExtraParser().get_extra_markers( - self._marker_string - ) - if extra_markers: - good_names = MarkerExtraCleaner().clean_marker_extras( - extra_markers + try: + self._markers = _coerce_parse_result( + MARKER_EXTRA.parseString(marker) ) - self._markers = good_names - else: - self._markers = self.get_marker_not_extra(self._marker_string) + except ParseException: + try: + self._markers = _coerce_parse_result( + MARKER.parseString(marker) + ) + except ParseException as e2: + err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( + marker, marker[e2.loc:e2.loc + 8]) + raise InvalidMarker(err_str) def __str__(self): return _format_marker(self._markers) @@ -311,14 +315,6 @@ def __str__(self): def __repr__(self): return "".format(str(self)) - def get_marker_not_extra(self, marker): - try: - return _coerce_parse_result(MARKER.parseString(marker)) - except ParseException as e2: - err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( - marker, marker[e2.loc:e2.loc + 8]) - raise InvalidMarker(err_str) - def evaluate(self, environment=None): """Evaluate a marker. @@ -333,84 +329,3 @@ def evaluate(self, environment=None): current_environment.update(environment) return _evaluate_markers(self._markers, current_environment) - - -class MarkerExtraParser(object): - - @classmethod - def get_extra_markers(cls, marker): - try: - tmp_markers = _coerce_parse_result( - MARKER_EXTRA.parseString(marker) - ) - return tmp_markers - except ParseException: - return False - - -class MarkerExtraCleaner(object): - - @classmethod - def clean_marker_extras(cls, markers): - clean_markers = [] - for parsed_marker in markers: - clean_marker = cls._clean_marker_extra(parsed_marker) - clean_markers.append(clean_marker) - return clean_markers - - @classmethod - def _clean_marker_extra(cls, marker): - extra_locations = cls._get_extra_index_location(marker) - if extra_locations: - return cls._fix_extra_values(extra_locations, marker) - else: - return marker - - @classmethod - def _get_extra_index_location(cls, marker): - locations = [] - if len(marker) < 3: - return locations - for index in range(len(marker)): - if cls._is_variable(marker[index]): - if cls._is_op(marker[index + 1]): - if cls._is_value(marker[index + 2]): - locations.append(index) - return locations - - @classmethod - def _is_variable(cls, variable): - return cls.check_attribute(variable, Variable, 'value', 'extra') - - @classmethod - def _is_op(cls, op): - return cls.check_attribute(op, Op, 'value', ('==', '===', 'is')) - - @classmethod - def _is_value(cls, value): - return isinstance(value, Value) - - @staticmethod - def check_attribute(obj, object_types, attribute_names, attribute_values): - if not isinstance(attribute_values, (list, tuple)): - attribute_values = (attribute_values,) - if not isinstance(object_types, (list, tuple)): - object_types = (object_types,) - if not isinstance(attribute_names, (list, tuple)): - attribute_names = (attribute_names,) - for attribute_value in attribute_values: - for object_type in object_types: - for attribute_name in attribute_names: - if isinstance(obj, object_type): - if getattr(obj, attribute_name) == attribute_value: - return True - return False - - @classmethod - def _fix_extra_values(cls, extra_locations, marker): - parsed_marker = list(marker) - for extra_location in extra_locations: - parsed_marker[extra_location + 2].value = canonicalize_name( - parsed_marker[extra_location + 2].value - ) - return tuple(parsed_marker) diff --git a/packaging/utils.py b/packaging/utils.py index 942387ce..90461bac 100644 --- a/packaging/utils.py +++ b/packaging/utils.py @@ -5,6 +5,7 @@ import re +from .version import Version, InvalidVersion _canonicalize_regex = re.compile(r"[-_.]+") @@ -12,3 +13,32 @@ def canonicalize_name(name): # This is taken from PEP 503. return _canonicalize_regex.sub("-", name).lower() + + +def safe_extra(extra): + """Convert an arbitrary string to a standard 'extra' name + + Any runs of non-alphanumeric characters are replaced with a single '_', + and the result is always lowercased. + """ + return re.sub('[^A-Za-z0-9.-]+', '_', extra).lower() + + +def safe_name(name): + """Convert an arbitrary string to a standard distribution name + + Any runs of non-alphanumeric/. characters are replaced with a single '-'. + """ + return re.sub('[^A-Za-z0-9.]+', '-', name) + + +def safe_version(version): + """ + Convert an arbitrary string to a standard version string + """ + try: + # normalize the version + return str(Version(version)) + except InvalidVersion: + version = version.replace(' ', '.') + return re.sub('[^A-Za-z0-9.]+', '-', version) diff --git a/tests/test_markers.py b/tests/test_markers.py index 5b9f9612..dcee5528 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -13,9 +13,7 @@ import pytest from packaging.markers import ( - Node, Variable, Op, Value, - InvalidMarker, UndefinedComparison, UndefinedEnvironmentName, - Marker, MarkerExtraParser, MarkerExtraCleaner, + Node, InvalidMarker, UndefinedComparison, UndefinedEnvironmentName, Marker, default_environment, format_full_version, ) @@ -201,8 +199,7 @@ class TestMarker: ], ) def test_parses_valid(self, marker_string): - marker = Marker(marker_string) - assert marker._marker_string == marker_string + Marker(marker_string) @pytest.mark.parametrize( "marker_string", @@ -315,53 +312,7 @@ def test_extra_with_no_extra_in_environment(self): ) def test_evaluates(self, marker_string, environment, expected): args = [] if environment is None else [environment] - marker = Marker(marker_string) - assert marker.evaluate(*args) == expected - - @pytest.mark.parametrize( - ("marker_string", "expected"), - [ - ("os_name == '{0}'".format(os.name), True), - ("os_name == 'foo'", True), - ("os_name == 'foo'", True), - ("'2.7' in python_version", True), - ("'2.7' not in python_version", True), - ( - "os_name == 'foo' and python_version ~= '2.7.0'", - True, - ), - ( - "python_version ~= '2.7.0' and (os_name == 'foo' or " - "os_name == 'bar')", - True, - ), - ( - "python_version ~= '2.7.0' and (os_name == 'foo' or " - "os_name == 'bar')", - True, - ), - ( - "python_version ~= '2.7.0' and (os_name == 'foo' or " - "os_name == 'bar')", - True, - ), - ( - "extra == 'security'", - True, - ), - ( - "extra == 'security'", - True, - ), - ( - "extra == 'SECURITY'", - True, - ), - ], - ) - def test_parse_marker_not_extra(self, marker_string, expected): - result = Marker(marker_string).get_marker_not_extra(marker_string) - assert bool(result) == expected + assert Marker(marker_string).evaluate(*args) == expected @pytest.mark.parametrize( "marker_string", @@ -413,14 +364,10 @@ def test_evaluate_pep345_markers(self, marker_string, environment, "marker_string", [ "{0} {1} {2!r}".format(*i) - for i in itertools.product( - SETUPTOOLS_VARIABLES, OPERATORS, VALUES - ) + for i in itertools.product(SETUPTOOLS_VARIABLES, OPERATORS, VALUES) ] + [ "{2!r} {1} {0}".format(*i) - for i in itertools.product( - SETUPTOOLS_VARIABLES, OPERATORS, VALUES - ) + for i in itertools.product(SETUPTOOLS_VARIABLES, OPERATORS, VALUES) ], ) def test_parses_setuptools_legacy_valid(self, marker_string): @@ -430,204 +377,3 @@ def test_evaluate_setuptools_legacy_markers(self): marker_string = "python_implementation=='Jython'" args = [{"platform_python_implementation": "CPython"}] assert Marker(marker_string).evaluate(*args) is False - - -class TestMarkerExtraParser: - @pytest.mark.parametrize( - ("marker_string", "expected"), - [ - ("os_name == '{0}'".format(os.name), False), - ("os_name == 'foo'", False), - ("os_name == 'foo'", False), - ("'2.7' in python_version", False), - ("'2.7' not in python_version", False), - ( - "os_name == 'foo' and python_version ~= '2.7.0'", - False, - ), - ( - "python_version ~= '2.7.0' and (os_name == 'foo' or " - "os_name == 'bar')", - False, - ), - ( - "python_version ~= '2.7.0' and (os_name == 'foo' or " - "os_name == 'bar')", - False, - ), - ( - "python_version ~= '2.7.0' and (os_name == 'foo' or " - "os_name == 'bar')", - False, - ), - ( - "extra == 'security'", - True, - ), - ( - "extra == 'security'", - True, - ), - ( - "extra == 'SECURITY'", - True, - ), - ], - ) - def test_parse_extra_markers(self, marker_string, expected): - result = MarkerExtraParser.get_extra_markers(marker_string) - assert bool(result) == expected - - -class TestExtraMarkerCleaner(object): - @pytest.mark.parametrize( - ("markers", "value"), - [ - ( - [((Variable('extra')), Op('=='), Value('Security'),)], - 'security' - ), - - ], - ) - def test_clean_marker_extras(self, markers, value): - cleaner = MarkerExtraCleaner() - result = cleaner.clean_marker_extras(markers) - assert result[0][2].value == value - - @pytest.mark.parametrize( - ("markers", "value"), - [ - ( - ((Variable('extra')), Op('=='), Value('Security'),), - 'security' - ), - - ], - ) - def test_clean_marker_extra(self, markers, value): - cleaner = MarkerExtraCleaner() - result = cleaner._clean_marker_extra(markers) - assert result[2].value == value - - @pytest.mark.parametrize( - ("markers", "locations"), - [ - ( - ((Variable('extra')), Op('=='), Value('Security'),), - [0] - ), - ( - ((Variable('extra')), Op('is'), Value('Security'),), - [0] - ), - ( - ((Variable('extra')), Op('<'), Value('Security'),), - [] - ), - ( - ( - (Variable('extra')), - Op('=='), - Value('Security'), - (Variable('extra')), - Op('=='), - Value('Security') - ), - [0, 3] - ), - ( - ( - (Variable('extra')), - Op('=='), - (Variable('extra')), - (Variable('extra')), - Op('=='), - Value('Security'), - ), - [3] - ), - ( - ((Variable('security')), Op('<'), Value('extra'),), - [] - ), - ( - ((Variable('extra')), Value('Security'),), - [] - ), - ( - ((Variable('security')), Value('extra'), Op('<'), ), - [] - ), - ( - ((Variable('security')), Op('<'), Op('<'),), - [] - ), - ( - tuple(), - [] - ), - ( - ( - (Variable('extra')), - Op('=='), - Value('Security'), - Value('security') - ), - [0] - ), - ], - ) - def test_get_extra_index_location(self, markers, locations): - cleaner = MarkerExtraCleaner() - result = cleaner._get_extra_index_location(markers) - assert result == locations - - @pytest.mark.parametrize( - ("obj", "object_types", "attribute_names", - "attribute_values", "expect"), - [ - ( - Variable('extra'), - Variable, - 'value', - 'extra', - True - ), - ( - Variable('extra'), - (Variable, Marker), - 'value', - 'extra', - True - ), - ( - Variable('extra'), - Variable, - ('value', 'extra'), - 'extra', - True - ), - ( - Variable('extra'), - Variable, - 'value', - ('extra', 'bad value'), - True - ), - - ], - ) - def test_check_attribute( - self, - obj, - object_types, - attribute_names, - attribute_values, - expect - ): - cleaner = MarkerExtraCleaner() - result = cleaner.check_attribute( - obj, object_types, attribute_names, attribute_values - ) - assert result == expect