From fc7d3db30ffedccaab4061b48ec7be6fc2407111 Mon Sep 17 00:00:00 2001 From: Gabriel Curio Date: Tue, 5 Dec 2017 22:54:01 -0600 Subject: [PATCH 01/10] Introduce parsing and normalization for markers with an extra value. Add unit tests. --- packaging/markers.py | 129 ++++++++++++++++++-- tests/test_markers.py | 269 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 386 insertions(+), 12 deletions(-) diff --git a/packaging/markers.py b/packaging/markers.py index 5fdf510c..6743bbe9 100644 --- a/packaging/markers.py +++ b/packaging/markers.py @@ -14,6 +14,7 @@ from ._compat import string_types from .specifiers import Specifier, InvalidSpecifier +from .utils import canonicalize_name __all__ = [ @@ -91,7 +92,7 @@ def serialize(self): L("platform.version") | # PEP-345 L("platform.machine") | # PEP-345 L("platform.python_implementation") | # PEP-345 - L("python_implementation") | # undocumented setuptools legacy + L("python_implementation") | # undocumented setuptools legacy L("extra") ) ALIASES = { @@ -115,6 +116,12 @@ def serialize(self): L("<") ) +MARKER_EXTRA_VARIABLE = L('extra') +MARKER_EXTRA_VARIABLE.setParseAction(lambda s, l, t: Variable(t[0])) +MARKER_EXTRA_OP = (L("==") | L("===")) | L("not in") | L("in") +MARKER_EXTRA_OP.setParseAction(lambda s, l, t: Op(t[0])) + + MARKER_OP = VERSION_CMP | L("not in") | L("in") MARKER_OP.setParseAction(lambda s, l, t: Op(t[0])) @@ -138,6 +145,20 @@ def serialize(self): MARKER = stringStart + MARKER_EXPR + stringEnd +MARKER_EXTRA_ITEM = Group( + MARKER_EXTRA_VARIABLE + MARKER_EXTRA_OP + MARKER_VALUE +) +MARKER_EXTRA_ITEM.setParseAction(lambda s, l, t: tuple(t[0])) +MARKER_EXTRA_EXPR = Forward() + +MARKER_EXTRA_GROUP = Group(LPAREN + MARKER_EXTRA_EXPR + RPAREN) +MARKER_EXTRA_ATOM = MARKER_EXTRA_ITEM | MARKER_EXTRA_GROUP + + +MARKER_EXTRA_EXPR << MARKER_EXTRA_ATOM + ZeroOrMore(BOOLOP + MARKER_EXTRA_EXPR) +MARKER_EXTRA = stringStart + MARKER_EXTRA_EXPR + stringEnd + + def _coerce_parse_result(results): if isinstance(results, ParseResults): return [_coerce_parse_result(i) for i in results] @@ -272,12 +293,17 @@ def default_environment(): class Marker(object): def __init__(self, marker): - try: - self._markers = _coerce_parse_result(MARKER.parseString(marker)) - except ParseException as e: - err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( - marker, marker[e.loc:e.loc + 8]) - raise InvalidMarker(err_str) + self._marker_string = marker + extra_markers = MarkerExtraParser().get_extra_markers( + self._marker_string + ) + if extra_markers: + good_names = MarkerExtraCleaner().clean_marker_extras( + extra_markers + ) + self._markers = good_names + else: + self._markers = self.get_marker_not_extra(self._marker_string) def __str__(self): return _format_marker(self._markers) @@ -285,6 +311,14 @@ def __str__(self): def __repr__(self): return "".format(str(self)) + def get_marker_not_extra(self, marker): + try: + return _coerce_parse_result(MARKER.parseString(marker)) + except ParseException as e2: + err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( + marker, marker[e2.loc:e2.loc + 8]) + raise InvalidMarker(err_str) + def evaluate(self, environment=None): """Evaluate a marker. @@ -299,3 +333,84 @@ def evaluate(self, environment=None): current_environment.update(environment) return _evaluate_markers(self._markers, current_environment) + + +class MarkerExtraParser(object): + + @classmethod + def get_extra_markers(cls, marker): + try: + tmp_markers = _coerce_parse_result( + MARKER_EXTRA.parseString(marker) + ) + return tmp_markers + except ParseException: + return False + + +class MarkerExtraCleaner(object): + + @classmethod + def clean_marker_extras(cls, markers): + clean_markers = [] + for parsed_marker in markers: + clean_marker = cls._clean_marker_extra(parsed_marker) + clean_markers.append(clean_marker) + return clean_markers + + @classmethod + def _clean_marker_extra(cls, marker): + extra_locations = cls._get_extra_index_location(marker) + if extra_locations: + return cls._fix_extra_values(extra_locations, marker) + else: + return marker + + @classmethod + def _get_extra_index_location(cls, marker): + locations = [] + if len(marker) < 3: + return locations + for index in range(len(marker)): + if cls._is_variable(marker[index]): + if cls._is_op(marker[index + 1]): + if cls._is_value(marker[index + 2]): + locations.append(index) + return locations + + @classmethod + def _is_variable(cls, variable): + return cls.check_attribute(variable, Variable, 'value', 'extra') + + @classmethod + def _is_op(cls, op): + return cls.check_attribute(op, Op, 'value', ('==', '===', 'is')) + + @classmethod + def _is_value(cls, value): + return isinstance(value, Value) + + @staticmethod + def check_attribute(obj, object_types, attribute_names, attribute_values): + if not isinstance(attribute_values, (list, tuple)): + attribute_values = (attribute_values,) + if not isinstance(object_types, (list, tuple)): + object_types = (object_types,) + if not isinstance(attribute_names, (list, tuple)): + attribute_names = (attribute_names,) + for attribute_value in attribute_values: + for object_type in object_types: + for attribute_name in attribute_names: + if isinstance(obj, object_type): + if getattr(obj, attribute_name) == attribute_value: + return True + return False + + @classmethod + def _fix_extra_values(cls, extra_locations, marker): + parsed_marker = list(marker) + for extra_location in extra_locations: + parsed_marker[extra_location + 2].value = canonicalize_name( + parsed_marker[extra_location + 2].value + ) + return tuple(parsed_marker) diff --git a/tests/test_markers.py b/tests/test_markers.py index 0e2d96a9..5b9f9612 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -13,7 +13,9 @@ import pytest from packaging.markers import ( - Node, InvalidMarker, UndefinedComparison, UndefinedEnvironmentName, Marker, + Node, Variable, Op, Value, + InvalidMarker, UndefinedComparison, UndefinedEnvironmentName, + Marker, MarkerExtraParser, MarkerExtraCleaner, default_environment, format_full_version, ) @@ -199,7 +201,8 @@ class TestMarker: ], ) def test_parses_valid(self, marker_string): - Marker(marker_string) + marker = Marker(marker_string) + assert marker._marker_string == marker_string @pytest.mark.parametrize( "marker_string", @@ -303,11 +306,62 @@ def test_extra_with_no_extra_in_environment(self): {"extra": "security"}, True, ), + ( + "extra == 'SECURITY'", + {"extra": "security"}, + True, + ), ], ) def test_evaluates(self, marker_string, environment, expected): args = [] if environment is None else [environment] - assert Marker(marker_string).evaluate(*args) == expected + marker = Marker(marker_string) + assert marker.evaluate(*args) == expected + + @pytest.mark.parametrize( + ("marker_string", "expected"), + [ + ("os_name == '{0}'".format(os.name), True), + ("os_name == 'foo'", True), + ("os_name == 'foo'", True), + ("'2.7' in python_version", True), + ("'2.7' not in python_version", True), + ( + "os_name == 'foo' and python_version ~= '2.7.0'", + True, + ), + ( + "python_version ~= '2.7.0' and (os_name == 'foo' or " + "os_name == 'bar')", + True, + ), + ( + "python_version ~= '2.7.0' and (os_name == 'foo' or " + "os_name == 'bar')", + True, + ), + ( + "python_version ~= '2.7.0' and (os_name == 'foo' or " + "os_name == 'bar')", + True, + ), + ( + "extra == 'security'", + True, + ), + ( + "extra == 'security'", + True, + ), + ( + "extra == 'SECURITY'", + True, + ), + ], + ) + def test_parse_marker_not_extra(self, marker_string, expected): + result = Marker(marker_string).get_marker_not_extra(marker_string) + assert bool(result) == expected @pytest.mark.parametrize( "marker_string", @@ -359,10 +413,14 @@ def test_evaluate_pep345_markers(self, marker_string, environment, "marker_string", [ "{0} {1} {2!r}".format(*i) - for i in itertools.product(SETUPTOOLS_VARIABLES, OPERATORS, VALUES) + for i in itertools.product( + SETUPTOOLS_VARIABLES, OPERATORS, VALUES + ) ] + [ "{2!r} {1} {0}".format(*i) - for i in itertools.product(SETUPTOOLS_VARIABLES, OPERATORS, VALUES) + for i in itertools.product( + SETUPTOOLS_VARIABLES, OPERATORS, VALUES + ) ], ) def test_parses_setuptools_legacy_valid(self, marker_string): @@ -372,3 +430,204 @@ def test_evaluate_setuptools_legacy_markers(self): marker_string = "python_implementation=='Jython'" args = [{"platform_python_implementation": "CPython"}] assert Marker(marker_string).evaluate(*args) is False + + +class TestMarkerExtraParser: + @pytest.mark.parametrize( + ("marker_string", "expected"), + [ + ("os_name == '{0}'".format(os.name), False), + ("os_name == 'foo'", False), + ("os_name == 'foo'", False), + ("'2.7' in python_version", False), + ("'2.7' not in python_version", False), + ( + "os_name == 'foo' and python_version ~= '2.7.0'", + False, + ), + ( + "python_version ~= '2.7.0' and (os_name == 'foo' or " + "os_name == 'bar')", + False, + ), + ( + "python_version ~= '2.7.0' and (os_name == 'foo' or " + "os_name == 'bar')", + False, + ), + ( + "python_version ~= '2.7.0' and (os_name == 'foo' or " + "os_name == 'bar')", + False, + ), + ( + "extra == 'security'", + True, + ), + ( + "extra == 'security'", + True, + ), + ( + "extra == 'SECURITY'", + True, + ), + ], + ) + def test_parse_extra_markers(self, marker_string, expected): + result = MarkerExtraParser.get_extra_markers(marker_string) + assert bool(result) == expected + + +class TestExtraMarkerCleaner(object): + @pytest.mark.parametrize( + ("markers", "value"), + [ + ( + [((Variable('extra')), Op('=='), Value('Security'),)], + 'security' + ), + + ], + ) + def test_clean_marker_extras(self, markers, value): + cleaner = MarkerExtraCleaner() + result = cleaner.clean_marker_extras(markers) + assert result[0][2].value == value + + @pytest.mark.parametrize( + ("markers", "value"), + [ + ( + ((Variable('extra')), Op('=='), Value('Security'),), + 'security' + ), + + ], + ) + def test_clean_marker_extra(self, markers, value): + cleaner = MarkerExtraCleaner() + result = cleaner._clean_marker_extra(markers) + assert result[2].value == value + + @pytest.mark.parametrize( + ("markers", "locations"), + [ + ( + ((Variable('extra')), Op('=='), Value('Security'),), + [0] + ), + ( + ((Variable('extra')), Op('is'), Value('Security'),), + [0] + ), + ( + ((Variable('extra')), Op('<'), Value('Security'),), + [] + ), + ( + ( + (Variable('extra')), + Op('=='), + Value('Security'), + (Variable('extra')), + Op('=='), + Value('Security') + ), + [0, 3] + ), + ( + ( + (Variable('extra')), + Op('=='), + (Variable('extra')), + (Variable('extra')), + Op('=='), + Value('Security'), + ), + [3] + ), + ( + ((Variable('security')), Op('<'), Value('extra'),), + [] + ), + ( + ((Variable('extra')), Value('Security'),), + [] + ), + ( + ((Variable('security')), Value('extra'), Op('<'), ), + [] + ), + ( + ((Variable('security')), Op('<'), Op('<'),), + [] + ), + ( + tuple(), + [] + ), + ( + ( + (Variable('extra')), + Op('=='), + Value('Security'), + Value('security') + ), + [0] + ), + ], + ) + def test_get_extra_index_location(self, markers, locations): + cleaner = MarkerExtraCleaner() + result = cleaner._get_extra_index_location(markers) + assert result == locations + + @pytest.mark.parametrize( + ("obj", "object_types", "attribute_names", + "attribute_values", "expect"), + [ + ( + Variable('extra'), + Variable, + 'value', + 'extra', + True + ), + ( + Variable('extra'), + (Variable, Marker), + 'value', + 'extra', + True + ), + ( + Variable('extra'), + Variable, + ('value', 'extra'), + 'extra', + True + ), + ( + Variable('extra'), + Variable, + 'value', + ('extra', 'bad value'), + True + ), + + ], + ) + def test_check_attribute( + self, + obj, + object_types, + attribute_names, + attribute_values, + expect + ): + cleaner = MarkerExtraCleaner() + result = cleaner.check_attribute( + obj, object_types, attribute_names, attribute_values + ) + assert result == expect From 9f233a8c859a0e61952f71055f0d5225b681bb18 Mon Sep 17 00:00:00 2001 From: Gabriel Curio Date: Mon, 11 Dec 2017 17:57:57 -0600 Subject: [PATCH 02/10] Use pyparsing to control extra normalization. --- packaging/markers.py | 119 +++---------------- packaging/utils.py | 30 +++++ tests/test_markers.py | 264 +----------------------------------------- 3 files changed, 52 insertions(+), 361 deletions(-) diff --git a/packaging/markers.py b/packaging/markers.py index 6743bbe9..4061c532 100644 --- a/packaging/markers.py +++ b/packaging/markers.py @@ -14,7 +14,7 @@ from ._compat import string_types from .specifiers import Specifier, InvalidSpecifier -from .utils import canonicalize_name +from .utils import safe_extra __all__ = [ @@ -92,7 +92,7 @@ def serialize(self): L("platform.version") | # PEP-345 L("platform.machine") | # PEP-345 L("platform.python_implementation") | # PEP-345 - L("python_implementation") | # undocumented setuptools legacy + L("python_implementation") | # undocumented setuptools legacy L("extra") ) ALIASES = { @@ -144,9 +144,11 @@ def serialize(self): MARKER = stringStart + MARKER_EXPR + stringEnd +MARKER_EXTRA_VALUE = QuotedString("'") | QuotedString('"') +MARKER_EXTRA_VALUE.setParseAction(lambda s, l, t: Value(safe_extra(t[0]))) MARKER_EXTRA_ITEM = Group( - MARKER_EXTRA_VARIABLE + MARKER_EXTRA_OP + MARKER_VALUE + MARKER_EXTRA_VARIABLE + MARKER_EXTRA_OP + MARKER_EXTRA_VALUE ) MARKER_EXTRA_ITEM.setParseAction(lambda s, l, t: tuple(t[0])) MARKER_EXTRA_EXPR = Forward() @@ -293,17 +295,19 @@ def default_environment(): class Marker(object): def __init__(self, marker): - self._marker_string = marker - extra_markers = MarkerExtraParser().get_extra_markers( - self._marker_string - ) - if extra_markers: - good_names = MarkerExtraCleaner().clean_marker_extras( - extra_markers + try: + self._markers = _coerce_parse_result( + MARKER_EXTRA.parseString(marker) ) - self._markers = good_names - else: - self._markers = self.get_marker_not_extra(self._marker_string) + except ParseException: + try: + self._markers = _coerce_parse_result( + MARKER.parseString(marker) + ) + except ParseException as e2: + err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( + marker, marker[e2.loc:e2.loc + 8]) + raise InvalidMarker(err_str) def __str__(self): return _format_marker(self._markers) @@ -311,14 +315,6 @@ def __str__(self): def __repr__(self): return "".format(str(self)) - def get_marker_not_extra(self, marker): - try: - return _coerce_parse_result(MARKER.parseString(marker)) - except ParseException as e2: - err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( - marker, marker[e2.loc:e2.loc + 8]) - raise InvalidMarker(err_str) - def evaluate(self, environment=None): """Evaluate a marker. @@ -333,84 +329,3 @@ def evaluate(self, environment=None): current_environment.update(environment) return _evaluate_markers(self._markers, current_environment) - - -class MarkerExtraParser(object): - - @classmethod - def get_extra_markers(cls, marker): - try: - tmp_markers = _coerce_parse_result( - MARKER_EXTRA.parseString(marker) - ) - return tmp_markers - except ParseException: - return False - - -class MarkerExtraCleaner(object): - - @classmethod - def clean_marker_extras(cls, markers): - clean_markers = [] - for parsed_marker in markers: - clean_marker = cls._clean_marker_extra(parsed_marker) - clean_markers.append(clean_marker) - return clean_markers - - @classmethod - def _clean_marker_extra(cls, marker): - extra_locations = cls._get_extra_index_location(marker) - if extra_locations: - return cls._fix_extra_values(extra_locations, marker) - else: - return marker - - @classmethod - def _get_extra_index_location(cls, marker): - locations = [] - if len(marker) < 3: - return locations - for index in range(len(marker)): - if cls._is_variable(marker[index]): - if cls._is_op(marker[index + 1]): - if cls._is_value(marker[index + 2]): - locations.append(index) - return locations - - @classmethod - def _is_variable(cls, variable): - return cls.check_attribute(variable, Variable, 'value', 'extra') - - @classmethod - def _is_op(cls, op): - return cls.check_attribute(op, Op, 'value', ('==', '===', 'is')) - - @classmethod - def _is_value(cls, value): - return isinstance(value, Value) - - @staticmethod - def check_attribute(obj, object_types, attribute_names, attribute_values): - if not isinstance(attribute_values, (list, tuple)): - attribute_values = (attribute_values,) - if not isinstance(object_types, (list, tuple)): - object_types = (object_types,) - if not isinstance(attribute_names, (list, tuple)): - attribute_names = (attribute_names,) - for attribute_value in attribute_values: - for object_type in object_types: - for attribute_name in attribute_names: - if isinstance(obj, object_type): - if getattr(obj, attribute_name) == attribute_value: - return True - return False - - @classmethod - def _fix_extra_values(cls, extra_locations, marker): - parsed_marker = list(marker) - for extra_location in extra_locations: - parsed_marker[extra_location + 2].value = canonicalize_name( - parsed_marker[extra_location + 2].value - ) - return tuple(parsed_marker) diff --git a/packaging/utils.py b/packaging/utils.py index 942387ce..90461bac 100644 --- a/packaging/utils.py +++ b/packaging/utils.py @@ -5,6 +5,7 @@ import re +from .version import Version, InvalidVersion _canonicalize_regex = re.compile(r"[-_.]+") @@ -12,3 +13,32 @@ def canonicalize_name(name): # This is taken from PEP 503. return _canonicalize_regex.sub("-", name).lower() + + +def safe_extra(extra): + """Convert an arbitrary string to a standard 'extra' name + + Any runs of non-alphanumeric characters are replaced with a single '_', + and the result is always lowercased. + """ + return re.sub('[^A-Za-z0-9.-]+', '_', extra).lower() + + +def safe_name(name): + """Convert an arbitrary string to a standard distribution name + + Any runs of non-alphanumeric/. characters are replaced with a single '-'. + """ + return re.sub('[^A-Za-z0-9.]+', '-', name) + + +def safe_version(version): + """ + Convert an arbitrary string to a standard version string + """ + try: + # normalize the version + return str(Version(version)) + except InvalidVersion: + version = version.replace(' ', '.') + return re.sub('[^A-Za-z0-9.]+', '-', version) diff --git a/tests/test_markers.py b/tests/test_markers.py index 5b9f9612..dcee5528 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -13,9 +13,7 @@ import pytest from packaging.markers import ( - Node, Variable, Op, Value, - InvalidMarker, UndefinedComparison, UndefinedEnvironmentName, - Marker, MarkerExtraParser, MarkerExtraCleaner, + Node, InvalidMarker, UndefinedComparison, UndefinedEnvironmentName, Marker, default_environment, format_full_version, ) @@ -201,8 +199,7 @@ class TestMarker: ], ) def test_parses_valid(self, marker_string): - marker = Marker(marker_string) - assert marker._marker_string == marker_string + Marker(marker_string) @pytest.mark.parametrize( "marker_string", @@ -315,53 +312,7 @@ def test_extra_with_no_extra_in_environment(self): ) def test_evaluates(self, marker_string, environment, expected): args = [] if environment is None else [environment] - marker = Marker(marker_string) - assert marker.evaluate(*args) == expected - - @pytest.mark.parametrize( - ("marker_string", "expected"), - [ - ("os_name == '{0}'".format(os.name), True), - ("os_name == 'foo'", True), - ("os_name == 'foo'", True), - ("'2.7' in python_version", True), - ("'2.7' not in python_version", True), - ( - "os_name == 'foo' and python_version ~= '2.7.0'", - True, - ), - ( - "python_version ~= '2.7.0' and (os_name == 'foo' or " - "os_name == 'bar')", - True, - ), - ( - "python_version ~= '2.7.0' and (os_name == 'foo' or " - "os_name == 'bar')", - True, - ), - ( - "python_version ~= '2.7.0' and (os_name == 'foo' or " - "os_name == 'bar')", - True, - ), - ( - "extra == 'security'", - True, - ), - ( - "extra == 'security'", - True, - ), - ( - "extra == 'SECURITY'", - True, - ), - ], - ) - def test_parse_marker_not_extra(self, marker_string, expected): - result = Marker(marker_string).get_marker_not_extra(marker_string) - assert bool(result) == expected + assert Marker(marker_string).evaluate(*args) == expected @pytest.mark.parametrize( "marker_string", @@ -413,14 +364,10 @@ def test_evaluate_pep345_markers(self, marker_string, environment, "marker_string", [ "{0} {1} {2!r}".format(*i) - for i in itertools.product( - SETUPTOOLS_VARIABLES, OPERATORS, VALUES - ) + for i in itertools.product(SETUPTOOLS_VARIABLES, OPERATORS, VALUES) ] + [ "{2!r} {1} {0}".format(*i) - for i in itertools.product( - SETUPTOOLS_VARIABLES, OPERATORS, VALUES - ) + for i in itertools.product(SETUPTOOLS_VARIABLES, OPERATORS, VALUES) ], ) def test_parses_setuptools_legacy_valid(self, marker_string): @@ -430,204 +377,3 @@ def test_evaluate_setuptools_legacy_markers(self): marker_string = "python_implementation=='Jython'" args = [{"platform_python_implementation": "CPython"}] assert Marker(marker_string).evaluate(*args) is False - - -class TestMarkerExtraParser: - @pytest.mark.parametrize( - ("marker_string", "expected"), - [ - ("os_name == '{0}'".format(os.name), False), - ("os_name == 'foo'", False), - ("os_name == 'foo'", False), - ("'2.7' in python_version", False), - ("'2.7' not in python_version", False), - ( - "os_name == 'foo' and python_version ~= '2.7.0'", - False, - ), - ( - "python_version ~= '2.7.0' and (os_name == 'foo' or " - "os_name == 'bar')", - False, - ), - ( - "python_version ~= '2.7.0' and (os_name == 'foo' or " - "os_name == 'bar')", - False, - ), - ( - "python_version ~= '2.7.0' and (os_name == 'foo' or " - "os_name == 'bar')", - False, - ), - ( - "extra == 'security'", - True, - ), - ( - "extra == 'security'", - True, - ), - ( - "extra == 'SECURITY'", - True, - ), - ], - ) - def test_parse_extra_markers(self, marker_string, expected): - result = MarkerExtraParser.get_extra_markers(marker_string) - assert bool(result) == expected - - -class TestExtraMarkerCleaner(object): - @pytest.mark.parametrize( - ("markers", "value"), - [ - ( - [((Variable('extra')), Op('=='), Value('Security'),)], - 'security' - ), - - ], - ) - def test_clean_marker_extras(self, markers, value): - cleaner = MarkerExtraCleaner() - result = cleaner.clean_marker_extras(markers) - assert result[0][2].value == value - - @pytest.mark.parametrize( - ("markers", "value"), - [ - ( - ((Variable('extra')), Op('=='), Value('Security'),), - 'security' - ), - - ], - ) - def test_clean_marker_extra(self, markers, value): - cleaner = MarkerExtraCleaner() - result = cleaner._clean_marker_extra(markers) - assert result[2].value == value - - @pytest.mark.parametrize( - ("markers", "locations"), - [ - ( - ((Variable('extra')), Op('=='), Value('Security'),), - [0] - ), - ( - ((Variable('extra')), Op('is'), Value('Security'),), - [0] - ), - ( - ((Variable('extra')), Op('<'), Value('Security'),), - [] - ), - ( - ( - (Variable('extra')), - Op('=='), - Value('Security'), - (Variable('extra')), - Op('=='), - Value('Security') - ), - [0, 3] - ), - ( - ( - (Variable('extra')), - Op('=='), - (Variable('extra')), - (Variable('extra')), - Op('=='), - Value('Security'), - ), - [3] - ), - ( - ((Variable('security')), Op('<'), Value('extra'),), - [] - ), - ( - ((Variable('extra')), Value('Security'),), - [] - ), - ( - ((Variable('security')), Value('extra'), Op('<'), ), - [] - ), - ( - ((Variable('security')), Op('<'), Op('<'),), - [] - ), - ( - tuple(), - [] - ), - ( - ( - (Variable('extra')), - Op('=='), - Value('Security'), - Value('security') - ), - [0] - ), - ], - ) - def test_get_extra_index_location(self, markers, locations): - cleaner = MarkerExtraCleaner() - result = cleaner._get_extra_index_location(markers) - assert result == locations - - @pytest.mark.parametrize( - ("obj", "object_types", "attribute_names", - "attribute_values", "expect"), - [ - ( - Variable('extra'), - Variable, - 'value', - 'extra', - True - ), - ( - Variable('extra'), - (Variable, Marker), - 'value', - 'extra', - True - ), - ( - Variable('extra'), - Variable, - ('value', 'extra'), - 'extra', - True - ), - ( - Variable('extra'), - Variable, - 'value', - ('extra', 'bad value'), - True - ), - - ], - ) - def test_check_attribute( - self, - obj, - object_types, - attribute_names, - attribute_values, - expect - ): - cleaner = MarkerExtraCleaner() - result = cleaner.check_attribute( - obj, object_types, attribute_names, attribute_values - ) - assert result == expect From 4cbca83b5a1c1921e336c45797a1bc788780bb8e Mon Sep 17 00:00:00 2001 From: Gabriel Curio Date: Mon, 11 Dec 2017 18:19:49 -0600 Subject: [PATCH 03/10] Remove unused utilities. --- packaging/utils.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/packaging/utils.py b/packaging/utils.py index 90461bac..8fc1c5f9 100644 --- a/packaging/utils.py +++ b/packaging/utils.py @@ -5,7 +5,6 @@ import re -from .version import Version, InvalidVersion _canonicalize_regex = re.compile(r"[-_.]+") @@ -22,23 +21,3 @@ def safe_extra(extra): and the result is always lowercased. """ return re.sub('[^A-Za-z0-9.-]+', '_', extra).lower() - - -def safe_name(name): - """Convert an arbitrary string to a standard distribution name - - Any runs of non-alphanumeric/. characters are replaced with a single '-'. - """ - return re.sub('[^A-Za-z0-9.]+', '-', name) - - -def safe_version(version): - """ - Convert an arbitrary string to a standard version string - """ - try: - # normalize the version - return str(Version(version)) - except InvalidVersion: - version = version.replace(' ', '.') - return re.sub('[^A-Za-z0-9.]+', '-', version) From ee769ee514a5454cf04554bc172db79e814a1c5c Mon Sep 17 00:00:00 2001 From: Gabriel Curio Date: Mon, 11 Dec 2017 18:41:13 -0600 Subject: [PATCH 04/10] Marker v Marker Extra difference made by pyparser. --- packaging/markers.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/packaging/markers.py b/packaging/markers.py index 4061c532..eb94dac6 100644 --- a/packaging/markers.py +++ b/packaging/markers.py @@ -160,6 +160,8 @@ def serialize(self): MARKER_EXTRA_EXPR << MARKER_EXTRA_ATOM + ZeroOrMore(BOOLOP + MARKER_EXTRA_EXPR) MARKER_EXTRA = stringStart + MARKER_EXTRA_EXPR + stringEnd +MARKER = MARKER_EXTRA | MARKER + def _coerce_parse_result(results): if isinstance(results, ParseResults): @@ -296,18 +298,11 @@ class Marker(object): def __init__(self, marker): try: - self._markers = _coerce_parse_result( - MARKER_EXTRA.parseString(marker) - ) - except ParseException: - try: - self._markers = _coerce_parse_result( - MARKER.parseString(marker) - ) - except ParseException as e2: - err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( - marker, marker[e2.loc:e2.loc + 8]) - raise InvalidMarker(err_str) + self._markers = _coerce_parse_result(MARKER.parseString(marker)) + except ParseException as e: + err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( + marker, marker[e.loc:e.loc + 8]) + raise InvalidMarker(err_str) def __str__(self): return _format_marker(self._markers) From ba7a1557eb3fae120061a019583e25f6c6e6ed1d Mon Sep 17 00:00:00 2001 From: Gabriel Curio Date: Mon, 11 Dec 2017 18:45:37 -0600 Subject: [PATCH 05/10] Fix marker extra operators. --- packaging/markers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/markers.py b/packaging/markers.py index eb94dac6..d9965e23 100644 --- a/packaging/markers.py +++ b/packaging/markers.py @@ -118,7 +118,7 @@ def serialize(self): MARKER_EXTRA_VARIABLE = L('extra') MARKER_EXTRA_VARIABLE.setParseAction(lambda s, l, t: Variable(t[0])) -MARKER_EXTRA_OP = (L("==") | L("===")) | L("not in") | L("in") +MARKER_EXTRA_OP = (L("==") | L("===")) | L("!=") MARKER_EXTRA_OP.setParseAction(lambda s, l, t: Op(t[0])) From 72cf54397fe519e092824543f3ce271dc6c1a23a Mon Sep 17 00:00:00 2001 From: Gabriel Curio Date: Mon, 11 Dec 2017 18:47:54 -0600 Subject: [PATCH 06/10] Move marker extra definitions for clarity. --- packaging/markers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packaging/markers.py b/packaging/markers.py index d9965e23..587911d2 100644 --- a/packaging/markers.py +++ b/packaging/markers.py @@ -116,11 +116,6 @@ def serialize(self): L("<") ) -MARKER_EXTRA_VARIABLE = L('extra') -MARKER_EXTRA_VARIABLE.setParseAction(lambda s, l, t: Variable(t[0])) -MARKER_EXTRA_OP = (L("==") | L("===")) | L("!=") -MARKER_EXTRA_OP.setParseAction(lambda s, l, t: Op(t[0])) - MARKER_OP = VERSION_CMP | L("not in") | L("in") MARKER_OP.setParseAction(lambda s, l, t: Op(t[0])) @@ -144,6 +139,11 @@ def serialize(self): MARKER = stringStart + MARKER_EXPR + stringEnd +MARKER_EXTRA_VARIABLE = L('extra') +MARKER_EXTRA_VARIABLE.setParseAction(lambda s, l, t: Variable(t[0])) +MARKER_EXTRA_OP = (L("==") | L("===")) | L("!=") +MARKER_EXTRA_OP.setParseAction(lambda s, l, t: Op(t[0])) + MARKER_EXTRA_VALUE = QuotedString("'") | QuotedString('"') MARKER_EXTRA_VALUE.setParseAction(lambda s, l, t: Value(safe_extra(t[0]))) From 65dd17614b583e003a03f3d034fb8abd25803c8b Mon Sep 17 00:00:00 2001 From: Gabriel Curio Date: Mon, 11 Dec 2017 18:49:07 -0600 Subject: [PATCH 07/10] remove whitespace. --- packaging/markers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packaging/markers.py b/packaging/markers.py index 587911d2..8fd39428 100644 --- a/packaging/markers.py +++ b/packaging/markers.py @@ -116,7 +116,6 @@ def serialize(self): L("<") ) - MARKER_OP = VERSION_CMP | L("not in") | L("in") MARKER_OP.setParseAction(lambda s, l, t: Op(t[0])) @@ -156,7 +155,6 @@ def serialize(self): MARKER_EXTRA_GROUP = Group(LPAREN + MARKER_EXTRA_EXPR + RPAREN) MARKER_EXTRA_ATOM = MARKER_EXTRA_ITEM | MARKER_EXTRA_GROUP - MARKER_EXTRA_EXPR << MARKER_EXTRA_ATOM + ZeroOrMore(BOOLOP + MARKER_EXTRA_EXPR) MARKER_EXTRA = stringStart + MARKER_EXTRA_EXPR + stringEnd From f666e0b398d9fce01f8e67b2712be54cb75ff24f Mon Sep 17 00:00:00 2001 From: Gabriel Curio Date: Mon, 11 Dec 2017 21:39:01 -0600 Subject: [PATCH 08/10] Better parsing of marker extras. --- packaging/markers.py | 35 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/packaging/markers.py b/packaging/markers.py index 8fd39428..3120a8b0 100644 --- a/packaging/markers.py +++ b/packaging/markers.py @@ -126,7 +126,18 @@ def serialize(self): MARKER_VAR = VARIABLE | MARKER_VALUE -MARKER_ITEM = Group(MARKER_VAR + MARKER_OP + MARKER_VAR) +# Special parsing rules apply for markers that evaluate extras. +EXTRA_VARIABLE = L('extra') +EXTRA_VARIABLE.setParseAction(lambda s, l, t: Variable(t[0])) + +EXTRA_OP = (L("==") | L("===")) | L("!=") +EXTRA_OP.setParseAction(lambda s, l, t: Op(t[0])) + +EXTRA_VALUE = QuotedString("'") | QuotedString('"') +EXTRA_VALUE.setParseAction(lambda s, l, t: Value(safe_extra(t[0]))) + +MARKER_ITEM = Group(EXTRA_VARIABLE + EXTRA_OP + EXTRA_VALUE) | \ + Group(MARKER_VAR + MARKER_OP + MARKER_VAR) MARKER_ITEM.setParseAction(lambda s, l, t: tuple(t[0])) LPAREN = L("(").suppress() @@ -138,28 +149,6 @@ def serialize(self): MARKER = stringStart + MARKER_EXPR + stringEnd -MARKER_EXTRA_VARIABLE = L('extra') -MARKER_EXTRA_VARIABLE.setParseAction(lambda s, l, t: Variable(t[0])) -MARKER_EXTRA_OP = (L("==") | L("===")) | L("!=") -MARKER_EXTRA_OP.setParseAction(lambda s, l, t: Op(t[0])) - -MARKER_EXTRA_VALUE = QuotedString("'") | QuotedString('"') -MARKER_EXTRA_VALUE.setParseAction(lambda s, l, t: Value(safe_extra(t[0]))) - -MARKER_EXTRA_ITEM = Group( - MARKER_EXTRA_VARIABLE + MARKER_EXTRA_OP + MARKER_EXTRA_VALUE -) -MARKER_EXTRA_ITEM.setParseAction(lambda s, l, t: tuple(t[0])) -MARKER_EXTRA_EXPR = Forward() - -MARKER_EXTRA_GROUP = Group(LPAREN + MARKER_EXTRA_EXPR + RPAREN) -MARKER_EXTRA_ATOM = MARKER_EXTRA_ITEM | MARKER_EXTRA_GROUP - -MARKER_EXTRA_EXPR << MARKER_EXTRA_ATOM + ZeroOrMore(BOOLOP + MARKER_EXTRA_EXPR) -MARKER_EXTRA = stringStart + MARKER_EXTRA_EXPR + stringEnd - -MARKER = MARKER_EXTRA | MARKER - def _coerce_parse_result(results): if isinstance(results, ParseResults): From 7edbcef329b8eb95912d8e01d0b9be21c8fdd32b Mon Sep 17 00:00:00 2001 From: Gabriel Curio Date: Tue, 12 Dec 2017 08:35:17 -0600 Subject: [PATCH 09/10] Minor refactoring for clarity. No functional changes. --- packaging/markers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packaging/markers.py b/packaging/markers.py index 3120a8b0..21d7d55a 100644 --- a/packaging/markers.py +++ b/packaging/markers.py @@ -126,6 +126,8 @@ def serialize(self): MARKER_VAR = VARIABLE | MARKER_VALUE +MARKER_ITEM = Group(MARKER_VAR + MARKER_OP + MARKER_VAR) + # Special parsing rules apply for markers that evaluate extras. EXTRA_VARIABLE = L('extra') EXTRA_VARIABLE.setParseAction(lambda s, l, t: Variable(t[0])) @@ -136,8 +138,9 @@ def serialize(self): EXTRA_VALUE = QuotedString("'") | QuotedString('"') EXTRA_VALUE.setParseAction(lambda s, l, t: Value(safe_extra(t[0]))) -MARKER_ITEM = Group(EXTRA_VARIABLE + EXTRA_OP + EXTRA_VALUE) | \ - Group(MARKER_VAR + MARKER_OP + MARKER_VAR) +EXTRA_ITEM = Group(EXTRA_VARIABLE + EXTRA_OP + EXTRA_VALUE) + +MARKER_ITEM = EXTRA_ITEM | MARKER_ITEM MARKER_ITEM.setParseAction(lambda s, l, t: tuple(t[0])) LPAREN = L("(").suppress() From b501c05b760836fc89d65340dfdf5bf7a615a601 Mon Sep 17 00:00:00 2001 From: Gabriel Curio Date: Tue, 12 Dec 2017 13:52:09 -0600 Subject: [PATCH 10/10] All marker extra environment args should be normalized prior to evaluation. --- packaging/markers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packaging/markers.py b/packaging/markers.py index 21d7d55a..b431e75a 100644 --- a/packaging/markers.py +++ b/packaging/markers.py @@ -312,5 +312,9 @@ def evaluate(self, environment=None): current_environment = default_environment() if environment is not None: current_environment.update(environment) + if current_environment.get('extra'): + current_environment['extra'] = safe_extra( + current_environment['extra'] + ) return _evaluate_markers(self._markers, current_environment)