Merge pull request #59 from sarugaku/fix-marker-duplication

Fix marker duplication

setup.cfg

@@ -37,8 +37,9 @@ python_requires = >=2.7,!=3.0,!=3.1,!=3.2,!=3.3
 setup_requires = setuptools>=36.2.2
 install_requires =
     appdirs
+    backports.functools_lru_cache; python_version <= "3.4"
     cached-property
-    distlib>=0.2.8
+    distlib
     installer
     packagebuilder
     packaging
@@ -55,11 +56,14 @@ install_requires =
 pack =
     invoke
     parver
+virtualenv =
+    mork
 tests =
     pytest-xdist
     pytest-timeout
     pytest-cov
     pytest-mock
+    virtualenv
     pytest
 
 [options.entry_points]
@@ -104,8 +108,6 @@ norecursedirs = .* build dist news tasks docs
 filterwarnings =
     ignore::DeprecationWarning
     ignore::PendingDeprecationWarning
-markers =
-    needs_internet: test cases that need access to the internet
 
 [build-system]
 requires = ["setuptools", "wheel"]

src/passa/internals/markers.py

@@ -3,27 +3,56 @@
 from __future__ import absolute_import, unicode_literals
 
 from packaging.markers import Marker
+from .specifiers import PySpecs, gen_marker
+import six
+import distlib.markers
+
+six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc"))
+from six.moves import Mapping, reduce
+
+try:
+    from functools import lru_cache
+except ImportError:
+    from backports.functools_lru_cache import lru_cache
+
+
+def _ensure_marker(marker):
+    if not isinstance(marker, Marker):
+        return Marker(str(marker))
+    return marker
 
 
 def _strip_extra(elements):
-    """Remove the "extra == ..." operands from the list.
+    """Remove the "extra == ..." operands from the list."""
+
+    return _strip_marker_elem("extra", elements)
+
+
+def _strip_pyversion(elements):
+    return _strip_marker_elem("python_version", elements)
+
+
+def _strip_marker_elem(elem_name, elements):
+    """Remove the supplied element from the marker.
 
     This is not a comprehensive implementation, but relies on an important
-    characteristic of metadata generation: The "extra == ..." operand is always
+    characteristic of metadata generation: The element's operand is always
     associated with an "and" operator. This means that we can simply remove the
     operand and the "and" operator associated with it.
     """
+
     extra_indexes = []
+    preceding_operators = ["and"] if elem_name == "extra" else ["and", "or"]
     for i, element in enumerate(elements):
         if isinstance(element, list):
-            cancelled = _strip_extra(element)
+            cancelled = _strip_marker_elem(elem_name, element)
             if cancelled:
                 extra_indexes.append(i)
-        elif isinstance(element, tuple) and element[0].value == "extra":
+        elif isinstance(element, tuple) and element[0].value == elem_name:
             extra_indexes.append(i)
     for i in reversed(extra_indexes):
         del elements[i]
-        if i > 0 and elements[i - 1] == "and":
+        if i > 0 and elements[i - 1] in preceding_operators:
             # Remove the "and" before it.
             del elements[i - 1]
         elif elements:
@@ -34,6 +63,19 @@ def _strip_extra(elements):
     return (not elements)
 
 
+def _get_stripped_marker(marker, strip_func):
+    """Build a new marker which is cleaned according to `strip_func`"""
+
+    if not marker:
+        return None
+    marker = _ensure_marker(marker)
+    elements = marker._markers
+    strip_func(elements)
+    if elements:
+        return marker
+    return None
+
+
 def get_without_extra(marker):
     """Build a new marker without the `extra == ...` part.
 
@@ -43,18 +85,18 @@ def get_without_extra(marker):
     This could return `None` if the `extra == ...` part is the only one in the
     input marker.
     """
-    # TODO: Why is this very deep in the internals? Why is a better solution
-    # implementing it yourself when someone is already maintaining a codebase
-    # for this? It's literally a grammar implementation that is required to
-    # meet the demands of a pep... -d
-    if not marker:
-        return None
-    marker = Marker(str(marker))
-    elements = marker._markers
-    _strip_extra(elements)
-    if elements:
-        return marker
-    return None
+
+    return _get_stripped_marker(marker, _strip_extra)
+
+
+def get_without_pyversion(marker):
+    """Built a new marker without the `python_version` part.
+
+    This could return `None` if the `python_version` section is the only section in the
+    marker.
+    """
+
+    return _get_stripped_marker(marker, _strip_pyversion)
 
 
 def _markers_collect_extras(markers, collection):
@@ -70,34 +112,140 @@ def _markers_collect_extras(markers, collection):
             _markers_collect_extras(el, collection)
 
 
+def _markers_collect_pyversions(markers, collection):
+    local_collection = []
+    marker_format_str = "{0}"
+    for i, el in enumerate(reversed(markers)):
+        if isinstance(el, tuple) and el[0].value == "python_version":
+            new_marker = str(gen_marker(el))
+            local_collection.append(marker_format_str.format(new_marker))
+        elif isinstance(el, list):
+            _markers_collect_pyversions(el, local_collection)
+    if local_collection:
+        # local_collection = "{0}".format(" ".join(local_collection))
+        collection.extend(local_collection)
+
+
+@lru_cache(maxsize=128)
 def get_contained_extras(marker):
     """Collect "extra == ..." operands from a marker.
 
     Returns a list of str. Each str is a speficied extra in this marker.
     """
     if not marker:
         return set()
-    marker = Marker(str(marker))
     extras = set()
+    marker = _ensure_marker(marker)
     _markers_collect_extras(marker._markers, extras)
     return extras
 
 
+def get_contained_pyversions(marker):
+    """Collect all `python_version` operands from a marker.
+
+    Returns a set of :class:`~passa.internals.specifiers.PySpecs` instances.
+    """
+
+    collection = []
+    if not marker:
+        return set()
+    marker = _ensure_marker(marker)
+    # Collect the (Variable, Op, Value) tuples and string joiners from the marker
+    _markers_collect_pyversions(marker._markers, collection)
+    marker_str = " ".join(collection)
+    if not marker_str:
+        return set()
+    # Use the distlib dictionary parser to create a dictionary 'trie' which is a bit
+    # easier to reason about
+    marker_dict = distlib.markers.parse_marker(marker_str)[0]
+    version_set = set()
+    pyversions = parse_marker_dict(marker_dict)
+    if isinstance(pyversions, set):
+        version_set.update(pyversions)
+    else:
+        version_set.add(pyversions)
+    # Each distinct element in the set was separated by an "and" operator in the marker
+    # So we will need to reduce them with an intersection here rather than a union
+    # in order to find the boundaries
+    versions = reduce(lambda x, y: x & y, version_set)
+    if not versions:
+        return PySpecs()
+    return versions
+
+
 def _markers_contains_extra(markers):
     # Optimization: the marker element is usually appended at the end.
+    return _markers_contains_key(markers, "extra")
+
+
+def _markers_contains_pyversion(markers):
+    return _markers_contains_key(markers, "python_version")
+
+
+def _markers_contains_key(markers, key):
     for element in reversed(markers):
-        if isinstance(element, tuple) and element[0].value == "extra":
+        if isinstance(element, tuple) and element[0].value == key:
             return True
         elif isinstance(element, list):
-            if _markers_contains_extra(element):
+            if _markers_contains_key(element, key):
                 return True
     return False
 
 
+@lru_cache(maxsize=128)
 def contains_extra(marker):
     """Check whehter a marker contains an "extra == ..." operand.
     """
     if not marker:
         return False
-    marker = Marker(str(marker))
+    marker = _ensure_marker(marker)
     return _markers_contains_extra(marker._markers)
+
+
+@lru_cache(maxsize=128)
+def contains_pyversion(marker):
+    """Check whether a marker contains a python_version operand.
+    """
+
+    if not marker:
+        return False
+    marker = _ensure_marker(marker)
+    return _markers_contains_pyversion(marker._markers)
+
+
+def parse_marker_dict(marker_dict):
+    op = marker_dict["op"]
+    lhs = marker_dict["lhs"]
+    rhs = marker_dict["rhs"]
+    # This is where the spec sets for each side land if we have an "or" operator
+    sides = set()
+    # And if we hit the end of the parse tree we use this format string to make a marker
+    format_string = "{lhs} {op} {rhs}"
+    # Essentially we will iterate over each side of the parsed marker if either one is
+    # A mapping instance (i.e. a dictionary) and recursively parse and reduce the specset
+    # Union the "and" specs, intersect the "or"s to find the most appropriate range
+    if any(issubclass(type(side), Mapping) for side in (lhs, rhs)):
+        for side in (lhs, rhs):
+            specs = PySpecs()
+            if issubclass(type(side), Mapping):
+                specs.add(parse_marker_dict(side))
+            else:
+                # This is the easiest way to go from a string to a PySpec instance
+                specs.add(PySpecs.from_marker(Marker(side)))
+            sides.add(specs)
+        if op == "and":
+            # When we are "and"-ing things together, it probably makes the most sense
+            # to reduce them here into a single PySpec instance
+            if not sides:
+                sides = [lhs, rhs]
+            sides = reduce(lambda x, y: x | y, sides)
+            if not sides:
+                return sides
+            return PySpecs.from_marker(Marker(str(sides)))
+        # Actually when we "or" things as well we can also just turn them into a reduced
+        # set using this logic now
+        return reduce(lambda x, y: x & y, sides)
+    else:
+        # At the tip of the tree we are dealing with strings all around and they just need
+        # to be smashed together
+        return PySpecs.from_marker(Marker(format_string.format(**marker_dict)))