Implement backjumping over unrelated states

news/113.feature.rst

@@ -0,0 +1 @@
+Implement backjumping to significantly speed up the resolution process by skipping over irrelevant parts of the resolution search space.

src/resolvelib/resolvers.py

@@ -266,8 +266,8 @@ def _attempt_to_pin_criterion(self, name):
         # end, signal for backtracking.
         return causes
 
-    def _backtrack(self):
-        """Perform backtracking.
+    def _backjump(self, causes):
+        """Perform backjumping.
 
         When we enter here, the stack is like this::
 
@@ -283,22 +283,44 @@ def _backtrack(self):
 
         Each iteration of the loop will:
 
-        1.  Discard Z.
-        2.  Discard Y but remember its incompatibility information gathered
+        1.  Identify Z. The incompatibility is not always caused by the latest state.
+            For example, given three requirements A, B and C, with dependencies
+            A1, B1 and C1, where A1 and B1 are incompatible: the last state
+            might be related to C, so we want to discard the previous state.
+        2.  Discard Z.
+        3.  Discard Y but remember its incompatibility information gathered
             previously, and the failure we're dealing with right now.
-        3.  Push a new state Y' based on X, and apply the incompatibility
+        4.  Push a new state Y' based on X, and apply the incompatibility
             information from Y to Y'.
-        4a. If this causes Y' to conflict, we need to backtrack again. Make Y'
+        5a. If this causes Y' to conflict, we need to backtrack again. Make Y'
             the new Z and go back to step 2.
-        4b. If the incompatibilities apply cleanly, end backtracking.
+        5b. If the incompatibilities apply cleanly, end backtracking.
         """
+        incompatible_deps = set(
+            [c.parent.name for c in causes if c.parent is not None]
+            + [c.requirement.name for c in causes]
+        )
         while len(self._states) >= 3:
             # Remove the state that triggered backtracking.
             del self._states[-1]
 
-            # Retrieve the last candidate pin and known incompatibilities.
-            broken_state = self._states.pop()
-            name, candidate = broken_state.mapping.popitem()
+            # Ensure to backtrack to a state that caused the incompatibility
+            incompatible_state = False
+            while not incompatible_state:
+                # Retrieve the last candidate pin and known incompatibilities.
+                try:
+                    broken_state = self._states.pop()
+                    name, candidate = broken_state.mapping.popitem()
+                except (IndexError, KeyError):
+                    raise ResolutionImpossible(causes)
+                current_dependencies = {
+                    self._p.identify(d)
+                    for d in self._p.get_dependencies(candidate)
+                }
+                incompatible_state = not current_dependencies.isdisjoint(
+                    incompatible_deps
+                )
+
             incompatibilities_from_broken = [
                 (k, list(v.incompatibilities))
                 for k, v in broken_state.criteria.items()
@@ -403,10 +425,10 @@ def resolve(self, requirements, max_rounds):
 
             if failure_causes:
                 causes = [i for c in failure_causes for i in c.information]
-                # Backtrack if pinning fails. The backtrack process puts us in
+                # Backjump if pinning fails. The backjump process puts us in
                 # an unpinned state, so we can work on it in the next round.
                 self._r.resolving_conflicts(causes=causes)
-                success = self._backtrack()
+                success = self._backjump(causes)
                 self.state.backtrack_causes[:] = causes
 
                 # Dead ends everywhere. Give up.

tests/test_resolvers.py

@@ -1,14 +1,17 @@
-from typing import (
-    Any,
-    Iterable,
-    Iterator,
-    List,
-    Mapping,
-    Sequence,
-    Set,
-    Tuple,
-    Union,
-)
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import (
+        Any,
+        Iterable,
+        Iterator,
+        List,
+        Mapping,
+        Sequence,
+        Set,
+        Tuple,
+        Union,
+    )
 
 import pytest
 from packaging.requirements import Requirement
@@ -21,12 +24,17 @@
     ResolutionImpossible,
     Resolver,
 )
-from resolvelib.resolvers import (
-    Criterion,
-    RequirementInformation,
-    RequirementsConflicted,
-    Resolution,
-)
+
+if TYPE_CHECKING:
+    from resolvelib.resolvers import (
+        Criterion,
+        RequirementInformation,
+        RequirementsConflicted,
+    )
+
+from collections import namedtuple
+
+from resolvelib.resolvers import Resolution
 
 
 def test_candidate_inconsistent_error():
@@ -115,10 +123,21 @@ def is_satisfied_by(self, requirement, candidate):
 
 
 def test_resolving_conflicts():
+    Candidate = namedtuple(
+        "Candidate", ["name", "version", "requirements"]
+    )  # name, version, requirements
+    _Requirement = namedtuple(
+        "Requirement", ["name", "versions"]
+    )  # name, versions
+    a1 = Candidate("a", 1, [_Requirement("q", {1})])
+    a2 = Candidate("a", 2, [_Requirement("q", {2})])
+    b = Candidate("b", 1, [_Requirement("q", {1})])
+    q1 = Candidate("q", 1, [])
+    q2 = Candidate("q", 2, [])
     all_candidates = {
-        "a": [("a", 1, [("q", {1})]), ("a", 2, [("q", {2})])],
-        "b": [("b", 1, [("q", {1})])],
-        "q": [("q", 1, []), ("q", 2, [])],
+        "a": [a1, a2],
+        "b": [b],
+        "q": [q1, q2],
     }
 
     class Reporter(BaseReporter):
@@ -136,20 +155,22 @@ def get_preference(self, **_):
             return 0
 
         def get_dependencies(self, candidate):
-            return candidate[2]
+            return candidate.requirements
 
         def find_matches(self, identifier, requirements, incompatibilities):
-            bad_versions = {c[1] for c in incompatibilities[identifier]}
+            bad_versions = {c.version for c in incompatibilities[identifier]}
             candidates = [
                 c
                 for c in all_candidates[identifier]
-                if all(c[1] in r[1] for r in requirements[identifier])
-                and c[1] not in bad_versions
+                if all(
+                    c.version in r.versions for r in requirements[identifier]
+                )
+                and c.version not in bad_versions
             ]
-            return sorted(candidates, key=lambda c: c[1], reverse=True)
+            return sorted(candidates, key=lambda c: c.version, reverse=True)
 
         def is_satisfied_by(self, requirement, candidate):
-            return candidate[1] in requirement[1]
+            return candidate.version in requirement.versions
 
     def run_resolver(*args):
         reporter = Reporter()
@@ -160,8 +181,12 @@ def run_resolver(*args):
         except ResolutionImpossible as e:
             return e.causes
 
-    backtracking_causes = run_resolver([("a", {1, 2}), ("b", {1})])
-    exception_causes = run_resolver([("a", {2}), ("b", {1})])
+    backtracking_causes = run_resolver(
+        [_Requirement("a", {1, 2}), _Requirement("b", {1})]
+    )
+    exception_causes = run_resolver(
+        [_Requirement("a", {2}), _Requirement("b", {1})]
+    )
     assert exception_causes == backtracking_causes
 
 
@@ -171,9 +196,10 @@ def test_pin_conflict_with_self(monkeypatch, reporter):
     Verify correct behavior of attempting to pin a candidate version that conflicts
     with a previously pinned (now invalidated) version for that same candidate (#91).
     """
-    Candidate = Tuple[  # noqa: F841
-        str, Version, Sequence[str]
-    ]  # name, version, requirements
+    if TYPE_CHECKING:
+        Candidate = Tuple[  # noqa: F841
+            str, Version, Sequence[str]
+        ]  # name, version, requirements
     all_candidates = {
         "parent": [("parent", Version("1"), ["child<2"])],
         "child": [