Cut off and minor adjustmets

tests/data/graph/db_1.yaml

@@ -25,7 +25,6 @@ b:
           resolved:
             - '3.1.0'
             - '4.1.0'
-            - '5.1.0'
   2.0.0:
     -  index_url: aicoe-index
        depends_on:

tests/python/test_dependency_graph.py

@@ -38,10 +38,20 @@ def test_construct_project0(self):
 
         dependency_graph = DependencyGraph.from_project(graph, project)
         count = 0
+        stacks = set()
         for reasoning, generated_project in dependency_graph.walk(self.always_true):
             assert isinstance(generated_project, Project)
+            generated_project = generated_project.to_dict()
+            pipfile_lock = generated_project['requirements_locked']
+            stack = tuple((k, v['version'], v['index']) for k, v in pipfile_lock['default'].items())
+            assert stack not in stacks
+            stacks.add(stack)
+
+            assert not pipfile_lock['develop']
+
             assert isinstance(reasoning, tuple)
             assert reasoning == (True, [{"foo": "bar"}])
             count += 1
 
-        assert count == 9
+        # 7 stacks for a/b, 8 stacks for b/d => 56 as they do not interfere
+        assert count == 56

tests/python/test_solver.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+    #!/usr/bin/env python3
 # thoth-adviser
 # Copyright(C) 2018 Fridolin Pokorny
 #
@@ -156,3 +156,17 @@ def test_db_0_package_multiple(self, graph):
             "==2.0.0",
             "==3.0.0",
         }
+
+    @pytest.mark.parametrize("graph", [MockedGraphDatabase("db_0.yaml")])
+    def test_db_0_multiple_times_error(self, graph):
+        """Check that resolving can resolve multiple Python packages."""
+        from thoth.solver.python.base import SolverException
+        with pytest.raises(SolverException):
+            PythonPackageGraphSolver(graph_db=graph).solve(
+                [
+                    PackageVersion(name="a", version="<=1.2.0", index=None, develop=False),
+                    PackageVersion(name="a", version=">1.0.0", index=None, develop=False),
+                ],
+                graceful=False,
+                all_versions=True,
+            )

thoth/adviser/python/advise.py

@@ -20,17 +20,12 @@
 
 import logging
 import heapq
-import operator
 import typing
 
 import attr
-import random
 
 from thoth.adviser import RuntimeEnvironment
-from thoth.solver.python.base import SolverException
 from thoth.python import Project
-from thoth.adviser.python import DECISISON_FUNCTIONS
-from thoth.adviser.python import DEFAULT_DECISION_FUNCTION
 from thoth.adviser.python import DependencyGraph
 from thoth.adviser.enums import RecommendationType
 from thoth.adviser.python.helpers import fill_package_digests_from_graph
@@ -65,6 +60,9 @@ class Adviser:
 
     count = attr.ib(type=int, default=None)
     limit = attr.ib(type=int, default=None)
+    recommendation_type = attr.ib(
+        type=RecommendationType, default=RecommendationType.STABLE
+    )
     _computed_stacks_heap = attr.ib(type=RuntimeEnvironment, default=attr.Factory(list))
     _visited = attr.ib(type=int, default=0)
 
@@ -74,7 +72,7 @@ def compute(
         project: Project,
         scoring_function: typing.Callable,
         dry_run: bool = False,
-    ) -> typing.List[Project]:
+    ) -> typing.Union[typing.List[Project], int]:
         """Compute recommendations for the given project."""
         dependency_graph = DependencyGraph.from_project(graph, project)
 
@@ -83,6 +81,7 @@ def compute(
                 scoring_function
             ):
                 score, reasoning = decision_function_result
+                reasoning.append({'score': score})
                 self._visited += 1
 
                 if dry_run:
@@ -104,6 +103,7 @@ def compute(
                 return self._visited
 
             # Sort computed stacks based on score and return them.
+            # TODO: heap pop (?)
             result = (
                 item.get_entries()
                 for item in sorted(self._computed_stacks_heap, reverse=True)
@@ -129,7 +129,9 @@ def compute_on_project(
         dry_run: bool = False
     ) -> list:
         """Compute recommendations for the given project, a syntax sugar for the compute method."""
-        instance = cls(count=count, limit=limit)
+        instance = cls(
+            count=count, limit=limit, recommendation_type=recommendation_type
+        )
 
         graph = GraphDatabase()
         graph.connect()
@@ -138,5 +140,6 @@ def compute_on_project(
             graph=graph,
             runtime_environment=runtime_environment,
             recommendation_type=recommendation_type,
+            python_version=project.python_version,
         )
         return instance.compute(graph, project, scoring_function, dry_run=dry_run)

thoth/adviser/python/decision.py

@@ -35,13 +35,15 @@ class DecisionFunction:
     graph = attr.ib(type=GraphDatabase)
     runtime_environment = attr.ib(type=RuntimeEnvironment)
 
-    def random_uniform(self, _: typing.Sequence[PackageVersion]):
+    @staticmethod
+    def random_uniform(_: typing.Sequence[PackageVersion]):
         """Retrieve a random stack."""
-        return random.getrandbits(1), None
+        return random.getrandbits(1), []
 
-    def everything(self, _: typing.Sequence[PackageVersion]):
+    @staticmethod
+    def everything(_: typing.Sequence[PackageVersion]):
         """Decide to include everything."""
-        return 1.0, None
+        return 1.0, []
 
     @classmethod
     def get_decision_function(

thoth/adviser/python/dependency_graph.py

@@ -36,7 +36,6 @@
 from collections import deque
 from functools import reduce
 from itertools import chain
-from itertools import chain
 from itertools import product
 import operator
 
@@ -165,11 +164,53 @@ def _prepare_direct_dependencies(
 
     @classmethod
     def _cut_off_dependencies(
-        cls, graph: GraphDatabase, transitive_paths: typing.List[dict]
+        cls,
+        graph: GraphDatabase,
+        transitive_paths: typing.List[dict],
+        core_packages: typing.Dict[str, dict],
+        allow_prereleases: bool,
     ) -> typing.List[dict]:
         """Cut off paths that have not relevant dependencies - dependencies we don't want to include in the stack."""
-        _LOGGER.debug("Cutting off not relevant paths")
-        return transitive_paths
+        result = []
+        for core_package_name, versions in core_packages.items():
+            if not versions:
+                continue
+
+            versions = [
+                (PackageVersion.parse_semantic_version(version), version)
+                for version in versions.keys()
+            ]
+            # Always pick the latest for now, later ask for observations.
+            version = sorted(versions, key=operator.itemgetter(0), reverse=True)[0]
+            index_url = core_packages[core_package_name][version[1]]
+            core_packages[core_package_name] = {version[1]: index_url}
+
+        for path in transitive_paths:
+            include_path = True
+            for package_tuple in path:
+
+                if package_tuple[0] in core_packages.keys() and (
+                    package_tuple[1] not in core_packages[package_tuple[0]]
+                    or package_tuple[2]
+                    not in core_packages[package_tuple[0]][package_tuple[1]]
+                ):
+                    _LOGGER.debug("Excluding the given stack due to core package %r", package_tuple)
+                    include_path = False
+                    break
+
+                if not allow_prereleases:
+                    semantic_version = PackageVersion.parse_semantic_version(
+                        package_tuple[1]
+                    )
+                    if semantic_version.build or semantic_version.prerelease:
+                        _LOGGER.debug("Excluding path with package %r as pre-releases were disabled", package_tuple)
+                        include_path = False
+                        break
+
+            if include_path:
+                result.append(path)
+
+        return result
 
     @staticmethod
     def _get_python_package_tuples(
@@ -181,7 +222,7 @@ def _get_python_package_tuples(
         seen_ids = set(ids_map.keys())
         unseen_ids = python_package_ids - seen_ids
 
-        _LOGGER.info(
+        _LOGGER.debug(
             "Retrieving package tuples for transitive dependencies (count: %d)",
             len(unseen_ids),
         )
@@ -221,9 +262,13 @@ def from_project(
         source_dependencies = {}
         # All paths that should be included for computed stacks.
         all_transitive_dependencies_to_include = []
+        # Core python packages as pip, setuptools, six - we need to reduce the amount of software stacks
+        # generated, pick the most suitable ones when cutting off. These packages are also pre-analyzed by our init-job.
+        core_packages = dict.fromkeys(("pip", "setuptools", "six"), {})
 
+        _LOGGER.info("Retrieving transitive dependencies of direct dependencies")
         for graph_item in all_direct_dependencies:
-            _LOGGER.info(
+            _LOGGER.debug(
                 "Retrieving transitive dependencies for %r in version %r from %r",
                 graph_item.package_version.name,
                 graph_item.package_version.locked_version,
@@ -246,10 +291,16 @@ def from_project(
             for entry in transitive_dependencies:
                 exclude = False
                 new_entry = []
-                for idx in range(0, len(entry), 2):
+                for idx in range(0, len(entry)):
                     item = entry[idx]
                     item = package_tuples_map[item]
                     package, version, index_url = item[0], item[1], item[2]
+
+                    if package in core_packages.keys():
+                        if version not in core_packages[package]:
+                            core_packages[package][version] = []
+                        core_packages[package][version].append(index_url)
+
                     new_entry.append((package, version, index_url))
                     direct_packages_of_this = [
                         dep
@@ -289,10 +340,15 @@ def from_project(
                 transitive_dependencies_to_include
             )
 
+        _LOGGER.info("Cutting off unwanted dependencies")
         all_transitive_dependencies_to_include = cls._cut_off_dependencies(
-            graph, all_transitive_dependencies_to_include
+            graph,
+            all_transitive_dependencies_to_include,
+            core_packages,
+            project.prereleases_allowed,
         )
 
+        _LOGGER.info("Creating in-memory hierarchical structures")
         for entry in all_transitive_dependencies_to_include:
             for idx in range(0, len(entry) - 1):
                 source_idx = idx
@@ -505,7 +561,7 @@ def walk(
                 decision_function_result = decision_function(
                     (graph_item.package_version for graph_item in state[0].values())
                 )
-                if decision_function_result[0]:
+                if decision_function_result[0] is not False:
                     _LOGGER.info(
                         "Decision function included the computed stack - result was %r",
                         decision_function_result[0],