New dependencies handling

.github/workflows/test.yml

@@ -38,6 +38,7 @@ jobs:
     - name: Unit tests
       run: |
         pytest tests
+        pytest --doctest-modules komodo
 
     - name: Lint examples
       run: |

ci/repository.yml

@@ -3,16 +3,10 @@ numpy:
     source: pypi
     make: pip
     maintainer: ci
-    depends:
-      - setuptools
-      - python
   1.23.5:
     source: pypi
     make: pip
     maintainer: ci
-    depends:
-      - setuptools
-      - python
 
 python:
   3-builtin:
@@ -26,14 +20,9 @@ setuptools:
     source: pypi
     make: pip
     maintainer: ci
-    depends:
-      - wheel
-      - python
 
 wheel:
   0.42.0:
     source: pypi
     make: pip
     maintainer: ci
-    depends:
-      - python

komodo/build.py

@@ -4,50 +4,13 @@
 import os
 import re
 import stat
-import sys
 from pathlib import Path
-from typing import Dict, List, Optional, Set
+from typing import Dict
 
 import requests
 
 from komodo.shell import pushd, shell
 
-
-def full_dfs(
-    all_packages: Dict[str, str],
-    repo: Dict[str, Dict],
-    packages_to_check: Optional[List[str]] = None,
-) -> List[str]:
-    if packages_to_check is None:
-        packages_to_check = list(all_packages.keys())
-    all_packages_set = set(all_packages)
-
-    def dfs(package_name: str, pkg_order: List[str], visited: Set[str]):
-        if package_name in visited:
-            return
-        visited.add(package_name)
-        ver = all_packages[package_name]
-        dependencies = repo[package_name][ver].get("depends", [])
-        missing_depends = set(dependencies) - all_packages_set
-        if missing_depends:
-            print(
-                "error: "
-                + ",".join(missing_depends)
-                + f" required as dependency for {package_name}, is not in distribution",
-                file=sys.stderr,
-            )
-            sys.exit(1)
-        for dep in dependencies:
-            dfs(dep, pkg_order, visited)
-        pkg_order.append(package_name)
-
-    visited = set()
-    pkg_order = []
-    for package in packages_to_check:
-        dfs(package, pkg_order, visited)
-    return pkg_order
-
-
 # When running cmake we pass the option -DDEST_PREFIX=fakeroot, this is an
 # absolute hack to be able to build opm-common and sunbeam with the ~fakeroot
 # implementation used by komodo.
@@ -232,8 +195,7 @@ def make(
     pip="pip",
     fakeroot=".",
 ):
-    pkgorder = full_dfs(pkgs, repo)
-    assert len(set(pkgorder)) == len(pkgs)
+    pkgorder = list(pkgs.keys())
     fakeprefix = fakeroot + prefix
     shell(["mkdir -p", fakeprefix])
     prefix = os.path.abspath(prefix)

komodo/check_unused_package.py

@@ -4,38 +4,39 @@
 import os
 import sys
 
-from komodo.build import full_dfs
 from komodo.prettier import load_yaml
 from komodo.yaml_file_types import ReleaseFile, RepositoryFile
 
+from .pypi_dependencies import PypiDependencies
+
 
 def check_for_unused_package(
     release_file: ReleaseFile, package_status_file: str, repository: RepositoryFile
 ):
     package_status = load_yaml(package_status_file)
-    private_packages = [
-        pkg
-        for pkg in release_file.content
-        if package_status[pkg]["visibility"] == "private"
-    ]
     public_and_plugin_packages = [
-        pkg
-        for pkg in release_file.content
+        (pkg, version)
+        for pkg, version in release_file.content.items()
         if package_status[pkg]["visibility"] in ("public", "private-plugin")
     ]
-    public_and_plugin_dependencies = full_dfs(
-        release_file.content,
-        repository.content,
-        public_and_plugin_packages,
-    )
-    diff_packages = set(private_packages).difference(
-        set(public_and_plugin_dependencies)
-    )
-    if diff_packages:
+    python_version = release_file.content["python"]
+    # For pypi we need to change '3.8.6-builtin' -> '3.8.6'
+    python_version = python_version[: python_version.rindex("-")]
+    dependencies = PypiDependencies(release_file.content, python_version=python_version)
+    for name, version in release_file.content.items():
+        metadata = repository.content.get(name, {}).get(version, {})
+        if metadata.get("source") != "pypi":
+            dependencies.add_user_specified(name, metadata.get("depends", []))
+    unused_private_packages = set(
+        pkg
+        for pkg in release_file.content
+        if package_status[pkg]["visibility"] == "private"
+    ).difference(dependencies.used_packages(public_and_plugin_packages))
+    if unused_private_packages:
         print(
-            f"The following {len(diff_packages)} private packages are not dependencies of any public or private-plugin packages:"
+            f"The following {len(unused_private_packages)} private packages are not dependencies of any public or private-plugin packages:"
         )
-        print(", ".join(sorted(list(diff_packages))))
+        print(", ".join(sorted(list(unused_private_packages))))
         print(
             "If you have added or removed any packages check that the dependencies in repository.yml are correct."
         )

komodo/lint.py

@@ -8,7 +8,8 @@
 
 from packaging.version import parse
 
-from komodo.yaml_file_types import KomodoException, ReleaseFile, RepositoryFile
+from .pypi_dependencies import PypiDependencies
+from .yaml_file_types import KomodoException, ReleaseFile, RepositoryFile
 
 KomodoError = namedtuple(
     "KomodoError",
@@ -66,8 +67,12 @@ def lint_version_numbers(package, version, repo):
     return None
 
 
-def lint(release_file: ReleaseFile, repository_file: RepositoryFile):
-    maintainers, deps, versions = [], [], []
+def lint(
+    release_file: ReleaseFile,
+    repository_file: RepositoryFile,
+    check_dependencies: bool = False,
+) -> Report:
+    maintainers, versions = [], []
     for package_name, package_version in release_file.content.items():
         try:
             lint_maintainer = repository_file.lint_maintainer(
@@ -84,27 +89,55 @@ def lint(release_file: ReleaseFile, repository_file: RepositoryFile):
             )
             if lint_version_number:
                 versions.append(lint_version_number)
-            missing = []
-            repository_file_package_version_data = repository_file.content.get(
-                package_name,
-            ).get(package_version)
-            for dependency in repository_file_package_version_data.get("depends", []):
-                if dependency not in release_file.content:
-                    missing.append(dependency)
-            if missing:
-                deps.append(
-                    _komodo_error(
-                        package=package_name,
-                        version=package_version,
-                        depends=missing,
-                        err=(
-                            f"{MISSING_DEPENDENCY} for {package_name} {package_version}"
-                        ),
-                    ),
-                )
         except KomodoException as komodo_exception:
             maintainers.append(komodo_exception.error)
 
+    if check_dependencies:
+        pypi_dependencies = {
+            name: version
+            for name, version in release_file.content.items()
+            if repository_file.content.get(name, {}).get(version, {}).get("source")
+            == "pypi"
+        }
+
+        python_version = release_file.content["python"]
+        # For pypi we need to change '3.8.6-builtin' -> '3.8.6'
+        python_version = python_version[: python_version.rindex("-")]
+        dependencies = PypiDependencies(
+            pypi_dependencies, python_version=python_version
+        )
+        for name, version in release_file.content.items():
+            if (
+                repository_file.content.get(name, {}).get(version, {}).get("source")
+                != "pypi"
+            ):
+                if (
+                    name not in repository_file.content
+                    or version not in repository_file.content[name]
+                ):
+                    raise ValueError(
+                        f"Missing package in repository file: {name}=={version}"
+                    )
+                depends = repository_file.content[name][version].get("depends", [])
+                if depends:
+                    dependencies.add_user_specified(
+                        name, repository_file.content[name][version]["depends"]
+                    )
+
+        failed_requirements = dependencies.failed_requirements()
+        if failed_requirements:
+            deps = [
+                _komodo_error(
+                    err="Failed requirements ",
+                    depends=[str(r) for r in failed_requirements],
+                )
+            ]
+        else:
+            deps = []
+        dependencies.dump_cache()
+    else:
+        deps = []
+
     return Report(
         release_name=[],
         maintainers=maintainers,
@@ -135,31 +168,38 @@ def get_args():
         dest="loglevel",
         const=logging.INFO,
     )
+    parser.add_argument(
+        "--check-pypi-dependencies",
+        dest="check_pypi_dependencies",
+        help="Checks package metadata",
+        action="store_true",
+        default=False,
+    )
     return parser.parse_args()
 
 
 def lint_main():
     args = get_args()
     logging.basicConfig(format="%(message)s", level=args.loglevel)
 
-    try:
-        report = lint(args.packagefile, args.repofile)
-        maintainers, deps, versions = (
-            report.maintainers,
-            report.dependencies,
-            report.versions,
-        )
-    except ValueError as err:
-        sys.exit(str(err))
+    report = lint(
+        args.packagefile, args.repofile, check_dependencies=args.check_pypi_dependencies
+    )
+    maintainers, deps, versions = (
+        report.maintainers,
+        report.dependencies,
+        report.versions,
+    )
     print(f"{len(maintainers)} packages")
     if not any(err.err for err in maintainers + deps + versions):
         print("No errors found")
         sys.exit(0)
 
     for err in maintainers + deps + versions:
         if err.err:
-            dep = f": {', '.join(err.depends)}" if err.depends else ""
-            print(f"{err.err}{dep}")
+            print(f"{err.err}")
+            if err.depends:
+                print("\n  ".join(err.depends))
 
     if not any(err.err for err in maintainers + deps):
         sys.exit(0)  # currently we allow erronous version numbers