Python requirements.txt support (#56)

* Various refactoring to support Python requirements.txt with forward-looking infra for other formats.

* WIP.

* Refactor comparisons to support more complex specifications.

* Fix dependencies.

* Update integration tests.

* Add some coverage.

* Add Python support.

README.md

@@ -5,6 +5,7 @@
 A Python CLI to encourage (😅) people to update their dependencies!
 
 Supported package managers:
+* Python (requirements.txt)
 * Jsonnet
 * _More coming soon!_
 
@@ -51,6 +52,11 @@ The `rubrical.yaml` file is used to configure checks for your application.
 ```yaml
 version: 1
 package_managers:
+  - name: python
+    packages:
+      - name: Mopidy-Dirble
+        block: v1.2.1
+        warn: v1.2.2
   - name: jsonnet
     packages:
       - name: "xunleii/vector_jsonnet" # Name of the dependency

Taskfile.yaml

@@ -16,9 +16,10 @@ tasks:
     cmds:
       - poetry install
 
-  fix:
+  fmt:
     desc: Run linting and fix issues.
     cmds:
+      - poetry run black .
       - poetry run ruff rubrical tests --fix
 
   lint:

pyproject.toml

@@ -53,6 +53,7 @@ semver = "^2.13.0"
 pygithub = "^1.58.0"
 python-benedict = {extras = ["all"], version = "^0.30.0"}
 pyyaml = "^6.0"
+requirements-parser = "^0.5.0"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.0.0"

rubrical/comparisons/__init__.py

@@ -0,0 +1,24 @@
+from rubrical.enum import PackageCheck, PackageTypes
+from rubrical.schemas.configuration import PackageRequirement
+from rubrical.schemas.package import Package
+
+from . import semversion, string
+
+
+def check_package(
+    package_requirement: PackageRequirement, package: Package
+) -> PackageCheck:
+    result = None
+
+    if package_requirement.type == PackageTypes.SEMVER:
+        result = semversion.compare_package_semver(
+            package=package, package_requirement=package_requirement
+        )
+    elif package_requirement.type == PackageTypes.GENERIC:
+        result = string.compare_package_str(
+            package=package, package_requirement=package_requirement
+        )
+    else:
+        raise NotImplementedError("Comparison type not implemented.")
+
+    return result

rubrical/comparisons/semversion.py

@@ -0,0 +1,56 @@
+import semver
+
+from rubrical.comparisons.utils import results_to_status
+from rubrical.enum import (
+    DependencySpecifications,
+    PackageCheck,
+    SemverComparison,
+)
+from rubrical.schemas.configuration import PackageRequirement
+from rubrical.schemas.package import Package
+
+
+def __semver_conversion(version: str):
+    return version[1:] if version.startswith("v") else version
+
+
+def compare_package_semver(
+    package_requirement: PackageRequirement, package: Package
+) -> PackageCheck:
+    result = None
+
+    if package.specifier in [
+        DependencySpecifications.EQ,
+        DependencySpecifications.LT,
+        DependencySpecifications.LTE,
+    ]:
+        package_semver = __semver_conversion(package.version)
+    elif package.specifier in [
+        DependencySpecifications.NE,
+        DependencySpecifications.GT,
+        DependencySpecifications.GTE,
+    ]:
+        result = PackageCheck.NOOP
+    elif package.specifier == DependencySpecifications.COMPATIBLE:
+        parsed_version = __semver_conversion(package.version).split(".")
+        if len(parsed_version) < 3:
+            parsed_version += ["999999"] * (3 - len(parsed_version))
+        package_semver = ".".join(parsed_version)
+
+    if not result:
+        try:
+            warn_signal = SemverComparison.GT.value != semver.compare(
+                package_semver,
+                __semver_conversion(package_requirement.warn),
+            )
+            block_signal = SemverComparison.GT.value != semver.compare(
+                package_semver,
+                __semver_conversion(package_requirement.block),
+            )
+        except ValueError:
+            warn_signal = False
+            block_signal = False
+
+        result = results_to_status(warn_signal=warn_signal, block_signal=block_signal)
+
+    return result

rubrical/comparisons/string.py

@@ -0,0 +1,13 @@
+from rubrical.comparisons.utils import results_to_status
+from rubrical.enum import PackageCheck
+from rubrical.schemas.configuration import PackageRequirement
+from rubrical.schemas.package import Package
+
+
+def compare_package_str(
+    package_requirement: PackageRequirement, package: Package
+) -> PackageCheck:
+    warn_signal = package_requirement.warn >= package.version
+    block_signal = package_requirement.block >= package.version
+
+    return results_to_status(warn_signal=warn_signal, block_signal=block_signal)

rubrical/comparisons/utils.py

@@ -0,0 +1,12 @@
+from rubrical.enum import PackageCheck
+
+
+def results_to_status(warn_signal: bool, block_signal: bool) -> PackageCheck:
+    if block_signal:
+        status = PackageCheck.BLOCK
+    elif warn_signal:
+        status = PackageCheck.WARN
+    else:
+        status = PackageCheck.OK
+
+    return status

rubrical/enum.py

@@ -1,19 +1,22 @@
 from enum import Enum
 
 
-class ComparisonOperators(Enum):
-    GT = ">"
-    GTE = ">="
-    LT = "<"
-    LTE = "<="
-
-
 class SemverComparison(Enum):
     LT = -1
     GT = 1
     EQ = 0
 
 
+class DependencySpecifications(Enum):
+    EQ = "EQ"
+    GT = "GT"
+    GTE = "GTE"
+    LT = "LT"
+    LTE = "LTE"
+    NE = "NE"
+    COMPATIBLE = "COMPATIBLE"
+
+
 class PackageTypes(Enum):
     SEMVER = "semver"
     GENERIC = "generic"
@@ -23,7 +26,9 @@ class PackageCheck(Enum):
     OK = "ok"
     WARN = "warn"
     BLOCK = "block"
+    NOOP = "noop"
 
 
 class SupportedPackageManagers(Enum):
     JSONNET = "jsonnet"
+    PYTHON = "python"

rubrical/package_managers/base_package_manager.py

@@ -2,6 +2,7 @@
 from pathlib import Path
 from typing import Dict, List
 
+from rubrical.enum import DependencySpecifications
 from rubrical.schemas.package import Package
 
 
@@ -10,6 +11,15 @@ class BasePackageManager(abc.ABC):
     target_file: str = ""
     found_files: Dict[str, str]
     packages: Dict[str, List[Package]]
+    specification_symbols: Dict[str, List[str]] = {
+        DependencySpecifications.EQ.value: ["=="],
+        DependencySpecifications.GT.value: [">"],
+        DependencySpecifications.GTE.value: [">=", "=>"],
+        DependencySpecifications.LT.value: ["<"],
+        DependencySpecifications.LTE.value: ["<=", "=<"],
+        DependencySpecifications.NE.value: ["!="],
+        DependencySpecifications.COMPATIBLE.value: [],
+    }
 
     def __init__(self) -> None:
         super().__init__()
@@ -30,6 +40,9 @@ def parse_package_manager_files(self):
         for file in self.found_files.items():
             self.parse_package_manager_file(*file)
 
+    def append_package(self, package_file_filename: str, package: Package):
+        self.packages[package_file_filename].append(package)
+
     @abc.abstractmethod
     def parse_package_manager_file(self, package_file_filename: str, package_file: str):
         pass

rubrical/package_managers/jsonnet.py

@@ -3,7 +3,7 @@
 
 from pydantic import BaseModel
 
-from rubrical.enum import SupportedPackageManagers
+from rubrical.enum import DependencySpecifications, SupportedPackageManagers
 from rubrical.package_managers.base_package_manager import BasePackageManager
 from rubrical.package_managers.utilities import git
 from rubrical.schemas.package import Package
@@ -48,5 +48,6 @@ def parse_package_manager_file(
                 Package(
                     name=git.repository_from_url(dependency.source.git.remote),
                     version=dependency.version,
+                    specifier=DependencySpecifications.EQ,
                 )
             )

rubrical/package_managers/python.py

@@ -0,0 +1,53 @@
+import requirements
+
+from rubrical.enum import DependencySpecifications, SupportedPackageManagers
+from rubrical.package_managers.base_package_manager import BasePackageManager
+from rubrical.schemas.package import Package
+
+
+class Python(BasePackageManager):
+    target_file = "requirements.txt"
+
+    def __init__(self) -> None:
+        super().__init__()
+
+        self.name = SupportedPackageManagers.PYTHON.value
+
+        self.specification_symbols = self.specification_symbols | {
+            DependencySpecifications.COMPATIBLE.value: ["~="]
+        }
+
+    def _set_package(self, package_file_filename: str, requirement, spec):
+        for key, value in self.specification_symbols.items():
+            if spec[0] in value:
+                self.append_package(
+                    package_file_filename,
+                    Package(
+                        name=requirement.name,
+                        version=spec[1],
+                        specifier=DependencySpecifications(key),
+                    ),
+                )
+
+    def parse_package_manager_file(
+        self, package_file_filename: str, package_file_contents: str
+    ) -> None:
+        self.packages[package_file_filename] = []
+
+        for req in requirements.parse(package_file_contents):
+            if req.specifier:
+                # Handle cases for single specifiers.
+                if len(req.specs) == 1:
+                    [spec] = req.specs
+                    self._set_package(package_file_filename, req, spec)
+                elif len(req.specs) > 1:
+                    [spec] = [
+                        x
+                        for x in req.specs
+                        if x[0]
+                        in self.specification_symbols["GT"]
+                        + self.specification_symbols["GTE"]
+                    ]
+                    self._set_package(package_file_filename, req, spec)
+                else:
+                    pass

rubrical/reporters/gh.py

@@ -17,7 +17,9 @@ def _generate_report(reporting_data: Dict[str, List[PackageCheckResult]]):
         test += f"### {package_manager}\n\n"
 
         not_ok_results = [
-            x for x in reporting_data[package_manager] if x.check != PackageCheck.OK
+            x
+            for x in reporting_data[package_manager]
+            if x.check in [PackageCheck.BLOCK, PackageCheck.WARN]
         ]
         if not_ok_results:
             test += "| File | Dependency | Result |\n"

rubrical/reporters/terminal.py

@@ -12,7 +12,7 @@ def terminal_report(
     package_manager_name: str, check_results: List[PackageCheckResult]
 ) -> None:
     # If any unsuccessful checks.
-    if [x for x in check_results if x.check != PackageCheck.OK]:
+    if [x for x in check_results if x.check in [PackageCheck.BLOCK, PackageCheck.WARN]]:
         console.print_message(
             f"[bold][dark_orange]{package_manager_name}[/dark_orange][/bold] checks completed with violations!"
         )

rubrical/rubrical.py

@@ -1,16 +1,20 @@
 from pathlib import Path
 from typing import Dict, List, Tuple
 
-from rubrical.comparison import check_package
+from rubrical.comparisons import check_package
 from rubrical.enum import PackageCheck, SupportedPackageManagers
 from rubrical.package_managers.base_package_manager import BasePackageManager
 from rubrical.package_managers.jsonnet import Jsonnet
+from rubrical.package_managers.python import Python
 from rubrical.reporters import terminal
 from rubrical.schemas.configuration import RubricalConfig
 from rubrical.schemas.results import PackageCheckResult
 from rubrical.utilities import console
 
-PACKAGE_MANAGER_MAPPING = {SupportedPackageManagers.JSONNET.value: Jsonnet}
+PACKAGE_MANAGER_MAPPING = {
+    SupportedPackageManagers.JSONNET.value: Jsonnet,
+    SupportedPackageManagers.PYTHON.value: Python,
+}
 
 
 class Rubrical:

rubrical/schemas/package.py

@@ -1,6 +1,11 @@
+from typing import Optional
+
 from pydantic import BaseModel
 
+from rubrical.enum import DependencySpecifications
+
 
 class Package(BaseModel):
     name: str
     version: str
+    specifier: Optional[DependencySpecifications] = DependencySpecifications.EQ

tests/files/python/requirements.txt

@@ -8,3 +8,4 @@ docopt == 0.6.1             # Version Matching. Must be version 0.6.1
 keyring >= 4.1.1            # Minimum version 4.1.1
 coverage != 3.5             # Version Exclusion. Anything except version 3.5
 Mopidy-Dirble ~= 1.1        # Compatible release. Same as >= 1.1, == 1.*
+something-something <1.6,>1.5