0.2.0 - fixes to YAML read/writes

.github/workflows/ci.yml

@@ -19,7 +19,7 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
         # Empty is latest, head is latest from GitHub
-        pdm-version: ["", "head", "2.7.4"]
+        pdm-version: ["", "head", "2.7.4", "2.8.0", "2.8.1"]
 
     steps:
     - uses: actions/checkout@v3

.pre-commit-config.yaml

@@ -1,4 +1,3 @@
----
 default_language_version:
   python: python3.11
 repos:
@@ -15,12 +14,12 @@ repos:
       - id: fix-byte-order-marker
 
   - repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: 23.7.0
     hooks:
       - id: black
 
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.276
+  - rev: v0.0.280
+    repo: https://github.com/charliermarsh/ruff-pre-commit
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

.tool-versions

@@ -1,2 +1,2 @@
-pdm    2.7.4
+pdm    2.8.1
 python 3.11.4

README.md

@@ -103,8 +103,9 @@ Feel free to open an issue or a PR if you have any idea, or if you want to help!
 - [ ] Support reordering DB inputs (file/global config/python module/cli)?
 - [ ] Test using SSH/file dependencies?
 - [ ] Check ref existence before writing?
-- [ ] Support multiple hooks repos for the same dependency?
+- [ ] Support multiple hooks repos for the same dependency? E.g. ruff new and old repo
 - [ ] New feature to convert from pre-commit online to local?
+- [ ] Warning if pre-commit CI auto update is also set?
 
 ## Inspiration
 

pyproject.toml

@@ -4,10 +4,13 @@ build-backend = "pdm.backend"
 
 [project]
 name = "sync-pre-commit-lock"
-version = "0.1.2"
+version = "0.2.0"
 description = "PDM plugin to sync your pre-commit versions with your lockfile, and install them, all automatically."
 authors = [{ name = "Gabriel Dugny", email = "sync-pre-commit-lock@dugny.me" }]
-dependencies = ["PyYAML>=6.0", "tomli>=2.0.0; python_version < \"3.11\""]
+dependencies = [
+    "tomli>=2.0.0; python_version < \"3.11\"",
+    "strictyaml>=1.7.3",
+]
 requires-python = ">=3.10"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -48,6 +51,7 @@ pdm = [
 ]
 [tool.pdm.dev-dependencies]
 dev = [
+    "PyYAML>=6.0",
     "black>=23.3.0",
     "mypy>=1.4.1",
     "ruff>=0.0.275",
@@ -73,6 +77,10 @@ target-version = "py310"
 [tool.mypy]
 strict = true
 
+[[tool.mypy.overrides]]
+module = "strictyaml"
+ignore_missing_imports = true
+
 [tool.black]
 line-length = 120
 target-version = ["py310"]

src/sync_pre_commit_lock/actions/install_hooks.py

@@ -65,7 +65,7 @@ def _is_pre_commit_package_installed(self) -> bool:
                 self.check_pre_commit_version_command,  # noqa: S603
             ).decode()
             return "pre-commit" in output
-        except FileNotFoundError:
+        except (subprocess.CalledProcessError, FileNotFoundError):
             return False
 
     @staticmethod

src/sync_pre_commit_lock/actions/sync_hooks.py

@@ -1,14 +1,9 @@
 from __future__ import annotations
 
-import difflib
-import re
-from functools import cached_property
-from typing import TYPE_CHECKING, Any, NamedTuple
-
-import yaml
+from typing import TYPE_CHECKING, NamedTuple
 
 from sync_pre_commit_lock.db import DEPENDENCY_MAPPING, PackageRepoMapping, RepoInfo
-from sync_pre_commit_lock.utils import normalize_git_url
+from sync_pre_commit_lock.pre_commit_config import PreCommitHookConfig, PreCommitRepo
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -23,71 +18,6 @@ class GenericLockedPackage(NamedTuple):
     # Add original data here?
 
 
-class PreCommitRepo(NamedTuple):
-    repo: str
-    rev: str  # Check if is not loaded as float/int/other yolo
-
-
-class PreCommitHookConfig:
-    def __init__(self, data: dict[str, Any], pre_commit_config_file_path: Path, original_file_lines: list[str]) -> None:
-        self._data = data
-        self.pre_commit_config_file_path = pre_commit_config_file_path
-        self.original_file_lines: list[str] = original_file_lines
-
-    @property
-    def data(self) -> dict[str, Any]:
-        return self._data
-
-    @data.setter
-    def data(self, value: dict[str, Any]) -> None:
-        raise NotImplementedError("This should not be used. Recreate the object instead.")
-
-    @classmethod
-    def from_yaml_file(cls, file_path: Path) -> PreCommitHookConfig:
-        with file_path.open("r") as stream:
-            file_contents = stream.read()
-
-        data = yaml.safe_load(file_contents)
-        if not isinstance(data, dict):
-            raise ValueError(f"Invalid pre-commit config file: {file_path}. Expected a dict, got {type(data)}")
-        if "repos" in data and not isinstance(data["repos"], list):
-            raise ValueError(
-                f"Invalid pre-commit config file: {file_path}. Expected a list for `repos`, got {type(data['repos'])}"
-            )
-        return PreCommitHookConfig(data, file_path, original_file_lines=file_contents.splitlines(keepends=True))
-
-    @cached_property
-    def repos(self) -> list[PreCommitRepo]:
-        return [PreCommitRepo(repo=repo["repo"], rev=repo["rev"]) for repo in (self.data["repos"] or [])]
-
-    @cached_property
-    def repos_normalized(self) -> set[PreCommitRepo]:
-        return {PreCommitRepo(repo=normalize_git_url(repo.repo), rev=repo.rev) for repo in self.repos}
-
-    def update_pre_commit_repo_versions(self, new_versions: dict[PreCommitRepo, str]) -> None:
-        """Fix the pre-commit hooks to match the lockfile. Preserve comments and formatting as much as possible."""
-
-        original_lines = self.original_file_lines
-        updated_lines = original_lines[:]
-        pre_commit_data = self.data
-
-        for repo, rev in new_versions.items():
-            for pre_commit_repo in pre_commit_data["repos"]:
-                if pre_commit_repo["repo"] != repo.repo:
-                    continue
-                rev_line_number = [i for i, line in enumerate(original_lines) if f"repo: {repo.repo}" in line][0] + 1
-                original_rev_line = updated_lines[rev_line_number]
-                updated_lines[rev_line_number] = re.sub(r"(?<=rev: )\S*", rev, original_rev_line)
-
-        changes = difflib.ndiff(original_lines, updated_lines)
-        change_count = sum(1 for change in changes if change[0] in ["+", "-"])
-
-        if change_count == 0:
-            return
-        with self.pre_commit_config_file_path.open("w") as stream:
-            stream.writelines(updated_lines)
-
-
 class SyncPreCommitHooksVersion:
     def __init__(
         self,
@@ -127,12 +57,12 @@ def execute(self) -> None:
         to_fix = self.analyze_repos(pre_commit_config_data.repos_normalized, mapping, mapping_reverse_by_url)
 
         if len(to_fix) == 0:
-            self.printer.success("All matched pre-commit hooks already in sync with the lockfile!")
+            self.printer.info("All matched pre-commit hooks already in sync with the lockfile!")
             return
 
         self.printer.info("Detected pre-commit hooks that can be updated to match the lockfile:")
         for repo, rev in to_fix.items():
-            self.printer.info(f"  - {repo.repo}: {repo.rev} -> {rev}")
+            self.printer.info(f" - {repo.repo}: {repo.rev} -> {rev}")
         pre_commit_config_data.update_pre_commit_repo_versions(to_fix)
         self.printer.success("Pre-commit hooks have been updated to match the lockfile!")
 

src/sync_pre_commit_lock/pre_commit_config.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import difflib
+from functools import cached_property
+from typing import TYPE_CHECKING, Any, NamedTuple
+
+import strictyaml as yaml
+from strictyaml import Any as AnyStrictYaml
+from strictyaml import MapCombined, Optional, Seq, Str
+
+from sync_pre_commit_lock.utils import normalize_git_url
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+schema = MapCombined(
+    {
+        Optional("repos"): Seq(
+            MapCombined(
+                {
+                    "repo": Str(),
+                    Optional("rev"): Str(),
+                },
+                Str(),
+                AnyStrictYaml(),
+            ),
+        )
+    },
+    Str(),
+    AnyStrictYaml(),
+)
+
+
+class PreCommitRepo(NamedTuple):
+    repo: str
+    rev: str  # Check if is not loaded as float/int/other yolo
+
+
+class PreCommitHookConfig:
+    def __init__(
+        self,
+        raw_file_contents: str,
+        pre_commit_config_file_path: Path,
+    ) -> None:
+        self.raw_file_contents = raw_file_contents
+        self.yaml = yaml.dirty_load(
+            raw_file_contents, schema=schema, allow_flow_style=True, label=str(pre_commit_config_file_path)
+        )
+
+        self.pre_commit_config_file_path = pre_commit_config_file_path
+
+    @cached_property
+    def original_file_lines(self) -> list[str]:
+        return self.raw_file_contents.splitlines(keepends=True)
+
+    @property
+    def data(self) -> Any:
+        return self.yaml.data
+
+    @classmethod
+    def from_yaml_file(cls, file_path: Path) -> PreCommitHookConfig:
+        with file_path.open("r") as stream:
+            file_contents = stream.read()
+
+        return PreCommitHookConfig(file_contents, file_path)
+
+    @cached_property
+    def repos(self) -> list[PreCommitRepo]:
+        """Return the repos, excluding local repos."""
+        return [
+            PreCommitRepo(repo=repo["repo"], rev=repo["rev"]) for repo in (self.data["repos"] or []) if "rev" in repo
+        ]
+
+    @cached_property
+    def repos_normalized(self) -> set[PreCommitRepo]:
+        return {PreCommitRepo(repo=normalize_git_url(repo.repo), rev=repo.rev) for repo in self.repos}
+
+    @cached_property
+    def document_start_offset(self) -> int:
+        """Return the line number where the YAML document starts."""
+
+        lines = self.raw_file_contents.split("\n")
+        for i, line in enumerate(lines):
+            # Trim leading/trailing whitespaces
+            line = line.rstrip()
+            # Skip if line is a comment or empty/whitespace
+            if line.startswith("#") or line == "":
+                continue
+            # If line is '---', return line number + 1
+            if line == "---":
+                return i + 1
+        return 0
+
+    def update_pre_commit_repo_versions(self, new_versions: dict[PreCommitRepo, str]) -> None:
+        """Fix the pre-commit hooks to match the lockfile. Preserve comments and formatting as much as possible."""
+
+        if len(new_versions) == 0:
+            return
+
+        original_lines = self.original_file_lines
+        updated_lines = original_lines[:]
+
+        for repo_rev in self.yaml["repos"]:
+            if "rev" not in repo_rev:
+                continue
+
+            repo, rev = repo_rev["repo"], repo_rev["rev"]
+            normalized_repo = PreCommitRepo(normalize_git_url(str(repo)), str(rev))
+            if normalized_repo not in new_versions:
+                continue
+
+            rev_line_number: int = rev.end_line + self.document_start_offset
+            rev_line_idx: int = rev_line_number - 1
+            original_rev_line: str = updated_lines[rev_line_idx]
+            updated_lines[rev_line_idx] = original_rev_line.replace(str(rev), new_versions[normalized_repo])
+
+        changes = difflib.ndiff(original_lines, updated_lines)
+        change_count = sum(1 for change in changes if change[0] in ["+", "-"])
+
+        if change_count == 0:
+            raise RuntimeError("No changes to write, this should not happen")
+        with self.pre_commit_config_file_path.open("w") as stream:
+            stream.writelines(updated_lines)

tests/fixtures/sample_pre_commit_config/pre-commit-config-document-separator.yaml

@@ -0,0 +1,33 @@
+
+# Many unused lines before document separator
+
+---
+default_language_version:
+  python: python3.11
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-toml
+
+  - repo: https://github.com/psf/black
+    rev: 23.2.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: 'v0.0.275'
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+
+  - repo: local
+    hooks:
+      - id: mypy
+        name: mypy
+        entry: mypy
+        args: [src, tests, --color-output]
+        language: system
+        types: [python]
+        pass_filenames: false
+        require_serial: true

tests/fixtures/sample_pre_commit_config/pre-commit-config-start-empty-lines.yaml

@@ -0,0 +1,30 @@
+
+
+
+default_language_version:
+  python: python3.11
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-toml
+      - id: trailing-whitespace
+      - id: check-executables-have-shebangs
+      - id: debug-statements
+      - id: end-of-file-fixer
+      - id: check-added-large-files
+      - id: check-merge-conflict
+      - id: fix-byte-order-marker
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.277'
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+  - repo: https://github.com/psf/black
+    rev: 23.3.0
+    hooks:
+      - id: black
+
+# XXX Fix the issue with documents