Add a hook to run shellcheck on all "runs" steps in melange pipelines

.github/workflows/lint.yaml

@@ -0,0 +1,21 @@
+name: Lint
+
+on:
+  pull_request:
+    branches: ['main']
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+
+    steps:
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # ratchet:actions/checkout@v4.2.2
+    - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+    - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1

.pre-commit-config.yaml

@@ -1,25 +1,6 @@
-# In general, this file (when sourced from https://github.com/chainguard-dev/pre-commit-hooks)
-# should be able to be used as-is in other repos such as wolfi-dev/os, enterprise-packages, and extra-packages,
-# and can be assumed to be the "source of truth" for our pre-commit rules.
-# See https://eng.inky.wtf/docs/technical/git/pre-commit for info on how to edit this file.
+# Update with `pre-commit autoupdate --freeze` which
+# pins all repos using commit hashes, not mutable references
 repos:
-  - repo: https://github.com/chainguard-dev/pre-commit-hooks
-    rev: e4f3bba353cc583ce73f660dcf217e245fd681d3
-    hooks:
-      - id: check-for-epoch-bump
-  - repo: https://github.com/chainguard-dev/yam
-    rev: 498642e77997ba79709f43a7ee2c84b12b2145bb # v0.2.12
-    hooks:
-      - id: yam
-  # TODO: Swap with CG repo: - repo: https://github.com/wolfi-dev/wolfictl
-  - repo: https://github.com/amberarcadia/wolfictl
-    rev: 85b1301c4d17fcd0c8f0ce455724941cae815d68
-    hooks:
-      - id: wolfictl-lint
-  - repo: https://github.com/golangci/misspell
-    rev: 528d713e620bdf4b41849db93cb489c4fef9f5c5 # v0.6.0
-    hooks:
-      - id: misspell
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: cef0300fd0fc4d2a87a85fa2093c6b283ea36f4b # v5.0.0
     hooks:
@@ -30,4 +11,38 @@ repos:
       - id: check-merge-conflict
       - id: check-symlinks
       - id: detect-private-key
-        exclude: ^ruby-3\.0/0001-ruby-3\.0\.6-openssl-patch\.patch$
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: 12753357c00c3fb8615100354c9fdc6ab80b044d # frozen: v0.11.10
+    hooks:
+      - id: ruff-check
+      - id: ruff-format
+  - repo: https://github.com/asottile/setup-cfg-fmt
+    rev: 79cc0ae5abfa1ba092b5938cd811a6069710ad77  # frozen: v2.8.0
+    hooks:
+      - id: setup-cfg-fmt
+  - repo: https://github.com/asottile/reorder-python-imports
+    rev: fd0b4e1292716bcd12a396b86af1d1271aaaa62c  # frozen: v3.14.0
+    hooks:
+      - id: reorder-python-imports
+        args: [--py39-plus, --add-import, 'from __future__ import annotations']
+  - repo: https://github.com/asottile/add-trailing-comma
+    rev: d2e6adc1665e461a764e2f38edfa2ef61f41be20  # frozen: v3.1.0
+    hooks:
+      - id: add-trailing-comma
+  - repo: https://github.com/asottile/pyupgrade
+    rev: ce40a160603ab0e7d9c627ae33d7ef3906e2d2b2  # frozen: v3.19.1
+    hooks:
+      - id: pyupgrade
+        args: [--py39-plus]
+  - repo: https://github.com/hhatto/autopep8
+    rev: 4046ad49e25b7fa1db275bf66b1b7d60600ac391  # frozen: v2.3.2
+    hooks:
+      - id: autopep8
+  - repo: https://github.com/PyCQA/flake8
+    rev: 4b5e89b4b108a6c1a000c591d334a99a80d34c7b  # frozen: 7.2.0
+    hooks:
+      - id: flake8
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: f40886d54c729f533f864ed6ce584e920feb0af7  # frozen: v1.15.0
+    hooks:
+      - id: mypy

.pre-commit-hooks.yaml

@@ -10,3 +10,13 @@
     - manual
   types:
     - yaml
+- id: shellcheck-run-steps
+  name: shellcheck run steps
+  description: run shellcheck on each "run" step in a melange pipeline
+  entry: shellcheck-run-steps
+  language: python
+  stages:
+    - pre-commit
+    - manual
+  types:
+    - yaml

example.pre-commit-config.yaml

@@ -0,0 +1,34 @@
+# In general, this file (when sourced from https://github.com/chainguard-dev/pre-commit-hooks)
+# should be able to be used as-is in other repos such as wolfi-dev/os, enterprise-packages, and extra-packages,
+# and can be assumed to be the "source of truth" for our pre-commit rules.
+# See https://eng.inky.wtf/docs/technical/git/pre-commit for info on how to edit this file.
+repos:
+  - repo: https://github.com/chainguard-dev/pre-commit-hooks
+    rev: e4f3bba353cc583ce73f660dcf217e245fd681d3
+    hooks:
+      - id: check-for-epoch-bump
+      - id: shellcheck-run-steps
+  - repo: https://github.com/chainguard-dev/yam
+    rev: 498642e77997ba79709f43a7ee2c84b12b2145bb # v0.2.12
+    hooks:
+      - id: yam
+  # TODO: Swap with CG repo: - repo: https://github.com/wolfi-dev/wolfictl
+  - repo: https://github.com/amberarcadia/wolfictl
+    rev: 85b1301c4d17fcd0c8f0ce455724941cae815d68
+    hooks:
+      - id: wolfictl-lint
+  - repo: https://github.com/golangci/misspell
+    rev: 528d713e620bdf4b41849db93cb489c4fef9f5c5 # v0.6.0
+    hooks:
+      - id: misspell
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: cef0300fd0fc4d2a87a85fa2093c6b283ea36f4b # v5.0.0
+    hooks:
+      - id: check-yaml
+      - id: forbid-submodules
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: detect-private-key
+        exclude: ^ruby-3\.0/0001-ruby-3\.0\.6-openssl-patch\.patch$

pre_commit_hooks/shellcheck_run_steps.py

@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+import argparse
+import contextlib
+import os
+import subprocess
+import tempfile
+from collections.abc import Generator
+from collections.abc import Mapping
+from collections.abc import Sequence
+from typing import Any
+from typing import NamedTuple
+
+import ruamel.yaml
+
+yaml = ruamel.yaml.YAML(typ="safe")
+
+
+def _exhaust(gen: Generator[str]) -> None:
+    for _ in gen:
+        pass
+
+
+def _parse_unsafe(*args: Any, **kwargs: Any) -> None:
+    _exhaust(yaml.parse(*args, **kwargs))
+
+
+def _load_all(*args: Any, **kwargs: Any) -> None:
+    _exhaust(yaml.load_all(*args, **kwargs))
+
+
+class Key(NamedTuple):
+    multi: bool
+    unsafe: bool
+
+
+def do_shellcheck(
+    melange_cfg: Mapping[str, Any],
+    shellcheck: list[str],
+) -> None:
+    if melange_cfg == {}:
+        return
+
+    pkgs = [melange_cfg]
+    pkgs.extend(melange_cfg.get("subpackages", []))
+    pipelines: list[Mapping[str, Any]] = []
+    for pkg in pkgs:
+        pipelines.extend(pkg.get("pipeline", []))
+        if "test" in pkg.keys():
+            test_pipeline = pkg["test"].get("pipeline", [])
+            pipelines.extend(test_pipeline)
+    name = melange_cfg["package"]["name"]
+    all_run_files = []
+    with contextlib.ExitStack() as stack:
+        for step in pipelines:
+            if "runs" not in step.keys():
+                continue
+            all_run_files.extend(
+                [
+                    stack.enter_context(
+                        tempfile.NamedTemporaryFile(
+                            mode="w",
+                            prefix=name,
+                            dir=os.getcwd(),
+                            delete_on_close=False,
+                        ),
+                    ),
+                ],
+            )
+        for shfile in all_run_files:
+            shfile.write(step["runs"])
+            shfile.close()
+        subprocess.check_call(
+            ["/usr/bin/shellcheck"]
+            + ["--shell=busybox", "--"]
+            + [os.path.basename(f.name) for f in all_run_files],
+            cwd=os.getcwd(),
+        )
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("filenames", nargs="*", help="Filenames to check.")
+    parser.add_argument(
+        "--shellcheck",
+        default=[
+            "docker",
+            "run",
+            f"--volume={os.getcwd()}:/mnt",
+            "--rm",
+            "-it",
+            "koalaman/shellcheck:latest",
+        ],
+        nargs="*",
+        help="shellcheck command",
+    )
+    args = parser.parse_args(argv)
+
+    melange_cfg = {}
+    for filename in args.filenames:
+        with tempfile.NamedTemporaryFile(
+            "w",
+            delete_on_close=False,
+        ) as compiled_out:
+            subprocess.check_call(
+                [
+                    "melange",
+                    "compile",
+                    "--arch=x86_64",
+                    "--pipeline-dir=./pipelines",
+                    filename,
+                ],
+                stdout=compiled_out,
+            )
+            compiled_out.close()
+            try:
+                with open(compiled_out.name) as compiled_in:
+                    melange_cfg = yaml.load(compiled_in)
+                    do_shellcheck(melange_cfg, args.shellcheck)
+            except ruamel.yaml.YAMLError as exc:
+                print(exc)
+                return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ruff.toml

@@ -0,0 +1 @@
+line-length = 80

setup.cfg

@@ -0,0 +1,44 @@
+[metadata]
+name = pre_commit_hooks
+version = 0.0.1
+description = chainguard hooks for pre-commit
+long_description = file: README.md
+long_description_content_type = text/markdown
+url = https://github.com/chainguard-dev/pre-commit-hooks
+license = MIT
+license_files = LICENSE
+classifiers =
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: Implementation :: CPython
+    Programming Language :: Python :: Implementation :: PyPy
+
+[options]
+packages = find:
+install_requires =
+    ruamel.yaml>=0.15
+python_requires = >=3.9
+
+[options.entry_points]
+console_scripts =
+    shellcheck-run-steps = pre_commit_hooks.shellcheck_run_steps:main
+
+[bdist_wheel]
+universal = True
+
+[coverage:run]
+plugins = covdefaults
+
+[mypy]
+check_untyped_defs = true
+disallow_any_generics = true
+disallow_incomplete_defs = true
+disallow_untyped_defs = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+
+[mypy-testing.*]
+disallow_untyped_defs = false
+
+[mypy-tests.*]
+disallow_untyped_defs = false

setup.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from setuptools import setup
+
+setup()