Control dependencies more carefully and ensure frequent updates

Author: sbp

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
 - repo: meta
   hooks:
     - id: check-hooks-apply
-      name: run check hooks apply
+      name: check that hooks apply
       description: check that all the hooks apply to the repository
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v6.0.0
@@ -30,7 +30,7 @@ repos:
   rev: v1.5.5
   hooks:
     - id: insert-license
-      name: Add license for all Python files
+      name: add a license for all Python files
       files: ^(atr|scripts|tests)/(.*\.py$|.*\.pyi)$
       args:
         - --comment-style
@@ -39,7 +39,7 @@ repos:
         - scripts/ci/LICENSE-template.txt
         - --fuzzy-match-generates-todo
     - id: insert-license
-      name: Add license for JS and TS files
+      name: add a license for JS and TS files
       files: ^atr/static/(js/src/.*\.js|ts/.*\.ts)$
       args:
         - --comment-style
@@ -51,7 +51,7 @@ repos:
   rev: v0.47.0
   hooks:
     - id: markdownlint
-      name: run markdownlint
+      name: run Markdown lints
       description: check Markdown files with markdownlint
       args: [--config=.github/linters/.markdown-lint.yml]
       types: [markdown]
@@ -105,7 +105,7 @@ repos:
   rev: v1.33.0
   hooks:
     - id: oxlint
-      name: Oxlint JS Linter
+      name: lint JS files with Oxlint
       types: [javascript]
       files: ^atr/static/js/src/.*\.js$
       args:
@@ -134,21 +134,21 @@ repos:
 - repo: local
   hooks:
     - id: ruff
-      name: Ruff Linter
-      entry: uv run ruff check --fix
+      name: lint Python files with Ruff
+      entry: uv run --frozen ruff check --fix
       language: system
       types: [python]
 
     - id: ruff-format
-      name: Ruff Formatter
-      entry: uv run ruff format --force-exclude
+      name: format Python files with Ruff
+      entry: uv run --frozen ruff format --force-exclude
       language: system
       types: [python]
 
     - id: biome-format
       # Install biome from https://biomejs.dev/guides/manual-installation/
       # Avoid using NPM to install it until #359 is resolved
-      name: Biome JS Formatter
+      name: format JS files with Biome
       entry: biome format --write --files-ignore-unknown=true --no-errors-on-unmatched --colors=off
       language: system
       types: [javascript]
@@ -157,24 +157,32 @@ repos:
     - id: biome-check
       # Install biome from https://biomejs.dev/guides/manual-installation/
       # Avoid using NPM to install it until #359 is resolved
-      name: Biome JS Linter
+      name: lint JS files with Biome
       entry: biome check --write --error-on-warnings --files-ignore-unknown=true --no-errors-on-unmatched --colors=off
       language: system
       types: [javascript]
       files: ^atr/static/js/src/.*\.js$
 
     - id: pyright
-      name: Pyright Type Check
-      entry: uv run pyright
+      name: type check Python files with Pyright
+      entry: uv run --frozen pyright
       language: system
       require_serial: true
       types: [python]
       exclude: ^tests/
 
     - id: jinja-route-check
-      name: Jinja Route Checker
+      name: check Jinja2 routes
       description: Check whether routes used in Jinja2 templates actually exist
-      entry: uv run python scripts/lint/jinja_route_checker.py
+      entry: uv run --frozen python scripts/lint/jinja_route_checker.py
+      language: system
+      pass_filenames: false
+      always_run: true
+
+    - id: check-when-dependencies-updated
+      name: check when dependencies were updated
+      description: Verify that dependencies were updated within the configured timeframe, for ASVS 15.2.1
+      entry: uv run --frozen python scripts/check_when_dependencies_updated.py
       language: system
       pass_filenames: false
       always_run: true

.pre-commit-light.yaml

@@ -3,19 +3,19 @@ repos:
 - repo: local
   hooks:
     - id: ruff
-      name: Ruff Linter
+      name: ruff linter
       entry: ruff check --fix
       language: system
       types: [python]
 
     - id: ruff-format
-      name: Ruff Formatter
+      name: ruff formatter
       entry: ruff format --force-exclude
       language: system
       types: [python]
 
     - id: pyright
-      name: Pyright Type Check
+      name: pyright type check
       entry: pyright
       language: system
       require_serial: true

Makefile

@@ -37,15 +37,15 @@ bump-bootstrap:
 
 certs:
 	if test ! -f state/cert.pem || test ! -f state/key.pem; \
-	then uv run scripts/generate-certificates; \
+	then uv run --frozen scripts/generate-certificates; \
 	fi
 
 certs-local:
 	cd state && mkcert localhost.apache.org localhost 127.0.0.1 ::1
 
 check:
 	git add -A
-	uv run pre-commit run --all-files
+	uv run --frozen pre-commit run --all-files
 
 check-extra:
 	@git add -A
@@ -54,11 +54,11 @@ check-extra:
 
 check-heavy:
 	git add -A
-	uv run pre-commit run --all-files --config .pre-commit-heavy.yaml
+	uv run --frozen pre-commit run --all-files --config .pre-commit-heavy.yaml
 
 check-light:
 	git add -A
-	uv run pre-commit run --all-files --config .pre-commit-light.yaml
+	uv run --frozen pre-commit run --all-files --config .pre-commit-light.yaml
 
 commit:
 	git add -A
@@ -68,16 +68,16 @@ commit:
 
 docs:
 	mkdir -p docs
-	uv run python3 scripts/docs_check.py
+	uv run --frozen python3 scripts/docs_check.py
 	rm -f docs/*.html
-	uv run python3 scripts/docs_build.py
+	uv run --frozen python3 scripts/docs_build.py
 	for fn in atr/docs/*.md; do out=$${fn#atr/}; cmark "$$fn" > "$${out%.md}.html"; done
-	uv run python3 scripts/docs_post_process.py docs/*.html
-	uv run python3 scripts/docs_check.py
+	uv run --frozen python3 scripts/docs_post_process.py docs/*.html
+	uv run --frozen python3 scripts/docs_check.py
 
 generate-version:
 	@rm -f atr/version.py
-	@uv run python3 atr/metadata.py > /tmp/version.py
+	@uv run --frozen python3 atr/metadata.py > /tmp/version.py
 	@mv /tmp/version.py atr/version.py
 	@cat atr/version.py
 
@@ -101,23 +101,23 @@ run-playwright-slow:
 	docker run --net=host -it atr-playwright python3 test.py --tidy
 
 serve:
-	SSH_HOST=127.0.0.1 uv run hypercorn --bind $(BIND) \
+	SSH_HOST=127.0.0.1 uv run --frozen hypercorn --bind $(BIND) \
 	  --keyfile localhost.apache.org+3-key.pem --certfile localhost.apache.org+3.pem \
 	  atr.server:app --debug --reload
 
 serve-local:
 	APP_HOST=localhost.apache.org:8080 SECRET_KEY=insecure-local-key \
-	  ALLOW_TESTS=1 SSH_HOST=127.0.0.1 uv run hypercorn --bind $(BIND) \
+	  ALLOW_TESTS=1 SSH_HOST=127.0.0.1 uv run --frozen hypercorn --bind $(BIND) \
 	  --keyfile localhost.apache.org+3-key.pem --certfile localhost.apache.org+3.pem \
 	  atr.server:app --debug --reload
 
 sync:
-	uv sync --no-dev
+	uv sync --frozen --no-dev
 
 sync-all:
-	uv sync --all-groups
+	uv sync --frozen --all-groups
 
 update-deps:
 	pre-commit autoupdate || :
-	uv lock --upgrade
-	uv sync --all-groups
+	uv lock --upgrade --exclude-newer "$$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+	uv sync --frozen --all-groups

notes/development.md

@@ -5,14 +5,14 @@ You will need to have a working [Python 3.13](https://www.python.org/downloads/r
 Ensure that you have the pre-commit hook installed:
 
 ```shell
-make sync PYTHON="$(which python3)"
-poetry run pre-commit install
+make sync-all
+uv run --frozen pre-commit install
 ```
 
-To run the project, use the following commands, which will add a local CA root to your OS and browser certificate qstore if using Firefox:
+To run the project, use the following commands, which will add a local CA root to your OS and browser certificate store if using Firefox:
 
 ```shell
-uv sync --all-groups
+make sync-all
 make certs-local
 make serve
 ```

scripts/check_when_dependencies_updated.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import datetime
+import pathlib
+import sys
+from typing import Final
+
+_MAX_AGE_DAYS: Final[int] = 30
+
+
+def main() -> None:
+    lock_path = pathlib.Path("uv.lock")
+    if not lock_path.exists():
+        print("ERROR: uv.lock not found", file=sys.stderr)
+        sys.exit(1)
+
+    exclude_newer = _parse_exclude_newer(lock_path)
+    if exclude_newer is None:
+        print("ERROR: No exclude-newer timestamp in uv.lock", file=sys.stderr)
+        print("Run: make update-deps", file=sys.stderr)
+        sys.exit(1)
+
+    timestamp = _parse_timestamp(exclude_newer)
+    if timestamp is None:
+        print(f"ERROR: Could not parse timestamp: {exclude_newer}", file=sys.stderr)
+        sys.exit(1)
+
+    now = datetime.datetime.now(datetime.UTC)
+    age = now - timestamp
+
+    if age > datetime.timedelta(days=_MAX_AGE_DAYS):
+        print(f"ERROR: Dependencies are {age.days} days old (the limit is {_MAX_AGE_DAYS} days)", file=sys.stderr)
+        print(f"Last updated: {exclude_newer}", file=sys.stderr)
+        print("Run: make update-deps", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"OK: Dependencies are {age.days} days old (the limit is {_MAX_AGE_DAYS} days)")
+
+
+def _parse_exclude_newer(lock_path: pathlib.Path) -> str | None:
+    for line in lock_path.read_text(encoding="utf-8").splitlines():
+        if line.startswith("exclude-newer"):
+            _, _, value = line.partition("=")
+            return value.strip().strip('"')
+    return None
+
+
+def _parse_timestamp(timestamp_str: str) -> datetime.datetime | None:
+    try:
+        return datetime.datetime.fromisoformat(timestamp_str.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+
+
+if __name__ == "__main__":
+    main()