Enable dynamic schema refresh

.github/workflows/tox.yml

@@ -163,7 +163,7 @@ jobs:
       WSLENV: FORCE_COLOR:PYTEST_REQPASS:TOXENV:TOX_PARALLEL_NO_SPINNER
       # Number of expected test passes, safety measure for accidental skip of
       # tests. Update value if you add/remove tests.
-      PYTEST_REQPASS: 713
+      PYTEST_REQPASS: 715
 
     steps:
       - name: Activate WSL1

.pre-commit-config.yaml

@@ -188,7 +188,7 @@ repos:
     hooks:
       - id: schemas
         name: update json schemas
-        entry: python3 src/ansiblelint/schemas/__init__.py
+        entry: python3 src/ansiblelint/schemas/__main__.py
         language: python
         stages: [manual]
       - id: pip-compile

src/ansiblelint/__main__.py

@@ -170,7 +170,7 @@ def _do_transform(result: LintResult, opts: Namespace) -> None:
     transformer.run()
 
 
-# pylint: disable=too-many-branches
+# pylint: disable=too-many-branches,too-many-statements
 def main(argv: list[str] | None = None) -> int:  # noqa: C901
     """Linter CLI entry point."""
     # alter PATH if needed (venv support)
@@ -196,6 +196,15 @@ def main(argv: list[str] | None = None) -> int:  # noqa: C901
     _logger.debug("Options: %s", options)
     _logger.debug(os.getcwd())
 
+    if not options.offline:
+        # refresh schemas must happen before loading rules
+        if "ansiblelint.schemas" in sys.modules:
+            raise RuntimeError("ansiblelint.schemas should not be loaded yet")
+        # pylint: disable=import-outside-toplevel
+        from ansiblelint.schemas import refresh_schemas
+
+        refresh_schemas()
+
     # pylint: disable=import-outside-toplevel
     from ansiblelint.rules import RulesCollection
     from ansiblelint.runner import _get_matches

src/ansiblelint/schemas/__init__.py

@@ -1,63 +1,4 @@
 """Module containing cached JSON schemas."""
-import logging
-import os
-import sys
-import urllib.request
+from ansiblelint.schemas.main import JSON_SCHEMAS, refresh_schemas
 
-_logger = logging.getLogger(__package__)
-
-
-# Maps kinds to JSON schemas
-# See https://www.schemastore.org/json/
-JSON_SCHEMAS = {
-    # Do not use anchors in these URLs because python-jsonschema does not support them:
-    "playbook": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-playbook.json",
-    "tasks": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-tasks.json",
-    "vars": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-vars.json",
-    "requirements": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-requirements.json",
-    "meta": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-meta.json",
-    "galaxy": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-galaxy.json",
-    "execution-environment": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-ee.json",
-    "meta-runtime": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-meta-runtime.json",
-    "inventory": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-inventory.json",
-    "ansible-lint-config": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible-lint-config.json",
-    "ansible-navigator-config": "https://raw.githubusercontent.com/ansible/ansible-navigator/main/src/ansible_navigator/data/ansible-navigator.json",
-    "arg_specs": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-argument-specs.json",
-}
-
-
-def refresh_schemas() -> int:
-    """Refresh JSON schemas by downloading latest versions.
-
-    Returns number of changed schemas.
-    """
-    changed = 0
-    for kind, url in sorted(JSON_SCHEMAS.items()):
-        if url.startswith("https://raw.githubusercontent.com/ansible/ansible-lint"):
-            _logger.warning(
-                "Skipped updating schema that is part of the ansible-lint repository: %s",
-                url,
-            )
-            continue
-        path = f"{os.path.relpath(os.path.dirname(__file__))}/{kind}.json"
-        print(f"Refreshing {path} ...")
-        with urllib.request.urlopen(url) as response:
-            content = response.read().decode("utf-8")
-            with open(f"{path}", "r+", encoding="utf-8") as f_out:
-                if f_out.read() != content:
-                    f_out.seek(0)
-                    f_out.write(content)
-                    f_out.truncate()
-                    changed += 1
-    return changed
-
-
-if __name__ == "__main__":
-
-    if refresh_schemas():
-        print(
-            "Schemas are outdated, please update them in a separate pull request.",
-        )
-        sys.exit(1)
-    else:
-        print("Schemas already updated", 0)
+__all__ = ("JSON_SCHEMAS", "refresh_schemas")

src/ansiblelint/schemas/__main__.py

@@ -0,0 +1,12 @@
+"""Module containing cached JSON schemas."""
+import sys
+
+from .main import refresh_schemas
+
+if __name__ == "__main__":
+
+    if refresh_schemas():
+        print("Schemas were updated.")
+        sys.exit(1)
+    else:
+        print("Schemas not updated", 0)

src/ansiblelint/schemas/__store__.json

@@ -0,0 +1,49 @@
+{
+    "ansible-lint-config": {
+        "url": "https://raw.githubusercontent.com/ansible/ansible-lint/main/src/ansiblelint/schemas/ansible-lint-config.json"
+    },
+    "ansible-navigator-config": {
+        "etag": "259a168c473dc88b5def7ea8c7ed28365e46a8abf894cd08fcd57613508e1f26",
+        "url": "https://raw.githubusercontent.com/ansible/ansible-navigator/main/src/ansible_navigator/data/ansible-navigator.json"
+    },
+    "arg_specs": {
+        "etag": "cf3e180b9f9a1980ce491905db37bae4968be10bd0e45fb2096e0e360d4a4912",
+        "url": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-argument-specs.json"
+    },
+    "execution-environment": {
+        "etag": "b2c0e183aabbef8fd83e61e26af2fb1d9a2c8f6549599a24ebf7ccada4c11a81",
+        "url": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-ee.json"
+    },
+    "galaxy": {
+        "etag": "2bf1bc8df72c9063463daabddcb707e9ec6923633e4b66343237e39be102f63b",
+        "url": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-galaxy.json"
+    },
+    "inventory": {
+        "etag": "4bf635a87d034c9316c441ab441ef63689059bf96ecd652339ee86200a707f0f",
+        "url": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-inventory.json"
+    },
+    "meta": {
+        "etag": "ffa1e3f48e1f3dbc2b6a0f18fd27676980bac4c94b39e6c072d55ee5ec27449e",
+        "url": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-meta.json"
+    },
+    "meta-runtime": {
+        "etag": "21c8e4c37930a1c9b135101fcac6c4ef52cfbfe5a724d27612e7ec067cfc5a6a",
+        "url": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-meta-runtime.json"
+    },
+    "playbook": {
+        "etag": "3d3d803a7bba95bb159ad65c2b22c8961264308ef32c7e379d9d30ba3efeecae",
+        "url": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-playbook.json"
+    },
+    "requirements": {
+        "etag": "9805cd5cc2eb54de699fb9bc7a81db6321ab916d068bdd04e98ea58c8a7855e5",
+        "url": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-requirements.json"
+    },
+    "tasks": {
+        "etag": "68f5b5e1a568199617f777438c434006d9d6cfcd468a0f8b6bf8d1ce8d74af70",
+        "url": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-tasks.json"
+    },
+    "vars": {
+        "etag": "91faa0cc8adc0adf365a095efa948f8df21fcfc86ac742013b3429d5ea9a6092",
+        "url": "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible-vars.json"
+    }
+}

src/ansiblelint/schemas/main.py

@@ -0,0 +1,99 @@
+"""Module containing cached JSON schemas."""
+import json
+import logging
+import os
+import sys
+import time
+import urllib.request
+from pathlib import Path
+from urllib.request import Request
+
+_logger = logging.getLogger(__package__)
+
+
+# Maps kinds to JSON schemas
+# See https://www.schemastore.org/json/
+store_file = Path(f"{__file__}/../__store__.json").resolve()
+with open(store_file, encoding="utf-8") as json_file:
+    JSON_SCHEMAS = json.load(json_file)
+
+
+# pylint: disable=too-many-branches
+def refresh_schemas(min_age_seconds: int = 3600 * 24) -> int:
+    """Refresh JSON schemas by downloading latest versions.
+
+    Returns number of changed schemas.
+    """
+    age = int(time.time() - store_file.stat().st_mtime)
+
+    # never check for updated schemas more than once a day
+    if min_age_seconds < age:
+        return 0
+    if not os.access(store_file, os.W_OK):
+        _logger.debug(
+            "Skipping schema update due to lack of writing rights on %s", store_file
+        )
+        return 0
+    _logger.debug("Checking for updated schemas...")
+
+    changed = 0
+    for kind, data in JSON_SCHEMAS.items():
+        url = data["url"]
+        if "#" in url:
+            raise RuntimeError(
+                f"Schema URLs cannot contain # due to python-jsonschema limitation: {url}"
+            )
+        if url.startswith("https://raw.githubusercontent.com/ansible/ansible-lint"):
+            _logger.debug(
+                "Skipped updating schema that is part of the ansible-lint repository: %s",
+                url,
+            )
+            continue
+        path = Path(f"{os.path.relpath(os.path.dirname(__file__))}/{kind}.json")
+        _logger.debug("Refreshing %s schema ...", kind)
+        request = Request(url)
+        etag = data.get("etag", "")
+        if etag:
+            request.add_header("If-None-Match", f'"{data.get("etag")}"')
+        try:
+            with urllib.request.urlopen(request) as response:
+                if response.status == 200:
+                    content = response.read().decode("utf-8")
+                    etag = response.headers["etag"].strip('"')
+                    if etag != data.get("etag", ""):
+                        data["etag"] = etag
+                        changed += 1
+                        print(etag)
+                with open(f"{path}", "r+", encoding="utf-8") as f_out:
+                    _logger.info("Schema %s was updated", kind)
+                    if f_out.read() != content:
+                        f_out.seek(0)
+                        f_out.write(content)
+                        f_out.truncate()
+                    else:
+                        path.touch()
+                    changed += 1
+        except urllib.error.HTTPError as exc:
+            if exc.code == 304:
+                _logger.debug("Schema %s is not modified", url)
+                continue
+            _logger.warning(
+                "Skipped schema refresh due to unexpected exception: %s", exc
+            )
+            return 0
+    if changed:
+        with open(store_file, "w", encoding="utf-8") as f_out:
+            json.dump(JSON_SCHEMAS, f_out, indent=4, sort_keys=True)
+    else:
+        store_file.touch()
+        changed = 1
+    return changed
+
+
+if __name__ == "__main__":
+
+    if refresh_schemas():
+        print("Schemas were updated.")
+        sys.exit(1)
+    else:
+        print("Schemas not updated", 0)

test/test_schemas.py

@@ -0,0 +1,12 @@
+"""Test schemas modules."""
+from ansiblelint.schemas import refresh_schemas
+
+
+def test_refresh_schemas_skip() -> None:
+    """Test for schema update skip."""
+    assert refresh_schemas(min_age_seconds=0) == 0
+
+
+def test_refresh_schemas_forced() -> None:
+    """Test for forced refresh."""
+    assert refresh_schemas(min_age_seconds=3600 * 24 * 365 * 10) == 1