Regenerate stale provider dependencies instead of silently using them

dev/breeze/src/airflow_breeze/utils/provider_dependencies.py

@@ -179,24 +179,30 @@ def _calculate_provider_deps_hash():
 def get_provider_dependencies() -> dict:
     if not PROVIDER_DEPENDENCIES_JSON_PATH.exists():
         calculated_hash = _calculate_provider_deps_hash()
-        PROVIDER_DEPENDENCIES_JSON_HASH_PATH.write_text(calculated_hash + "\n")
         # We use regular print there as rich console might not be initialized yet here
         print("Regenerating provider dependencies file")
         regenerate_provider_dependencies_once()
+        # Only record the hash once regeneration succeeded, otherwise a failed run would
+        # leave a sidecar claiming that a missing/stale file is up to date.
+        PROVIDER_DEPENDENCIES_JSON_HASH_PATH.write_text(calculated_hash + "\n")
     return json.loads(PROVIDER_DEPENDENCIES_JSON_PATH.read_text())
 
 
+def _force_regenerate_provider_dependencies() -> None:
+    # get_provider_dependencies() only regenerates when the JSON is absent, so the file has
+    # to be removed for it to pick up changed provider.yaml/pyproject.toml contents.
+    PROVIDER_DEPENDENCIES_JSON_PATH.unlink(missing_ok=True)
+    get_provider_dependencies.cache_clear()
+    get_provider_dependencies()
+
+
 def generate_provider_dependencies_if_needed():
     if not PROVIDER_DEPENDENCIES_JSON_PATH.exists() or not PROVIDER_DEPENDENCIES_JSON_HASH_PATH.exists():
-        get_provider_dependencies.cache_clear()
-        get_provider_dependencies()
+        _force_regenerate_provider_dependencies()
     else:
         calculated_hash = _calculate_provider_deps_hash()
         if calculated_hash.strip() != PROVIDER_DEPENDENCIES_JSON_HASH_PATH.read_text().strip():
-            # Force re-generation
-            PROVIDER_DEPENDENCIES_JSON_PATH.unlink(missing_ok=True)
-            get_provider_dependencies.cache_clear()
-            get_provider_dependencies()
+            _force_regenerate_provider_dependencies()
 
 
 def get_related_providers(

dev/breeze/tests/test_provider_dependencies.py

@@ -16,9 +16,13 @@
 # under the License.
 from __future__ import annotations
 
+from unittest import mock
+
 import pytest
 
+from airflow_breeze.utils import provider_dependencies as provider_dependencies_module
 from airflow_breeze.utils.provider_dependencies import (
+    generate_provider_dependencies_if_needed,
     get_related_providers,
 )
 
@@ -47,3 +51,80 @@ def test_both():
 def test_none():
     with pytest.raises(ValueError, match=r".*must be.*"):
         get_related_providers("trino", upstream_dependencies=False, downstream_dependencies=False)
+
+
+@pytest.fixture
+def provider_deps_files(tmp_path):
+    """Point the module at a throwaway json/sha256sum pair and clear the lru_cache around it."""
+    json_path = tmp_path / "provider_dependencies.json"
+    hash_path = tmp_path / "provider_dependencies.json.sha256sum"
+    provider_dependencies_module.get_provider_dependencies.cache_clear()
+    with mock.patch.multiple(
+        provider_dependencies_module,
+        PROVIDER_DEPENDENCIES_JSON_PATH=json_path,
+        PROVIDER_DEPENDENCIES_JSON_HASH_PATH=hash_path,
+    ):
+        yield json_path, hash_path
+    provider_dependencies_module.get_provider_dependencies.cache_clear()
+
+
+@pytest.mark.parametrize(
+    "hash_sidecar_present",
+    [
+        pytest.param(False, id="missing-sidecar"),
+        pytest.param(True, id="stale-sidecar"),
+    ],
+)
+@mock.patch.object(provider_dependencies_module, "regenerate_provider_dependencies_once")
+@mock.patch.object(provider_dependencies_module, "_calculate_provider_deps_hash")
+def test_stale_dependencies_are_regenerated(
+    mock_hash, mock_regenerate, provider_deps_files, hash_sidecar_present
+):
+    """A present-but-outdated json must be regenerated, and the sidecar refreshed.
+
+    Without this, breeze silently enumerates providers from stale state - which dropped
+    common.ai from the 2026-08-01 provider release wave.
+    """
+    json_path, hash_path = provider_deps_files
+    json_path.write_text('{"common.ai": {"state": "not-ready"}}')
+    if hash_sidecar_present:
+        hash_path.write_text("stale-hash\n")
+    mock_hash.return_value = "fresh-hash"
+
+    def regenerate():
+        json_path.write_text('{"common.ai": {"state": "ready"}}')
+
+    mock_regenerate.side_effect = regenerate
+
+    generate_provider_dependencies_if_needed()
+
+    mock_regenerate.assert_called_once()
+    assert provider_dependencies_module.get_provider_dependencies() == {"common.ai": {"state": "ready"}}
+    assert hash_path.read_text().strip() == "fresh-hash"
+
+
+@mock.patch.object(provider_dependencies_module, "regenerate_provider_dependencies_once")
+@mock.patch.object(provider_dependencies_module, "_calculate_provider_deps_hash")
+def test_up_to_date_dependencies_are_not_regenerated(mock_hash, mock_regenerate, provider_deps_files):
+    json_path, hash_path = provider_deps_files
+    json_path.write_text('{"common.ai": {"state": "ready"}}')
+    hash_path.write_text("fresh-hash\n")
+    mock_hash.return_value = "fresh-hash"
+
+    generate_provider_dependencies_if_needed()
+
+    mock_regenerate.assert_not_called()
+
+
+@mock.patch.object(provider_dependencies_module, "regenerate_provider_dependencies_once")
+@mock.patch.object(provider_dependencies_module, "_calculate_provider_deps_hash")
+def test_hash_not_written_when_regeneration_fails(mock_hash, mock_regenerate, provider_deps_files):
+    """A failed regeneration must not leave a sidecar vouching for stale content."""
+    _, hash_path = provider_deps_files
+    mock_hash.return_value = "fresh-hash"
+    mock_regenerate.side_effect = RuntimeError("boom")
+
+    with pytest.raises(RuntimeError, match="boom"):
+        generate_provider_dependencies_if_needed()
+
+    assert not hash_path.exists()