From 3f83bb5e6ce40ac55512d9973142b2956a005045 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 3 Aug 2026 02:44:46 +0200 Subject: [PATCH] Regenerate stale provider dependencies instead of silently using them Release commands enumerate providers from generated/provider_dependencies.json, filtering on each provider's state. The freshness check runs on every breeze invocation, but when the json was present and its sha256sum sidecar was not, it delegated to a helper that only regenerates when the json is missing. The stale file was therefore returned as-is and no sidecar was ever written, leaving the check permanently disabled for that checkout. A provider whose state changed then stays invisible to the release: common.ai was flipped to ready in 682cf06525 but a checkout carrying an older json still called it not-ready, so it was dropped from the 2026-08-01 wave with no warning and was only caught by an unrelated package count mismatch. --- .../utils/provider_dependencies.py | 20 +++-- .../tests/test_provider_dependencies.py | 81 +++++++++++++++++++ 2 files changed, 94 insertions(+), 7 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py b/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py index d67e0ec9bfc1b..02aa337de3725 100644 --- a/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py +++ b/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py @@ -179,24 +179,30 @@ def _calculate_provider_deps_hash(): def get_provider_dependencies() -> dict: if not PROVIDER_DEPENDENCIES_JSON_PATH.exists(): calculated_hash = _calculate_provider_deps_hash() - PROVIDER_DEPENDENCIES_JSON_HASH_PATH.write_text(calculated_hash + "\n") # We use regular print there as rich console might not be initialized yet here print("Regenerating provider dependencies file") regenerate_provider_dependencies_once() + # Only record the hash once regeneration succeeded, otherwise a failed run would + # leave a sidecar claiming that a missing/stale file is up to date. + PROVIDER_DEPENDENCIES_JSON_HASH_PATH.write_text(calculated_hash + "\n") return json.loads(PROVIDER_DEPENDENCIES_JSON_PATH.read_text()) +def _force_regenerate_provider_dependencies() -> None: + # get_provider_dependencies() only regenerates when the JSON is absent, so the file has + # to be removed for it to pick up changed provider.yaml/pyproject.toml contents. + PROVIDER_DEPENDENCIES_JSON_PATH.unlink(missing_ok=True) + get_provider_dependencies.cache_clear() + get_provider_dependencies() + + def generate_provider_dependencies_if_needed(): if not PROVIDER_DEPENDENCIES_JSON_PATH.exists() or not PROVIDER_DEPENDENCIES_JSON_HASH_PATH.exists(): - get_provider_dependencies.cache_clear() - get_provider_dependencies() + _force_regenerate_provider_dependencies() else: calculated_hash = _calculate_provider_deps_hash() if calculated_hash.strip() != PROVIDER_DEPENDENCIES_JSON_HASH_PATH.read_text().strip(): - # Force re-generation - PROVIDER_DEPENDENCIES_JSON_PATH.unlink(missing_ok=True) - get_provider_dependencies.cache_clear() - get_provider_dependencies() + _force_regenerate_provider_dependencies() def get_related_providers( diff --git a/dev/breeze/tests/test_provider_dependencies.py b/dev/breeze/tests/test_provider_dependencies.py index 4ddc16a60cfa7..bf77f6dd29d4c 100644 --- a/dev/breeze/tests/test_provider_dependencies.py +++ b/dev/breeze/tests/test_provider_dependencies.py @@ -16,9 +16,13 @@ # under the License. from __future__ import annotations +from unittest import mock + import pytest +from airflow_breeze.utils import provider_dependencies as provider_dependencies_module from airflow_breeze.utils.provider_dependencies import ( + generate_provider_dependencies_if_needed, get_related_providers, ) @@ -47,3 +51,80 @@ def test_both(): def test_none(): with pytest.raises(ValueError, match=r".*must be.*"): get_related_providers("trino", upstream_dependencies=False, downstream_dependencies=False) + + +@pytest.fixture +def provider_deps_files(tmp_path): + """Point the module at a throwaway json/sha256sum pair and clear the lru_cache around it.""" + json_path = tmp_path / "provider_dependencies.json" + hash_path = tmp_path / "provider_dependencies.json.sha256sum" + provider_dependencies_module.get_provider_dependencies.cache_clear() + with mock.patch.multiple( + provider_dependencies_module, + PROVIDER_DEPENDENCIES_JSON_PATH=json_path, + PROVIDER_DEPENDENCIES_JSON_HASH_PATH=hash_path, + ): + yield json_path, hash_path + provider_dependencies_module.get_provider_dependencies.cache_clear() + + +@pytest.mark.parametrize( + "hash_sidecar_present", + [ + pytest.param(False, id="missing-sidecar"), + pytest.param(True, id="stale-sidecar"), + ], +) +@mock.patch.object(provider_dependencies_module, "regenerate_provider_dependencies_once") +@mock.patch.object(provider_dependencies_module, "_calculate_provider_deps_hash") +def test_stale_dependencies_are_regenerated( + mock_hash, mock_regenerate, provider_deps_files, hash_sidecar_present +): + """A present-but-outdated json must be regenerated, and the sidecar refreshed. + + Without this, breeze silently enumerates providers from stale state - which dropped + common.ai from the 2026-08-01 provider release wave. + """ + json_path, hash_path = provider_deps_files + json_path.write_text('{"common.ai": {"state": "not-ready"}}') + if hash_sidecar_present: + hash_path.write_text("stale-hash\n") + mock_hash.return_value = "fresh-hash" + + def regenerate(): + json_path.write_text('{"common.ai": {"state": "ready"}}') + + mock_regenerate.side_effect = regenerate + + generate_provider_dependencies_if_needed() + + mock_regenerate.assert_called_once() + assert provider_dependencies_module.get_provider_dependencies() == {"common.ai": {"state": "ready"}} + assert hash_path.read_text().strip() == "fresh-hash" + + +@mock.patch.object(provider_dependencies_module, "regenerate_provider_dependencies_once") +@mock.patch.object(provider_dependencies_module, "_calculate_provider_deps_hash") +def test_up_to_date_dependencies_are_not_regenerated(mock_hash, mock_regenerate, provider_deps_files): + json_path, hash_path = provider_deps_files + json_path.write_text('{"common.ai": {"state": "ready"}}') + hash_path.write_text("fresh-hash\n") + mock_hash.return_value = "fresh-hash" + + generate_provider_dependencies_if_needed() + + mock_regenerate.assert_not_called() + + +@mock.patch.object(provider_dependencies_module, "regenerate_provider_dependencies_once") +@mock.patch.object(provider_dependencies_module, "_calculate_provider_deps_hash") +def test_hash_not_written_when_regeneration_fails(mock_hash, mock_regenerate, provider_deps_files): + """A failed regeneration must not leave a sidecar vouching for stale content.""" + _, hash_path = provider_deps_files + mock_hash.return_value = "fresh-hash" + mock_regenerate.side_effect = RuntimeError("boom") + + with pytest.raises(RuntimeError, match="boom"): + generate_provider_dependencies_if_needed() + + assert not hash_path.exists()