From 034e1e4ac6811e35d7cadcaed36a19535c9d3436 Mon Sep 17 00:00:00 2001 From: Shahar Epstein <60007259+shahar1@users.noreply.github.com> Date: Fri, 5 Jun 2026 19:17:03 +0300 Subject: [PATCH 1/2] [v3-2-test] Only force the full test matrix on API changes when the contract changes (#68060) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Any change under the API directory (airflow-core/src/airflow/api_fastapi/, the legacy api/, or their test dirs) forced full-tests-needed=true — the whole ~135-job matrix including every provider's tests. The API tree is large and churns constantly, so this fired on ~1 in 8 PRs and was, by measurement, the single largest source of unnecessary full-matrix runs. But only the API *contract* changing ripples broadly: the generated OpenAPI spec (consumed by the UI codegen and the generated clients). Plain API source/test edits that leave the committed spec untouched do not — and a prek hook regenerates and verifies the spec, so an unchanged spec reliably means an unchanged contract. Narrow the full-tests trigger from API_FILES to API_CODEGEN_FILES (the generated spec / client generator). API source edits still run the `API` test type and the `fab` provider via run_api_tests; they just no longer drag in the full provider matrix. Spec changes still force the full matrix. In a 30-day sample (750 merged PRs) this spares ~63 full-matrix runs while preserving full coverage on the 12 PRs that changed the contract. (cherry picked from commit d452d2769dfe395f02c8fa48630de2c082319b6f) Co-authored-by: Shahar Epstein <60007259+shahar1@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 --- .../airflow_breeze/utils/selective_checks.py | 15 +- dev/breeze/tests/test_selective_checks.py | 159 ++++++++++++++---- 2 files changed, 135 insertions(+), 39 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 87960c07c3124..784eaef122f5a 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -663,10 +663,21 @@ def full_tests_needed(self) -> bool: console_print("[warning]Running full set of tests because env files changed[/]") return True if self._matching_files( - FileGroupForCi.API_FILES, + FileGroupForCi.API_CODEGEN_FILES, CI_FILE_GROUP_MATCHES, ): - console_print("[warning]Running full set of tests because api files changed[/]") + # Only the API *contract* changing (the generated OpenAPI spec, or the + # client generator) ripples broadly — to the UI codegen, the generated + # clients, and every consumer — so it warrants the full matrix. Plain + # API source/test edits that leave the committed spec untouched do not: + # a prek hook regenerates and verifies the spec, so an unchanged spec + # reliably means an unchanged contract. Those edits still run the `API` + # test type and the `fab` provider (via `run_api_tests`); they just no + # longer drag in the whole provider matrix. + console_print( + "[warning]Running full set of tests because the API contract " + "(generated OpenAPI spec / client generator) changed[/]" + ) return True if self._matching_files( FileGroupForCi.GIT_PROVIDER_FILES, diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index ee5f9ead4f528..36fdd0f1d2565 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -146,6 +146,22 @@ "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) +# API source/test change with NO OpenAPI spec change: the full matrix is no longer +# forced. airflow-core Python changed (so mypy-airflow-core + flynt run); no +# provider.yaml, helm, or UI files changed, so those checks stay skipped. +ALL_SKIPPED_COMMITS_IF_ONLY_API_SOURCE_CHANGED = ( + "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones," + "mypy-task-sdk,mypy-task-sdk-integration-tests," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" +) + ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_AND_UI = ( "check-provider-yaml-valid,identity," "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," @@ -314,72 +330,114 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("airflow-core/src/airflow/api/file.py",), { - "selected-providers-list-as-string": "", + "full-tests-needed": "false", + "selected-providers-list-as-string": "common.compat fab", "all-python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']", "all-python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION, "python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']", "python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION, "ci-image-build": "true", - "prod-image-build": "true", - "run-helm-tests": "true", + "prod-image-build": "false", + "run-helm-tests": "false", "run-unit-tests": "true", + "run-amazon-tests": "false", + "run-api-tests": "true", "docs-build": "true", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED, + "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_ONLY_API_SOURCE_CHANGED, "upgrade-to-newer-dependencies": "false", - "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "individual-providers-test-types-list-as-strings-in-json": LIST_OF_ALL_PROVIDER_TESTS_AS_JSON, - "run-mypy-providers": "true", + "core-test-types-list-as-strings-in-json": json.dumps( + [{"description": "API...Always", "test_types": "API Always"}] + ), + "providers-test-types-list-as-strings-in-json": json.dumps( + [{"description": "common.compat,fab", "test_types": "Providers[common.compat,fab]"}] + ), + "individual-providers-test-types-list-as-strings-in-json": json.dumps( + [ + { + "description": "common.compat...fab", + "test_types": "Providers[common.compat] Providers[fab]", + } + ] + ), + "run-mypy-providers": "false", }, - id="All tests should be run when API file changed", + id="API source change (no spec) runs API + fab only, not the full matrix", ) ), ( pytest.param( ("airflow-core/src/airflow/api_fastapi/file.py",), { + "full-tests-needed": "false", + "selected-providers-list-as-string": "common.compat fab", "all-python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']", "all-python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION, "python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']", "python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION, "ci-image-build": "true", - "prod-image-build": "true", - "run-helm-tests": "true", + "prod-image-build": "false", + "run-helm-tests": "false", "run-unit-tests": "true", - "run-amazon-tests": "true", + "run-amazon-tests": "false", + "run-api-tests": "true", "docs-build": "true", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED, + "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_ONLY_API_SOURCE_CHANGED, "upgrade-to-newer-dependencies": "false", - "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "individual-providers-test-types-list-as-strings-in-json": LIST_OF_ALL_PROVIDER_TESTS_AS_JSON, - "run-mypy-providers": "true", + "core-test-types-list-as-strings-in-json": json.dumps( + [{"description": "API...Always", "test_types": "API Always"}] + ), + "providers-test-types-list-as-strings-in-json": json.dumps( + [{"description": "common.compat,fab", "test_types": "Providers[common.compat,fab]"}] + ), + "individual-providers-test-types-list-as-strings-in-json": json.dumps( + [ + { + "description": "common.compat...fab", + "test_types": "Providers[common.compat] Providers[fab]", + } + ] + ), + "run-mypy-providers": "false", }, - id="All tests should be run when fastapi files change", + id="fastapi source change (no spec) runs API + fab only, not the full matrix", ) ), ( pytest.param( ("airflow-core/tests/unit/api/file.py",), { + "full-tests-needed": "false", + "selected-providers-list-as-string": "common.compat fab", "all-python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']", "all-python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION, "python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']", "python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION, "ci-image-build": "true", - "prod-image-build": "true", - "run-helm-tests": "true", + "prod-image-build": "false", + "run-helm-tests": "false", "run-unit-tests": "true", - "run-amazon-tests": "true", - "docs-build": "true", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED, + "run-amazon-tests": "false", + "run-api-tests": "true", + "docs-build": "false", + "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_ONLY_API_SOURCE_CHANGED, "upgrade-to-newer-dependencies": "false", - "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "individual-providers-test-types-list-as-strings-in-json": LIST_OF_ALL_PROVIDER_TESTS_AS_JSON, - "run-mypy-providers": "true", + "core-test-types-list-as-strings-in-json": json.dumps( + [{"description": "API...Always", "test_types": "API Always"}] + ), + "providers-test-types-list-as-strings-in-json": json.dumps( + [{"description": "common.compat,fab", "test_types": "Providers[common.compat,fab]"}] + ), + "individual-providers-test-types-list-as-strings-in-json": json.dumps( + [ + { + "description": "common.compat...fab", + "test_types": "Providers[common.compat] Providers[fab]", + } + ] + ), + "run-mypy-providers": "false", }, - id="All tests should run when API test files change", + id="API test change (no spec) runs API + fab only, not the full matrix", ) ), ( @@ -429,25 +487,49 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "providers/postgres/tests/unit/postgres/file.py", ), { - "selected-providers-list-as-string": "", + "full-tests-needed": "false", + "selected-providers-list-as-string": "amazon common.compat common.sql fab google " + "microsoft.azure openlineage pgvector postgres", "all-python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']", "all-python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION, "python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']", "python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION, "ci-image-build": "true", - "prod-image-build": "true", - "run-helm-tests": "true", + "prod-image-build": "false", + "run-helm-tests": "false", "run-unit-tests": "true", "run-amazon-tests": "true", + "run-api-tests": "true", "docs-build": "true", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED, + "skip-prek-hooks": ( + "identity,lint-helm-chart," + "mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering," + "mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "upgrade-to-newer-dependencies": "false", - "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "individual-providers-test-types-list-as-strings-in-json": LIST_OF_ALL_PROVIDER_TESTS_AS_JSON, + "core-test-types-list-as-strings-in-json": json.dumps( + [{"description": "API...Always", "test_types": "API Always"}] + ), + "providers-test-types-list-as-strings-in-json": json.dumps( + [ + { + "description": "amazon...google", + "test_types": "Providers[amazon] " + "Providers[common.compat,common.sql,fab,microsoft.azure,openlineage,pgvector,postgres] " + "Providers[google]", + } + ] + ), "run-mypy-providers": "true", }, - id="All tests and docs should run on API change", + id="API source + provider change runs API + affected providers, not the full matrix", ) ), ( @@ -2261,6 +2343,9 @@ def test_expected_output_push( pytest.param( ("airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml",), { + # The OpenAPI spec IS the API contract — changing it ripples to the UI + # codegen and generated clients, so it still forces the full matrix. + "full-tests-needed": "true", "selected-providers-list-as-string": "", "all-python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']", "all-python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION, @@ -2275,7 +2360,7 @@ def test_expected_output_push( "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "run-mypy-providers": "true", }, - id="pre commit ts-compile-format-lint should not be ignored if openapi spec changed.", + id="OpenAPI spec change still forces the full matrix", ), pytest.param( ( From 9d561506af8f1686b153cbdd3f47b2e390181907 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 5 Jun 2026 22:46:40 +0200 Subject: [PATCH 2/2] Fix selective-checks test expectations for v3-2-test (no shared/state) The backported tests expected mypy-shared-state in the skip-prek-hooks output, but the shared/state distribution does not exist on v3-2-test, so that hook is never generated. Drop mypy-shared-state from the two expected skip-prek-hooks strings so the API-matrix selective-check tests pass on this branch. --- dev/breeze/tests/test_selective_checks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 36fdd0f1d2565..719d6cc1d8a23 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -157,7 +157,7 @@ "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," - "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones," "mypy-task-sdk,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) @@ -509,7 +509,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," - "mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering," + "mypy-shared-serialization,mypy-shared-template_rendering," "mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ),