Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions airflow-core/src/airflow/provider.yaml.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,13 @@
"type": "string"
}
},
"excluded-platforms": {
"description": "List of platforms (e.g. linux/arm64) excluded for that provider. Used to skip providers whose native dependencies are unavailable on a given architecture.",
"type": "array",
"items": {
"type": "string"
}
},
"integrations": {
"description": "List of integrations supported by the provider.",
"type": "array",
Expand Down
49 changes: 49 additions & 0 deletions dev/breeze/src/airflow_breeze/utils/selective_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -1319,6 +1319,53 @@ def core_test_types_list_as_strings_in_json(self) -> str | None:
list_of_list_of_types = _split_list(current_test_types, NUMBER_OF_CORE_SLICES)
return json.dumps(_get_test_list_as_json(list_of_list_of_types))

@cached_property
def _platform_excluded_providers(self) -> set[str]:
"""Provider ids that opt out of the current ``self.platform`` via provider.yaml.

Mirrors the ``excluded-python-versions`` mechanism but keyed by Docker platform
string (e.g. ``linux/arm64``) so providers whose native dependencies are unavailable
on a given architecture can be removed from the test matrix at planning time.
"""
excluded: set[str] = set()
for provider_id, provider_info in get_provider_dependencies().items():
if self.platform in provider_info.get("excluded-platforms", []):
excluded.add(provider_id)
return excluded

def _filter_platform_excluded_test_types(self, current_test_types: set[str]) -> None:
"""Rewrite ``Providers[...]`` entries in-place to honor ``excluded-platforms``.

Handles three shapes produced upstream:

* ``Providers[foo]`` — dropped entirely when ``foo`` is excluded.
* ``Providers[a,foo,b]`` — rewritten to ``Providers[a,b]``; dropped if empty.
* ``Providers[-amazon,celery,google,standard]`` (negative, "all except") —
excluded providers are appended to the negation so they remain skipped.
"""
excluded = self._platform_excluded_providers
if not excluded:
return
for original in tuple(current_test_types):
if not original.startswith("Providers[") or not original.endswith("]"):
continue
inner = original[len("Providers[") : -1]
if inner.startswith("-"):
negated = [p for p in inner[1:].split(",") if p]
additions = sorted(p for p in excluded if p not in negated)
if not additions:
continue
current_test_types.remove(original)
current_test_types.add(f"Providers[-{','.join(sorted(negated + additions))}]")
continue
providers_in = [p for p in inner.split(",") if p]
kept = [p for p in providers_in if p not in excluded]
if kept == providers_in:
continue
current_test_types.remove(original)
if kept:
current_test_types.add(f"Providers[{','.join(kept)}]")

@cached_property
def providers_test_types_list_as_strings_in_json(self) -> str:
if not self.run_unit_tests:
Expand All @@ -1335,6 +1382,7 @@ def providers_test_types_list_as_strings_in_json(self) -> str:
test_types_to_remove.add(test_type)
current_test_types = current_test_types - test_types_to_remove
self._extract_long_provider_tests(current_test_types)
self._filter_platform_excluded_test_types(current_test_types)
return json.dumps(_get_test_list_as_json([sorted(current_test_types)]))

def _get_individual_providers_list(self):
Expand All @@ -1344,6 +1392,7 @@ def _get_individual_providers_list(self):
current_test_types.update(
{f"Providers[{provider}]" for provider in get_available_distributions(include_not_ready=True)}
)
self._filter_platform_excluded_test_types(current_test_types)
return current_test_types

@cached_property
Expand Down
72 changes: 72 additions & 0 deletions dev/breeze/tests/test_selective_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -3178,6 +3178,78 @@ def test_testable_providers_integrations_excludes_arm_disabled_on_arm():
assert "ydb" not in result


def test_individual_providers_excludes_platform_excluded_on_arm():
"""ibm.mq declares `excluded-platforms: [linux/arm64]`, so it must be absent from
the ARM individual-providers matrix (used by the Low-dep ARM canary job) and
present on AMD."""
arm_checks = SelectiveChecks(
files=("airflow-core/tests/test_example.py",),
commit_ref=NEUTRAL_COMMIT,
github_event=GithubEvents.SCHEDULE,
github_context_dict={"ref_name": "main"},
default_branch="main",
pr_labels=("full tests needed",),
)
with patch.object(
SelectiveChecks, "runner_type", new_callable=lambda: property(lambda self: '["ubuntu-22.04-arm"]')
):
assert arm_checks.platform == "linux/arm64"
arm_output = arm_checks.individual_providers_test_types_list_as_strings_in_json
assert arm_output is not None
assert "Providers[ibm.mq]" not in arm_output

amd_checks = SelectiveChecks(
files=("airflow-core/tests/test_example.py",),
commit_ref=NEUTRAL_COMMIT,
github_event=GithubEvents.SCHEDULE,
github_context_dict={"ref_name": "main"},
default_branch="main",
pr_labels=("full tests needed",),
)
with patch.object(
SelectiveChecks, "runner_type", new_callable=lambda: property(lambda self: PUBLIC_AMD_RUNNERS)
):
assert amd_checks.platform == "linux/amd64"
amd_output = amd_checks.individual_providers_test_types_list_as_strings_in_json
assert amd_output is not None
assert "Providers[ibm.mq]" in amd_output


def test_filter_platform_excluded_test_types_handles_all_shapes():
"""Direct unit check of the in-place filter for the three Providers[...] shapes."""
checks = SelectiveChecks(
files=(),
commit_ref=NEUTRAL_COMMIT,
github_event=GithubEvents.SCHEDULE,
github_context_dict={"ref_name": "main"},
default_branch="main",
)
with patch.object(
SelectiveChecks,
"_platform_excluded_providers",
new_callable=lambda: property(lambda self: {"ibm.mq"}),
):
# Bare match drops entry.
ts = {"Providers[ibm.mq]"}
checks._filter_platform_excluded_test_types(ts)
assert ts == set()

# Combined positive form drops just the excluded id.
ts = {"Providers[amazon,ibm.mq,google]"}
checks._filter_platform_excluded_test_types(ts)
assert ts == {"Providers[amazon,google]"}

# Negative form gets the excluded id appended.
ts = {"Providers[-amazon,celery,google,standard]"}
checks._filter_platform_excluded_test_types(ts)
assert ts == {"Providers[-amazon,celery,google,ibm.mq,standard]"}

# Non-Providers entries are untouched.
ts = {"Core", "Always"}
checks._filter_platform_excluded_test_types(ts)
assert ts == {"Core", "Always"}


@patch("airflow_breeze.utils.selective_checks.run_command")
def test_provider_dependency_bump_check_no_changes(mock_run_command):
"""Test that provider dependency bump check passes when no pyproject.toml files are changed."""
Expand Down
8 changes: 8 additions & 0 deletions providers/ibm/mq/provider.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,14 @@ lifecycle: incubation
source-date-epoch: 1758787200
description: |
`IBM MQ <https://www.ibm.com/products/mq/>`__
# IBM MQ ships its native C client library only for Linux x86_64, Windows x64 and Java
# (https://www.ibm.com/support/pages/downloading-ibm-mq-94 — Redist clients). The
# `ibmmq` Python bindings link against that C client at build time, so `uv sync
# --resolution lowest-direct --all-extras` fails to build the sdist on aarch64.
# Exclude the provider from the linux/arm64 test matrix until IBM publishes an aarch64
# redistributable.
excluded-platforms:
- linux/arm64
# Note that those versions are maintained by release manager - do not update them manually
# with the exception of case where other provider in sources has >= new provider version.
# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
Expand Down
2 changes: 2 additions & 0 deletions scripts/ci/prek/update_providers_dependencies.py
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,8 @@ def check_if_different_provider_used(file_path: Path) -> None:
)
excluded_versions = ALL_PROVIDERS[key].get("excluded-python-versions")
unique_sorted_dependencies[key]["excluded-python-versions"] = excluded_versions or []
excluded_platforms = ALL_PROVIDERS[key].get("excluded-platforms")
unique_sorted_dependencies[key]["excluded-platforms"] = excluded_platforms or []
unique_sorted_dependencies[key]["state"] = STATES[key]
if errors:
console.print()
Expand Down
Loading