From e55fa2f85082be3b4753d7e4b7a9d08112930638 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 14:24:45 +0300 Subject: [PATCH 001/309] [v3-2-test] Load hook metadata from YAML without importing Hook class (#63826) (#64723) * Load hook metadata from YAML without importing Hook class * Add hook-name to all provider.yaml connection-types * Add hook-name to connection types and regenerate get_provider_info.py * Fix ruff import order in connections.py * fix: import ProvidersManager at top level per review * Fix provider connection hook display names * Add iter_connection_type_hook_ui_metadata for connection UI hook metadata (cherry picked from commit c4a209ba0ec1749ce0d0ac561fb22feba5bad567) Co-authored-by: Yuseok Jo --- .../core_api/services/ui/connections.py | 32 +++++------ .../src/airflow/provider.yaml.schema.json | 4 ++ .../src/airflow/provider_info.schema.json | 4 ++ airflow-core/src/airflow/providers_manager.py | 55 ++++++++++++++++++- .../unit/always/test_providers_manager.py | 8 +++ providers/airbyte/provider.yaml | 1 + .../providers/airbyte/get_provider_info.py | 1 + providers/alibaba/provider.yaml | 4 ++ .../providers/alibaba/get_provider_info.py | 4 ++ providers/amazon/provider.yaml | 5 ++ .../providers/amazon/get_provider_info.py | 5 ++ providers/apache/cassandra/provider.yaml | 1 + .../apache/cassandra/get_provider_info.py | 1 + providers/apache/drill/provider.yaml | 1 + .../apache/drill/get_provider_info.py | 1 + providers/apache/druid/provider.yaml | 1 + .../apache/druid/get_provider_info.py | 1 + providers/apache/hdfs/provider.yaml | 1 + .../apache/hdfs/get_provider_info.py | 1 + providers/apache/hive/provider.yaml | 3 + .../apache/hive/get_provider_info.py | 3 + providers/apache/iceberg/provider.yaml | 1 + .../apache/iceberg/get_provider_info.py | 1 + providers/apache/impala/provider.yaml | 1 + .../apache/impala/get_provider_info.py | 1 + providers/apache/kafka/provider.yaml | 1 + .../apache/kafka/get_provider_info.py | 1 + providers/apache/kylin/provider.yaml | 1 + .../apache/kylin/get_provider_info.py | 1 + providers/apache/livy/provider.yaml | 1 + .../apache/livy/get_provider_info.py | 1 + providers/apache/pig/provider.yaml | 1 + .../providers/apache/pig/get_provider_info.py | 1 + providers/apache/pinot/provider.yaml | 2 + .../apache/pinot/get_provider_info.py | 2 + providers/apache/spark/provider.yaml | 4 ++ .../apache/spark/get_provider_info.py | 4 ++ providers/apache/tinkerpop/provider.yaml | 1 + .../apache/tinkerpop/get_provider_info.py | 1 + providers/apprise/provider.yaml | 1 + .../providers/apprise/get_provider_info.py | 1 + providers/arangodb/provider.yaml | 1 + .../providers/arangodb/get_provider_info.py | 1 + providers/asana/provider.yaml | 1 + .../providers/asana/get_provider_info.py | 1 + providers/atlassian/jira/provider.yaml | 1 + .../atlassian/jira/get_provider_info.py | 1 + providers/cloudant/provider.yaml | 1 + .../providers/cloudant/get_provider_info.py | 1 + providers/cncf/kubernetes/provider.yaml | 1 + .../cncf/kubernetes/get_provider_info.py | 1 + providers/cohere/provider.yaml | 1 + .../providers/cohere/get_provider_info.py | 1 + providers/common/ai/provider.yaml | 5 ++ .../providers/common/ai/get_provider_info.py | 5 ++ providers/databricks/provider.yaml | 1 + .../providers/databricks/get_provider_info.py | 1 + providers/datadog/provider.yaml | 1 + .../providers/datadog/get_provider_info.py | 1 + providers/dbt/cloud/provider.yaml | 1 + .../providers/dbt/cloud/get_provider_info.py | 1 + providers/dingding/provider.yaml | 1 + .../providers/dingding/get_provider_info.py | 1 + providers/discord/provider.yaml | 1 + .../providers/discord/get_provider_info.py | 1 + providers/docker/provider.yaml | 1 + .../providers/docker/get_provider_info.py | 1 + providers/elasticsearch/provider.yaml | 1 + .../elasticsearch/get_provider_info.py | 1 + providers/exasol/provider.yaml | 1 + .../providers/exasol/get_provider_info.py | 1 + providers/facebook/provider.yaml | 1 + .../providers/facebook/get_provider_info.py | 1 + providers/ftp/provider.yaml | 1 + .../providers/ftp/get_provider_info.py | 6 +- providers/git/provider.yaml | 1 + .../providers/git/get_provider_info.py | 1 + providers/github/provider.yaml | 1 + .../providers/github/get_provider_info.py | 1 + providers/google/provider.yaml | 9 +++ .../providers/google/get_provider_info.py | 9 +++ providers/grpc/provider.yaml | 1 + .../providers/grpc/get_provider_info.py | 1 + providers/hashicorp/provider.yaml | 1 + .../providers/hashicorp/get_provider_info.py | 1 + providers/http/provider.yaml | 1 + .../providers/http/get_provider_info.py | 1 + providers/imap/provider.yaml | 1 + .../providers/imap/get_provider_info.py | 6 +- providers/influxdb/provider.yaml | 1 + .../providers/influxdb/get_provider_info.py | 1 + providers/informatica/provider.yaml | 1 + .../informatica/get_provider_info.py | 1 + providers/jdbc/provider.yaml | 1 + .../providers/jdbc/get_provider_info.py | 1 + providers/jenkins/provider.yaml | 1 + .../providers/jenkins/get_provider_info.py | 1 + providers/microsoft/azure/provider.yaml | 17 ++++++ .../microsoft/azure/get_provider_info.py | 17 ++++++ .../microsoft/azure/hooks/data_lake.py | 2 +- providers/microsoft/mssql/provider.yaml | 1 + .../microsoft/mssql/get_provider_info.py | 1 + providers/microsoft/psrp/provider.yaml | 1 + .../microsoft/psrp/get_provider_info.py | 1 + providers/microsoft/winrm/provider.yaml | 1 + .../microsoft/winrm/get_provider_info.py | 1 + providers/mongo/provider.yaml | 1 + .../providers/mongo/get_provider_info.py | 1 + providers/mysql/provider.yaml | 1 + .../providers/mysql/get_provider_info.py | 6 +- providers/neo4j/provider.yaml | 1 + .../providers/neo4j/get_provider_info.py | 6 +- providers/odbc/provider.yaml | 1 + .../providers/odbc/get_provider_info.py | 6 +- providers/openai/provider.yaml | 1 + .../providers/openai/get_provider_info.py | 1 + providers/openfaas/provider.yaml | 1 + .../providers/openfaas/get_provider_info.py | 1 + providers/opensearch/provider.yaml | 1 + .../providers/opensearch/get_provider_info.py | 1 + providers/opsgenie/provider.yaml | 1 + .../providers/opsgenie/get_provider_info.py | 1 + providers/oracle/provider.yaml | 1 + .../providers/oracle/get_provider_info.py | 1 + providers/pagerduty/provider.yaml | 2 + .../providers/pagerduty/get_provider_info.py | 2 + providers/papermill/provider.yaml | 1 + .../providers/papermill/get_provider_info.py | 1 + providers/pinecone/provider.yaml | 1 + .../providers/pinecone/get_provider_info.py | 1 + providers/postgres/provider.yaml | 1 + .../providers/postgres/get_provider_info.py | 1 + providers/presto/provider.yaml | 1 + .../providers/presto/get_provider_info.py | 1 + providers/qdrant/provider.yaml | 1 + .../providers/qdrant/get_provider_info.py | 1 + providers/redis/provider.yaml | 1 + .../providers/redis/get_provider_info.py | 1 + providers/salesforce/provider.yaml | 1 + .../providers/salesforce/get_provider_info.py | 1 + providers/samba/provider.yaml | 1 + .../providers/samba/get_provider_info.py | 1 + providers/segment/provider.yaml | 1 + .../providers/segment/get_provider_info.py | 1 + providers/sftp/provider.yaml | 1 + .../providers/sftp/get_provider_info.py | 1 + providers/slack/provider.yaml | 2 + .../providers/slack/get_provider_info.py | 2 + providers/smtp/provider.yaml | 1 + .../providers/smtp/get_provider_info.py | 1 + providers/snowflake/provider.yaml | 1 + .../providers/snowflake/get_provider_info.py | 1 + providers/sqlite/provider.yaml | 1 + .../providers/sqlite/get_provider_info.py | 1 + providers/ssh/provider.yaml | 1 + .../providers/ssh/get_provider_info.py | 1 + providers/standard/provider.yaml | 2 + .../providers/standard/get_provider_info.py | 2 + providers/tableau/provider.yaml | 1 + .../providers/tableau/get_provider_info.py | 1 + providers/telegram/provider.yaml | 1 + .../providers/telegram/get_provider_info.py | 1 + providers/teradata/provider.yaml | 1 + .../providers/teradata/get_provider_info.py | 1 + providers/trino/provider.yaml | 1 + .../providers/trino/get_provider_info.py | 1 + providers/vertica/provider.yaml | 1 + .../providers/vertica/get_provider_info.py | 1 + providers/weaviate/provider.yaml | 1 + .../providers/weaviate/get_provider_info.py | 1 + providers/yandex/provider.yaml | 1 + .../providers/yandex/get_provider_info.py | 1 + .../airflow/providers/yandex/hooks/yandex.py | 4 +- providers/ydb/provider.yaml | 1 + .../providers/ydb/get_provider_info.py | 1 + providers/zendesk/provider.yaml | 1 + .../providers/zendesk/get_provider_info.py | 1 + 177 files changed, 365 insertions(+), 27 deletions(-) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/services/ui/connections.py b/airflow-core/src/airflow/api_fastapi/core_api/services/ui/connections.py index 96480555c4000..1cf45b7f7e1d0 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/services/ui/connections.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/services/ui/connections.py @@ -27,10 +27,11 @@ ConnectionHookMetaData, StandardHookFields, ) +from airflow.providers_manager import HookInfo, ProvidersManager from airflow.serialization.definitions.param import SerializedParam if TYPE_CHECKING: - from airflow.providers_manager import ConnectionFormWidgetInfo, HookInfo + from airflow.providers_manager import ConnectionFormWidgetInfo log = logging.getLogger(__name__) @@ -125,8 +126,6 @@ def _get_hooks_with_mocked_fab() -> tuple[ """Get hooks with all details w/o FAB needing to be installed.""" from unittest import mock - from airflow.providers_manager import ProvidersManager - def mock_lazy_gettext(txt: str) -> str: """Mock for flask_babel.lazy_gettext.""" return txt @@ -225,19 +224,16 @@ def _convert_extra_fields(form_widgets: dict[str, ConnectionFormWidgetInfo]) -> @staticmethod @cache def hook_meta_data() -> list[ConnectionHookMetaData]: - hooks, connection_form_widgets, field_behaviours = HookMetaService._get_hooks_with_mocked_fab() - result: list[ConnectionHookMetaData] = [] - widgets = HookMetaService._convert_extra_fields(connection_form_widgets) - for hook_key, hook_info in hooks.items(): - if not hook_info: - continue - hook_meta = ConnectionHookMetaData( - connection_type=hook_key, - hook_class_name=hook_info.hook_class_name, - default_conn_name=None, # TODO: later - hook_name=hook_info.hook_name, - standard_fields=HookMetaService._make_standard_fields(field_behaviours.get(hook_key)), - extra_fields=widgets.get(hook_key), + pm = ProvidersManager() + widgets = HookMetaService._convert_extra_fields(pm._connection_form_widgets_from_metadata) + return [ + ConnectionHookMetaData( + connection_type=meta.connection_type, + hook_class_name=meta.hook_class_name, + default_conn_name=None, + hook_name=meta.hook_name, + standard_fields=HookMetaService._make_standard_fields(meta.field_behaviour), + extra_fields=widgets.get(meta.connection_type), ) - result.append(hook_meta) - return result + for meta in pm.iter_connection_type_hook_ui_metadata() + ] diff --git a/airflow-core/src/airflow/provider.yaml.schema.json b/airflow-core/src/airflow/provider.yaml.schema.json index ac6b05f30c87b..5714b8db658c5 100644 --- a/airflow-core/src/airflow/provider.yaml.schema.json +++ b/airflow-core/src/airflow/provider.yaml.schema.json @@ -378,6 +378,10 @@ "description": "Hook class name that implements the connection type", "type": "string" }, + "hook-name": { + "description": "Display name for the connection type in the UI (e.g. 'File (path)', 'Slack')", + "type": "string" + }, "ui-field-behaviour": { "description": "Customizations for standard connection form fields", "type": "object", diff --git a/airflow-core/src/airflow/provider_info.schema.json b/airflow-core/src/airflow/provider_info.schema.json index 7c3eea12591dd..86fc726a05168 100644 --- a/airflow-core/src/airflow/provider_info.schema.json +++ b/airflow-core/src/airflow/provider_info.schema.json @@ -298,6 +298,10 @@ "hook-class-name": { "description": "Hook class name that implements the connection type", "type": "string" + }, + "hook-name": { + "description": "Display name for the connection type in the UI", + "type": "string" } }, "required": [ diff --git a/airflow-core/src/airflow/providers_manager.py b/airflow-core/src/airflow/providers_manager.py index b8d48a31b9c34..6fefcbc39b06d 100644 --- a/airflow-core/src/airflow/providers_manager.py +++ b/airflow-core/src/airflow/providers_manager.py @@ -26,7 +26,7 @@ import logging import traceback import warnings -from collections.abc import Callable, MutableMapping +from collections.abc import Callable, Iterator, MutableMapping from dataclasses import dataclass from functools import wraps from importlib.resources import files as resource_files @@ -243,6 +243,15 @@ class HookInfo(NamedTuple): dialects: list[str] = [] +class ConnectionTypeHookUIMetadata(NamedTuple): + """Hook metadata for one connection type (connection UI); ``field_behaviour`` is standard fields.""" + + connection_type: str + hook_name: str + hook_class_name: str | None + field_behaviour: dict | None + + class ConnectionFormWidgetInfo(NamedTuple): """Connection Form Widget information.""" @@ -413,6 +422,8 @@ def __init__(self): self._dialect_provider_dict: dict[str, DialectInfo] = {} # Keeps dict of hooks keyed by connection type. They are lazy evaluated at access time self._hooks_lazy_dict: LazyDictWithCache[str, HookInfo | Callable] = LazyDictWithCache() + # Keeps hook display names read from provider.yaml (hook-name field) + self._hook_name_dict: dict[str, str] = {} # Keeps methods that should be used to add custom widgets tuple of keyed by name of the extra field self._connection_form_widgets: dict[str, ConnectionFormWidgetInfo] = {} # Customizations for javascript fields are kept here @@ -979,6 +990,9 @@ def _load_ui_metadata(self) -> None: if not connection_type or not hook_class_name: continue + if hook_name := conn_config.get("hook-name"): + self._hook_name_dict[connection_type] = hook_name + if conn_fields := conn_config.get("conn-fields"): self._add_widgets(package_name, hook_class_name, connection_type, conn_fields) @@ -1349,6 +1363,45 @@ def hooks(self) -> MutableMapping[str, HookInfo | None]: # When we return hooks here it will only be used to retrieve hook information return self._hooks_lazy_dict + def iter_connection_type_hook_ui_metadata(self) -> Iterator[ConnectionTypeHookUIMetadata]: + """ + Yield hook metadata per connection type for the connection UI. + + Does not import hook classes. + """ + self.initialize_providers_hooks() + all_types = frozenset(self._hooks_lazy_dict) | frozenset(self._hook_provider_dict) + for conn_type in sorted(all_types): + raw_entry = self._hooks_lazy_dict._raw_dict.get(conn_type) + provider_entry = self._hook_provider_dict.get(conn_type) + if isinstance(raw_entry, HookInfo): + hook_name = raw_entry.hook_name + hook_class_name = raw_entry.hook_class_name + elif provider_entry: + hook_name = self._hook_name_dict.get(conn_type, conn_type) + hook_class_name = provider_entry.hook_class_name + else: + hook_name = self._hook_name_dict.get(conn_type, conn_type) + hook_class_name = None + yield ConnectionTypeHookUIMetadata( + connection_type=conn_type, + hook_name=hook_name, + hook_class_name=hook_class_name, + field_behaviour=self._field_behaviours.get(conn_type), + ) + + @property + def _connection_form_widgets_from_metadata(self) -> dict[str, ConnectionFormWidgetInfo]: + """Return connection form widgets from metadata without importing every hook.""" + self.initialize_providers_hooks() + return self._connection_form_widgets + + @property + def _field_behaviours_from_metadata(self) -> dict[str, dict]: + """Return field behaviour dicts from metadata without importing every hook.""" + self.initialize_providers_hooks() + return self._field_behaviours + @property def dialects(self) -> MutableMapping[str, DialectInfo]: """Return dictionary of connection_type-to-dialect mapping.""" diff --git a/airflow-core/tests/unit/always/test_providers_manager.py b/airflow-core/tests/unit/always/test_providers_manager.py index 580676d18b3bf..afa473e80a4f0 100644 --- a/airflow-core/tests/unit/always/test_providers_manager.py +++ b/airflow-core/tests/unit/always/test_providers_manager.py @@ -428,6 +428,14 @@ def test_load_ui_for_http_provider(self): assert "relabeling" in behaviour assert "placeholders" in behaviour + def test_iter_connection_type_hook_ui_metadata_matches_field_behaviours(self): + """iter_connection_type_hook_ui_metadata should expose the same standard-field behaviour dict.""" + pm = ProvidersManager() + pm.initialize_providers_hooks() + by_type = {m.connection_type: m for m in pm.iter_connection_type_hook_ui_metadata()} + assert "http" in by_type + assert by_type["http"].field_behaviour == pm._field_behaviours["http"] + def test_ui_metadata_loading_without_hook_import(self): """Test that UI metadata loads from provider info without importing hook classes.""" with patch("airflow.providers_manager.import_string") as mock_import: diff --git a/providers/airbyte/provider.yaml b/providers/airbyte/provider.yaml index b60aa43473fb0..fcef42aaeedde 100644 --- a/providers/airbyte/provider.yaml +++ b/providers/airbyte/provider.yaml @@ -97,6 +97,7 @@ triggers: connection-types: - hook-class-name: airflow.providers.airbyte.hooks.airbyte.AirbyteHook + hook-name: "Airbyte" connection-type: airbyte ui-field-behaviour: hidden-fields: diff --git a/providers/airbyte/src/airflow/providers/airbyte/get_provider_info.py b/providers/airbyte/src/airflow/providers/airbyte/get_provider_info.py index 33be5eaa8bf24..6ca1af004d5a9 100644 --- a/providers/airbyte/src/airflow/providers/airbyte/get_provider_info.py +++ b/providers/airbyte/src/airflow/providers/airbyte/get_provider_info.py @@ -50,6 +50,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.airbyte.hooks.airbyte.AirbyteHook", + "hook-name": "Airbyte", "connection-type": "airbyte", "ui-field-behaviour": { "hidden-fields": ["extra", "port"], diff --git a/providers/alibaba/provider.yaml b/providers/alibaba/provider.yaml index e02be8f9a676f..6d5dd6aad4304 100644 --- a/providers/alibaba/provider.yaml +++ b/providers/alibaba/provider.yaml @@ -123,10 +123,13 @@ hooks: connection-types: - hook-class-name: airflow.providers.alibaba.cloud.hooks.oss.OSSHook + hook-name: "OSS" connection-type: oss - hook-class-name: airflow.providers.alibaba.cloud.hooks.analyticdb_spark.AnalyticDBSparkHook + hook-name: "AnalyticDB Spark" connection-type: adb_spark - hook-class-name: airflow.providers.alibaba.cloud.hooks.base_alibaba.AlibabaBaseHook + hook-name: "Alibaba Cloud" connection-type: alibaba_cloud conn-fields: access_key_id: @@ -144,6 +147,7 @@ connection-types: - 'null' format: password - hook-class-name: airflow.providers.alibaba.cloud.hooks.maxcompute.MaxComputeHook + hook-name: "MaxCompute" connection-type: maxcompute ui-field-behaviour: hidden-fields: diff --git a/providers/alibaba/src/airflow/providers/alibaba/get_provider_info.py b/providers/alibaba/src/airflow/providers/alibaba/get_provider_info.py index 0b8a5ab971ec0..27e01cd0e74a2 100644 --- a/providers/alibaba/src/airflow/providers/alibaba/get_provider_info.py +++ b/providers/alibaba/src/airflow/providers/alibaba/get_provider_info.py @@ -91,14 +91,17 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.alibaba.cloud.hooks.oss.OSSHook", + "hook-name": "OSS", "connection-type": "oss", }, { "hook-class-name": "airflow.providers.alibaba.cloud.hooks.analyticdb_spark.AnalyticDBSparkHook", + "hook-name": "AnalyticDB Spark", "connection-type": "adb_spark", }, { "hook-class-name": "airflow.providers.alibaba.cloud.hooks.base_alibaba.AlibabaBaseHook", + "hook-name": "Alibaba Cloud", "connection-type": "alibaba_cloud", "conn-fields": { "access_key_id": { @@ -113,6 +116,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.alibaba.cloud.hooks.maxcompute.MaxComputeHook", + "hook-name": "MaxCompute", "connection-type": "maxcompute", "ui-field-behaviour": { "hidden-fields": ["host", "schema", "login", "password", "port", "extra"], diff --git a/providers/amazon/provider.yaml b/providers/amazon/provider.yaml index a7e2a9f7b0b47..0390a3fca6dec 100644 --- a/providers/amazon/provider.yaml +++ b/providers/amazon/provider.yaml @@ -931,6 +931,7 @@ extra-links: connection-types: - hook-class-name: airflow.providers.amazon.aws.hooks.base_aws.AwsGenericHook + hook-name: "Amazon Web Services" connection-type: aws ui-field-behaviour: hidden-fields: @@ -955,6 +956,7 @@ connection-types: "endpoint_url": "http://localhost:4566" } - hook-class-name: airflow.providers.amazon.aws.hooks.chime.ChimeWebhookHook + hook-name: "Amazon Chime Webhook" connection-type: chime ui-field-behaviour: hidden-fields: @@ -969,6 +971,7 @@ connection-types: host: hooks.chime.aws/incomingwebhook/ password: T00000000?token=XXXXXXXXXXXXXXXXXXXXXXXX - hook-class-name: airflow.providers.amazon.aws.hooks.emr.EmrHook + hook-name: "Amazon Elastic MapReduce" connection-type: emr ui-field-behaviour: hidden-fields: @@ -999,12 +1002,14 @@ connection-types: "StepConcurrencyLevel": 2 } - hook-class-name: airflow.providers.amazon.aws.hooks.redshift_sql.RedshiftSQLHook + hook-name: "Amazon Redshift" connection-type: redshift ui-field-behaviour: relabeling: login: User schema: Database - hook-class-name: airflow.providers.amazon.aws.hooks.athena_sql.AthenaSQLHook + hook-name: "Amazon Athena" connection-type: athena ui-field-behaviour: hidden-fields: diff --git a/providers/amazon/src/airflow/providers/amazon/get_provider_info.py b/providers/amazon/src/airflow/providers/amazon/get_provider_info.py index d29d1099506e9..f2eac44a3bde2 100644 --- a/providers/amazon/src/airflow/providers/amazon/get_provider_info.py +++ b/providers/amazon/src/airflow/providers/amazon/get_provider_info.py @@ -1084,6 +1084,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.amazon.aws.hooks.base_aws.AwsGenericHook", + "hook-name": "Amazon Web Services", "connection-type": "aws", "ui-field-behaviour": { "hidden-fields": ["host", "schema", "port"], @@ -1097,6 +1098,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.amazon.aws.hooks.chime.ChimeWebhookHook", + "hook-name": "Amazon Chime Webhook", "connection-type": "chime", "ui-field-behaviour": { "hidden-fields": ["login", "port", "extra"], @@ -1110,6 +1112,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.amazon.aws.hooks.emr.EmrHook", + "hook-name": "Amazon Elastic MapReduce", "connection-type": "emr", "ui-field-behaviour": { "hidden-fields": ["host", "schema", "port", "login", "password"], @@ -1121,11 +1124,13 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.amazon.aws.hooks.redshift_sql.RedshiftSQLHook", + "hook-name": "Amazon Redshift", "connection-type": "redshift", "ui-field-behaviour": {"relabeling": {"login": "User", "schema": "Database"}}, }, { "hook-class-name": "airflow.providers.amazon.aws.hooks.athena_sql.AthenaSQLHook", + "hook-name": "Amazon Athena", "connection-type": "athena", "ui-field-behaviour": { "hidden-fields": ["host", "port"], diff --git a/providers/apache/cassandra/provider.yaml b/providers/apache/cassandra/provider.yaml index 6e7362bc721d3..3c66d42965c4f 100644 --- a/providers/apache/cassandra/provider.yaml +++ b/providers/apache/cassandra/provider.yaml @@ -85,4 +85,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.apache.cassandra.hooks.cassandra.CassandraHook + hook-name: "Cassandra" connection-type: cassandra diff --git a/providers/apache/cassandra/src/airflow/providers/apache/cassandra/get_provider_info.py b/providers/apache/cassandra/src/airflow/providers/apache/cassandra/get_provider_info.py index 263641e35877c..6418967562902 100644 --- a/providers/apache/cassandra/src/airflow/providers/apache/cassandra/get_provider_info.py +++ b/providers/apache/cassandra/src/airflow/providers/apache/cassandra/get_provider_info.py @@ -53,6 +53,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.apache.cassandra.hooks.cassandra.CassandraHook", + "hook-name": "Cassandra", "connection-type": "cassandra", } ], diff --git a/providers/apache/drill/provider.yaml b/providers/apache/drill/provider.yaml index 3290c5e2cfa4c..f8f32d4e552c2 100644 --- a/providers/apache/drill/provider.yaml +++ b/providers/apache/drill/provider.yaml @@ -81,4 +81,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.apache.drill.hooks.drill.DrillHook + hook-name: "Drill" connection-type: drill diff --git a/providers/apache/drill/src/airflow/providers/apache/drill/get_provider_info.py b/providers/apache/drill/src/airflow/providers/apache/drill/get_provider_info.py index c767a12b370a5..6a5241392edce 100644 --- a/providers/apache/drill/src/airflow/providers/apache/drill/get_provider_info.py +++ b/providers/apache/drill/src/airflow/providers/apache/drill/get_provider_info.py @@ -44,6 +44,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.apache.drill.hooks.drill.DrillHook", + "hook-name": "Drill", "connection-type": "drill", } ], diff --git a/providers/apache/druid/provider.yaml b/providers/apache/druid/provider.yaml index ca302b5981045..bb890f8ed000d 100644 --- a/providers/apache/druid/provider.yaml +++ b/providers/apache/druid/provider.yaml @@ -95,6 +95,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.apache.druid.hooks.druid.DruidDbApiHook + hook-name: "Druid" connection-type: druid transfers: diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/get_provider_info.py b/providers/apache/druid/src/airflow/providers/apache/druid/get_provider_info.py index c9c501e8f3898..0093743726464 100644 --- a/providers/apache/druid/src/airflow/providers/apache/druid/get_provider_info.py +++ b/providers/apache/druid/src/airflow/providers/apache/druid/get_provider_info.py @@ -50,6 +50,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.apache.druid.hooks.druid.DruidDbApiHook", + "hook-name": "Druid", "connection-type": "druid", } ], diff --git a/providers/apache/hdfs/provider.yaml b/providers/apache/hdfs/provider.yaml index 25536b7e4ac6f..9da1a398cc380 100644 --- a/providers/apache/hdfs/provider.yaml +++ b/providers/apache/hdfs/provider.yaml @@ -97,4 +97,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook + hook-name: "Apache WebHDFS" connection-type: webhdfs diff --git a/providers/apache/hdfs/src/airflow/providers/apache/hdfs/get_provider_info.py b/providers/apache/hdfs/src/airflow/providers/apache/hdfs/get_provider_info.py index 9a9a7973caec0..ebe28966df6a9 100644 --- a/providers/apache/hdfs/src/airflow/providers/apache/hdfs/get_provider_info.py +++ b/providers/apache/hdfs/src/airflow/providers/apache/hdfs/get_provider_info.py @@ -52,6 +52,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook", + "hook-name": "Apache WebHDFS", "connection-type": "webhdfs", } ], diff --git a/providers/apache/hive/provider.yaml b/providers/apache/hive/provider.yaml index cae016e6718e6..e0eda25c985ce 100644 --- a/providers/apache/hive/provider.yaml +++ b/providers/apache/hive/provider.yaml @@ -143,6 +143,7 @@ transfers: connection-types: - hook-class-name: airflow.providers.apache.hive.hooks.hive.HiveCliHook + hook-name: "Hive Client Wrapper" connection-type: hive_cli ui-field-behaviour: hidden-fields: @@ -178,8 +179,10 @@ connection-types: - 'null' default: false - hook-class-name: airflow.providers.apache.hive.hooks.hive.HiveServer2Hook + hook-name: "Hive Server 2 Thrift" connection-type: hiveserver2 - hook-class-name: airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook + hook-name: "Hive Metastore Thrift" connection-type: hive_metastore plugins: diff --git a/providers/apache/hive/src/airflow/providers/apache/hive/get_provider_info.py b/providers/apache/hive/src/airflow/providers/apache/hive/get_provider_info.py index 9929536d96f8b..7fe5d92770496 100644 --- a/providers/apache/hive/src/airflow/providers/apache/hive/get_provider_info.py +++ b/providers/apache/hive/src/airflow/providers/apache/hive/get_provider_info.py @@ -95,6 +95,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.apache.hive.hooks.hive.HiveCliHook", + "hook-name": "Hive Client Wrapper", "connection-type": "hive_cli", "ui-field-behaviour": {"hidden-fields": ["extra"], "relabeling": {}}, "conn-fields": { @@ -118,10 +119,12 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.apache.hive.hooks.hive.HiveServer2Hook", + "hook-name": "Hive Server 2 Thrift", "connection-type": "hiveserver2", }, { "hook-class-name": "airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook", + "hook-name": "Hive Metastore Thrift", "connection-type": "hive_metastore", }, ], diff --git a/providers/apache/iceberg/provider.yaml b/providers/apache/iceberg/provider.yaml index 53bb94abf6bd3..84dbffeb560c7 100644 --- a/providers/apache/iceberg/provider.yaml +++ b/providers/apache/iceberg/provider.yaml @@ -55,6 +55,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.apache.iceberg.hooks.iceberg.IcebergHook + hook-name: "Iceberg" connection-type: iceberg ui-field-behaviour: hidden-fields: diff --git a/providers/apache/iceberg/src/airflow/providers/apache/iceberg/get_provider_info.py b/providers/apache/iceberg/src/airflow/providers/apache/iceberg/get_provider_info.py index 56bd75339fbaf..df1285577b9cc 100644 --- a/providers/apache/iceberg/src/airflow/providers/apache/iceberg/get_provider_info.py +++ b/providers/apache/iceberg/src/airflow/providers/apache/iceberg/get_provider_info.py @@ -43,6 +43,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.apache.iceberg.hooks.iceberg.IcebergHook", + "hook-name": "Iceberg", "connection-type": "iceberg", "ui-field-behaviour": { "hidden-fields": ["schema", "port"], diff --git a/providers/apache/impala/provider.yaml b/providers/apache/impala/provider.yaml index 2bbfe9515d6ec..d3c56b5cc404e 100644 --- a/providers/apache/impala/provider.yaml +++ b/providers/apache/impala/provider.yaml @@ -69,4 +69,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.apache.impala.hooks.impala.ImpalaHook + hook-name: "Impala" connection-type: impala diff --git a/providers/apache/impala/src/airflow/providers/apache/impala/get_provider_info.py b/providers/apache/impala/src/airflow/providers/apache/impala/get_provider_info.py index f3d98c255f563..c8976240b9e69 100644 --- a/providers/apache/impala/src/airflow/providers/apache/impala/get_provider_info.py +++ b/providers/apache/impala/src/airflow/providers/apache/impala/get_provider_info.py @@ -44,6 +44,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.apache.impala.hooks.impala.ImpalaHook", + "hook-name": "Impala", "connection-type": "impala", } ], diff --git a/providers/apache/kafka/provider.yaml b/providers/apache/kafka/provider.yaml index 3b5bacc3cf6da..e2bc8d4eb136d 100644 --- a/providers/apache/kafka/provider.yaml +++ b/providers/apache/kafka/provider.yaml @@ -95,6 +95,7 @@ triggers: connection-types: - hook-class-name: airflow.providers.apache.kafka.hooks.base.KafkaBaseHook + hook-name: "Apache Kafka" connection-type: kafka ui-field-behaviour: hidden-fields: diff --git a/providers/apache/kafka/src/airflow/providers/apache/kafka/get_provider_info.py b/providers/apache/kafka/src/airflow/providers/apache/kafka/get_provider_info.py index 41076d2b4c29b..11d8b678304e4 100644 --- a/providers/apache/kafka/src/airflow/providers/apache/kafka/get_provider_info.py +++ b/providers/apache/kafka/src/airflow/providers/apache/kafka/get_provider_info.py @@ -73,6 +73,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.apache.kafka.hooks.base.KafkaBaseHook", + "hook-name": "Apache Kafka", "connection-type": "kafka", "ui-field-behaviour": { "hidden-fields": ["schema", "login", "password", "port", "host"], diff --git a/providers/apache/kylin/provider.yaml b/providers/apache/kylin/provider.yaml index 2003e3be2a859..67d7e20bd5b49 100644 --- a/providers/apache/kylin/provider.yaml +++ b/providers/apache/kylin/provider.yaml @@ -79,4 +79,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.apache.kylin.hooks.kylin.KylinHook + hook-name: "Apache Kylin" connection-type: kylin diff --git a/providers/apache/kylin/src/airflow/providers/apache/kylin/get_provider_info.py b/providers/apache/kylin/src/airflow/providers/apache/kylin/get_provider_info.py index c989259d8576d..cb09228b7755f 100644 --- a/providers/apache/kylin/src/airflow/providers/apache/kylin/get_provider_info.py +++ b/providers/apache/kylin/src/airflow/providers/apache/kylin/get_provider_info.py @@ -50,6 +50,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.apache.kylin.hooks.kylin.KylinHook", + "hook-name": "Apache Kylin", "connection-type": "kylin", } ], diff --git a/providers/apache/livy/provider.yaml b/providers/apache/livy/provider.yaml index ef6a79d0a6916..43ecfc460378d 100644 --- a/providers/apache/livy/provider.yaml +++ b/providers/apache/livy/provider.yaml @@ -106,4 +106,5 @@ triggers: connection-types: - hook-class-name: airflow.providers.apache.livy.hooks.livy.LivyHook + hook-name: "Apache Livy" connection-type: livy diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/get_provider_info.py b/providers/apache/livy/src/airflow/providers/apache/livy/get_provider_info.py index d5c3f982efd26..0597c61659ad8 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/get_provider_info.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/get_provider_info.py @@ -62,6 +62,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.apache.livy.hooks.livy.LivyHook", + "hook-name": "Apache Livy", "connection-type": "livy", } ], diff --git a/providers/apache/pig/provider.yaml b/providers/apache/pig/provider.yaml index de7ad5ac07a7a..f16b1b6ebe8b7 100644 --- a/providers/apache/pig/provider.yaml +++ b/providers/apache/pig/provider.yaml @@ -78,3 +78,4 @@ hooks: connection-types: - connection-type: pig_cli hook-class-name: airflow.providers.apache.pig.hooks.pig.PigCliHook + hook-name: "Pig Client Wrapper" diff --git a/providers/apache/pig/src/airflow/providers/apache/pig/get_provider_info.py b/providers/apache/pig/src/airflow/providers/apache/pig/get_provider_info.py index 5104f7af41cbb..c05c3f3e10001 100644 --- a/providers/apache/pig/src/airflow/providers/apache/pig/get_provider_info.py +++ b/providers/apache/pig/src/airflow/providers/apache/pig/get_provider_info.py @@ -48,6 +48,7 @@ def get_provider_info(): { "connection-type": "pig_cli", "hook-class-name": "airflow.providers.apache.pig.hooks.pig.PigCliHook", + "hook-name": "Pig Client Wrapper", } ], } diff --git a/providers/apache/pinot/provider.yaml b/providers/apache/pinot/provider.yaml index 55794ffdf528d..8eb762343040e 100644 --- a/providers/apache/pinot/provider.yaml +++ b/providers/apache/pinot/provider.yaml @@ -82,6 +82,8 @@ hooks: connection-types: - hook-class-name: airflow.providers.apache.pinot.hooks.pinot.PinotDbApiHook + hook-name: "Pinot Broker" connection-type: pinot - hook-class-name: airflow.providers.apache.pinot.hooks.pinot.PinotAdminHook + hook-name: "Pinot Admin" connection-type: pinot_admin diff --git a/providers/apache/pinot/src/airflow/providers/apache/pinot/get_provider_info.py b/providers/apache/pinot/src/airflow/providers/apache/pinot/get_provider_info.py index 798860656c7d1..738a30142221d 100644 --- a/providers/apache/pinot/src/airflow/providers/apache/pinot/get_provider_info.py +++ b/providers/apache/pinot/src/airflow/providers/apache/pinot/get_provider_info.py @@ -44,10 +44,12 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.apache.pinot.hooks.pinot.PinotDbApiHook", + "hook-name": "Pinot Broker", "connection-type": "pinot", }, { "hook-class-name": "airflow.providers.apache.pinot.hooks.pinot.PinotAdminHook", + "hook-name": "Pinot Admin", "connection-type": "pinot_admin", }, ], diff --git a/providers/apache/spark/provider.yaml b/providers/apache/spark/provider.yaml index 33b212f73be33..1f5882454d1d7 100644 --- a/providers/apache/spark/provider.yaml +++ b/providers/apache/spark/provider.yaml @@ -114,6 +114,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.apache.spark.hooks.spark_connect.SparkConnectHook + hook-name: "Spark Connect" connection-type: spark_connect ui-field-behaviour: hidden-fields: @@ -130,8 +131,10 @@ connection-types: - 'null' default: false - hook-class-name: airflow.providers.apache.spark.hooks.spark_jdbc.SparkJDBCHook + hook-name: "Spark JDBC" connection-type: spark_jdbc - hook-class-name: airflow.providers.apache.spark.hooks.spark_sql.SparkSqlHook + hook-name: "Spark SQL" connection-type: spark_sql ui-field-behaviour: hidden-fields: @@ -149,6 +152,7 @@ connection-types: - string - 'null' - hook-class-name: airflow.providers.apache.spark.hooks.spark_submit.SparkSubmitHook + hook-name: "Spark" connection-type: spark ui-field-behaviour: hidden-fields: diff --git a/providers/apache/spark/src/airflow/providers/apache/spark/get_provider_info.py b/providers/apache/spark/src/airflow/providers/apache/spark/get_provider_info.py index bf9f4b2f8a7b3..b987115625719 100644 --- a/providers/apache/spark/src/airflow/providers/apache/spark/get_provider_info.py +++ b/providers/apache/spark/src/airflow/providers/apache/spark/get_provider_info.py @@ -63,6 +63,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.apache.spark.hooks.spark_connect.SparkConnectHook", + "hook-name": "Spark Connect", "connection-type": "spark_connect", "ui-field-behaviour": { "hidden-fields": ["schema"], @@ -74,10 +75,12 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.apache.spark.hooks.spark_jdbc.SparkJDBCHook", + "hook-name": "Spark JDBC", "connection-type": "spark_jdbc", }, { "hook-class-name": "airflow.providers.apache.spark.hooks.spark_sql.SparkSqlHook", + "hook-name": "Spark SQL", "connection-type": "spark_sql", "ui-field-behaviour": { "hidden-fields": ["schema", "login", "password", "extra"], @@ -93,6 +96,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.apache.spark.hooks.spark_submit.SparkSubmitHook", + "hook-name": "Spark", "connection-type": "spark", "ui-field-behaviour": { "hidden-fields": ["schema", "login", "password", "extra"], diff --git a/providers/apache/tinkerpop/provider.yaml b/providers/apache/tinkerpop/provider.yaml index aacd8289f0507..ea641284f83e4 100644 --- a/providers/apache/tinkerpop/provider.yaml +++ b/providers/apache/tinkerpop/provider.yaml @@ -51,4 +51,5 @@ hooks: - airflow.providers.apache.tinkerpop.hooks.gremlin connection-types: - hook-class-name: airflow.providers.apache.tinkerpop.hooks.gremlin.GremlinHook + hook-name: "Gremlin" connection-type: gremlin diff --git a/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/get_provider_info.py b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/get_provider_info.py index d7358d1578353..0466362719261 100644 --- a/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/get_provider_info.py +++ b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/get_provider_info.py @@ -50,6 +50,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.apache.tinkerpop.hooks.gremlin.GremlinHook", + "hook-name": "Gremlin", "connection-type": "gremlin", } ], diff --git a/providers/apprise/provider.yaml b/providers/apprise/provider.yaml index 41ce053e23277..2143551686976 100644 --- a/providers/apprise/provider.yaml +++ b/providers/apprise/provider.yaml @@ -67,6 +67,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.apprise.hooks.apprise.AppriseHook + hook-name: "Apprise" connection-type: apprise conn-fields: config: diff --git a/providers/apprise/src/airflow/providers/apprise/get_provider_info.py b/providers/apprise/src/airflow/providers/apprise/get_provider_info.py index ce0346c29b877..4677d56d42bb4 100644 --- a/providers/apprise/src/airflow/providers/apprise/get_provider_info.py +++ b/providers/apprise/src/airflow/providers/apprise/get_provider_info.py @@ -39,6 +39,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.apprise.hooks.apprise.AppriseHook", + "hook-name": "Apprise", "connection-type": "apprise", "conn-fields": { "config": { diff --git a/providers/arangodb/provider.yaml b/providers/arangodb/provider.yaml index 273896bbcf82d..dd9355093369c 100644 --- a/providers/arangodb/provider.yaml +++ b/providers/arangodb/provider.yaml @@ -78,6 +78,7 @@ sensors: connection-types: - hook-class-name: airflow.providers.arangodb.hooks.arangodb.ArangoDBHook + hook-name: "ArangoDB" connection-type: arangodb ui-field-behaviour: hidden-fields: diff --git a/providers/arangodb/src/airflow/providers/arangodb/get_provider_info.py b/providers/arangodb/src/airflow/providers/arangodb/get_provider_info.py index 5d90c27124327..b93b01b7bea19 100644 --- a/providers/arangodb/src/airflow/providers/arangodb/get_provider_info.py +++ b/providers/arangodb/src/airflow/providers/arangodb/get_provider_info.py @@ -52,6 +52,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.arangodb.hooks.arangodb.ArangoDBHook", + "hook-name": "ArangoDB", "connection-type": "arangodb", "ui-field-behaviour": { "hidden-fields": ["port", "extra"], diff --git a/providers/asana/provider.yaml b/providers/asana/provider.yaml index 5b43857ec19cd..e7e09ff1fd32f 100644 --- a/providers/asana/provider.yaml +++ b/providers/asana/provider.yaml @@ -78,6 +78,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.asana.hooks.asana.AsanaHook + hook-name: "Asana" connection-type: asana ui-field-behaviour: hidden-fields: diff --git a/providers/asana/src/airflow/providers/asana/get_provider_info.py b/providers/asana/src/airflow/providers/asana/get_provider_info.py index 6e0f10b1c90e8..57a81694e338b 100644 --- a/providers/asana/src/airflow/providers/asana/get_provider_info.py +++ b/providers/asana/src/airflow/providers/asana/get_provider_info.py @@ -42,6 +42,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.asana.hooks.asana.AsanaHook", + "hook-name": "Asana", "connection-type": "asana", "ui-field-behaviour": { "hidden-fields": ["port", "host", "login", "schema"], diff --git a/providers/atlassian/jira/provider.yaml b/providers/atlassian/jira/provider.yaml index 40f19f4120366..b61ce78aca252 100644 --- a/providers/atlassian/jira/provider.yaml +++ b/providers/atlassian/jira/provider.yaml @@ -78,6 +78,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.atlassian.jira.hooks.jira.JiraHook + hook-name: "JIRA" connection-type: jira notifications: diff --git a/providers/atlassian/jira/src/airflow/providers/atlassian/jira/get_provider_info.py b/providers/atlassian/jira/src/airflow/providers/atlassian/jira/get_provider_info.py index 70ccfcd06eac3..4b36780e8fb9d 100644 --- a/providers/atlassian/jira/src/airflow/providers/atlassian/jira/get_provider_info.py +++ b/providers/atlassian/jira/src/airflow/providers/atlassian/jira/get_provider_info.py @@ -55,6 +55,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.atlassian.jira.hooks.jira.JiraHook", + "hook-name": "JIRA", "connection-type": "jira", } ], diff --git a/providers/cloudant/provider.yaml b/providers/cloudant/provider.yaml index f62bf66e2213b..f3ac4dd883d8d 100644 --- a/providers/cloudant/provider.yaml +++ b/providers/cloudant/provider.yaml @@ -74,6 +74,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.cloudant.hooks.cloudant.CloudantHook + hook-name: "Cloudant" connection-type: cloudant ui-field-behaviour: hidden-fields: diff --git a/providers/cloudant/src/airflow/providers/cloudant/get_provider_info.py b/providers/cloudant/src/airflow/providers/cloudant/get_provider_info.py index e806ab26ce3bd..3db1ab77642f8 100644 --- a/providers/cloudant/src/airflow/providers/cloudant/get_provider_info.py +++ b/providers/cloudant/src/airflow/providers/cloudant/get_provider_info.py @@ -43,6 +43,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.cloudant.hooks.cloudant.CloudantHook", + "hook-name": "Cloudant", "connection-type": "cloudant", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "extra"], diff --git a/providers/cncf/kubernetes/provider.yaml b/providers/cncf/kubernetes/provider.yaml index 4e5c48ea830d4..0c05d7d65c6fe 100644 --- a/providers/cncf/kubernetes/provider.yaml +++ b/providers/cncf/kubernetes/provider.yaml @@ -168,6 +168,7 @@ secrets-backends: connection-types: - hook-class-name: airflow.providers.cncf.kubernetes.hooks.kubernetes.KubernetesHook + hook-name: "Kubernetes Cluster Connection" connection-type: kubernetes conn-fields: in_cluster: diff --git a/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/get_provider_info.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/get_provider_info.py index 9e4d433827e88..f0cb9ccf9a2e0 100644 --- a/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/get_provider_info.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/get_provider_info.py @@ -81,6 +81,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.cncf.kubernetes.hooks.kubernetes.KubernetesHook", + "hook-name": "Kubernetes Cluster Connection", "connection-type": "kubernetes", "conn-fields": { "in_cluster": { diff --git a/providers/cohere/provider.yaml b/providers/cohere/provider.yaml index 31a6a7827b94e..370845dfebcd3 100644 --- a/providers/cohere/provider.yaml +++ b/providers/cohere/provider.yaml @@ -72,6 +72,7 @@ operators: connection-types: - hook-class-name: airflow.providers.cohere.hooks.cohere.CohereHook + hook-name: "Cohere" connection-type: cohere ui-field-behaviour: hidden-fields: ["schema", "login", "port", "extra"] diff --git a/providers/cohere/src/airflow/providers/cohere/get_provider_info.py b/providers/cohere/src/airflow/providers/cohere/get_provider_info.py index a91c98224cb8a..ad445d1afbdcc 100644 --- a/providers/cohere/src/airflow/providers/cohere/get_provider_info.py +++ b/providers/cohere/src/airflow/providers/cohere/get_provider_info.py @@ -43,6 +43,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.cohere.hooks.cohere.CohereHook", + "hook-name": "Cohere", "connection-type": "cohere", "ui-field-behaviour": { "hidden-fields": ["schema", "login", "port", "extra"], diff --git a/providers/common/ai/provider.yaml b/providers/common/ai/provider.yaml index 43a98af32a3a2..f8a0761ffea89 100644 --- a/providers/common/ai/provider.yaml +++ b/providers/common/ai/provider.yaml @@ -61,6 +61,7 @@ plugins: connection-types: - hook-class-name: airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIHook + hook-name: "Pydantic AI" connection-type: pydanticai ui-field-behaviour: hidden-fields: @@ -80,6 +81,7 @@ connection-types: - string - 'null' - hook-class-name: airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIAzureHook + hook-name: "Pydantic AI (Azure OpenAI)" connection-type: pydanticai-azure ui-field-behaviour: hidden-fields: @@ -107,6 +109,7 @@ connection-types: - string - 'null' - hook-class-name: airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIBedrockHook + hook-name: "Pydantic AI (AWS Bedrock)" connection-type: pydanticai-bedrock ui-field-behaviour: hidden-fields: @@ -189,6 +192,7 @@ connection-types: - number - 'null' - hook-class-name: airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIVertexHook + hook-name: "Pydantic AI (Google Vertex AI)" connection-type: pydanticai-vertex ui-field-behaviour: hidden-fields: @@ -250,6 +254,7 @@ connection-types: - string - 'null' - hook-class-name: airflow.providers.common.ai.hooks.mcp.MCPHook + hook-name: "MCP Server" connection-type: mcp ui-field-behaviour: hidden-fields: diff --git a/providers/common/ai/src/airflow/providers/common/ai/get_provider_info.py b/providers/common/ai/src/airflow/providers/common/ai/get_provider_info.py index 90ae393d64d90..574bdde6a81c8 100644 --- a/providers/common/ai/src/airflow/providers/common/ai/get_provider_info.py +++ b/providers/common/ai/src/airflow/providers/common/ai/get_provider_info.py @@ -66,6 +66,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIHook", + "hook-name": "Pydantic AI", "connection-type": "pydanticai", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "login"], @@ -82,6 +83,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIAzureHook", + "hook-name": "Pydantic AI (Azure OpenAI)", "connection-type": "pydanticai-azure", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "login"], @@ -103,6 +105,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIBedrockHook", + "hook-name": "Pydantic AI (AWS Bedrock)", "connection-type": "pydanticai-bedrock", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "login", "host", "password"], @@ -164,6 +167,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIVertexHook", + "hook-name": "Pydantic AI (Google Vertex AI)", "connection-type": "pydanticai-vertex", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "login", "host", "password"], @@ -210,6 +214,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.common.ai.hooks.mcp.MCPHook", + "hook-name": "MCP Server", "connection-type": "mcp", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "login"], diff --git a/providers/databricks/provider.yaml b/providers/databricks/provider.yaml index 41ab271bcbcd1..9daebf79a4cea 100644 --- a/providers/databricks/provider.yaml +++ b/providers/databricks/provider.yaml @@ -167,6 +167,7 @@ sensors: connection-types: - hook-class-name: airflow.providers.databricks.hooks.databricks.DatabricksHook + hook-name: "Databricks" connection-type: databricks plugins: diff --git a/providers/databricks/src/airflow/providers/databricks/get_provider_info.py b/providers/databricks/src/airflow/providers/databricks/get_provider_info.py index 6ba5f8ed20945..5f12cb02ddbe8 100644 --- a/providers/databricks/src/airflow/providers/databricks/get_provider_info.py +++ b/providers/databricks/src/airflow/providers/databricks/get_provider_info.py @@ -117,6 +117,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.databricks.hooks.databricks.DatabricksHook", + "hook-name": "Databricks", "connection-type": "databricks", } ], diff --git a/providers/datadog/provider.yaml b/providers/datadog/provider.yaml index 02215069a3945..e15ad55a48337 100644 --- a/providers/datadog/provider.yaml +++ b/providers/datadog/provider.yaml @@ -78,6 +78,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.datadog.hooks.datadog.DatadogHook + hook-name: "Datadog" connection-type: datadog conn-fields: api_host: diff --git a/providers/datadog/src/airflow/providers/datadog/get_provider_info.py b/providers/datadog/src/airflow/providers/datadog/get_provider_info.py index a87ee7fb0e6d0..5d6d598c57f01 100644 --- a/providers/datadog/src/airflow/providers/datadog/get_provider_info.py +++ b/providers/datadog/src/airflow/providers/datadog/get_provider_info.py @@ -43,6 +43,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.datadog.hooks.datadog.DatadogHook", + "hook-name": "Datadog", "connection-type": "datadog", "conn-fields": { "api_host": {"label": "API endpoint", "schema": {"type": ["string", "null"]}}, diff --git a/providers/dbt/cloud/provider.yaml b/providers/dbt/cloud/provider.yaml index c0da84706dd17..2ad844dcbcb73 100644 --- a/providers/dbt/cloud/provider.yaml +++ b/providers/dbt/cloud/provider.yaml @@ -112,6 +112,7 @@ triggers: connection-types: - hook-class-name: airflow.providers.dbt.cloud.hooks.dbt.DbtCloudHook + hook-name: "dbt Cloud" connection-type: dbt_cloud ui-field-behaviour: hidden-fields: diff --git a/providers/dbt/cloud/src/airflow/providers/dbt/cloud/get_provider_info.py b/providers/dbt/cloud/src/airflow/providers/dbt/cloud/get_provider_info.py index f0119e2ea98bb..16ed9a37e722f 100644 --- a/providers/dbt/cloud/src/airflow/providers/dbt/cloud/get_provider_info.py +++ b/providers/dbt/cloud/src/airflow/providers/dbt/cloud/get_provider_info.py @@ -50,6 +50,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.dbt.cloud.hooks.dbt.DbtCloudHook", + "hook-name": "dbt Cloud", "connection-type": "dbt_cloud", "ui-field-behaviour": { "hidden-fields": ["schema", "port"], diff --git a/providers/dingding/provider.yaml b/providers/dingding/provider.yaml index 10e79df0d0841..c5d075099ec82 100644 --- a/providers/dingding/provider.yaml +++ b/providers/dingding/provider.yaml @@ -78,4 +78,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.dingding.hooks.dingding.DingdingHook + hook-name: "DingTalk Custom Robot (Dingding)" connection-type: dingding diff --git a/providers/dingding/src/airflow/providers/dingding/get_provider_info.py b/providers/dingding/src/airflow/providers/dingding/get_provider_info.py index 181903d94e106..8eb35c62525f3 100644 --- a/providers/dingding/src/airflow/providers/dingding/get_provider_info.py +++ b/providers/dingding/src/airflow/providers/dingding/get_provider_info.py @@ -47,6 +47,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.dingding.hooks.dingding.DingdingHook", + "hook-name": "DingTalk Custom Robot (Dingding)", "connection-type": "dingding", } ], diff --git a/providers/discord/provider.yaml b/providers/discord/provider.yaml index ff726615a1263..160c005f7f622 100644 --- a/providers/discord/provider.yaml +++ b/providers/discord/provider.yaml @@ -80,6 +80,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.discord.hooks.discord_webhook.DiscordWebhookHook + hook-name: "Discord" connection-type: discord conn-fields: webhook_endpoint: diff --git a/providers/discord/src/airflow/providers/discord/get_provider_info.py b/providers/discord/src/airflow/providers/discord/get_provider_info.py index 04521b10150b4..ee4c9503f82a8 100644 --- a/providers/discord/src/airflow/providers/discord/get_provider_info.py +++ b/providers/discord/src/airflow/providers/discord/get_provider_info.py @@ -49,6 +49,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.discord.hooks.discord_webhook.DiscordWebhookHook", + "hook-name": "Discord", "connection-type": "discord", "conn-fields": { "webhook_endpoint": {"label": "Webhook Endpoint", "schema": {"type": ["string", "null"]}} diff --git a/providers/docker/provider.yaml b/providers/docker/provider.yaml index 7018ce1cdf7d9..19e91dd09f493 100644 --- a/providers/docker/provider.yaml +++ b/providers/docker/provider.yaml @@ -116,6 +116,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.docker.hooks.docker.DockerHook + hook-name: "Docker" connection-type: docker conn-fields: reauth: diff --git a/providers/docker/src/airflow/providers/docker/get_provider_info.py b/providers/docker/src/airflow/providers/docker/get_provider_info.py index 3b1556f1c7bcd..fd466e185a1cc 100644 --- a/providers/docker/src/airflow/providers/docker/get_provider_info.py +++ b/providers/docker/src/airflow/providers/docker/get_provider_info.py @@ -53,6 +53,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.docker.hooks.docker.DockerHook", + "hook-name": "Docker", "connection-type": "docker", "conn-fields": { "reauth": { diff --git a/providers/elasticsearch/provider.yaml b/providers/elasticsearch/provider.yaml index 91a0b7ab23bf9..1d0e4b51dd4d8 100644 --- a/providers/elasticsearch/provider.yaml +++ b/providers/elasticsearch/provider.yaml @@ -102,6 +102,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.elasticsearch.hooks.elasticsearch.ElasticsearchSQLHook + hook-name: "Elasticsearch" connection-type: elasticsearch logging: diff --git a/providers/elasticsearch/src/airflow/providers/elasticsearch/get_provider_info.py b/providers/elasticsearch/src/airflow/providers/elasticsearch/get_provider_info.py index a527aa7cc5035..2d357cecb1944 100644 --- a/providers/elasticsearch/src/airflow/providers/elasticsearch/get_provider_info.py +++ b/providers/elasticsearch/src/airflow/providers/elasticsearch/get_provider_info.py @@ -43,6 +43,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.elasticsearch.hooks.elasticsearch.ElasticsearchSQLHook", + "hook-name": "Elasticsearch", "connection-type": "elasticsearch", } ], diff --git a/providers/exasol/provider.yaml b/providers/exasol/provider.yaml index 1ead105818ae2..d99fe1828141b 100644 --- a/providers/exasol/provider.yaml +++ b/providers/exasol/provider.yaml @@ -99,4 +99,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.exasol.hooks.exasol.ExasolHook + hook-name: "Exasol" connection-type: exasol diff --git a/providers/exasol/src/airflow/providers/exasol/get_provider_info.py b/providers/exasol/src/airflow/providers/exasol/get_provider_info.py index 61aa7124a92fc..8dec9e2ba2049 100644 --- a/providers/exasol/src/airflow/providers/exasol/get_provider_info.py +++ b/providers/exasol/src/airflow/providers/exasol/get_provider_info.py @@ -44,6 +44,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.exasol.hooks.exasol.ExasolHook", + "hook-name": "Exasol", "connection-type": "exasol", } ], diff --git a/providers/facebook/provider.yaml b/providers/facebook/provider.yaml index 5e7227afac6bd..3c7b2dfceeb7f 100644 --- a/providers/facebook/provider.yaml +++ b/providers/facebook/provider.yaml @@ -75,4 +75,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.facebook.ads.hooks.ads.FacebookAdsReportingHook + hook-name: "Facebook Ads" connection-type: facebook_social diff --git a/providers/facebook/src/airflow/providers/facebook/get_provider_info.py b/providers/facebook/src/airflow/providers/facebook/get_provider_info.py index a378b1066a817..c5e0842036ef7 100644 --- a/providers/facebook/src/airflow/providers/facebook/get_provider_info.py +++ b/providers/facebook/src/airflow/providers/facebook/get_provider_info.py @@ -43,6 +43,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.facebook.ads.hooks.ads.FacebookAdsReportingHook", + "hook-name": "Facebook Ads", "connection-type": "facebook_social", } ], diff --git a/providers/ftp/provider.yaml b/providers/ftp/provider.yaml index 8c3b518a8a194..4599e61158fc3 100644 --- a/providers/ftp/provider.yaml +++ b/providers/ftp/provider.yaml @@ -87,4 +87,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.ftp.hooks.ftp.FTPHook + hook-name: "FTP" connection-type: ftp diff --git a/providers/ftp/src/airflow/providers/ftp/get_provider_info.py b/providers/ftp/src/airflow/providers/ftp/get_provider_info.py index 5c2039744e7a0..8658c11981f08 100644 --- a/providers/ftp/src/airflow/providers/ftp/get_provider_info.py +++ b/providers/ftp/src/airflow/providers/ftp/get_provider_info.py @@ -53,6 +53,10 @@ def get_provider_info(): } ], "connection-types": [ - {"hook-class-name": "airflow.providers.ftp.hooks.ftp.FTPHook", "connection-type": "ftp"} + { + "hook-class-name": "airflow.providers.ftp.hooks.ftp.FTPHook", + "hook-name": "FTP", + "connection-type": "ftp", + } ], } diff --git a/providers/git/provider.yaml b/providers/git/provider.yaml index 69d1951c2f2a8..65e0a311d132b 100644 --- a/providers/git/provider.yaml +++ b/providers/git/provider.yaml @@ -64,6 +64,7 @@ bundles: connection-types: - hook-class-name: airflow.providers.git.hooks.git.GitHook + hook-name: "GIT" connection-type: git ui-field-behaviour: hidden-fields: diff --git a/providers/git/src/airflow/providers/git/get_provider_info.py b/providers/git/src/airflow/providers/git/get_provider_info.py index 21dcec07984e6..82a3bb35096b9 100644 --- a/providers/git/src/airflow/providers/git/get_provider_info.py +++ b/providers/git/src/airflow/providers/git/get_provider_info.py @@ -41,6 +41,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.git.hooks.git.GitHook", + "hook-name": "GIT", "connection-type": "git", "ui-field-behaviour": { "hidden-fields": ["schema"], diff --git a/providers/github/provider.yaml b/providers/github/provider.yaml index 1c6a20692cb55..c4c5c0b8bb940 100644 --- a/providers/github/provider.yaml +++ b/providers/github/provider.yaml @@ -83,6 +83,7 @@ sensors: connection-types: - hook-class-name: airflow.providers.github.hooks.github.GithubHook + hook-name: "GitHub" connection-type: github ui-field-behaviour: hidden-fields: diff --git a/providers/github/src/airflow/providers/github/get_provider_info.py b/providers/github/src/airflow/providers/github/get_provider_info.py index 1bc6637ad71e0..c46aca1feb3fb 100644 --- a/providers/github/src/airflow/providers/github/get_provider_info.py +++ b/providers/github/src/airflow/providers/github/get_provider_info.py @@ -46,6 +46,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.github.hooks.github.GithubHook", + "hook-name": "GitHub", "connection-type": "github", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "login", "extra"], diff --git a/providers/google/provider.yaml b/providers/google/provider.yaml index 04541d5569d92..63a09c1487c2e 100644 --- a/providers/google/provider.yaml +++ b/providers/google/provider.yaml @@ -1133,6 +1133,7 @@ transfers: connection-types: - hook-class-name: airflow.providers.google.common.hooks.base_google.GoogleBaseHook + hook-name: "Google Cloud" connection-type: google_cloud_platform ui-field-behaviour: hidden-fields: ["host", "schema", "login", "password", "port", "extra"] @@ -1200,12 +1201,16 @@ connection-types: type: ["boolean", "null"] default: false - hook-class-name: airflow.providers.google.cloud.hooks.dataprep.GoogleDataprepHook + hook-name: "Google Dataprep" connection-type: dataprep - hook-class-name: airflow.providers.google.cloud.hooks.cloud_sql.CloudSQLHook + hook-name: "Google Cloud SQL" connection-type: gcpcloudsql - hook-class-name: airflow.providers.google.cloud.hooks.cloud_sql.CloudSQLDatabaseHook + hook-name: "Google Cloud SQL Database" connection-type: gcpcloudsqldb - hook-class-name: airflow.providers.google.cloud.hooks.bigquery.BigQueryHook + hook-name: "Google Bigquery" connection-type: gcpbigquery ui-field-behaviour: hidden-fields: ["host", "schema", "login", "password", "port", "extra"] @@ -1294,14 +1299,17 @@ connection-types: schema: type: ["string", "null"] - hook-class-name: airflow.providers.google.cloud.hooks.compute_ssh.ComputeEngineSSHHook + hook-name: "Google Cloud SSH" connection-type: gcpssh ui-field-behaviour: hidden-fields: ["host", "schema", "login", "password", "port", "extra"] relabeling: {} placeholders: {} - hook-class-name: airflow.providers.google.leveldb.hooks.leveldb.LevelDBHook + hook-name: "LevelDB" connection-type: leveldb - hook-class-name: airflow.providers.google.ads.hooks.ads.GoogleAdsHook + hook-name: "Google Ads" connection-type: google_ads ui-field-behaviour: hidden-fields: ["host", "login", "schema", "port"] @@ -1328,6 +1336,7 @@ connection-types: type: ["string", "null"] format: password - hook-class-name: airflow.providers.google.cloud.hooks.looker.LookerHook + hook-name: "Google Looker" connection-type: gcp_looker extra-links: diff --git a/providers/google/src/airflow/providers/google/get_provider_info.py b/providers/google/src/airflow/providers/google/get_provider_info.py index fec76eb9d5f3b..3b1aa5fac9a5f 100644 --- a/providers/google/src/airflow/providers/google/get_provider_info.py +++ b/providers/google/src/airflow/providers/google/get_provider_info.py @@ -1380,6 +1380,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.google.common.hooks.base_google.GoogleBaseHook", + "hook-name": "Google Cloud", "connection-type": "google_cloud_platform", "ui-field-behaviour": { "hidden-fields": ["host", "schema", "login", "password", "port", "extra"], @@ -1438,18 +1439,22 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.google.cloud.hooks.dataprep.GoogleDataprepHook", + "hook-name": "Google Dataprep", "connection-type": "dataprep", }, { "hook-class-name": "airflow.providers.google.cloud.hooks.cloud_sql.CloudSQLHook", + "hook-name": "Google Cloud SQL", "connection-type": "gcpcloudsql", }, { "hook-class-name": "airflow.providers.google.cloud.hooks.cloud_sql.CloudSQLDatabaseHook", + "hook-name": "Google Cloud SQL Database", "connection-type": "gcpcloudsqldb", }, { "hook-class-name": "airflow.providers.google.cloud.hooks.bigquery.BigQueryHook", + "hook-name": "Google Bigquery", "connection-type": "gcpbigquery", "ui-field-behaviour": { "hidden-fields": ["host", "schema", "login", "password", "port", "extra"], @@ -1519,6 +1524,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.google.cloud.hooks.compute_ssh.ComputeEngineSSHHook", + "hook-name": "Google Cloud SSH", "connection-type": "gcpssh", "ui-field-behaviour": { "hidden-fields": ["host", "schema", "login", "password", "port", "extra"], @@ -1528,10 +1534,12 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.google.leveldb.hooks.leveldb.LevelDBHook", + "hook-name": "LevelDB", "connection-type": "leveldb", }, { "hook-class-name": "airflow.providers.google.ads.hooks.ads.GoogleAdsHook", + "hook-name": "Google Ads", "connection-type": "google_ads", "ui-field-behaviour": { "hidden-fields": ["host", "login", "schema", "port"], @@ -1553,6 +1561,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.google.cloud.hooks.looker.LookerHook", + "hook-name": "Google Looker", "connection-type": "gcp_looker", }, ], diff --git a/providers/grpc/provider.yaml b/providers/grpc/provider.yaml index 3842b5c7fb3cb..f86892d0a0a7b 100644 --- a/providers/grpc/provider.yaml +++ b/providers/grpc/provider.yaml @@ -77,6 +77,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.grpc.hooks.grpc.GrpcHook + hook-name: "GRPC Connection" connection-type: grpc ui-field-behaviour: hidden-fields: diff --git a/providers/grpc/src/airflow/providers/grpc/get_provider_info.py b/providers/grpc/src/airflow/providers/grpc/get_provider_info.py index 267bdfcf6793e..4323b63fc35b3 100644 --- a/providers/grpc/src/airflow/providers/grpc/get_provider_info.py +++ b/providers/grpc/src/airflow/providers/grpc/get_provider_info.py @@ -41,6 +41,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.grpc.hooks.grpc.GrpcHook", + "hook-name": "GRPC Connection", "connection-type": "grpc", "ui-field-behaviour": { "hidden-fields": ["login", "password", "schema", "extra"], diff --git a/providers/hashicorp/provider.yaml b/providers/hashicorp/provider.yaml index 69330eb9c9fb4..1b47b264b5cdc 100644 --- a/providers/hashicorp/provider.yaml +++ b/providers/hashicorp/provider.yaml @@ -86,6 +86,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.hashicorp.hooks.vault.VaultHook + hook-name: "Hashicorp Vault" connection-type: vault ui-field-behaviour: hidden-fields: ["extra"] diff --git a/providers/hashicorp/src/airflow/providers/hashicorp/get_provider_info.py b/providers/hashicorp/src/airflow/providers/hashicorp/get_provider_info.py index 1fd65f01b98fd..28020342f9710 100644 --- a/providers/hashicorp/src/airflow/providers/hashicorp/get_provider_info.py +++ b/providers/hashicorp/src/airflow/providers/hashicorp/get_provider_info.py @@ -43,6 +43,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.hashicorp.hooks.vault.VaultHook", + "hook-name": "Hashicorp Vault", "connection-type": "vault", "ui-field-behaviour": {"hidden-fields": ["extra"]}, "conn-fields": { diff --git a/providers/http/provider.yaml b/providers/http/provider.yaml index b3f0610e0123b..9373fbcec3498 100644 --- a/providers/http/provider.yaml +++ b/providers/http/provider.yaml @@ -115,6 +115,7 @@ triggers: connection-types: - hook-class-name: airflow.providers.http.hooks.http.HttpHook + hook-name: "HTTP" connection-type: http ui-field-behaviour: hidden-fields: [] diff --git a/providers/http/src/airflow/providers/http/get_provider_info.py b/providers/http/src/airflow/providers/http/get_provider_info.py index 1c95246e12dd6..93d137842dea8 100644 --- a/providers/http/src/airflow/providers/http/get_provider_info.py +++ b/providers/http/src/airflow/providers/http/get_provider_info.py @@ -63,6 +63,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.http.hooks.http.HttpHook", + "hook-name": "HTTP", "connection-type": "http", "ui-field-behaviour": {"hidden-fields": [], "relabeling": {}, "placeholders": {}}, "conn-fields": {}, diff --git a/providers/imap/provider.yaml b/providers/imap/provider.yaml index 18754a51ed177..7ba1b323ac979 100644 --- a/providers/imap/provider.yaml +++ b/providers/imap/provider.yaml @@ -85,6 +85,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.imap.hooks.imap.ImapHook + hook-name: "IMAP" connection-type: imap config: diff --git a/providers/imap/src/airflow/providers/imap/get_provider_info.py b/providers/imap/src/airflow/providers/imap/get_provider_info.py index ac0c1a2220948..3623dabfa9db2 100644 --- a/providers/imap/src/airflow/providers/imap/get_provider_info.py +++ b/providers/imap/src/airflow/providers/imap/get_provider_info.py @@ -47,7 +47,11 @@ def get_provider_info(): } ], "connection-types": [ - {"hook-class-name": "airflow.providers.imap.hooks.imap.ImapHook", "connection-type": "imap"} + { + "hook-class-name": "airflow.providers.imap.hooks.imap.ImapHook", + "hook-name": "IMAP", + "connection-type": "imap", + } ], "config": { "imap": { diff --git a/providers/influxdb/provider.yaml b/providers/influxdb/provider.yaml index 65e9c30f26e97..3da9e9b249c25 100644 --- a/providers/influxdb/provider.yaml +++ b/providers/influxdb/provider.yaml @@ -80,6 +80,7 @@ operators: connection-types: - hook-class-name: airflow.providers.influxdb.hooks.influxdb.InfluxDBHook + hook-name: "Influxdb" connection-type: influxdb ui-field-behaviour: hidden-fields: diff --git a/providers/influxdb/src/airflow/providers/influxdb/get_provider_info.py b/providers/influxdb/src/airflow/providers/influxdb/get_provider_info.py index fca723981e99d..f9f0a30e5a8ec 100644 --- a/providers/influxdb/src/airflow/providers/influxdb/get_provider_info.py +++ b/providers/influxdb/src/airflow/providers/influxdb/get_provider_info.py @@ -46,6 +46,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.influxdb.hooks.influxdb.InfluxDBHook", + "hook-name": "Influxdb", "connection-type": "influxdb", "ui-field-behaviour": { "hidden-fields": ["login", "password"], diff --git a/providers/informatica/provider.yaml b/providers/informatica/provider.yaml index 83786c7ade4a5..caedf21072412 100644 --- a/providers/informatica/provider.yaml +++ b/providers/informatica/provider.yaml @@ -44,6 +44,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.informatica.hooks.edc.InformaticaEDCHook + hook-name: "Informatica EDC" connection-type: informatica_edc plugins: diff --git a/providers/informatica/src/airflow/providers/informatica/get_provider_info.py b/providers/informatica/src/airflow/providers/informatica/get_provider_info.py index 98840280755f9..175bab466f4aa 100644 --- a/providers/informatica/src/airflow/providers/informatica/get_provider_info.py +++ b/providers/informatica/src/airflow/providers/informatica/get_provider_info.py @@ -40,6 +40,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.informatica.hooks.edc.InformaticaEDCHook", + "hook-name": "Informatica EDC", "connection-type": "informatica_edc", } ], diff --git a/providers/jdbc/provider.yaml b/providers/jdbc/provider.yaml index c89933f3bd860..ef25b75549770 100644 --- a/providers/jdbc/provider.yaml +++ b/providers/jdbc/provider.yaml @@ -87,6 +87,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.jdbc.hooks.jdbc.JdbcHook + hook-name: "JDBC Connection" connection-type: jdbc ui-field-behaviour: hidden-fields: diff --git a/providers/jdbc/src/airflow/providers/jdbc/get_provider_info.py b/providers/jdbc/src/airflow/providers/jdbc/get_provider_info.py index 2cac3a4e2f902..9d64b469f0b0c 100644 --- a/providers/jdbc/src/airflow/providers/jdbc/get_provider_info.py +++ b/providers/jdbc/src/airflow/providers/jdbc/get_provider_info.py @@ -44,6 +44,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.jdbc.hooks.jdbc.JdbcHook", + "hook-name": "JDBC Connection", "connection-type": "jdbc", "ui-field-behaviour": { "hidden-fields": ["port", "schema"], diff --git a/providers/jenkins/provider.yaml b/providers/jenkins/provider.yaml index 402e5b93c5152..14509df23b75c 100644 --- a/providers/jenkins/provider.yaml +++ b/providers/jenkins/provider.yaml @@ -93,6 +93,7 @@ sensors: connection-types: - hook-class-name: airflow.providers.jenkins.hooks.jenkins.JenkinsHook + hook-name: "Jenkins" connection-type: jenkins ui-field-behaviour: hidden-fields: diff --git a/providers/jenkins/src/airflow/providers/jenkins/get_provider_info.py b/providers/jenkins/src/airflow/providers/jenkins/get_provider_info.py index a5ea5c9b70db5..b62a2a10fd98c 100644 --- a/providers/jenkins/src/airflow/providers/jenkins/get_provider_info.py +++ b/providers/jenkins/src/airflow/providers/jenkins/get_provider_info.py @@ -49,6 +49,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.jenkins.hooks.jenkins.JenkinsHook", + "hook-name": "Jenkins", "connection-type": "jenkins", "ui-field-behaviour": { "hidden-fields": ["extra"], diff --git a/providers/microsoft/azure/provider.yaml b/providers/microsoft/azure/provider.yaml index fdf39ed0d2c96..12454120af9e4 100644 --- a/providers/microsoft/azure/provider.yaml +++ b/providers/microsoft/azure/provider.yaml @@ -351,6 +351,7 @@ transfers: connection-types: - hook-class-name: airflow.providers.microsoft.azure.hooks.base_azure.AzureBaseHook + hook-name: "Azure" connection-type: azure ui-field-behaviour: hidden-fields: ["schema", "port", "host"] @@ -381,6 +382,7 @@ connection-types: schema: type: ["string", "null"] - hook-class-name: airflow.providers.microsoft.azure.hooks.compute.AzureComputeHook + hook-name: "Azure Compute" connection-type: azure_compute ui-field-behaviour: hidden-fields: ["schema", "port", "host"] @@ -411,6 +413,7 @@ connection-types: schema: type: ["string", "null"] - hook-class-name: airflow.providers.microsoft.azure.hooks.adx.AzureDataExplorerHook + hook-name: "Azure Data Explorer" connection-type: azure_data_explorer ui-field-behaviour: hidden-fields: ["schema", "port", "extra"] @@ -452,6 +455,7 @@ connection-types: schema: type: ["string", "null"] - hook-class-name: airflow.providers.microsoft.azure.hooks.batch.AzureBatchHook + hook-name: "Azure Batch Service" connection-type: azure_batch ui-field-behaviour: hidden-fields: ["schema", "port", "host", "extra"] @@ -473,6 +477,7 @@ connection-types: schema: type: ["string", "null"] - hook-class-name: airflow.providers.microsoft.azure.hooks.cosmos.AzureCosmosDBHook + hook-name: "Azure CosmosDB" connection-type: azure_cosmos ui-field-behaviour: hidden-fields: ["schema", "port", "host", "extra"] @@ -512,6 +517,7 @@ connection-types: schema: type: ["string", "null"] - hook-class-name: airflow.providers.microsoft.azure.hooks.data_lake.AzureDataLakeHook + hook-name: "Azure Data Lake" connection-type: azure_data_lake ui-field-behaviour: hidden-fields: ["schema", "port", "host", "extra"] @@ -541,6 +547,7 @@ connection-types: schema: type: ["string", "null"] - hook-class-name: airflow.providers.microsoft.azure.hooks.fileshare.AzureFileShareHook + hook-name: "Azure FileShare" connection-type: azure_fileshare ui-field-behaviour: hidden-fields: ["schema", "port", "host", "extra"] @@ -571,6 +578,7 @@ connection-types: schema: type: ["string", "null"] - hook-class-name: airflow.providers.microsoft.azure.hooks.container_volume.AzureContainerVolumeHook + hook-name: "Azure Container Volume" connection-type: azure_container_volume ui-field-behaviour: hidden-fields: ["schema", "port", "host", "extra"] @@ -606,8 +614,10 @@ connection-types: schema: type: ["string", "null"] - hook-class-name: airflow.providers.microsoft.azure.hooks.container_instance.AzureContainerInstanceHook + hook-name: "Azure Container Instance" connection-type: azure_container_instance - hook-class-name: airflow.providers.microsoft.azure.hooks.wasb.WasbHook + hook-name: "Azure Blob Storage" connection-type: wasb ui-field-behaviour: hidden-fields: ["schema", "port"] @@ -655,6 +665,7 @@ connection-types: schema: type: ["string", "null"] - hook-class-name: airflow.providers.microsoft.azure.hooks.data_factory.AzureDataFactoryHook + hook-name: "Azure Data Factory" connection-type: azure_data_factory ui-field-behaviour: hidden-fields: ["schema", "port", "host", "extra"] @@ -688,6 +699,7 @@ connection-types: schema: type: ["string", "null"] - hook-class-name: airflow.providers.microsoft.azure.hooks.container_registry.AzureContainerRegistryHook + hook-name: "Azure Container Registry" connection-type: azure_container_registry ui-field-behaviour: hidden-fields: ["schema", "port", "extra"] @@ -719,6 +731,7 @@ connection-types: schema: type: ["string", "null"] - hook-class-name: airflow.providers.microsoft.azure.hooks.asb.BaseAzureServiceBusHook + hook-name: "Azure Service Bus" connection-type: azure_service_bus ui-field-behaviour: hidden-fields: ["port", "host", "extra", "login", "password"] @@ -750,6 +763,7 @@ connection-types: schema: type: ["string", "null"] - hook-class-name: airflow.providers.microsoft.azure.hooks.synapse.BaseAzureSynapseHook + hook-name: "Azure Synapse" connection-type: azure_synapse ui-field-behaviour: hidden-fields: ["schema", "port", "extra"] @@ -776,6 +790,7 @@ connection-types: schema: type: ["string", "null"] - hook-class-name: airflow.providers.microsoft.azure.hooks.data_lake.AzureDataLakeStorageV2Hook + hook-name: "Azure Data Lake Storage V2" connection-type: adls ui-field-behaviour: hidden-fields: ["schema", "port"] @@ -809,6 +824,7 @@ connection-types: schema: type: ["string", "null"] - hook-class-name: airflow.providers.microsoft.azure.hooks.msgraph.KiotaRequestAdapterHook + hook-name: "Microsoft Graph API" connection-type: msgraph ui-field-behaviour: hidden-fields: ["extra"] @@ -875,6 +891,7 @@ connection-types: schema: type: ["string", "null"] - hook-class-name: airflow.providers.microsoft.azure.hooks.powerbi.PowerBIHook + hook-name: "Power BI" connection-type: powerbi ui-field-behaviour: hidden-fields: ["schema", "port", "host", "extra"] diff --git a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/get_provider_info.py b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/get_provider_info.py index a13e4e6229b1c..4ca9bc5c620d1 100644 --- a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/get_provider_info.py +++ b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/get_provider_info.py @@ -344,6 +344,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.microsoft.azure.hooks.base_azure.AzureBaseHook", + "hook-name": "Azure", "connection-type": "azure", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "host"], @@ -374,6 +375,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.microsoft.azure.hooks.compute.AzureComputeHook", + "hook-name": "Azure Compute", "connection-type": "azure_compute", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "host"], @@ -404,6 +406,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.microsoft.azure.hooks.adx.AzureDataExplorerHook", + "hook-name": "Azure Data Explorer", "connection-type": "azure_data_explorer", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "extra"], @@ -440,6 +443,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.microsoft.azure.hooks.batch.AzureBatchHook", + "hook-name": "Azure Batch Service", "connection-type": "azure_batch", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "host", "extra"], @@ -460,6 +464,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.microsoft.azure.hooks.cosmos.AzureCosmosDBHook", + "hook-name": "Azure CosmosDB", "connection-type": "azure_cosmos", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "host", "extra"], @@ -502,6 +507,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.microsoft.azure.hooks.data_lake.AzureDataLakeHook", + "hook-name": "Azure Data Lake", "connection-type": "azure_data_lake", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "host", "extra"], @@ -531,6 +537,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.microsoft.azure.hooks.fileshare.AzureFileShareHook", + "hook-name": "Azure FileShare", "connection-type": "azure_fileshare", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "host", "extra"], @@ -566,6 +573,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.microsoft.azure.hooks.container_volume.AzureContainerVolumeHook", + "hook-name": "Azure Container Volume", "connection-type": "azure_container_volume", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "host", "extra"], @@ -603,10 +611,12 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.microsoft.azure.hooks.container_instance.AzureContainerInstanceHook", + "hook-name": "Azure Container Instance", "connection-type": "azure_container_instance", }, { "hook-class-name": "airflow.providers.microsoft.azure.hooks.wasb.WasbHook", + "hook-name": "Azure Blob Storage", "connection-type": "wasb", "ui-field-behaviour": { "hidden-fields": ["schema", "port"], @@ -655,6 +665,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.microsoft.azure.hooks.data_factory.AzureDataFactoryHook", + "hook-name": "Azure Data Factory", "connection-type": "azure_data_factory", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "host", "extra"], @@ -681,6 +692,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.microsoft.azure.hooks.container_registry.AzureContainerRegistryHook", + "hook-name": "Azure Container Registry", "connection-type": "azure_container_registry", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "extra"], @@ -718,6 +730,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.microsoft.azure.hooks.asb.BaseAzureServiceBusHook", + "hook-name": "Azure Service Bus", "connection-type": "azure_service_bus", "ui-field-behaviour": { "hidden-fields": ["port", "host", "extra", "login", "password"], @@ -749,6 +762,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.microsoft.azure.hooks.synapse.BaseAzureSynapseHook", + "hook-name": "Azure Synapse", "connection-type": "azure_synapse", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "extra"], @@ -774,6 +788,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.microsoft.azure.hooks.data_lake.AzureDataLakeStorageV2Hook", + "hook-name": "Azure Data Lake Storage V2", "connection-type": "adls", "ui-field-behaviour": { "hidden-fields": ["schema", "port"], @@ -812,6 +827,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.microsoft.azure.hooks.msgraph.KiotaRequestAdapterHook", + "hook-name": "Microsoft Graph API", "connection-type": "msgraph", "ui-field-behaviour": { "hidden-fields": ["extra"], @@ -851,6 +867,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.microsoft.azure.hooks.powerbi.PowerBIHook", + "hook-name": "Power BI", "connection-type": "powerbi", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "host", "extra"], diff --git a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_lake.py b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_lake.py index 7535eaaaa3a46..8860c53fa2eda 100644 --- a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_lake.py +++ b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_lake.py @@ -278,7 +278,7 @@ class AzureDataLakeStorageV2Hook(BaseHook): conn_name_attr = "adls_conn_id" default_conn_name = "adls_default" conn_type = "adls" - hook_name = "Azure Date Lake Storage V2" + hook_name = "Azure Data Lake Storage V2" @classmethod @add_managed_identity_connection_widgets diff --git a/providers/microsoft/mssql/provider.yaml b/providers/microsoft/mssql/provider.yaml index 6416610194d83..5421345c09a40 100644 --- a/providers/microsoft/mssql/provider.yaml +++ b/providers/microsoft/mssql/provider.yaml @@ -89,4 +89,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.microsoft.mssql.hooks.mssql.MsSqlHook + hook-name: "Microsoft SQL Server" connection-type: mssql diff --git a/providers/microsoft/mssql/src/airflow/providers/microsoft/mssql/get_provider_info.py b/providers/microsoft/mssql/src/airflow/providers/microsoft/mssql/get_provider_info.py index 29c4ddb865a12..02c50438b3f26 100644 --- a/providers/microsoft/mssql/src/airflow/providers/microsoft/mssql/get_provider_info.py +++ b/providers/microsoft/mssql/src/airflow/providers/microsoft/mssql/get_provider_info.py @@ -50,6 +50,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.microsoft.mssql.hooks.mssql.MsSqlHook", + "hook-name": "Microsoft SQL Server", "connection-type": "mssql", } ], diff --git a/providers/microsoft/psrp/provider.yaml b/providers/microsoft/psrp/provider.yaml index 0ade5dc418931..f197e1e2acd4d 100644 --- a/providers/microsoft/psrp/provider.yaml +++ b/providers/microsoft/psrp/provider.yaml @@ -82,4 +82,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.microsoft.psrp.hooks.psrp.PsrpHook + hook-name: "PowerShell Remoting Protocol" connection-type: psrp diff --git a/providers/microsoft/psrp/src/airflow/providers/microsoft/psrp/get_provider_info.py b/providers/microsoft/psrp/src/airflow/providers/microsoft/psrp/get_provider_info.py index 1c6a7a448e503..bccab1b61fa11 100644 --- a/providers/microsoft/psrp/src/airflow/providers/microsoft/psrp/get_provider_info.py +++ b/providers/microsoft/psrp/src/airflow/providers/microsoft/psrp/get_provider_info.py @@ -48,6 +48,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.microsoft.psrp.hooks.psrp.PsrpHook", + "hook-name": "PowerShell Remoting Protocol", "connection-type": "psrp", } ], diff --git a/providers/microsoft/winrm/provider.yaml b/providers/microsoft/winrm/provider.yaml index d5dad062dd9c2..319a2d78419bd 100644 --- a/providers/microsoft/winrm/provider.yaml +++ b/providers/microsoft/winrm/provider.yaml @@ -92,4 +92,5 @@ triggers: connection-types: - hook-class-name: airflow.providers.microsoft.winrm.hooks.winrm.WinRMHook + hook-name: "WinRM" connection-type: winrm diff --git a/providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/get_provider_info.py b/providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/get_provider_info.py index 8c9d6c56f2f7e..6e8430e7a730a 100644 --- a/providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/get_provider_info.py +++ b/providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/get_provider_info.py @@ -56,6 +56,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.microsoft.winrm.hooks.winrm.WinRMHook", + "hook-name": "WinRM", "connection-type": "winrm", } ], diff --git a/providers/mongo/provider.yaml b/providers/mongo/provider.yaml index 6082719a7c4ca..9a28ecd4cd016 100644 --- a/providers/mongo/provider.yaml +++ b/providers/mongo/provider.yaml @@ -86,6 +86,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.mongo.hooks.mongo.MongoHook + hook-name: "MongoDB" connection-type: mongo conn-fields: srv: diff --git a/providers/mongo/src/airflow/providers/mongo/get_provider_info.py b/providers/mongo/src/airflow/providers/mongo/get_provider_info.py index 2a6eb2befe295..7e6cf3c0f60e1 100644 --- a/providers/mongo/src/airflow/providers/mongo/get_provider_info.py +++ b/providers/mongo/src/airflow/providers/mongo/get_provider_info.py @@ -41,6 +41,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.mongo.hooks.mongo.MongoHook", + "hook-name": "MongoDB", "connection-type": "mongo", "conn-fields": { "srv": {"label": "Srv", "schema": {"type": ["boolean", "null"]}}, diff --git a/providers/mysql/provider.yaml b/providers/mysql/provider.yaml index 85befd4333027..ee26d0856ba8d 100644 --- a/providers/mysql/provider.yaml +++ b/providers/mysql/provider.yaml @@ -117,6 +117,7 @@ transfers: connection-types: - hook-class-name: airflow.providers.mysql.hooks.mysql.MySqlHook + hook-name: "MySQL" connection-type: mysql asset-uris: diff --git a/providers/mysql/src/airflow/providers/mysql/get_provider_info.py b/providers/mysql/src/airflow/providers/mysql/get_provider_info.py index e4945605bf48f..f0503d851df1f 100644 --- a/providers/mysql/src/airflow/providers/mysql/get_provider_info.py +++ b/providers/mysql/src/airflow/providers/mysql/get_provider_info.py @@ -59,7 +59,11 @@ def get_provider_info(): }, ], "connection-types": [ - {"hook-class-name": "airflow.providers.mysql.hooks.mysql.MySqlHook", "connection-type": "mysql"} + { + "hook-class-name": "airflow.providers.mysql.hooks.mysql.MySqlHook", + "hook-name": "MySQL", + "connection-type": "mysql", + } ], "asset-uris": [ {"schemes": ["mysql", "mariadb"], "handler": "airflow.providers.mysql.assets.mysql.sanitize_uri"} diff --git a/providers/neo4j/provider.yaml b/providers/neo4j/provider.yaml index 9ddd0d1f96695..9f05b08897632 100644 --- a/providers/neo4j/provider.yaml +++ b/providers/neo4j/provider.yaml @@ -92,4 +92,5 @@ sensors: connection-types: - hook-class-name: airflow.providers.neo4j.hooks.neo4j.Neo4jHook + hook-name: "Neo4j" connection-type: neo4j diff --git a/providers/neo4j/src/airflow/providers/neo4j/get_provider_info.py b/providers/neo4j/src/airflow/providers/neo4j/get_provider_info.py index e3748c17e768b..6f9d603fd5be5 100644 --- a/providers/neo4j/src/airflow/providers/neo4j/get_provider_info.py +++ b/providers/neo4j/src/airflow/providers/neo4j/get_provider_info.py @@ -46,6 +46,10 @@ def get_provider_info(): {"integration-name": "Neo4j", "python-modules": ["airflow.providers.neo4j.sensors.neo4j"]} ], "connection-types": [ - {"hook-class-name": "airflow.providers.neo4j.hooks.neo4j.Neo4jHook", "connection-type": "neo4j"} + { + "hook-class-name": "airflow.providers.neo4j.hooks.neo4j.Neo4jHook", + "hook-name": "Neo4j", + "connection-type": "neo4j", + } ], } diff --git a/providers/odbc/provider.yaml b/providers/odbc/provider.yaml index 0c8505e980488..c0a012c20d1a3 100644 --- a/providers/odbc/provider.yaml +++ b/providers/odbc/provider.yaml @@ -85,4 +85,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.odbc.hooks.odbc.OdbcHook + hook-name: "ODBC" connection-type: odbc diff --git a/providers/odbc/src/airflow/providers/odbc/get_provider_info.py b/providers/odbc/src/airflow/providers/odbc/get_provider_info.py index afdec5ae09ea7..31f6fb9451b0f 100644 --- a/providers/odbc/src/airflow/providers/odbc/get_provider_info.py +++ b/providers/odbc/src/airflow/providers/odbc/get_provider_info.py @@ -37,6 +37,10 @@ def get_provider_info(): ], "hooks": [{"integration-name": "ODBC", "python-modules": ["airflow.providers.odbc.hooks.odbc"]}], "connection-types": [ - {"hook-class-name": "airflow.providers.odbc.hooks.odbc.OdbcHook", "connection-type": "odbc"} + { + "hook-class-name": "airflow.providers.odbc.hooks.odbc.OdbcHook", + "hook-name": "ODBC", + "connection-type": "odbc", + } ], } diff --git a/providers/openai/provider.yaml b/providers/openai/provider.yaml index e05e67c9d3e28..6bbf868a54e26 100644 --- a/providers/openai/provider.yaml +++ b/providers/openai/provider.yaml @@ -89,6 +89,7 @@ triggers: connection-types: - hook-class-name: airflow.providers.openai.hooks.openai.OpenAIHook + hook-name: "OpenAI" connection-type: openai ui-field-behaviour: hidden-fields: diff --git a/providers/openai/src/airflow/providers/openai/get_provider_info.py b/providers/openai/src/airflow/providers/openai/get_provider_info.py index 6b8eda581e2c6..47b5bba24f79f 100644 --- a/providers/openai/src/airflow/providers/openai/get_provider_info.py +++ b/providers/openai/src/airflow/providers/openai/get_provider_info.py @@ -56,6 +56,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.openai.hooks.openai.OpenAIHook", + "hook-name": "OpenAI", "connection-type": "openai", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "login"], diff --git a/providers/openfaas/provider.yaml b/providers/openfaas/provider.yaml index 97668473e843b..043414e0674c5 100644 --- a/providers/openfaas/provider.yaml +++ b/providers/openfaas/provider.yaml @@ -69,6 +69,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.openfaas.hooks.openfaas.OpenFaasHook + hook-name: "OpenFaaS" connection-type: openfaas ui-field-behaviour: hidden-fields: diff --git a/providers/openfaas/src/airflow/providers/openfaas/get_provider_info.py b/providers/openfaas/src/airflow/providers/openfaas/get_provider_info.py index c13d8ffb545dd..319cacb648a48 100644 --- a/providers/openfaas/src/airflow/providers/openfaas/get_provider_info.py +++ b/providers/openfaas/src/airflow/providers/openfaas/get_provider_info.py @@ -40,6 +40,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.openfaas.hooks.openfaas.OpenFaasHook", + "hook-name": "OpenFaaS", "connection-type": "openfaas", "ui-field-behaviour": {"hidden-fields": ["schema", "port", "login", "password", "extra"]}, } diff --git a/providers/opensearch/provider.yaml b/providers/opensearch/provider.yaml index a2fd318d72cfe..e56b20b6cc403 100644 --- a/providers/opensearch/provider.yaml +++ b/providers/opensearch/provider.yaml @@ -74,6 +74,7 @@ operators: connection-types: - hook-class-name: airflow.providers.opensearch.hooks.opensearch.OpenSearchHook + hook-name: "OpenSearch Hook" connection-type: opensearch ui-field-behaviour: hidden-fields: diff --git a/providers/opensearch/src/airflow/providers/opensearch/get_provider_info.py b/providers/opensearch/src/airflow/providers/opensearch/get_provider_info.py index f813b681baee2..70a3fac8ec291 100644 --- a/providers/opensearch/src/airflow/providers/opensearch/get_provider_info.py +++ b/providers/opensearch/src/airflow/providers/opensearch/get_provider_info.py @@ -50,6 +50,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.opensearch.hooks.opensearch.OpenSearchHook", + "hook-name": "OpenSearch Hook", "connection-type": "opensearch", "ui-field-behaviour": { "hidden-fields": ["schema"], diff --git a/providers/opsgenie/provider.yaml b/providers/opsgenie/provider.yaml index 3a14a6257eb16..845f446760a67 100644 --- a/providers/opsgenie/provider.yaml +++ b/providers/opsgenie/provider.yaml @@ -81,6 +81,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.opsgenie.hooks.opsgenie.OpsgenieAlertHook + hook-name: "Opsgenie" connection-type: opsgenie ui-field-behaviour: hidden-fields: diff --git a/providers/opsgenie/src/airflow/providers/opsgenie/get_provider_info.py b/providers/opsgenie/src/airflow/providers/opsgenie/get_provider_info.py index a74e1065c30d8..4e6a65f90f395 100644 --- a/providers/opsgenie/src/airflow/providers/opsgenie/get_provider_info.py +++ b/providers/opsgenie/src/airflow/providers/opsgenie/get_provider_info.py @@ -47,6 +47,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.opsgenie.hooks.opsgenie.OpsgenieAlertHook", + "hook-name": "Opsgenie", "connection-type": "opsgenie", "ui-field-behaviour": { "hidden-fields": ["port", "schema", "login", "extra"], diff --git a/providers/oracle/provider.yaml b/providers/oracle/provider.yaml index 49d3553037487..15adfad1ea69e 100644 --- a/providers/oracle/provider.yaml +++ b/providers/oracle/provider.yaml @@ -105,4 +105,5 @@ transfers: connection-types: - hook-class-name: airflow.providers.oracle.hooks.oracle.OracleHook + hook-name: "Oracle" connection-type: oracle diff --git a/providers/oracle/src/airflow/providers/oracle/get_provider_info.py b/providers/oracle/src/airflow/providers/oracle/get_provider_info.py index 30417195e31ff..91b3f870147c6 100644 --- a/providers/oracle/src/airflow/providers/oracle/get_provider_info.py +++ b/providers/oracle/src/airflow/providers/oracle/get_provider_info.py @@ -57,6 +57,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.oracle.hooks.oracle.OracleHook", + "hook-name": "Oracle", "connection-type": "oracle", } ], diff --git a/providers/pagerduty/provider.yaml b/providers/pagerduty/provider.yaml index 72b296dbc7fd5..3d546480b8f91 100644 --- a/providers/pagerduty/provider.yaml +++ b/providers/pagerduty/provider.yaml @@ -75,6 +75,7 @@ integrations: connection-types: - hook-class-name: airflow.providers.pagerduty.hooks.pagerduty.PagerdutyHook + hook-name: "Pagerduty" connection-type: pagerduty conn-fields: routing_key: @@ -94,6 +95,7 @@ connection-types: relabeling: password: Pagerduty API token - hook-class-name: airflow.providers.pagerduty.hooks.pagerduty_events.PagerdutyEventsHook + hook-name: "Pagerduty Events" connection-type: pagerduty_events ui-field-behaviour: hidden-fields: diff --git a/providers/pagerduty/src/airflow/providers/pagerduty/get_provider_info.py b/providers/pagerduty/src/airflow/providers/pagerduty/get_provider_info.py index 89cd71ea35561..f43651f64e37b 100644 --- a/providers/pagerduty/src/airflow/providers/pagerduty/get_provider_info.py +++ b/providers/pagerduty/src/airflow/providers/pagerduty/get_provider_info.py @@ -37,6 +37,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.pagerduty.hooks.pagerduty.PagerdutyHook", + "hook-name": "Pagerduty", "connection-type": "pagerduty", "conn-fields": { "routing_key": { @@ -51,6 +52,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.pagerduty.hooks.pagerduty_events.PagerdutyEventsHook", + "hook-name": "Pagerduty Events", "connection-type": "pagerduty_events", "ui-field-behaviour": { "hidden-fields": ["port", "login", "schema", "host", "extra"], diff --git a/providers/papermill/provider.yaml b/providers/papermill/provider.yaml index 0a9b7817c2c45..4dcc94277f804 100644 --- a/providers/papermill/provider.yaml +++ b/providers/papermill/provider.yaml @@ -88,4 +88,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.papermill.hooks.kernel.KernelHook + hook-name: "Jupyter Kernel" connection-type: jupyter_kernel diff --git a/providers/papermill/src/airflow/providers/papermill/get_provider_info.py b/providers/papermill/src/airflow/providers/papermill/get_provider_info.py index f9e1c3186e42c..1982205a98267 100644 --- a/providers/papermill/src/airflow/providers/papermill/get_provider_info.py +++ b/providers/papermill/src/airflow/providers/papermill/get_provider_info.py @@ -47,6 +47,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.papermill.hooks.kernel.KernelHook", + "hook-name": "Jupyter Kernel", "connection-type": "jupyter_kernel", } ], diff --git a/providers/pinecone/provider.yaml b/providers/pinecone/provider.yaml index fb25dcdc94315..a8761ef39bf24 100644 --- a/providers/pinecone/provider.yaml +++ b/providers/pinecone/provider.yaml @@ -68,6 +68,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.pinecone.hooks.pinecone.PineconeHook + hook-name: "Pinecone" connection-type: pinecone ui-field-behaviour: hidden-fields: diff --git a/providers/pinecone/src/airflow/providers/pinecone/get_provider_info.py b/providers/pinecone/src/airflow/providers/pinecone/get_provider_info.py index 507e41956afe7..9307960cf86e6 100644 --- a/providers/pinecone/src/airflow/providers/pinecone/get_provider_info.py +++ b/providers/pinecone/src/airflow/providers/pinecone/get_provider_info.py @@ -41,6 +41,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.pinecone.hooks.pinecone.PineconeHook", + "hook-name": "Pinecone", "connection-type": "pinecone", "ui-field-behaviour": { "hidden-fields": ["port", "schema"], diff --git a/providers/postgres/provider.yaml b/providers/postgres/provider.yaml index 23d5801ac4bd7..50e8988039f0c 100644 --- a/providers/postgres/provider.yaml +++ b/providers/postgres/provider.yaml @@ -108,6 +108,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.postgres.hooks.postgres.PostgresHook + hook-name: "Postgres" connection-type: postgres ui-field-behaviour: relabeling: diff --git a/providers/postgres/src/airflow/providers/postgres/get_provider_info.py b/providers/postgres/src/airflow/providers/postgres/get_provider_info.py index 50f4cfdb3b465..7919f57d4b977 100644 --- a/providers/postgres/src/airflow/providers/postgres/get_provider_info.py +++ b/providers/postgres/src/airflow/providers/postgres/get_provider_info.py @@ -50,6 +50,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.postgres.hooks.postgres.PostgresHook", + "hook-name": "Postgres", "connection-type": "postgres", "ui-field-behaviour": {"relabeling": {"schema": "Database"}}, } diff --git a/providers/presto/provider.yaml b/providers/presto/provider.yaml index 689b05f38800b..7375b569e9df9 100644 --- a/providers/presto/provider.yaml +++ b/providers/presto/provider.yaml @@ -102,4 +102,5 @@ transfers: connection-types: - hook-class-name: airflow.providers.presto.hooks.presto.PrestoHook + hook-name: "Presto" connection-type: presto diff --git a/providers/presto/src/airflow/providers/presto/get_provider_info.py b/providers/presto/src/airflow/providers/presto/get_provider_info.py index 75ef5c1bfb2b5..bc9975b7cba71 100644 --- a/providers/presto/src/airflow/providers/presto/get_provider_info.py +++ b/providers/presto/src/airflow/providers/presto/get_provider_info.py @@ -49,6 +49,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.presto.hooks.presto.PrestoHook", + "hook-name": "Presto", "connection-type": "presto", } ], diff --git a/providers/qdrant/provider.yaml b/providers/qdrant/provider.yaml index 7cd9461db8613..6c7fb1f369f78 100644 --- a/providers/qdrant/provider.yaml +++ b/providers/qdrant/provider.yaml @@ -65,6 +65,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.qdrant.hooks.qdrant.QdrantHook + hook-name: "Qdrant" connection-type: qdrant conn-fields: url: diff --git a/providers/qdrant/src/airflow/providers/qdrant/get_provider_info.py b/providers/qdrant/src/airflow/providers/qdrant/get_provider_info.py index f20c57b9daf91..6f892b167c6ad 100644 --- a/providers/qdrant/src/airflow/providers/qdrant/get_provider_info.py +++ b/providers/qdrant/src/airflow/providers/qdrant/get_provider_info.py @@ -41,6 +41,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.qdrant.hooks.qdrant.QdrantHook", + "hook-name": "Qdrant", "connection-type": "qdrant", "conn-fields": { "url": { diff --git a/providers/redis/provider.yaml b/providers/redis/provider.yaml index 5cd172b266233..67c03c45ecced 100644 --- a/providers/redis/provider.yaml +++ b/providers/redis/provider.yaml @@ -97,6 +97,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.redis.hooks.redis.RedisHook + hook-name: "Redis" connection-type: redis ui-field-behaviour: hidden-fields: diff --git a/providers/redis/src/airflow/providers/redis/get_provider_info.py b/providers/redis/src/airflow/providers/redis/get_provider_info.py index d93d7d7bea362..b685bfbea5811 100644 --- a/providers/redis/src/airflow/providers/redis/get_provider_info.py +++ b/providers/redis/src/airflow/providers/redis/get_provider_info.py @@ -60,6 +60,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.redis.hooks.redis.RedisHook", + "hook-name": "Redis", "connection-type": "redis", "ui-field-behaviour": {"hidden-fields": ["schema", "extra"], "relabeling": {}}, "conn-fields": { diff --git a/providers/salesforce/provider.yaml b/providers/salesforce/provider.yaml index e1b1035bf9a9a..1ee98526c3146 100644 --- a/providers/salesforce/provider.yaml +++ b/providers/salesforce/provider.yaml @@ -95,6 +95,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.salesforce.hooks.salesforce.SalesforceHook + hook-name: "Salesforce" connection-type: salesforce conn-fields: security_token: diff --git a/providers/salesforce/src/airflow/providers/salesforce/get_provider_info.py b/providers/salesforce/src/airflow/providers/salesforce/get_provider_info.py index 23ea607b0410a..9e59473f89b6c 100644 --- a/providers/salesforce/src/airflow/providers/salesforce/get_provider_info.py +++ b/providers/salesforce/src/airflow/providers/salesforce/get_provider_info.py @@ -56,6 +56,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.salesforce.hooks.salesforce.SalesforceHook", + "hook-name": "Salesforce", "connection-type": "salesforce", "conn-fields": { "security_token": { diff --git a/providers/samba/provider.yaml b/providers/samba/provider.yaml index 0e11b63ad6f38..d11e1f3b1d4eb 100644 --- a/providers/samba/provider.yaml +++ b/providers/samba/provider.yaml @@ -82,6 +82,7 @@ transfers: connection-types: - hook-class-name: airflow.providers.samba.hooks.samba.SambaHook + hook-name: "Samba" connection-type: samba conn-fields: share_type: diff --git a/providers/samba/src/airflow/providers/samba/get_provider_info.py b/providers/samba/src/airflow/providers/samba/get_provider_info.py index d46471380df81..fa6f98566f7b8 100644 --- a/providers/samba/src/airflow/providers/samba/get_provider_info.py +++ b/providers/samba/src/airflow/providers/samba/get_provider_info.py @@ -46,6 +46,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.samba.hooks.samba.SambaHook", + "hook-name": "Samba", "connection-type": "samba", "conn-fields": { "share_type": { diff --git a/providers/segment/provider.yaml b/providers/segment/provider.yaml index 59b794571a7fe..df2b724e9063a 100644 --- a/providers/segment/provider.yaml +++ b/providers/segment/provider.yaml @@ -74,4 +74,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.segment.hooks.segment.SegmentHook + hook-name: "Segment" connection-type: segment diff --git a/providers/segment/src/airflow/providers/segment/get_provider_info.py b/providers/segment/src/airflow/providers/segment/get_provider_info.py index f10f23ee7f761..a1813925b1671 100644 --- a/providers/segment/src/airflow/providers/segment/get_provider_info.py +++ b/providers/segment/src/airflow/providers/segment/get_provider_info.py @@ -46,6 +46,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.segment.hooks.segment.SegmentHook", + "hook-name": "Segment", "connection-type": "segment", } ], diff --git a/providers/sftp/provider.yaml b/providers/sftp/provider.yaml index f9082754991cf..0bcf5c89c033a 100644 --- a/providers/sftp/provider.yaml +++ b/providers/sftp/provider.yaml @@ -116,6 +116,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.sftp.hooks.sftp.SFTPHook + hook-name: "SFTP" connection-type: sftp ui-field-behaviour: hidden-fields: diff --git a/providers/sftp/src/airflow/providers/sftp/get_provider_info.py b/providers/sftp/src/airflow/providers/sftp/get_provider_info.py index 4e2048f42618a..f55230a5a499c 100644 --- a/providers/sftp/src/airflow/providers/sftp/get_provider_info.py +++ b/providers/sftp/src/airflow/providers/sftp/get_provider_info.py @@ -59,6 +59,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.sftp.hooks.sftp.SFTPHook", + "hook-name": "SFTP", "connection-type": "sftp", "ui-field-behaviour": { "hidden-fields": ["schema"], diff --git a/providers/slack/provider.yaml b/providers/slack/provider.yaml index 0f477e12658ec..cde93f7e1d165 100644 --- a/providers/slack/provider.yaml +++ b/providers/slack/provider.yaml @@ -136,6 +136,7 @@ transfers: connection-types: - hook-class-name: airflow.providers.slack.hooks.slack.SlackHook + hook-name: "Slack API" connection-type: slack ui-field-behaviour: hidden-fields: @@ -174,6 +175,7 @@ connection-types: - 'null' description: Optional. Proxy to make the Slack API call. - hook-class-name: airflow.providers.slack.hooks.slack_webhook.SlackWebhookHook + hook-name: "Slack Incoming Webhook" connection-type: slackwebhook ui-field-behaviour: hidden-fields: diff --git a/providers/slack/src/airflow/providers/slack/get_provider_info.py b/providers/slack/src/airflow/providers/slack/get_provider_info.py index e7306f1f07501..f7c686c97274c 100644 --- a/providers/slack/src/airflow/providers/slack/get_provider_info.py +++ b/providers/slack/src/airflow/providers/slack/get_provider_info.py @@ -82,6 +82,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.slack.hooks.slack.SlackHook", + "hook-name": "Slack API", "connection-type": "slack", "ui-field-behaviour": { "hidden-fields": ["login", "port", "host", "schema", "extra"], @@ -113,6 +114,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.slack.hooks.slack_webhook.SlackWebhookHook", + "hook-name": "Slack Incoming Webhook", "connection-type": "slackwebhook", "ui-field-behaviour": { "hidden-fields": ["login", "port", "extra"], diff --git a/providers/smtp/provider.yaml b/providers/smtp/provider.yaml index 6ace7528fc96e..7e3f675ff4da9 100644 --- a/providers/smtp/provider.yaml +++ b/providers/smtp/provider.yaml @@ -82,6 +82,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.smtp.hooks.smtp.SmtpHook + hook-name: "SMTP" connection-type: smtp conn-fields: from_email: diff --git a/providers/smtp/src/airflow/providers/smtp/get_provider_info.py b/providers/smtp/src/airflow/providers/smtp/get_provider_info.py index f8665d3df256d..46e46efb97056 100644 --- a/providers/smtp/src/airflow/providers/smtp/get_provider_info.py +++ b/providers/smtp/src/airflow/providers/smtp/get_provider_info.py @@ -49,6 +49,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.smtp.hooks.smtp.SmtpHook", + "hook-name": "SMTP", "connection-type": "smtp", "conn-fields": { "from_email": {"label": "From email", "schema": {"type": ["string", "null"]}}, diff --git a/providers/snowflake/provider.yaml b/providers/snowflake/provider.yaml index 3b466b0e3388e..c1b194a8c0c02 100644 --- a/providers/snowflake/provider.yaml +++ b/providers/snowflake/provider.yaml @@ -148,6 +148,7 @@ transfers: connection-types: - hook-class-name: airflow.providers.snowflake.hooks.snowflake.SnowflakeHook + hook-name: "Snowflake" connection-type: snowflake ui-field-behaviour: hidden-fields: ["port", "host"] diff --git a/providers/snowflake/src/airflow/providers/snowflake/get_provider_info.py b/providers/snowflake/src/airflow/providers/snowflake/get_provider_info.py index 9974117f789c9..44778761d8f53 100644 --- a/providers/snowflake/src/airflow/providers/snowflake/get_provider_info.py +++ b/providers/snowflake/src/airflow/providers/snowflake/get_provider_info.py @@ -85,6 +85,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.snowflake.hooks.snowflake.SnowflakeHook", + "hook-name": "Snowflake", "connection-type": "snowflake", "ui-field-behaviour": { "hidden-fields": ["port", "host"], diff --git a/providers/sqlite/provider.yaml b/providers/sqlite/provider.yaml index 4079a665bcae4..690e5ced54ced 100644 --- a/providers/sqlite/provider.yaml +++ b/providers/sqlite/provider.yaml @@ -84,4 +84,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.sqlite.hooks.sqlite.SqliteHook + hook-name: "Sqlite" connection-type: sqlite diff --git a/providers/sqlite/src/airflow/providers/sqlite/get_provider_info.py b/providers/sqlite/src/airflow/providers/sqlite/get_provider_info.py index ebe7ce1d3df31..2819ab3a6b767 100644 --- a/providers/sqlite/src/airflow/providers/sqlite/get_provider_info.py +++ b/providers/sqlite/src/airflow/providers/sqlite/get_provider_info.py @@ -41,6 +41,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.sqlite.hooks.sqlite.SqliteHook", + "hook-name": "Sqlite", "connection-type": "sqlite", } ], diff --git a/providers/ssh/provider.yaml b/providers/ssh/provider.yaml index 6424cd6a267a1..a85294233bda2 100644 --- a/providers/ssh/provider.yaml +++ b/providers/ssh/provider.yaml @@ -107,6 +107,7 @@ triggers: connection-types: - hook-class-name: airflow.providers.ssh.hooks.ssh.SSHHook + hook-name: "SSH" connection-type: ssh ui-field-behaviour: hidden-fields: diff --git a/providers/ssh/src/airflow/providers/ssh/get_provider_info.py b/providers/ssh/src/airflow/providers/ssh/get_provider_info.py index 1be00ce4d3417..fff6b35592938 100644 --- a/providers/ssh/src/airflow/providers/ssh/get_provider_info.py +++ b/providers/ssh/src/airflow/providers/ssh/get_provider_info.py @@ -56,6 +56,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.ssh.hooks.ssh.SSHHook", + "hook-name": "SSH", "connection-type": "ssh", "ui-field-behaviour": { "hidden-fields": ["schema"], diff --git a/providers/standard/provider.yaml b/providers/standard/provider.yaml index 91e2498118a46..2e8ea21e85e84 100644 --- a/providers/standard/provider.yaml +++ b/providers/standard/provider.yaml @@ -137,6 +137,7 @@ config: connection-types: - hook-class-name: airflow.providers.standard.hooks.filesystem.FSHook + hook-name: "File (path)" connection-type: fs ui-field-behaviour: hidden-fields: @@ -154,6 +155,7 @@ connection-types: - string - 'null' - hook-class-name: airflow.providers.standard.hooks.package_index.PackageIndexHook + hook-name: "Package Index (Python)" connection-type: package_index ui-field-behaviour: hidden-fields: diff --git a/providers/standard/src/airflow/providers/standard/get_provider_info.py b/providers/standard/src/airflow/providers/standard/get_provider_info.py index 5b80e66cf51dd..1f7b2049454d1 100644 --- a/providers/standard/src/airflow/providers/standard/get_provider_info.py +++ b/providers/standard/src/airflow/providers/standard/get_provider_info.py @@ -120,6 +120,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.standard.hooks.filesystem.FSHook", + "hook-name": "File (path)", "connection-type": "fs", "ui-field-behaviour": { "hidden-fields": ["host", "schema", "port", "login", "password", "extra"] @@ -128,6 +129,7 @@ def get_provider_info(): }, { "hook-class-name": "airflow.providers.standard.hooks.package_index.PackageIndexHook", + "hook-name": "Package Index (Python)", "connection-type": "package_index", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "extra"], diff --git a/providers/tableau/provider.yaml b/providers/tableau/provider.yaml index 68783d8d705e7..5c17ea3964028 100644 --- a/providers/tableau/provider.yaml +++ b/providers/tableau/provider.yaml @@ -94,4 +94,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.tableau.hooks.tableau.TableauHook + hook-name: "Tableau" connection-type: tableau diff --git a/providers/tableau/src/airflow/providers/tableau/get_provider_info.py b/providers/tableau/src/airflow/providers/tableau/get_provider_info.py index ec70a9b69212f..e72c0a913f2f7 100644 --- a/providers/tableau/src/airflow/providers/tableau/get_provider_info.py +++ b/providers/tableau/src/airflow/providers/tableau/get_provider_info.py @@ -47,6 +47,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.tableau.hooks.tableau.TableauHook", + "hook-name": "Tableau", "connection-type": "tableau", } ], diff --git a/providers/telegram/provider.yaml b/providers/telegram/provider.yaml index c17a768f05669..88db032a5f815 100644 --- a/providers/telegram/provider.yaml +++ b/providers/telegram/provider.yaml @@ -78,6 +78,7 @@ operators: connection-types: - hook-class-name: airflow.providers.telegram.hooks.telegram.TelegramHook + hook-name: "Telegram" connection-type: telegram ui-field-behaviour: hidden-fields: diff --git a/providers/telegram/src/airflow/providers/telegram/get_provider_info.py b/providers/telegram/src/airflow/providers/telegram/get_provider_info.py index 18ba5b1c1b4bf..960ec1ef90ec3 100644 --- a/providers/telegram/src/airflow/providers/telegram/get_provider_info.py +++ b/providers/telegram/src/airflow/providers/telegram/get_provider_info.py @@ -44,6 +44,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.telegram.hooks.telegram.TelegramHook", + "hook-name": "Telegram", "connection-type": "telegram", "ui-field-behaviour": {"hidden-fields": ["schema", "extra", "login", "port"]}, } diff --git a/providers/teradata/provider.yaml b/providers/teradata/provider.yaml index f8ab1b070cda0..573a8a3cfb0d6 100644 --- a/providers/teradata/provider.yaml +++ b/providers/teradata/provider.yaml @@ -125,6 +125,7 @@ transfers: connection-types: - hook-class-name: airflow.providers.teradata.hooks.teradata.TeradataHook + hook-name: "Teradata" connection-type: teradata ui-field-behaviour: hidden-fields: diff --git a/providers/teradata/src/airflow/providers/teradata/get_provider_info.py b/providers/teradata/src/airflow/providers/teradata/get_provider_info.py index df5c4ea4b3d64..5d90b0546f1ba 100644 --- a/providers/teradata/src/airflow/providers/teradata/get_provider_info.py +++ b/providers/teradata/src/airflow/providers/teradata/get_provider_info.py @@ -98,6 +98,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.teradata.hooks.teradata.TeradataHook", + "hook-name": "Teradata", "connection-type": "teradata", "ui-field-behaviour": { "hidden-fields": ["port"], diff --git a/providers/trino/provider.yaml b/providers/trino/provider.yaml index 77717ecf6cd27..640cea642360f 100644 --- a/providers/trino/provider.yaml +++ b/providers/trino/provider.yaml @@ -115,6 +115,7 @@ transfers: connection-types: - hook-class-name: airflow.providers.trino.hooks.trino.TrinoHook + hook-name: "Trino" connection-type: trino ui-field-behaviour: placeholders: diff --git a/providers/trino/src/airflow/providers/trino/get_provider_info.py b/providers/trino/src/airflow/providers/trino/get_provider_info.py index c2213e2c98ced..64e4d11a231c2 100644 --- a/providers/trino/src/airflow/providers/trino/get_provider_info.py +++ b/providers/trino/src/airflow/providers/trino/get_provider_info.py @@ -53,6 +53,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.trino.hooks.trino.TrinoHook", + "hook-name": "Trino", "connection-type": "trino", "ui-field-behaviour": { "placeholders": { diff --git a/providers/vertica/provider.yaml b/providers/vertica/provider.yaml index 4d5c30e10be2d..b3e1ad601d468 100644 --- a/providers/vertica/provider.yaml +++ b/providers/vertica/provider.yaml @@ -82,4 +82,5 @@ hooks: connection-types: - hook-class-name: airflow.providers.vertica.hooks.vertica.VerticaHook + hook-name: "Vertica" connection-type: vertica diff --git a/providers/vertica/src/airflow/providers/vertica/get_provider_info.py b/providers/vertica/src/airflow/providers/vertica/get_provider_info.py index eefd15120f263..986485f5f061c 100644 --- a/providers/vertica/src/airflow/providers/vertica/get_provider_info.py +++ b/providers/vertica/src/airflow/providers/vertica/get_provider_info.py @@ -41,6 +41,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.vertica.hooks.vertica.VerticaHook", + "hook-name": "Vertica", "connection-type": "vertica", } ], diff --git a/providers/weaviate/provider.yaml b/providers/weaviate/provider.yaml index 92b2ef9676b87..1d2f2954f33f5 100644 --- a/providers/weaviate/provider.yaml +++ b/providers/weaviate/provider.yaml @@ -73,6 +73,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.weaviate.hooks.weaviate.WeaviateHook + hook-name: "Weaviate" connection-type: weaviate ui-field-behaviour: hidden-fields: diff --git a/providers/weaviate/src/airflow/providers/weaviate/get_provider_info.py b/providers/weaviate/src/airflow/providers/weaviate/get_provider_info.py index 1098ddcc86e6f..9dcce2cf4d373 100644 --- a/providers/weaviate/src/airflow/providers/weaviate/get_provider_info.py +++ b/providers/weaviate/src/airflow/providers/weaviate/get_provider_info.py @@ -41,6 +41,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.weaviate.hooks.weaviate.WeaviateHook", + "hook-name": "Weaviate", "connection-type": "weaviate", "ui-field-behaviour": { "hidden-fields": ["schema"], diff --git a/providers/yandex/provider.yaml b/providers/yandex/provider.yaml index fc22e70ca983f..fbc92a1be406f 100644 --- a/providers/yandex/provider.yaml +++ b/providers/yandex/provider.yaml @@ -110,6 +110,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.yandex.hooks.yandex.YandexCloudBaseHook + hook-name: "Yandex Cloud" connection-type: yandexcloud ui-field-behaviour: hidden-fields: diff --git a/providers/yandex/src/airflow/providers/yandex/get_provider_info.py b/providers/yandex/src/airflow/providers/yandex/get_provider_info.py index 94dea772e1573..83a8e762cea79 100644 --- a/providers/yandex/src/airflow/providers/yandex/get_provider_info.py +++ b/providers/yandex/src/airflow/providers/yandex/get_provider_info.py @@ -69,6 +69,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.yandex.hooks.yandex.YandexCloudBaseHook", + "hook-name": "Yandex Cloud", "connection-type": "yandexcloud", "ui-field-behaviour": { "hidden-fields": ["host", "schema", "login", "password", "port", "extra"] diff --git a/providers/yandex/src/airflow/providers/yandex/hooks/yandex.py b/providers/yandex/src/airflow/providers/yandex/hooks/yandex.py index 86f20c8d5ea8c..db52fda501510 100644 --- a/providers/yandex/src/airflow/providers/yandex/hooks/yandex.py +++ b/providers/yandex/src/airflow/providers/yandex/hooks/yandex.py @@ -26,7 +26,7 @@ get_credentials, get_service_account_id, ) -from airflow.providers.yandex.utils.defaults import conn_name_attr, conn_type, default_conn_name, hook_name +from airflow.providers.yandex.utils.defaults import conn_name_attr, conn_type, default_conn_name from airflow.providers.yandex.utils.fields import get_field_from_extras from airflow.providers.yandex.utils.user_agent import provider_user_agent @@ -44,7 +44,7 @@ class YandexCloudBaseHook(BaseHook): conn_name_attr = conn_name_attr default_conn_name = default_conn_name conn_type = conn_type - hook_name = hook_name + hook_name = "Yandex Cloud" @classmethod def get_connection_form_widgets(cls) -> dict[str, Any]: diff --git a/providers/ydb/provider.yaml b/providers/ydb/provider.yaml index 72a3bd4b510f3..5f4f4058b7371 100644 --- a/providers/ydb/provider.yaml +++ b/providers/ydb/provider.yaml @@ -67,6 +67,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.ydb.hooks.ydb.YDBHook + hook-name: "YDB" connection-type: ydb ui-field-behaviour: hidden-fields: diff --git a/providers/ydb/src/airflow/providers/ydb/get_provider_info.py b/providers/ydb/src/airflow/providers/ydb/get_provider_info.py index 056b0cc58ebd8..5c7d2371caf3f 100644 --- a/providers/ydb/src/airflow/providers/ydb/get_provider_info.py +++ b/providers/ydb/src/airflow/providers/ydb/get_provider_info.py @@ -40,6 +40,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.ydb.hooks.ydb.YDBHook", + "hook-name": "YDB", "connection-type": "ydb", "ui-field-behaviour": { "hidden-fields": ["schema", "extra"], diff --git a/providers/zendesk/provider.yaml b/providers/zendesk/provider.yaml index 996c34f636121..2f76895a3dc60 100644 --- a/providers/zendesk/provider.yaml +++ b/providers/zendesk/provider.yaml @@ -72,6 +72,7 @@ hooks: connection-types: - hook-class-name: airflow.providers.zendesk.hooks.zendesk.ZendeskHook + hook-name: "Zendesk" connection-type: zendesk ui-field-behaviour: hidden-fields: diff --git a/providers/zendesk/src/airflow/providers/zendesk/get_provider_info.py b/providers/zendesk/src/airflow/providers/zendesk/get_provider_info.py index c5fae7782311a..3a6cdd295ce76 100644 --- a/providers/zendesk/src/airflow/providers/zendesk/get_provider_info.py +++ b/providers/zendesk/src/airflow/providers/zendesk/get_provider_info.py @@ -40,6 +40,7 @@ def get_provider_info(): "connection-types": [ { "hook-class-name": "airflow.providers.zendesk.hooks.zendesk.ZendeskHook", + "hook-name": "Zendesk", "connection-type": "zendesk", "ui-field-behaviour": { "hidden-fields": ["schema", "port", "extra"], From a28629026996ae2efcb478cfa436c88a9eafdb89 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 9 Apr 2026 01:28:00 +0300 Subject: [PATCH 002/309] [v3-2-test] Speed up cleanup_python_generated_files by skipping irrelevant dirs (#64927) (#64930) Replace two rglob calls with a single os.walk that prunes node_modules and hidden directories (e.g. .git, .venv) in-place, avoiding unnecessary traversal of large directory trees that never contain relevant .pyc files. (cherry picked from commit 27258d567887a93f43af25a9d79d773198751bd9) Co-authored-by: Jarek Potiuk --- .../src/airflow_breeze/utils/path_utils.py | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index 03877d6476163..7f2625a0bcca9 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -418,22 +418,26 @@ def cleanup_python_generated_files(): if get_verbose(): console_print("[info]Cleaning .pyc and __pycache__") permission_errors = [] - for path in AIRFLOW_ROOT_PATH.rglob("*.pyc"): - try: - path.unlink() - except FileNotFoundError: - # File has been removed in the meantime. - pass - except PermissionError: - permission_errors.append(path) - for path in AIRFLOW_ROOT_PATH.rglob("__pycache__"): - try: - shutil.rmtree(path) - except FileNotFoundError: - # File has been removed in the meantime. - pass - except PermissionError: - permission_errors.append(path) + for dirpath, dirnames, filenames in os.walk(AIRFLOW_ROOT_PATH): + # Skip node_modules and hidden directories (.*) — modify in place to prune os.walk + dirnames[:] = [d for d in dirnames if d != "node_modules" and not d.startswith(".")] + for filename in filenames: + if filename.endswith(".pyc"): + path = Path(dirpath) / filename + try: + path.unlink() + except FileNotFoundError: + pass + except PermissionError: + permission_errors.append(path) + if Path(dirpath).name == "__pycache__": + try: + shutil.rmtree(dirpath) + except FileNotFoundError: + pass + except PermissionError: + permission_errors.append(Path(dirpath)) + dirnames.clear() if permission_errors: if platform.uname().system.lower() == "linux": console_print("[warning]There were files that you could not clean-up:\n") From 9dade1ed30b6601767ffa5c6dc90a30adcf463b5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 18:09:05 +0200 Subject: [PATCH 003/309] [v3-2-test] Expose queueing/scheduled time in the Gantt Chart (#63372) (#65016) * Expose queueing time in the Gantt Chart * Also expose scheduled_dttm in the Gantt Chart * Simplify Gantt tooltip and ensure minimum bar visibility for short segments * Null safety for dayjs calls and add tests for timing segments (cherry picked from commit cd851646fba29e89c848a19ee78c9ee8f81ad238) Co-authored-by: Saumyajit Chowdhury <77187489+smyjt@users.noreply.github.com> --- .../core_api/datamodels/ui/gantt.py | 2 + .../core_api/openapi/_private_ui.yaml | 14 +++ .../api_fastapi/core_api/routes/ui/gantt.py | 6 ++ .../ui/openapi-gen/requests/schemas.gen.ts | 26 +++++- .../ui/openapi-gen/requests/types.gen.ts | 2 + .../src/layouts/Details/Gantt/utils.test.ts | 89 +++++++++++++++++++ .../ui/src/layouts/Details/Gantt/utils.ts | 50 +++++++++-- .../core_api/routes/ui/test_gantt.py | 24 +++++ 8 files changed, 207 insertions(+), 6 deletions(-) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/gantt.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/gantt.py index 57a96c8a0ad70..3b74e84b47f3d 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/gantt.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/gantt.py @@ -30,6 +30,8 @@ class GanttTaskInstance(BaseModel): task_display_name: str try_number: int state: TaskInstanceState | None + scheduled_dttm: datetime | None + queued_dttm: datetime | None start_date: datetime | None end_date: datetime | None is_group: bool = False diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml index 706cf64b492aa..87ed72c730be0 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml @@ -2249,6 +2249,18 @@ components: anyOf: - $ref: '#/components/schemas/TaskInstanceState' - type: 'null' + scheduled_dttm: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Scheduled Dttm + queued_dttm: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Queued Dttm start_date: anyOf: - type: string @@ -2275,6 +2287,8 @@ components: - task_display_name - try_number - state + - scheduled_dttm + - queued_dttm - start_date - end_date title: GanttTaskInstance diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/gantt.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/gantt.py index f33b12e6f7e8a..7807e3fd6bc0f 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/gantt.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/gantt.py @@ -67,6 +67,8 @@ def get_gantt_data( TaskInstance.task_display_name.label("task_display_name"), # type: ignore[attr-defined] TaskInstance.try_number.label("try_number"), TaskInstance.state.label("state"), + TaskInstance.scheduled_dttm.label("scheduled_dttm"), + TaskInstance.queued_dttm.label("queued_dttm"), TaskInstance.start_date.label("start_date"), TaskInstance.end_date.label("end_date"), ).where( @@ -81,6 +83,8 @@ def get_gantt_data( TaskInstanceHistory.task_display_name.label("task_display_name"), TaskInstanceHistory.try_number.label("try_number"), TaskInstanceHistory.state.label("state"), + TaskInstanceHistory.scheduled_dttm.label("scheduled_dttm"), + TaskInstanceHistory.queued_dttm.label("queued_dttm"), TaskInstanceHistory.start_date.label("start_date"), TaskInstanceHistory.end_date.label("end_date"), ).where( @@ -106,6 +110,8 @@ def get_gantt_data( task_display_name=row.task_display_name, try_number=row.try_number, state=row.state, + scheduled_dttm=row.scheduled_dttm, + queued_dttm=row.queued_dttm, start_date=row.start_date, end_date=row.end_date, ) diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts index 7ba43d2c93a93..a129de40e7d15 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -8130,6 +8130,30 @@ export const $GanttTaskInstance = { } ] }, + scheduled_dttm: { + anyOf: [ + { + type: 'string', + format: 'date-time' + }, + { + type: 'null' + } + ], + title: 'Scheduled Dttm' + }, + queued_dttm: { + anyOf: [ + { + type: 'string', + format: 'date-time' + }, + { + type: 'null' + } + ], + title: 'Queued Dttm' + }, start_date: { anyOf: [ { @@ -8166,7 +8190,7 @@ export const $GanttTaskInstance = { } }, type: 'object', - required: ['task_id', 'task_display_name', 'try_number', 'state', 'start_date', 'end_date'], + required: ['task_id', 'task_display_name', 'try_number', 'state', 'scheduled_dttm', 'queued_dttm', 'start_date', 'end_date'], title: 'GanttTaskInstance', description: 'Task instance data for Gantt chart.' } as const; diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts index 5b536a702a0f6..c4dba0647d308 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts @@ -1989,6 +1989,8 @@ export type GanttTaskInstance = { task_display_name: string; try_number: number; state: TaskInstanceState | null; + scheduled_dttm: string | null; + queued_dttm: string | null; start_date: string | null; end_date: string | null; is_group?: boolean; diff --git a/airflow-core/src/airflow/ui/src/layouts/Details/Gantt/utils.test.ts b/airflow-core/src/airflow/ui/src/layouts/Details/Gantt/utils.test.ts index b9fa6a491be54..a0e61cc857922 100644 --- a/airflow-core/src/airflow/ui/src/layouts/Details/Gantt/utils.test.ts +++ b/airflow-core/src/airflow/ui/src/layouts/Details/Gantt/utils.test.ts @@ -1,3 +1,5 @@ +/* eslint-disable max-lines */ + /*! * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -166,6 +168,10 @@ describe("transformGanttData", () => { end_date: null, is_mapped: false, // eslint-disable-next-line unicorn/no-null + queued_dttm: null, + // eslint-disable-next-line unicorn/no-null + scheduled_dttm: null, + // eslint-disable-next-line unicorn/no-null start_date: null, // eslint-disable-next-line unicorn/no-null state: null, @@ -189,6 +195,10 @@ describe("transformGanttData", () => { // eslint-disable-next-line unicorn/no-null end_date: null, is_mapped: false, + // eslint-disable-next-line unicorn/no-null + queued_dttm: null, + // eslint-disable-next-line unicorn/no-null + scheduled_dttm: null, start_date: "2024-03-14T10:00:00+00:00", state: "running", task_display_name: "task_1", @@ -237,6 +247,10 @@ describe("transformGanttData", () => { { end_date: "2024-03-14T10:05:00+00:00", is_mapped: false, + // eslint-disable-next-line unicorn/no-null + queued_dttm: null, + // eslint-disable-next-line unicorn/no-null + scheduled_dttm: null, start_date: "2024-03-14T10:00:00+00:00", state: "success", task_display_name: "task_1", @@ -258,4 +272,79 @@ describe("transformGanttData", () => { expect(Number.isNaN(start.valueOf())).toBe(false); expect(Number.isNaN(end.valueOf())).toBe(false); }); + + it("should produce 3 segments when scheduled_dttm and queued_dttm are present", () => { + const result = transformGanttData({ + allTries: [ + { + end_date: "2024-03-14T10:05:00+00:00", + is_mapped: false, + queued_dttm: "2024-03-14T09:59:00+00:00", + scheduled_dttm: "2024-03-14T09:58:00+00:00", + start_date: "2024-03-14T10:00:00+00:00", + state: "success", + task_display_name: "task_1", + task_id: "task_1", + try_number: 1, + }, + ], + flatNodes: [{ depth: 0, id: "task_1", is_mapped: false, label: "task_1" }], + gridSummaries: [], + }); + + expect(result).toHaveLength(3); + expect(result[0]?.state).toBe("scheduled"); + expect(result[1]?.state).toBe("queued"); + expect(result[2]?.state).toBe("success"); + }); + + it("should produce 2 segments when only queued_dttm is present", () => { + const result = transformGanttData({ + allTries: [ + { + end_date: "2024-03-14T10:05:00+00:00", + is_mapped: false, + queued_dttm: "2024-03-14T09:59:00+00:00", + // eslint-disable-next-line unicorn/no-null + scheduled_dttm: null, + start_date: "2024-03-14T10:00:00+00:00", + state: "success", + task_display_name: "task_1", + task_id: "task_1", + try_number: 1, + }, + ], + flatNodes: [{ depth: 0, id: "task_1", is_mapped: false, label: "task_1" }], + gridSummaries: [], + }); + + expect(result).toHaveLength(2); + expect(result[0]?.state).toBe("queued"); + expect(result[1]?.state).toBe("success"); + }); + + it("should produce 1 segment when scheduled_dttm and queued_dttm are null", () => { + const result = transformGanttData({ + allTries: [ + { + end_date: "2024-03-14T10:05:00+00:00", + is_mapped: false, + // eslint-disable-next-line unicorn/no-null + queued_dttm: null, + // eslint-disable-next-line unicorn/no-null + scheduled_dttm: null, + start_date: "2024-03-14T10:00:00+00:00", + state: "success", + task_display_name: "task_1", + task_id: "task_1", + try_number: 1, + }, + ], + flatNodes: [{ depth: 0, id: "task_1", is_mapped: false, label: "task_1" }], + gridSummaries: [], + }); + + expect(result).toHaveLength(1); + expect(result[0]?.state).toBe("success"); + }); }); diff --git a/airflow-core/src/airflow/ui/src/layouts/Details/Gantt/utils.ts b/airflow-core/src/airflow/ui/src/layouts/Details/Gantt/utils.ts index fab1d1bcf773a..621f22e8ab923 100644 --- a/airflow-core/src/airflow/ui/src/layouts/Details/Gantt/utils.ts +++ b/airflow-core/src/airflow/ui/src/layouts/Details/Gantt/utils.ts @@ -122,14 +122,47 @@ export const transformGanttData = ({ if (tries && tries.length > 0) { return tries .filter((tryInstance) => tryInstance.start_date !== null) - .map((tryInstance) => { + .flatMap((tryInstance) => { const hasTaskRunning = isStatePending(tryInstance.state); const endTime = hasTaskRunning || tryInstance.end_date === null ? dayjs().toISOString() : tryInstance.end_date; - - return { + const items: Array = []; + + // Scheduled segment: from scheduled_dttm to queued_dttm (or start_date if no queued_dttm) + if (tryInstance.scheduled_dttm !== null) { + const scheduledEnd = tryInstance.queued_dttm ?? tryInstance.start_date ?? undefined; + + items.push({ + isGroup: false, + isMapped: tryInstance.is_mapped, + state: "scheduled" as TaskInstanceState, + taskId: tryInstance.task_id, + tryNumber: tryInstance.try_number, + x: [dayjs(tryInstance.scheduled_dttm).toISOString(), dayjs(scheduledEnd).toISOString()], + y: tryInstance.task_display_name, + }); + } + + // Queue segment: from queued_dttm to start_date + if (tryInstance.queued_dttm !== null) { + items.push({ + isGroup: false, + isMapped: tryInstance.is_mapped, + state: "queued" as TaskInstanceState, + taskId: tryInstance.task_id, + tryNumber: tryInstance.try_number, + x: [ + dayjs(tryInstance.queued_dttm).toISOString(), + dayjs(tryInstance.start_date ?? undefined).toISOString(), + ], + y: tryInstance.task_display_name, + }); + } + + // Execution segment: from start_date to end_date + items.push({ isGroup: false, isMapped: tryInstance.is_mapped, state: tryInstance.state, @@ -137,7 +170,9 @@ export const transformGanttData = ({ tryNumber: tryInstance.try_number, x: [dayjs(tryInstance.start_date).toISOString(), dayjs(endTime).toISOString()], y: tryInstance.task_display_name, - }; + }); + + return items; }); } } @@ -259,6 +294,11 @@ export const createChartOptions = ({ duration: 150, easing: "linear" as const, }, + datasets: { + bar: { + minBarLength: 4, + }, + }, indexAxis: "y" as const, maintainAspectRatio: false, onClick: handleBarClick, @@ -331,7 +371,7 @@ export const createChartOptions = ({ label(tooltipItem: TooltipItem<"bar">) { const taskInstance = data[tooltipItem.dataIndex]; - return `${translate("state")}: ${translate(`states.${taskInstance?.state}`)}`; + return `${translate("state")}: ${translate(`common:states.${taskInstance?.state}`)}`; }, }, }, diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_gantt.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_gantt.py index 162c82682afcf..0e2be9e277cae 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_gantt.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_gantt.py @@ -51,6 +51,8 @@ "task_display_name": TASK_DISPLAY_NAME, "try_number": 1, "state": "success", + "scheduled_dttm": "2024-11-30T09:50:00Z", + "queued_dttm": "2024-11-30T09:55:00Z", "start_date": "2024-11-30T10:00:00Z", "end_date": "2024-11-30T10:05:00Z", "is_group": False, @@ -62,6 +64,8 @@ "task_display_name": TASK_DISPLAY_NAME_2, "try_number": 1, "state": "failed", + "scheduled_dttm": "2024-11-30T10:02:00Z", + "queued_dttm": "2024-11-30T10:03:00Z", "start_date": "2024-11-30T10:05:00Z", "end_date": "2024-11-30T10:10:00Z", "is_group": False, @@ -73,6 +77,8 @@ "task_display_name": TASK_DISPLAY_NAME_3, "try_number": 1, "state": "running", + "scheduled_dttm": None, + "queued_dttm": None, "start_date": "2024-11-30T10:10:00Z", "end_date": None, "is_group": False, @@ -116,16 +122,22 @@ def setup(dag_maker, session=None): if ti.task_id == TASK_ID: ti.state = TaskInstanceState.SUCCESS ti.try_number = 1 + ti.scheduled_dttm = pendulum.DateTime(2024, 11, 30, 9, 50, 0, tzinfo=pendulum.UTC) + ti.queued_dttm = pendulum.DateTime(2024, 11, 30, 9, 55, 0, tzinfo=pendulum.UTC) ti.start_date = pendulum.DateTime(2024, 11, 30, 10, 0, 0, tzinfo=pendulum.UTC) ti.end_date = pendulum.DateTime(2024, 11, 30, 10, 5, 0, tzinfo=pendulum.UTC) elif ti.task_id == TASK_ID_2: ti.state = TaskInstanceState.FAILED ti.try_number = 1 + ti.scheduled_dttm = pendulum.DateTime(2024, 11, 30, 10, 2, 0, tzinfo=pendulum.UTC) + ti.queued_dttm = pendulum.DateTime(2024, 11, 30, 10, 3, 0, tzinfo=pendulum.UTC) ti.start_date = pendulum.DateTime(2024, 11, 30, 10, 5, 0, tzinfo=pendulum.UTC) ti.end_date = pendulum.DateTime(2024, 11, 30, 10, 10, 0, tzinfo=pendulum.UTC) elif ti.task_id == TASK_ID_3: ti.state = TaskInstanceState.RUNNING ti.try_number = 1 + ti.scheduled_dttm = None + ti.queued_dttm = None ti.start_date = pendulum.DateTime(2024, 11, 30, 10, 10, 0, tzinfo=pendulum.UTC) ti.end_date = None @@ -306,6 +318,18 @@ def test_sorted_by_task_id_and_try_number(self, test_client): sorted_tis = sorted(task_instances, key=lambda x: (x["task_id"], x["try_number"])) assert task_instances == sorted_tis + def test_timing_fields_are_returned(self, test_client): + response = test_client.get(f"/gantt/{DAG_ID}/run_1") + assert response.status_code == 200 + data = response.json() + tis = {ti["task_id"]: ti for ti in data["task_instances"]} + assert tis[TASK_ID]["scheduled_dttm"] == "2024-11-30T09:50:00Z" + assert tis[TASK_ID]["queued_dttm"] == "2024-11-30T09:55:00Z" + assert tis[TASK_ID_2]["scheduled_dttm"] == "2024-11-30T10:02:00Z" + assert tis[TASK_ID_2]["queued_dttm"] == "2024-11-30T10:03:00Z" + assert tis[TASK_ID_3]["scheduled_dttm"] is None + assert tis[TASK_ID_3]["queued_dttm"] is None + def test_should_response_401(self, unauthenticated_test_client): response = unauthenticated_test_client.get(f"/gantt/{DAG_ID}/run_1") assert response.status_code == 401 From e3eeac777711ef6d21050bf3015fe0164d343dde Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 14 Apr 2026 20:09:30 +0200 Subject: [PATCH 004/309] [v3-2-test] Bump actions/github-script in the github-actions-updates group (#65150) (#65160) Bumps the github-actions-updates group with 1 update: [actions/github-script](https://github.com/actions/github-script). Updates `actions/github-script` from 8.0.0 to 9.0.0 - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/ed597411d8f924073f98dfc5c65a23a2325f34cd...3a2844b7e9c422d3c10d287c895573f7108da1b3) (cherry picked from commit e5a047ca8d23614cefeaad89e08eb1e3b0482e60) --- updated-dependencies: - dependency-name: actions/github-script dependency-version: 9.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions-updates ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> From 60684319faa30d424c7725c7768b0e090a48e07d Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 14 Apr 2026 20:20:55 +0200 Subject: [PATCH 005/309] [v3-2-test] Added breeze generate issue content for airflow-ctl (#65042) (#65241) * Add breeze generate issue content for airflow-ctl * add new command to doc (cherry picked from commit b24538b0bc3a53d8c666b62e52e023a17d102467) Co-authored-by: Justin Pakzad <114518232+justinpakzad@users.noreply.github.com> From c79580b0e7b2dd23d6f4f42144978a8200461e1c Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 14 Apr 2026 20:28:34 +0200 Subject: [PATCH 006/309] [v3-2-test] Run release calendar verification on its own schedule (#65118) (#65242) * Move release calendar verification to its own scheduled workflow Run dev/verify_release_calendar.py from a dedicated daily scheduled workflow instead of as a canary job in the main CI pipeline, and notify the #release-management Slack channel when the check fails so the issue is surfaced to release managers directly. * Include wiki and calendar links in release calendar Slack alert (cherry picked from commit 048e9a191b4a6625bb4bbad23fad537b256f4062) From 642197a15a192874c42fd6149078a35bbc3de21c Mon Sep 17 00:00:00 2001 From: Rahul Vats <43964496+vatsrahul1001@users.noreply.github.com> Date: Wed, 15 Apr 2026 19:43:27 +0530 Subject: [PATCH 007/309] Add dag runs filters (Consuming Asset) (#63624) (#65306) * Add dag runs filters (Consuming Asset) * Fix: correct consuming asset filter setup using association_table * Trigger CI rebuild * Rename consuming_asset filter to consuming_asset_pattern with database icon * Rename consuming_asset filter to consuming_asset_pattern with database icon * Trigger CI rebuild * Fix consuming_asset_pattern naming * Fix: rename consuming_asset to consuming_asset_pattern * Fix: rename consuming_asset to consuming_asset_pattern * Fix: Resolve PostgreSQL JSON comparison error in _ConsumingAssetFilter * Rebase and fix _ConsumingAssetFilter * Trigger CI * add consumingAsset and filters.searchAsset to en/common.json --------- (cherry picked from commit 5245419267b5892e7ff2e81042e1c4e21b48d6ca) Co-authored-by: fat-catTW <124506982+fat-catTW@users.noreply.github.com> Co-authored-by: Jens Scheffler <95105677+jscheffl@users.noreply.github.com> Co-authored-by: Jarek Potiuk --- .../airflow/api_fastapi/common/parameters.py | 42 +++++++++++++++++ .../openapi/v2-rest-api-generated.yaml | 10 +++++ .../core_api/routes/public/dag_run.py | 3 ++ .../airflow/ui/openapi-gen/queries/common.ts | 5 ++- .../ui/openapi-gen/queries/ensureQueryData.ts | 6 ++- .../ui/openapi-gen/queries/prefetch.ts | 6 ++- .../airflow/ui/openapi-gen/queries/queries.ts | 6 ++- .../ui/openapi-gen/queries/suspense.ts | 6 ++- .../ui/openapi-gen/requests/services.gen.ts | 4 +- .../ui/openapi-gen/requests/types.gen.ts | 4 ++ .../ui/public/i18n/locales/en/common.json | 2 + .../ui/src/constants/filterConfigs.tsx | 9 +++- .../airflow/ui/src/constants/searchParams.ts | 1 + .../src/airflow/ui/src/pages/DagRuns.tsx | 3 ++ .../airflow/ui/src/pages/DagRunsFilters.tsx | 1 + .../airflow/ui/src/utils/useFiltersHandler.ts | 1 + .../core_api/routes/public/test_dag_run.py | 45 +++++++++++++++++++ 17 files changed, 142 insertions(+), 12 deletions(-) diff --git a/airflow-core/src/airflow/api_fastapi/common/parameters.py b/airflow-core/src/airflow/api_fastapi/common/parameters.py index e963cdbff557f..065dff8cda8bb 100644 --- a/airflow-core/src/airflow/api_fastapi/common/parameters.py +++ b/airflow-core/src/airflow/api_fastapi/common/parameters.py @@ -46,11 +46,13 @@ from airflow.models import Base from airflow.models.asset import ( AssetAliasModel, + AssetEvent, AssetModel, AssetPartitionDagRun, DagScheduleAssetReference, TaskInletAssetReference, TaskOutletAssetReference, + association_table, ) from airflow.models.connection import Connection from airflow.models.dag import DagModel, DagTag @@ -785,6 +787,46 @@ def depends( QueryAssetDependencyFilter = Annotated[_AssetDependencyFilter, Depends(_AssetDependencyFilter.depends)] +class _ConsumingAssetFilter(BaseParam[str | None]): + """Filter DAG runs by consuming asset (name or URI).""" + + def to_orm(self, select: Select) -> Select: + if not self.value and self.skip_none: + return select + + event_subquery = ( + sql_select(AssetEvent.id) + .join(AssetModel, AssetEvent.asset_id == AssetModel.id) + .where( + or_( + AssetModel.name.ilike(f"%{self.value}%"), + AssetModel.uri.ilike(f"%{self.value}%"), + ) + ) + .distinct() + ) + + dagrun_subquery = ( + sql_select(association_table.c.dag_run_id) + .where(association_table.c.event_id.in_(event_subquery)) + .distinct() + ) + + return select.where(DagRun.id.in_(dagrun_subquery)) + + @classmethod + def depends( + cls, + consuming_asset_pattern: str | None = Query( + None, description="Filter by consuming asset name or URI using pattern matching" + ), + ) -> _ConsumingAssetFilter: + return cls().set_value(consuming_asset_pattern) + + +QueryConsumingAssetPatternSearch = Annotated[_ConsumingAssetFilter, Depends(_ConsumingAssetFilter.depends)] + + class _PendingActionsFilter(BaseParam[bool]): """Filter Dags by having pending HITL actions (more than 1).""" diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml index 582357bbfea89..6ed47e9fc2fb4 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml @@ -2375,6 +2375,16 @@ paths: description: "SQL LIKE expression \u2014 use `%` / `_` wildcards (e.g. `%customer_%`).\ \ or the pipe `|` operator for OR logic (e.g. `dag1 | dag2`). Regular expressions\ \ are **not** supported." + - name: consuming_asset_pattern + in: query + required: false + schema: + anyOf: + - type: string + - type: 'null' + description: Filter by consuming asset name or URI using pattern matching + title: Consuming Asset Pattern + description: Filter by consuming asset name or URI using pattern matching responses: '200': description: Successful Response diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_run.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_run.py index ff42238806b12..46dcd64a66e15 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_run.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_run.py @@ -46,6 +46,7 @@ FilterParam, LimitFilter, OffsetFilter, + QueryConsumingAssetPatternSearch, QueryDagRunPartitionKeySearch, QueryDagRunRunTypesFilter, QueryDagRunStateFilter, @@ -385,6 +386,7 @@ def get_dag_runs( ], dag_id_pattern: Annotated[_SearchParam, Depends(search_param_factory(DagRun.dag_id, "dag_id_pattern"))], partition_key_pattern: QueryDagRunPartitionKeySearch, + consuming_asset_pattern: QueryConsumingAssetPatternSearch, ) -> DAGRunCollectionResponse: """ Get all DAG Runs. @@ -420,6 +422,7 @@ def get_dag_runs( triggering_user_name_pattern, dag_id_pattern, partition_key_pattern, + consuming_asset_pattern, ], order_by=order_by, offset=offset, diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts index 612a3d56747e3..c02f01452b39e 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts @@ -143,9 +143,10 @@ export const UseDagRunServiceGetUpstreamAssetEventsKeyFn = ({ dagId, dagRunId }: export type DagRunServiceGetDagRunsDefaultResponse = Awaited>; export type DagRunServiceGetDagRunsQueryResult = UseQueryResult; export const useDagRunServiceGetDagRunsKey = "DagRunServiceGetDagRuns"; -export const UseDagRunServiceGetDagRunsKeyFn = ({ bundleVersion, confContains, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }: { +export const UseDagRunServiceGetDagRunsKeyFn = ({ bundleVersion, confContains, consumingAssetPattern, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }: { bundleVersion?: string; confContains?: string; + consumingAssetPattern?: string; dagId: string; dagIdPattern?: string; dagVersion?: number[]; @@ -181,7 +182,7 @@ export const UseDagRunServiceGetDagRunsKeyFn = ({ bundleVersion, confContains, d updatedAtGte?: string; updatedAtLt?: string; updatedAtLte?: string; -}, queryKey?: Array) => [useDagRunServiceGetDagRunsKey, ...(queryKey ?? [{ bundleVersion, confContains, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }])]; +}, queryKey?: Array) => [useDagRunServiceGetDagRunsKey, ...(queryKey ?? [{ bundleVersion, confContains, consumingAssetPattern, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }])]; export type DagRunServiceWaitDagRunUntilFinishedDefaultResponse = Awaited>; export type DagRunServiceWaitDagRunUntilFinishedQueryResult = UseQueryResult; export const useDagRunServiceWaitDagRunUntilFinishedKey = "DagRunServiceWaitDagRunUntilFinished"; diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts index 8bc9e9df8e6df..c0ca5feb6cbd4 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts @@ -299,12 +299,14 @@ export const ensureUseDagRunServiceGetUpstreamAssetEventsData = (queryClient: Qu * @param data.triggeringUserNamePattern SQL LIKE expression — use `%` / `_` wildcards (e.g. `%customer_%`). or the pipe `|` operator for OR logic (e.g. `dag1 | dag2`). Regular expressions are **not** supported. * @param data.dagIdPattern SQL LIKE expression — use `%` / `_` wildcards (e.g. `%customer_%`). or the pipe `|` operator for OR logic (e.g. `dag1 | dag2`). Regular expressions are **not** supported. * @param data.partitionKeyPattern SQL LIKE expression — use `%` / `_` wildcards (e.g. `%customer_%`). or the pipe `|` operator for OR logic (e.g. `dag1 | dag2`). Regular expressions are **not** supported. +* @param data.consumingAssetPattern Filter by consuming asset name or URI using pattern matching * @returns DAGRunCollectionResponse Successful Response * @throws ApiError */ -export const ensureUseDagRunServiceGetDagRunsData = (queryClient: QueryClient, { bundleVersion, confContains, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }: { +export const ensureUseDagRunServiceGetDagRunsData = (queryClient: QueryClient, { bundleVersion, confContains, consumingAssetPattern, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }: { bundleVersion?: string; confContains?: string; + consumingAssetPattern?: string; dagId: string; dagIdPattern?: string; dagVersion?: number[]; @@ -340,7 +342,7 @@ export const ensureUseDagRunServiceGetDagRunsData = (queryClient: QueryClient, { updatedAtGte?: string; updatedAtLt?: string; updatedAtLte?: string; -}) => queryClient.ensureQueryData({ queryKey: Common.UseDagRunServiceGetDagRunsKeyFn({ bundleVersion, confContains, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }), queryFn: () => DagRunService.getDagRuns({ bundleVersion, confContains, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }) }); +}) => queryClient.ensureQueryData({ queryKey: Common.UseDagRunServiceGetDagRunsKeyFn({ bundleVersion, confContains, consumingAssetPattern, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }), queryFn: () => DagRunService.getDagRuns({ bundleVersion, confContains, consumingAssetPattern, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }) }); /** * Experimental: Wait for a dag run to complete, and return task results if requested. * 🚧 This is an experimental endpoint and may change or be removed without notice.Successful response are streamed as newline-delimited JSON (NDJSON). Each line is a JSON object representing the DAG run state. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts index f4cb6f482bdf6..d7c8731e6a686 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts @@ -299,12 +299,14 @@ export const prefetchUseDagRunServiceGetUpstreamAssetEvents = (queryClient: Quer * @param data.triggeringUserNamePattern SQL LIKE expression — use `%` / `_` wildcards (e.g. `%customer_%`). or the pipe `|` operator for OR logic (e.g. `dag1 | dag2`). Regular expressions are **not** supported. * @param data.dagIdPattern SQL LIKE expression — use `%` / `_` wildcards (e.g. `%customer_%`). or the pipe `|` operator for OR logic (e.g. `dag1 | dag2`). Regular expressions are **not** supported. * @param data.partitionKeyPattern SQL LIKE expression — use `%` / `_` wildcards (e.g. `%customer_%`). or the pipe `|` operator for OR logic (e.g. `dag1 | dag2`). Regular expressions are **not** supported. +* @param data.consumingAssetPattern Filter by consuming asset name or URI using pattern matching * @returns DAGRunCollectionResponse Successful Response * @throws ApiError */ -export const prefetchUseDagRunServiceGetDagRuns = (queryClient: QueryClient, { bundleVersion, confContains, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }: { +export const prefetchUseDagRunServiceGetDagRuns = (queryClient: QueryClient, { bundleVersion, confContains, consumingAssetPattern, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }: { bundleVersion?: string; confContains?: string; + consumingAssetPattern?: string; dagId: string; dagIdPattern?: string; dagVersion?: number[]; @@ -340,7 +342,7 @@ export const prefetchUseDagRunServiceGetDagRuns = (queryClient: QueryClient, { b updatedAtGte?: string; updatedAtLt?: string; updatedAtLte?: string; -}) => queryClient.prefetchQuery({ queryKey: Common.UseDagRunServiceGetDagRunsKeyFn({ bundleVersion, confContains, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }), queryFn: () => DagRunService.getDagRuns({ bundleVersion, confContains, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }) }); +}) => queryClient.prefetchQuery({ queryKey: Common.UseDagRunServiceGetDagRunsKeyFn({ bundleVersion, confContains, consumingAssetPattern, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }), queryFn: () => DagRunService.getDagRuns({ bundleVersion, confContains, consumingAssetPattern, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }) }); /** * Experimental: Wait for a dag run to complete, and return task results if requested. * 🚧 This is an experimental endpoint and may change or be removed without notice.Successful response are streamed as newline-delimited JSON (NDJSON). Each line is a JSON object representing the DAG run state. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts index dac7a198e59bd..019d3ce4a9494 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts @@ -299,12 +299,14 @@ export const useDagRunServiceGetUpstreamAssetEvents = = unknown[]>({ bundleVersion, confContains, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }: { +export const useDagRunServiceGetDagRuns = = unknown[]>({ bundleVersion, confContains, consumingAssetPattern, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }: { bundleVersion?: string; confContains?: string; + consumingAssetPattern?: string; dagId: string; dagIdPattern?: string; dagVersion?: number[]; @@ -340,7 +342,7 @@ export const useDagRunServiceGetDagRuns = , "queryKey" | "queryFn">) => useQuery({ queryKey: Common.UseDagRunServiceGetDagRunsKeyFn({ bundleVersion, confContains, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }, queryKey), queryFn: () => DagRunService.getDagRuns({ bundleVersion, confContains, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }) as TData, ...options }); +}, queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">) => useQuery({ queryKey: Common.UseDagRunServiceGetDagRunsKeyFn({ bundleVersion, confContains, consumingAssetPattern, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }, queryKey), queryFn: () => DagRunService.getDagRuns({ bundleVersion, confContains, consumingAssetPattern, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }) as TData, ...options }); /** * Experimental: Wait for a dag run to complete, and return task results if requested. * 🚧 This is an experimental endpoint and may change or be removed without notice.Successful response are streamed as newline-delimited JSON (NDJSON). Each line is a JSON object representing the DAG run state. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts index c4a41691b1a2e..cfc50f2d88308 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts @@ -299,12 +299,14 @@ export const useDagRunServiceGetUpstreamAssetEventsSuspense = = unknown[]>({ bundleVersion, confContains, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }: { +export const useDagRunServiceGetDagRunsSuspense = = unknown[]>({ bundleVersion, confContains, consumingAssetPattern, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }: { bundleVersion?: string; confContains?: string; + consumingAssetPattern?: string; dagId: string; dagIdPattern?: string; dagVersion?: number[]; @@ -340,7 +342,7 @@ export const useDagRunServiceGetDagRunsSuspense = , "queryKey" | "queryFn">) => useSuspenseQuery({ queryKey: Common.UseDagRunServiceGetDagRunsKeyFn({ bundleVersion, confContains, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }, queryKey), queryFn: () => DagRunService.getDagRuns({ bundleVersion, confContains, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }) as TData, ...options }); +}, queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">) => useSuspenseQuery({ queryKey: Common.UseDagRunServiceGetDagRunsKeyFn({ bundleVersion, confContains, consumingAssetPattern, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }, queryKey), queryFn: () => DagRunService.getDagRuns({ bundleVersion, confContains, consumingAssetPattern, dagId, dagIdPattern, dagVersion, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, offset, orderBy, partitionKeyPattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, runType, startDateGt, startDateGte, startDateLt, startDateLte, state, triggeringUserNamePattern, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte }) as TData, ...options }); /** * Experimental: Wait for a dag run to complete, and return task results if requested. * 🚧 This is an experimental endpoint and may change or be removed without notice.Successful response are streamed as newline-delimited JSON (NDJSON). Each line is a JSON object representing the DAG run state. diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts index 6f0c2af82fb0e..1fbb2db01f520 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts @@ -1013,6 +1013,7 @@ export class DagRunService { * @param data.triggeringUserNamePattern SQL LIKE expression — use `%` / `_` wildcards (e.g. `%customer_%`). or the pipe `|` operator for OR logic (e.g. `dag1 | dag2`). Regular expressions are **not** supported. * @param data.dagIdPattern SQL LIKE expression — use `%` / `_` wildcards (e.g. `%customer_%`). or the pipe `|` operator for OR logic (e.g. `dag1 | dag2`). Regular expressions are **not** supported. * @param data.partitionKeyPattern SQL LIKE expression — use `%` / `_` wildcards (e.g. `%customer_%`). or the pipe `|` operator for OR logic (e.g. `dag1 | dag2`). Regular expressions are **not** supported. + * @param data.consumingAssetPattern Filter by consuming asset name or URI using pattern matching * @returns DAGRunCollectionResponse Successful Response * @throws ApiError */ @@ -1059,7 +1060,8 @@ export class DagRunService { run_id_pattern: data.runIdPattern, triggering_user_name_pattern: data.triggeringUserNamePattern, dag_id_pattern: data.dagIdPattern, - partition_key_pattern: data.partitionKeyPattern + partition_key_pattern: data.partitionKeyPattern, + consuming_asset_pattern: data.consumingAssetPattern }, errors: { 401: 'Unauthorized', diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts index c4dba0647d308..fdb6e663d4804 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts @@ -2562,6 +2562,10 @@ export type ClearDagRunResponse = TaskInstanceCollectionResponse | DAGRunRespons export type GetDagRunsData = { bundleVersion?: string | null; confContains?: string; + /** + * Filter by consuming asset name or URI using pattern matching + */ + consumingAssetPattern?: string | null; dagId: string; /** * SQL LIKE expression — use `%` / `_` wildcards (e.g. `%customer_%`). or the pipe `|` operator for OR logic (e.g. `dag1 | dag2`). Regular expressions are **not** supported. diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/en/common.json b/airflow-core/src/airflow/ui/public/i18n/locales/en/common.json index 0bbea8fc42d4f..39bfe2ee791ed 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/en/common.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/en/common.json @@ -28,6 +28,7 @@ }, "collapseAllExtra": "Collapse all extra JSON", "collapseDetailsPanel": "Collapse Details Panel", + "consumingAsset": "Consuming Asset", "createdAssetEvent_one": "Created Asset Event", "createdAssetEvent_other": "Created Asset Events", "dag_one": "Dag", @@ -131,6 +132,7 @@ "logicalDateTo": "Logical Date To", "runAfterFrom": "Run After From", "runAfterTo": "Run After To", + "searchAsset": "Search Asset", "selectDateRange": "Select Date Range", "startTime": "Start Time" }, diff --git a/airflow-core/src/airflow/ui/src/constants/filterConfigs.tsx b/airflow-core/src/airflow/ui/src/constants/filterConfigs.tsx index 691ed35c04272..4b06175db885e 100644 --- a/airflow-core/src/airflow/ui/src/constants/filterConfigs.tsx +++ b/airflow-core/src/airflow/ui/src/constants/filterConfigs.tsx @@ -21,7 +21,7 @@ import { Flex } from "@chakra-ui/react"; import { useTranslation } from "react-i18next"; import { BiTargetLock } from "react-icons/bi"; -import { FiBarChart, FiUser } from "react-icons/fi"; +import { FiBarChart, FiUser, FiDatabase } from "react-icons/fi"; import { LuBrackets } from "react-icons/lu"; import { MdDateRange, @@ -89,6 +89,13 @@ export const useFilterConfigs = () => { label: translate("common:dagRun.conf"), type: FilterTypes.TEXT, }, + [SearchParamsKeys.CONSUMING_ASSET_PATTERN]: { + hotkeyDisabled: true, + icon: , + label: translate("common:consumingAsset"), + placeholder: translate("common:filters.searchAsset"), + type: FilterTypes.TEXT, + }, [SearchParamsKeys.CREATED_AT_RANGE]: { endKey: SearchParamsKeys.CREATED_AT_LTE, icon: , diff --git a/airflow-core/src/airflow/ui/src/constants/searchParams.ts b/airflow-core/src/airflow/ui/src/constants/searchParams.ts index d37001e768802..7194fe69ceeb2 100644 --- a/airflow-core/src/airflow/ui/src/constants/searchParams.ts +++ b/airflow-core/src/airflow/ui/src/constants/searchParams.ts @@ -23,6 +23,7 @@ export enum SearchParamsKeys { BODY_SEARCH = "body_search", BUNDLE_VERSION = "bundle_version", CONF_CONTAINS = "conf_contains", + CONSUMING_ASSET_PATTERN = "consuming_asset_pattern", CREATED_AT_GTE = "created_at_gte", CREATED_AT_LTE = "created_at_lte", CREATED_AT_RANGE = "created_at_range", diff --git a/airflow-core/src/airflow/ui/src/pages/DagRuns.tsx b/airflow-core/src/airflow/ui/src/pages/DagRuns.tsx index 94b33640312e1..08eb817db5b70 100644 --- a/airflow-core/src/airflow/ui/src/pages/DagRuns.tsx +++ b/airflow-core/src/airflow/ui/src/pages/DagRuns.tsx @@ -47,6 +47,7 @@ type DagRunRow = { row: { original: DAGRunResponse } }; const { BUNDLE_VERSION: BUNDLE_VERSION_PARAM, CONF_CONTAINS: CONF_CONTAINS_PARAM, + CONSUMING_ASSET_PATTERN: CONSUMING_ASSET_PATTERN_PARAM, DAG_ID_PATTERN: DAG_ID_PATTERN_PARAM, DAG_VERSION: DAG_VERSION_PARAM, DURATION_GTE: DURATION_GTE_PARAM, @@ -214,6 +215,7 @@ export const DagRuns = () => { const filteredTriggeringUserNamePattern = searchParams.get(TRIGGERING_USER_NAME_PATTERN_PARAM); const filteredDagIdPattern = searchParams.get(DAG_ID_PATTERN_PARAM); const filteredDagVersion = searchParams.get(DAG_VERSION_PARAM); + const filteredConsumingAsset = searchParams.get(CONSUMING_ASSET_PATTERN_PARAM); const bundleVersion = searchParams.get(BUNDLE_VERSION_PARAM); const startDateGte = searchParams.get(START_DATE_GTE_PARAM); const startDateLte = searchParams.get(START_DATE_LTE_PARAM); @@ -234,6 +236,7 @@ export const DagRuns = () => { { bundleVersion: bundleVersion ?? undefined, confContains: confContains !== null && confContains !== "" ? confContains : undefined, + consumingAssetPattern: filteredConsumingAsset ?? undefined, dagId: dagId ?? "~", dagIdPattern: filteredDagIdPattern ?? undefined, dagVersion: diff --git a/airflow-core/src/airflow/ui/src/pages/DagRunsFilters.tsx b/airflow-core/src/airflow/ui/src/pages/DagRunsFilters.tsx index b0f90d66f9876..f39e66cab41ba 100644 --- a/airflow-core/src/airflow/ui/src/pages/DagRunsFilters.tsx +++ b/airflow-core/src/airflow/ui/src/pages/DagRunsFilters.tsx @@ -42,6 +42,7 @@ export const DagRunsFilters = ({ dagId }: DagRunsFiltersProps) => { SearchParamsKeys.DAG_VERSION, SearchParamsKeys.PARTITION_KEY_PATTERN, SearchParamsKeys.BUNDLE_VERSION, + SearchParamsKeys.CONSUMING_ASSET_PATTERN, ]; if (dagId === undefined) { diff --git a/airflow-core/src/airflow/ui/src/utils/useFiltersHandler.ts b/airflow-core/src/airflow/ui/src/utils/useFiltersHandler.ts index 17ab2fc35061f..6df86f6f6ab18 100644 --- a/airflow-core/src/airflow/ui/src/utils/useFiltersHandler.ts +++ b/airflow-core/src/airflow/ui/src/utils/useFiltersHandler.ts @@ -60,6 +60,7 @@ export type FilterableSearchParamsKeys = | SearchParamsKeys.BODY_SEARCH | SearchParamsKeys.BUNDLE_VERSION | SearchParamsKeys.CONF_CONTAINS + | SearchParamsKeys.CONSUMING_ASSET_PATTERN | SearchParamsKeys.CREATED_AT_RANGE | SearchParamsKeys.DAG_DISPLAY_NAME_PATTERN | SearchParamsKeys.DAG_ID diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py index 2e80a3501fe8e..9d27392de9251 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py @@ -40,6 +40,7 @@ from tests_common.test_utils.api_fastapi import _check_dag_run_note, _check_last_log from tests_common.test_utils.asserts import assert_queries_count from tests_common.test_utils.db import ( + clear_db_assets, clear_db_connections, clear_db_dag_bundles, clear_db_dags, @@ -130,6 +131,7 @@ def setup(request, dag_maker, session=None): clear_db_dag_bundles() clear_db_serialized_dags() clear_db_logs() + clear_db_assets() if "no_setup" in request.keywords: return @@ -220,6 +222,34 @@ def setup(request, dag_maker, session=None): session.merge(dag_maker.dag_model) session.commit() + asset1 = AssetModel(name="sales", uri="s3://bucket/sales") + asset2 = AssetModel(name="customer", uri="s3://bucket/customer") + session.add_all([asset1, asset2]) + session.flush() + + event1 = AssetEvent( + asset_id=asset1.id, + source_dag_id="source_dag", + source_run_id="source_run", + source_task_id="source_task", + ) + event2 = AssetEvent( + asset_id=asset2.id, + source_dag_id="source_dag", + source_run_id="source_run", + source_task_id="source_task", + ) + session.add_all([event1, event2]) + session.flush() + + dag_run1 = session.scalar(select(DagRun).filter(DagRun.id == dag_run1.id)) + dag_run2 = session.scalar(select(DagRun).filter(DagRun.id == dag_run2.id)) + + dag_run1.consumed_asset_events.append(event1) + dag_run2.consumed_asset_events.append(event2) + + session.commit() + def get_dag_versions_dict(dag_versions: list[DagVersion]) -> list[dict]: return [ @@ -710,6 +740,21 @@ def test_bad_limit_and_offset(self, test_client, query_params, expected_detail): ), # Test for debug key ("~", {"conf_contains": "version"}, [DAG1_RUN1_ID]), # Test for the key "version" ("~", {"conf_contains": "nonexistent_key"}, []), # Test for a key that doesn't exist + # Test consuming_asset_pattern filter + ("~", {"consuming_asset_pattern": "sales"}, [DAG1_RUN1_ID]), # Filter by asset name + ("~", {"consuming_asset_pattern": "s3://bucket/sales"}, [DAG1_RUN1_ID]), # Filter by asset URI + ("~", {"consuming_asset_pattern": "customer"}, [DAG1_RUN2_ID]), # Filter by another asset + ( + "~", + {"consuming_asset_pattern": "s3://bucket/customer"}, + [DAG1_RUN2_ID], + ), # Filter by customer URI + ( + "~", + {"consuming_asset_pattern": "s3://bucket"}, + [DAG1_RUN1_ID, DAG1_RUN2_ID], + ), # Partial URI match + ("~", {"consuming_asset_pattern": "nonexistent_asset"}, []), # Non-existent asset returns empty ], ) @pytest.mark.usefixtures("configure_git_connection_for_dag_bundle") From c568f08fd3593725ea0c833fad430c67a482cc03 Mon Sep 17 00:00:00 2001 From: Rahul Vats <43964496+vatsrahul1001@users.noreply.github.com> Date: Wed, 15 Apr 2026 19:45:30 +0530 Subject: [PATCH 008/309] fix(ui): invalidate task instances list query after clearing task instance (#63923) (#65304) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After clearing a task instance, the TaskInstances list page was not refreshing to show the updated state. This was because `useClearTaskInstances` was missing `[useTaskInstanceServiceGetTaskInstancesKey]` in the list of query keys to invalidate on success. Both `useClearRun` and `usePatchTaskInstance` correctly invalidate this query — this change brings `useClearTaskInstances` in line with them. Fixes: #60703 (cherry picked from commit f47038e9efb4aa1a193314b2df4c5f0c652be9d9) Co-authored-by: nagasrisai <59650078+nagasrisai@users.noreply.github.com> --- .../src/airflow/ui/src/queries/useClearTaskInstances.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airflow-core/src/airflow/ui/src/queries/useClearTaskInstances.ts b/airflow-core/src/airflow/ui/src/queries/useClearTaskInstances.ts index b02cc1bbac6fc..2d70c49bc5bd5 100644 --- a/airflow-core/src/airflow/ui/src/queries/useClearTaskInstances.ts +++ b/airflow-core/src/airflow/ui/src/queries/useClearTaskInstances.ts @@ -24,6 +24,7 @@ import { useDagRunServiceGetDagRunsKey, UseGanttServiceGetGanttDataKeyFn, UseTaskInstanceServiceGetMappedTaskInstanceKeyFn, + useTaskInstanceServiceGetTaskInstancesKey, useTaskInstanceServicePostClearTaskInstances, } from "openapi/queries"; import type { ApiError } from "openapi/requests"; @@ -114,6 +115,7 @@ export const useClearTaskInstances = ({ ...taskInstanceKeys, UseDagRunServiceGetDagRunKeyFn({ dagId, dagRunId }), [useDagRunServiceGetDagRunsKey], + [useTaskInstanceServiceGetTaskInstancesKey], [useClearTaskInstancesDryRunKey, dagId], [usePatchTaskInstanceDryRunKey, dagId, dagRunId], UseGanttServiceGetGanttDataKeyFn({ dagId, runId: dagRunId }), From 3b86dcb0046aecea1488271a2197c1f825d77522 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 02:12:37 +0100 Subject: [PATCH 009/309] [v3-2-test] Add Registry link to docs navbar (#65258) (#65338) The sphinx_airflow_theme default navbar_links includes Registry, but the docs override navbar_links in get_html_theme_options(), so the theme default never applies. (cherry picked from commit d988f75) Co-authored-by: Kaxil Naik --- devel-common/src/docs/utils/conf_constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/devel-common/src/docs/utils/conf_constants.py b/devel-common/src/docs/utils/conf_constants.py index 1bd89b3e8d655..6b75f3e899022 100644 --- a/devel-common/src/docs/utils/conf_constants.py +++ b/devel-common/src/docs/utils/conf_constants.py @@ -147,6 +147,7 @@ def get_html_theme_options(): {"href": "/community/", "text": "Community"}, {"href": "/meetups/", "text": "Meetups"}, {"href": "/docs/", "text": "Documentation"}, + {"href": "/registry/", "text": "Registry"}, {"href": "/use-cases/", "text": "Use Cases"}, {"href": "/announcements/", "text": "Announcements"}, {"href": "/blog/", "text": "Blog"}, From 0c4c2eb1ea50bbd17e5a89ed34a18954e2937010 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 03:58:44 +0200 Subject: [PATCH 010/309] [v3-2-test] Enforce per-file import-error authorization using relative_fileloc + bundle (#65329) (#65343) The public Import Errors API used to match ParseImportError.filename against DagModel.fileloc. In real deployments ``fileloc`` is an absolute path while ``filename`` is relative, so the file-to-DAG resolution often came back empty and the single endpoint fell through to returning the raw error. The list endpoint had a related gap: its CTE was pre-filtered by the caller-visible subset of DAGs, so the per-file authorization check only ever saw the DAGs the caller could already read -- a file containing a mix of readable and unreadable DAGs passed the check on the readable subset alone. * The single endpoint now matches ParseImportError.filename against DagModel.relative_fileloc + DagModel.bundle_name, which is the same key the list endpoint already uses for its join. When the resolved DAG set is empty (parse failed before any DAG was defined, or the name keys did not resolve), the stacktrace is now redacted rather than returned verbatim. * The list endpoint splits the previous ``visible_files_cte`` into two CTEs: ``readable_files_cte`` enumerates the ``(relative_fileloc, bundle_name)`` pairs where the caller can read at least one DAG, and ``file_dags_cte`` enumerates the full ``(relative_fileloc, dag_id, bundle_name)`` set for those files. The per-file authorization check in the groupby loop now receives the complete DAG set for each file and can correctly detect co-located DAGs outside the caller's scope. * The same fall-through in the list endpoint (file has no matching DAGs in DagModel) now redacts the stacktrace before appending. Add a test class that exercises the fix with distinct ``fileloc`` (absolute) and ``relative_fileloc`` (relative) string values, closing the test-fixture gap where both columns previously held the same relative string and the absolute-vs-relative mismatch could not manifest. One existing single-endpoint test documenting the previous fall-through behaviour is updated to assert the new redaction. (cherry picked from commit eba9b65a587a078134f3a9808c3fd6e18f92d3d0) Generated-by: Claude Opus 4.6 (1M context) following the guidelines at https: //github.com/apache/airflow/blob/main/contributing-docs/05_pull_requests.rst#gen-ai-assisted-contributions Co-authored-by: Jarek Potiuk --- .../core_api/routes/public/import_error.py | 72 ++++-- .../routes/public/test_import_error.py | 211 +++++++++++++++++- 2 files changed, 267 insertions(+), 16 deletions(-) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/import_error.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/import_error.py index 97bfc4c0691fc..57a937b4d455d 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/import_error.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/import_error.py @@ -80,13 +80,28 @@ def get_import_error( auth_manager = get_auth_manager() readable_dag_ids = auth_manager.get_authorized_dag_ids(user=user) - # We need file_dag_ids as a set for intersection, issubset operations + # ``ParseImportError.filename`` is a repository-relative path and + # ``DagModel.fileloc`` is typically the absolute path those files were + # loaded from, so matching on ``fileloc == filename`` would come back + # empty in most real deployments. Match on ``relative_fileloc`` (and the + # bundle name that scopes it) instead, which is the same key the list + # endpoint already uses for the join below. file_dag_ids = set( - session.scalars(select(DagModel.dag_id).where(DagModel.fileloc == error.filename)).all() + session.scalars( + select(DagModel.dag_id).where( + DagModel.relative_fileloc == error.filename, + DagModel.bundle_name == error.bundle_name, + ) + ).all() ) - # No DAGs in the file (failed to parse), nothing to check permissions against + # No DAGs matched for this file -- either the file genuinely contains + # no DAGs (parse failed before any DAG was defined), or the name keys + # did not resolve. Redact the stacktrace rather than returning the raw + # error, so the response stays on the deny-by-default side of the + # authorization check. if not file_dag_ids: + error.stacktrace = REDACTED_STACKTRACE return error # Can the user read any DAGs in the file? @@ -138,32 +153,55 @@ def get_import_errors( # Subquery for files that have any DAGs files_with_any_dags = select(DagModel.relative_fileloc).distinct().subquery() - # CTE for DAGs the user can read - visible_files_cte = ( - select(DagModel.relative_fileloc, DagModel.dag_id, DagModel.bundle_name) + # Files (identified by ``(relative_fileloc, bundle_name)``) where the + # user can read at least one DAG. Used to decide which import errors + # the user is allowed to see at all. + readable_files_cte = ( + select(DagModel.relative_fileloc, DagModel.bundle_name) .where(DagModel.dag_id.in_(readable_dag_ids)) + .distinct() + .cte() + ) + + # Full ``(relative_fileloc, dag_id, bundle_name)`` set for every file + # the user can see at least one DAG of. Crucially this is **not** + # filtered by ``readable_dag_ids`` -- the per-file authorization + # check in the loop below needs the complete DAG set so it can + # detect co-located DAGs that the caller is not authorized to read + # and redact the stacktrace accordingly. + file_dags_cte = ( + select(DagModel.relative_fileloc, DagModel.dag_id, DagModel.bundle_name) + .join( + readable_files_cte, + and_( + DagModel.relative_fileloc == readable_files_cte.c.relative_fileloc, + DagModel.bundle_name == readable_files_cte.c.bundle_name, + ), + ) .cte() ) - # Prepare the import errors query by joining with the cte. - # Each returned row will be a tuple: (ParseImportError, dag_id) + # Prepare the import errors query by joining with the CTE above. + # Each returned row will be a tuple: (ParseImportError, dag_id). + # ``dag_id`` is NULL for import errors whose file has no DAGs at all + # in ``DagModel`` (parse failed before any DAG was defined). import_errors_stmt = ( - select(ParseImportError, visible_files_cte.c.dag_id) + select(ParseImportError, file_dags_cte.c.dag_id) .outerjoin( files_with_any_dags, ParseImportError.filename == files_with_any_dags.c.relative_fileloc, ) .outerjoin( - visible_files_cte, + file_dags_cte, and_( - ParseImportError.filename == visible_files_cte.c.relative_fileloc, - ParseImportError.bundle_name == visible_files_cte.c.bundle_name, + ParseImportError.filename == file_dags_cte.c.relative_fileloc, + ParseImportError.bundle_name == file_dags_cte.c.bundle_name, ), ) .where( or_( files_with_any_dags.c.relative_fileloc.is_(None), - visible_files_cte.c.dag_id.isnot(None), + file_dags_cte.c.dag_id.isnot(None), ) ) .order_by(ParseImportError.id) @@ -186,8 +224,14 @@ def get_import_errors( for import_error, file_dag_ids_iter in import_errors_result: dag_ids = [dag_id for _, dag_id in file_dag_ids_iter if dag_id is not None] - # No DAGs in the file, nothing to check permissions against + # No DAGs matched for this file -- either the file genuinely has + # no DAGs yet (parse failed before any DAG was defined), or the + # name keys did not resolve. Redact the stacktrace before + # appending so the response stays on the deny-by-default side of + # the authorization check. if not dag_ids: + session.expunge(import_error) + import_error.stacktrace = REDACTED_STACKTRACE import_errors.append(import_error) continue diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_import_error.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_import_error.py index 469d746fb1467..ba0e07436977d 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_import_error.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_import_error.py @@ -23,6 +23,7 @@ import pytest from airflow.api_fastapi.auth.managers.models.resource_details import DagDetails +from airflow.api_fastapi.core_api.routes.public.import_error import REDACTED_STACKTRACE from airflow.models import DagModel from airflow.models.dagbundle import DagBundleModel from airflow.models.errors import ParseImportError @@ -276,7 +277,13 @@ def test_get_import_error__user_dont_have_read_permission_to_read_all_dags_in_fi @mock.patch("airflow.api_fastapi.core_api.routes.public.import_error.get_auth_manager") def test_get_import_error__no_dag_in_dagmodel(self, mock_get_auth_manager, test_client, import_errors): - """Test import error is returned when no DAG exists in DagModel.""" + """Import error is returned with a redacted stacktrace when no DAG + exists in ``DagModel`` for the file. + + When the file-to-DAG set resolves empty the endpoint cannot tell + which DAGs the caller is allowed to see, so the stacktrace is + redacted rather than returned verbatim. + """ import_error_id = import_errors[0].id set_mock_auth_manager__get_authorized_dag_ids(mock_get_auth_manager, set()) @@ -287,7 +294,7 @@ def test_get_import_error__no_dag_in_dagmodel(self, mock_get_auth_manager, test_ "import_error_id": import_error_id, "timestamp": from_datetime_to_zulu_without_ms(TIMESTAMP1), "filename": FILENAME1, - "stack_trace": STACKTRACE1, + "stack_trace": REDACTED_STACKTRACE, "bundle_name": BUNDLE_NAME, } @@ -525,3 +532,203 @@ def test_get_import_errors__no_dag_in_dagmodel(self, mock_get_auth_manager, test assert FILENAME1 in filenames assert FILENAME2 in filenames assert FILENAME3 in filenames + + +class TestImportErrorFileAuthorization: + """Tests that the import error endpoints apply per-file authorization + using ``relative_fileloc + bundle_name`` and redact stacktraces when the + resolved DAG set is empty or contains co-located DAGs outside the + caller's scope.""" + + LONELY_FILE_RELATIVE = "lonely_file.py" + LONELY_FILE_ABSOLUTE = "/opt/airflow/dags/lonely_file.py" + MIXED_FILE_RELATIVE = "mixed_file.py" + MIXED_FILE_ABSOLUTE = "/opt/airflow/dags/mixed_file.py" + LONELY_STACKTRACE = "stack trace for the lonely file" + MIXED_STACKTRACE = "stack trace for the mixed file" + + @pytest.fixture + @provide_session + def absolute_vs_relative_fileloc_dag( + self, + testing_dag_bundle, + session: Session = NEW_SESSION, + ) -> DagModel: + """DagModel whose ``fileloc`` is absolute and ``relative_fileloc`` is + the relative path that matches ``ParseImportError.filename``. + + The two columns deliberately hold different string values so that a + ``fileloc == filename`` match (the pre-fix behaviour) comes back + empty and a ``relative_fileloc == filename`` match (the fix) finds + the row. + """ + dag_model = DagModel( + fileloc=self.LONELY_FILE_ABSOLUTE, + relative_fileloc=self.LONELY_FILE_RELATIVE, + dag_id="lonely_dag", + is_paused=False, + bundle_name=BUNDLE_NAME, + ) + session.add(dag_model) + session.commit() + return dag_model + + @pytest.fixture + @provide_session + def mixed_file_dags( + self, + testing_dag_bundle, + session: Session = NEW_SESSION, + ) -> tuple[DagModel, DagModel]: + """Two DagModels pointing at the same ``(relative_fileloc, + bundle_name)`` pair so the per-file authorization check in the list + endpoint has a co-located DAG to redact against.""" + readable = DagModel( + fileloc=self.MIXED_FILE_ABSOLUTE, + relative_fileloc=self.MIXED_FILE_RELATIVE, + dag_id="readable_mixed_dag", + is_paused=False, + bundle_name=BUNDLE_NAME, + ) + colocated = DagModel( + fileloc=self.MIXED_FILE_ABSOLUTE, + relative_fileloc=self.MIXED_FILE_RELATIVE, + dag_id="colocated_mixed_dag", + is_paused=False, + bundle_name=BUNDLE_NAME, + ) + session.add_all([readable, colocated]) + session.commit() + return readable, colocated + + @pytest.fixture + @provide_session + def lonely_file_import_error( + self, + session: Session = NEW_SESSION, + ) -> ParseImportError: + error = ParseImportError( + bundle_name=BUNDLE_NAME, + filename=self.LONELY_FILE_RELATIVE, + stacktrace=self.LONELY_STACKTRACE, + timestamp=TIMESTAMP1, + ) + session.add(error) + session.commit() + return error + + @pytest.fixture + @provide_session + def mixed_file_import_error( + self, + session: Session = NEW_SESSION, + ) -> ParseImportError: + error = ParseImportError( + bundle_name=BUNDLE_NAME, + filename=self.MIXED_FILE_RELATIVE, + stacktrace=self.MIXED_STACKTRACE, + timestamp=TIMESTAMP2, + ) + session.add(error) + session.commit() + return error + + @mock.patch("airflow.api_fastapi.core_api.routes.public.import_error.get_auth_manager") + def test_single_endpoint_matches_file_via_relative_fileloc_not_fileloc( + self, + mock_get_auth_manager, + test_client, + absolute_vs_relative_fileloc_dag, + lonely_file_import_error, + ): + """Single endpoint resolves ``ParseImportError.filename`` against + ``DagModel.relative_fileloc`` (and ``bundle_name``), not + ``DagModel.fileloc``. + + The DagModel's ``fileloc`` is absolute while the ParseImportError's + ``filename`` is relative, so a ``fileloc == filename`` match comes + back empty in this fixture. The endpoint must still resolve the DAG + set via ``relative_fileloc`` and enforce the normal authorization + check. Here the caller has no DAG permissions at all, so the + response must be 403 -- not a 200 that returns the stack trace + verbatim via the empty-set fall-through. + """ + set_mock_auth_manager__get_authorized_dag_ids(mock_get_auth_manager, set()) + response = test_client.get(f"/importErrors/{lonely_file_import_error.id}") + assert response.status_code == 403 + + @mock.patch("airflow.api_fastapi.core_api.routes.public.import_error.get_auth_manager") + def test_single_endpoint_redacts_when_file_has_no_known_dags( + self, + mock_get_auth_manager, + test_client, + import_errors, + ): + """Single endpoint must redact the stacktrace when the + ``ParseImportError`` refers to a file with no matching ``DagModel`` + rows at all -- for example a file that failed to parse before any + DAG was defined. The response must be 200 with + ``REDACTED_STACKTRACE``, not a 200 with the raw error body. + """ + set_mock_auth_manager__get_authorized_dag_ids(mock_get_auth_manager, set()) + response = test_client.get(f"/importErrors/{import_errors[0].id}") + assert response.status_code == 200 + body = response.json() + assert body["filename"] == FILENAME1 + assert body["stack_trace"] == REDACTED_STACKTRACE + + @mock.patch("airflow.api_fastapi.core_api.routes.public.import_error.get_auth_manager") + def test_list_endpoint_redacts_mixed_file_with_colocated_dag_outside_callers_scope( + self, + mock_get_auth_manager, + test_client, + mixed_file_dags, + mixed_file_import_error, + ): + """List endpoint must redact the stacktrace for a file that + contains a DAG outside the caller's scope, even when the caller can + read another DAG in the same file. + + The ``side_effect`` below allows the call only when the request set + is a subset of the caller's readable set. Under the fixed code the + per-file authorization check receives the full DAG set for the + file (both ``readable_mixed_dag`` and ``colocated_mixed_dag``), so + the call is denied and the stacktrace is redacted. Under the + pre-fix code the check would only see the readable subset, the + call would be permitted, and the raw stacktrace would be returned. + """ + readable, _ = mixed_file_dags + set_mock_auth_manager__get_authorized_dag_ids(mock_get_auth_manager, {readable.dag_id}) + + def permit_only_readable(requests, user): + request_dag_ids = {req["details"].id for req in requests} + return request_dag_ids.issubset({readable.dag_id}) + + mock_get_auth_manager.return_value.batch_is_authorized_dag.side_effect = permit_only_readable + + response = test_client.get("/importErrors") + assert response.status_code == 200 + body = response.json() + mixed_entries = [err for err in body["import_errors"] if err["filename"] == self.MIXED_FILE_RELATIVE] + assert len(mixed_entries) == 1 + assert mixed_entries[0]["stack_trace"] == REDACTED_STACKTRACE + + @mock.patch("airflow.api_fastapi.core_api.routes.public.import_error.get_auth_manager") + def test_list_endpoint_redacts_when_file_has_no_known_dags( + self, + mock_get_auth_manager, + test_client, + import_errors, + ): + """List endpoint must redact the stacktrace for import errors + whose file has no matching ``DagModel`` rows -- closing the + ``if not dag_ids: import_errors.append(import_error)`` fall-through + that previously returned the raw error. + """ + set_mock_auth_manager__get_authorized_dag_ids(mock_get_auth_manager, set()) + response = test_client.get("/importErrors") + assert response.status_code == 200 + body = response.json() + assert body["total_entries"] == 3 + for entry in body["import_errors"]: + assert entry["stack_trace"] == REDACTED_STACKTRACE From 87568ea9e262e5020c81fb45623cc08534b9f912 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 04:55:19 +0200 Subject: [PATCH 011/309] [v3-2-test] Refuse to follow log symlinks that resolve outside the base log folder (#65325) (#65345) * Refuse to follow log symlinks that resolve outside the base log folder FileTaskHandler._read_from_local used to open every file that matched the task's log glob pattern, including symlinks whose real path was outside the configured base_log_folder. On deployments where worker logs are accessible from the api-server, that meant the log viewer could end up streaming content from files outside the configured log tree whenever a symlink in the task log directory happened to match the glob pattern. Canonicalise self.local_base once via os.path.realpath and, for every glob hit, resolve the path with os.path.realpath and skip it if the resolved form is not contained in the canonicalised base log folder (using os.path.commonpath, with a ValueError fallback for the different-drive case on Windows). Open the resolved path rather than the original glob hit so the file we open is the one we just validated. Append to sources only after a successful open so sources and log_streams stay aligned. Drop the @staticmethod decorator so the method can read self.local_base; existing call sites already invoke it via self. Add a test class covering: regular-file-inside-base is still streamed; a symlink whose real path is outside base_log_folder is skipped; a symlink that stays inside base_log_folder is followed (legitimate rotation case); and base_log_folder itself being a symlink works. Generated-by: Claude Opus 4.6 (1M context) following the guidelines at https://github.com/apache/airflow/blob/main/contributing-docs/05_pull_requests.rst#gen-ai-assisted-contributions * Fix test__read_from_local to use valid base_log_folder The existing test passed an empty string as base_log_folder, which after the containment check resolves to CWD via os.path.realpath(""), causing all files under tmp_path to be rejected. Use tmp_path instead. (cherry picked from commit 3eda84547e743397ec1027733f97d37ab2e628b4) Co-authored-by: Jarek Potiuk --- .../airflow/utils/log/file_task_handler.py | 28 +++++- .../unit/utils/log/test_file_task_handler.py | 99 +++++++++++++++++++ .../tests/unit/utils/test_log_handlers.py | 2 +- 3 files changed, 125 insertions(+), 4 deletions(-) diff --git a/airflow-core/src/airflow/utils/log/file_task_handler.py b/airflow-core/src/airflow/utils/log/file_task_handler.py index 1481d0a315e9b..13138d1cff575 100644 --- a/airflow-core/src/airflow/utils/log/file_task_handler.py +++ b/airflow-core/src/airflow/utils/log/file_task_handler.py @@ -857,20 +857,42 @@ def _init_file(self, ti, *, identifier: str | None = None): return full_path - @staticmethod def _read_from_local( + self, worker_log_path: Path, ) -> StreamingLogResponse: sources: LogSourceInfo = [] log_streams: list[RawLogStream] = [] + # The glob below can match symlinks as well as regular files, so + # resolve each hit and only open the ones that stay inside the base + # log folder. Canonicalising ``self.local_base`` once up front makes + # the containment check compare two already-resolved paths. + base_log_folder = os.path.realpath(self.local_base) paths = sorted(worker_log_path.parent.glob(worker_log_path.name + "*")) if not paths: return sources, log_streams for path in paths: + resolved_path = os.path.realpath(path) + try: + if os.path.commonpath([base_log_folder, resolved_path]) != base_log_folder: + continue + except ValueError: + # ``os.path.commonpath`` raises ``ValueError`` when the two + # paths have nothing in common (e.g. different drives on + # Windows); treat that as "not contained" and skip the file. + continue + + # Open the resolved path so the file we read is the same one we + # just validated above. Append to ``sources`` only after a + # successful ``open`` so ``sources`` and ``log_streams`` stay + # aligned. + try: + log_stream = _stream_lines_by_chunk(open(resolved_path, encoding="utf-8")) + except OSError: + continue sources.append(os.fspath(path)) - # Read the log file and yield lines - log_streams.append(_stream_lines_by_chunk(open(path, encoding="utf-8"))) + log_streams.append(log_stream) return sources, log_streams def _read_from_logs_server( diff --git a/airflow-core/tests/unit/utils/log/test_file_task_handler.py b/airflow-core/tests/unit/utils/log/test_file_task_handler.py index cdda0e66de84c..93606cffe1dd1 100644 --- a/airflow-core/tests/unit/utils/log/test_file_task_handler.py +++ b/airflow-core/tests/unit/utils/log/test_file_task_handler.py @@ -90,3 +90,102 @@ def test_403_shows_secret_key_message(self, mock_get_url, mock_fetch): assert len(sources) == 1 assert "secret_key" in sources[0] assert streams == [] + + +class TestFileTaskHandlerReadFromLocal: + """Tests for ``FileTaskHandler._read_from_local`` path containment.""" + + @staticmethod + def _drain(stream) -> str: + return "".join(list(stream)) + + def test_reads_regular_log_file_inside_base(self, tmp_path): + """A regular file under ``base_log_folder`` is streamed as before.""" + log_dir = tmp_path / "dag" / "run" / "task" + log_dir.mkdir(parents=True) + log_file = log_dir / "1.log" + log_file.write_text("legitimate log line\n") + + handler = FileTaskHandler(base_log_folder=str(tmp_path)) + sources, streams = handler._read_from_local(log_file) + + assert sources == [str(log_file)] + assert len(streams) == 1 + assert "legitimate log line" in self._drain(streams[0]) + + def test_skips_symlink_resolving_outside_base_log_folder(self, tmp_path): + """A glob hit that resolves outside ``base_log_folder`` is not streamed. + + This documents the intended containment behaviour: a file under the + task's log directory that is actually a symlink whose real path is + outside the configured base log folder must be skipped, even though + it matches the glob pattern used to discover the task's log files. + """ + base_log_folder = tmp_path / "logs" + log_dir = base_log_folder / "dag" / "run" / "task" + log_dir.mkdir(parents=True) + + # A regular log file inside the base log folder. + legit = log_dir / "1.log" + legit.write_text("legitimate log line\n") + + # A file that lives outside the base log folder. + external_dir = tmp_path / "external" + external_dir.mkdir() + external_file = external_dir / "other.txt" + external_file.write_text("external content\n") + + # A glob hit that matches ``1.log*`` but resolves outside the base. + escape_link = log_dir / "1.log.external" + escape_link.symlink_to(external_file) + + handler = FileTaskHandler(base_log_folder=str(base_log_folder)) + sources, streams = handler._read_from_local(legit) + + assert str(legit) in sources + assert str(escape_link) not in sources + content = "".join(self._drain(s) for s in streams) + assert "legitimate log line" in content + assert "external content" not in content + + def test_follows_symlink_within_base_log_folder(self, tmp_path): + """A symlink that resolves back into the base log folder is allowed. + + The containment check compares the realpath of the glob hit to the + realpath of the base log folder, so a symlink that stays entirely + inside the log tree (for example from log rotation) still works. + """ + base_log_folder = tmp_path / "logs" + log_dir = base_log_folder / "dag" / "run" / "task" + log_dir.mkdir(parents=True) + + real_file = log_dir / "real.log" + real_file.write_text("inner content\n") + + link = log_dir / "1.log.link" + link.symlink_to(real_file) + + handler = FileTaskHandler(base_log_folder=str(base_log_folder)) + sources, streams = handler._read_from_local(log_dir / "1.log") + + assert str(link) in sources + assert "inner content" in "".join(self._drain(s) for s in streams) + + def test_handles_base_log_folder_that_is_itself_a_symlink(self, tmp_path): + """``base_log_folder`` itself is realpath'd so a base that is a + symlink to the actual log directory is treated as contained.""" + real_base = tmp_path / "real_logs" + real_base.mkdir() + base_link = tmp_path / "logs" + base_link.symlink_to(real_base) + + log_dir = base_link / "dag" / "run" / "task" + log_dir.mkdir(parents=True) + log_file = log_dir / "1.log" + log_file.write_text("through-symlink content\n") + + handler = FileTaskHandler(base_log_folder=str(base_link)) + sources, streams = handler._read_from_local(log_file) + + assert len(sources) == 1 + assert "through-symlink content" in self._drain(streams[0]) diff --git a/airflow-core/tests/unit/utils/test_log_handlers.py b/airflow-core/tests/unit/utils/test_log_handlers.py index 6a93e940e7254..4c95327c9fb71 100644 --- a/airflow-core/tests/unit/utils/test_log_handlers.py +++ b/airflow-core/tests/unit/utils/test_log_handlers.py @@ -510,7 +510,7 @@ def test__read_from_local(self, tmp_path): path2 = tmp_path / "hello1.log.suffix.log" path1.write_text("file1 content\nfile1 content2") path2.write_text("file2 content\nfile2 content2") - fth = FileTaskHandler("") + fth = FileTaskHandler(str(tmp_path)) log_source_info, log_streams = fth._read_from_local(path1) assert log_source_info == [str(path1), str(path2)] assert len(log_streams) == 2 From 96af6f2d2f2fe6d907a85e74de97fb3587b15e18 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 12:00:47 +0200 Subject: [PATCH 012/309] [v3-2-test] Set JWT refresh cookie Secure flag when request is HTTPS (#65348) (#65363) JWTRefreshMiddleware derived the cookie Secure flag from the local api.ssl_cert config only. Deployments with TLS terminated at a reverse proxy (no local SSL cert on the Airflow process) therefore received the JWT refresh cookie without the Secure flag. Match the pattern already used by every other cookie-setting location in the codebase (auth.py, simple/routes/login.py, FAB and Keycloak login routes): treat secure as True when either the request came in over HTTPS or a local ssl_cert is configured. (cherry picked from commit 60db83fa80233c330933bb65123bdce14dfc5c26) Generated-by: Claude Opus 4.6 (1M context) following the guidelines at https: //github.com/apache/airflow/blob/main/contributing-docs/05_pull_requests.rst#gen-ai-assisted-contributions Co-authored-by: Jarek Potiuk --- .../auth/middlewares/refresh_token.py | 2 +- .../auth/middlewares/test_refresh_token.py | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/airflow-core/src/airflow/api_fastapi/auth/middlewares/refresh_token.py b/airflow-core/src/airflow/api_fastapi/auth/middlewares/refresh_token.py index b8a3a268ba855..179bb1ad07e3f 100644 --- a/airflow-core/src/airflow/api_fastapi/auth/middlewares/refresh_token.py +++ b/airflow-core/src/airflow/api_fastapi/auth/middlewares/refresh_token.py @@ -62,7 +62,7 @@ async def dispatch(self, request: Request, call_next): if new_token is not None: cookie_path = get_cookie_path() - secure = bool(conf.get("api", "ssl_cert", fallback="")) + secure = request.base_url.scheme == "https" or bool(conf.get("api", "ssl_cert", fallback="")) response.set_cookie( COOKIE_NAME_JWT_TOKEN, new_token, diff --git a/airflow-core/tests/unit/api_fastapi/auth/middlewares/test_refresh_token.py b/airflow-core/tests/unit/api_fastapi/auth/middlewares/test_refresh_token.py index 34b30ba1e7e4f..2ebfe2cc71b48 100644 --- a/airflow-core/tests/unit/api_fastapi/auth/middlewares/test_refresh_token.py +++ b/airflow-core/tests/unit/api_fastapi/auth/middlewares/test_refresh_token.py @@ -131,6 +131,51 @@ async def test_dispatch_with_refreshed_user( set_cookie_headers = response.headers.get("set-cookie", "") assert f"{COOKIE_NAME_JWT_TOKEN}=new_token" in set_cookie_headers + @pytest.mark.parametrize( + ("scheme", "ssl_cert", "expected_secure"), + [ + pytest.param("https", "", True, id="https-no-local-ssl-cert"), + pytest.param("http", "/etc/ssl/cert.pem", True, id="http-with-local-ssl-cert"), + pytest.param("https", "/etc/ssl/cert.pem", True, id="https-with-local-ssl-cert"), + pytest.param("http", "", False, id="http-no-local-ssl-cert"), + ], + ) + @patch("airflow.api_fastapi.auth.middlewares.refresh_token.get_auth_manager") + @patch("airflow.api_fastapi.auth.middlewares.refresh_token.resolve_user_from_token") + @patch("airflow.api_fastapi.auth.middlewares.refresh_token.conf") + @pytest.mark.asyncio + async def test_dispatch_cookie_secure_flag( + self, + mock_conf, + mock_resolve_user_from_token, + mock_get_auth_manager, + middleware, + mock_request, + mock_user, + scheme, + ssl_cert, + expected_secure, + ): + """The cookie Secure flag must follow the request scheme as well as the local ssl_cert.""" + refreshed_user = MagicMock(spec=BaseUser) + mock_request.cookies = {COOKIE_NAME_JWT_TOKEN: "valid_token"} + mock_request.base_url = MagicMock(scheme=scheme) + mock_resolve_user_from_token.return_value = mock_user + mock_auth_manager = MagicMock() + mock_get_auth_manager.return_value = mock_auth_manager + mock_auth_manager.refresh_user.return_value = refreshed_user + mock_auth_manager.generate_jwt.return_value = "new_token" + mock_conf.get.return_value = ssl_cert + + call_next = AsyncMock(return_value=Response()) + response = await middleware.dispatch(mock_request, call_next) + + set_cookie_headers = response.headers.get("set-cookie", "") + if expected_secure: + assert "Secure" in set_cookie_headers + else: + assert "Secure" not in set_cookie_headers + @patch("airflow.api_fastapi.auth.middlewares.refresh_token.get_cookie_path", return_value="/team-a/") @patch("airflow.api_fastapi.auth.middlewares.refresh_token.get_auth_manager") @patch("airflow.api_fastapi.auth.middlewares.refresh_token.resolve_user_from_token") From 6f350c318edffb830bcfa71828fca3fc379e015b Mon Sep 17 00:00:00 2001 From: Rahul Vats <43964496+vatsrahul1001@users.noreply.github.com> Date: Thu, 16 Apr 2026 21:13:24 +0530 Subject: [PATCH 013/309] [v3-2-test] Fix: PATCH /dags pagination bug and document wildcard dag_id_pattern (#65309) * [v3-2-test] Bump actions/github-script in the github-actions-updates group (#65150) (#65160) Bumps the github-actions-updates group with 1 update: [actions/github-script](https://github.com/actions/github-script). Updates `actions/github-script` from 8.0.0 to 9.0.0 - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/ed597411d8f924073f98dfc5c65a23a2325f34cd...3a2844b7e9c422d3c10d287c895573f7108da1b3) (cherry picked from commit e5a047ca8d23614cefeaad89e08eb1e3b0482e60) --- updated-dependencies: - dependency-name: actions/github-script dependency-version: 9.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions-updates ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * [v3-2-test] Added breeze generate issue content for airflow-ctl (#65042) (#65241) * Add breeze generate issue content for airflow-ctl * add new command to doc (cherry picked from commit b24538b0bc3a53d8c666b62e52e023a17d102467) Co-authored-by: Justin Pakzad <114518232+justinpakzad@users.noreply.github.com> * [v3-2-test] Run release calendar verification on its own schedule (#65118) (#65242) * Move release calendar verification to its own scheduled workflow Run dev/verify_release_calendar.py from a dedicated daily scheduled workflow instead of as a canary job in the main CI pipeline, and notify the #release-management Slack channel when the check fails so the issue is surfaced to release managers directly. * Include wiki and calendar links in release calendar Slack alert (cherry picked from commit 048e9a191b4a6625bb4bbad23fad537b256f4062) * Fix: PATCH /dags pagination bug and document wildcard dag_id_pattern (#63665) * fixed pagination bug and updated docstring to clarify dag_id_pattern wildcard usage * removed batch loop to update all dags in one shot and added additional test case * Fixed MySQL subquery issue (cherry picked from commit 9504886608f9b54398f757b4eb6fc3442141d28a) --------- Signed-off-by: dependabot[bot] Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jarek Potiuk Co-authored-by: Justin Pakzad <114518232+justinpakzad@users.noreply.github.com> --- .../openapi/v2-rest-api-generated.yaml | 9 ++- .../core_api/routes/public/dags.py | 29 ++++++--- .../airflow/ui/openapi-gen/queries/queries.ts | 4 ++ .../ui/openapi-gen/requests/services.gen.ts | 4 ++ .../core_api/routes/public/test_dags.py | 60 +++++++++++++++++++ 5 files changed, 98 insertions(+), 8 deletions(-) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml index 6ed47e9fc2fb4..24ab32f7113f7 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml @@ -3301,7 +3301,14 @@ paths: tags: - DAG summary: Patch Dags - description: Patch multiple DAGs. + description: 'Patch multiple DAGs. + + + If `dag_id_pattern` is not provided, no DAGs will be matched regardless + + of other filters. To match all DAGs, pass a wildcard value such as `~` + + or `%` for `dag_id_pattern`.' operationId: patch_dags security: - OAuth2PasswordBearer: [] diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dags.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dags.py index 6fbb7c831e9d9..b83cc1ef2236c 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dags.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dags.py @@ -26,10 +26,7 @@ from airflow.api.common import delete_dag as delete_dag_module from airflow.api_fastapi.common.dagbag import DagBagDep, get_latest_version_of_dag -from airflow.api_fastapi.common.db.common import ( - SessionDep, - paginated_select, -) +from airflow.api_fastapi.common.db.common import SessionDep, apply_filters_to_select, paginated_select from airflow.api_fastapi.common.db.dags import generate_dag_with_latest_run_query from airflow.api_fastapi.common.parameters import ( FilterOptionEnum, @@ -328,7 +325,13 @@ def patch_dags( session: SessionDep, update_mask: list[str] | None = Query(None), ) -> DAGCollectionResponse: - """Patch multiple DAGs.""" + """ + Patch multiple DAGs. + + If `dag_id_pattern` is not provided, no DAGs will be matched regardless + of other filters. To match all DAGs, pass a wildcard value such as `~` + or `%` for `dag_id_pattern`. + """ if update_mask: if update_mask != ["is_paused"]: raise HTTPException( @@ -356,10 +359,22 @@ def patch_dags( session=session, ) dags = session.scalars(dags_select).all() - dags_to_update = {dag.dag_id for dag in dags} + + filtered_dag_ids = apply_filters_to_select( + statement=select(DagModel.dag_id), + filters=[ + exclude_stale, + paused, + dag_id_pattern, + tags, + owners, + editable_dags_filter, + ], + ).subquery() + session.execute( update(DagModel) - .where(DagModel.dag_id.in_(dags_to_update)) + .where(DagModel.dag_id.in_(select(filtered_dag_ids.c.dag_id))) .values(is_paused=patch_body.is_paused) .execution_options(synchronize_session="fetch") ) diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts index 019d3ce4a9494..1af349e168998 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts @@ -2117,6 +2117,10 @@ export const useDagRunServicePatchDagRun = Date: Thu, 16 Apr 2026 17:10:20 +0100 Subject: [PATCH 014/309] [v3-2-test] [v3-2-test] Ensure that DB migrations run in a single connection. (#65231) (#65368) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was discovered by running with a custom External DB manager that had some gnarly queries that ended up being locked behind this transaction. `_single_connection_pool` replaces `settings.engine` with a SingletonThreadPool engine. But `work_session` was created before that — it still holds an internal reference to the old engine object. _single_connection_pool has no way to rebind work_session. So when _get_current_revision(session=work_session) runs on line 1203 — inside the _single_connection_pool() block — it calls session.connection() which goes through the old engine, not the SingletonThreadPool. The old engine's pool was disposed and recreated empty by engine.dispose(), so it creates a brand new connection. That connection is completely outside _single_connection_pool's control. _single_connection_pool guarantees one connection on the new engine. It can't prevent work_session from creating connections on the old one. The name is a bit of a lie — it's really "single connection pool for new code that uses settings.engine", not "single connection total." (cherry picked from commit e3fea3a55564fb4b0daeafe43c521f0ee2001f7a) (cherry picked from commit f8e0876) Co-authored-by: Ash Berlin-Taylor --- airflow-core/src/airflow/utils/db.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/airflow-core/src/airflow/utils/db.py b/airflow-core/src/airflow/utils/db.py index 9bc0608611b5a..b848764c558fc 100644 --- a/airflow-core/src/airflow/utils/db.py +++ b/airflow-core/src/airflow/utils/db.py @@ -1194,6 +1194,13 @@ def _run_upgradedb( with _configured_alembic_environment() as env: source_heads = env.script.get_heads() + # End the read-only transaction from _get_current_revision before + # external DB manager migrations, which may run DDL that is blocked by + # open transactions (e.g. CREATE INDEX CONCURRENTLY). The advisory lock + # from create_global_lock() is unaffected: it is session-level and held + # on a separate connection. + work_session.rollback() + if current_revision == source_heads[0] and not _SKIP_EXTERNAL_DB_MANAGERS_UPGRADE.get(): external_db_manager = RunDBManager() external_db_manager.upgradedb(work_session, use_migration_files=use_migration_files) From fa89a46a311a2778aea714b713403d4b23abe9bd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 18:25:07 +0200 Subject: [PATCH 015/309] [v3-2-test] fix(ui): register trigger and sensor graph node types (#65167) (#65321) * [v3-2-test] Bump actions/github-script in the github-actions-updates group (#65150) (#65160) Bumps the github-actions-updates group with 1 update: [actions/github-script](https://github.com/actions/github-script). Updates `actions/github-script` from 8.0.0 to 9.0.0 - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/ed597411d8f924073f98dfc5c65a23a2325f34cd...3a2844b7e9c422d3c10d287c895573f7108da1b3) (cherry picked from commit e5a047ca8d23614cefeaad89e08eb1e3b0482e60) --- updated-dependencies: - dependency-name: actions/github-script dependency-version: 9.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions-updates ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * [v3-2-test] Added breeze generate issue content for airflow-ctl (#65042) (#65241) * Add breeze generate issue content for airflow-ctl * add new command to doc (cherry picked from commit b24538b0bc3a53d8c666b62e52e023a17d102467) Co-authored-by: Justin Pakzad <114518232+justinpakzad@users.noreply.github.com> * [v3-2-test] Run release calendar verification on its own schedule (#65118) (#65242) * Move release calendar verification to its own scheduled workflow Run dev/verify_release_calendar.py from a dedicated daily scheduled workflow instead of as a canary job in the main CI pipeline, and notify the #release-management Slack channel when the check fails so the issue is surfaced to release managers directly. * Include wiki and calendar links in release calendar Slack alert (cherry picked from commit 048e9a191b4a6625bb4bbad23fad537b256f4062) * [v3-2-test] fix(ui): register trigger and sensor graph node types (#65167) * fix(ui): register trigger and sensor graph node types Adds missing Graph node type mappings for trigger/sensor and includes a focused unit test to prevent regressions where dependency graph rendering breaks for those node kinds. * docs(ui): add graph screenshot showing sensor and trigger nodes * chore(ui): keep PR scoped to graphTypes.ts only --------- (cherry picked from commit e0ed7950aa47cbb850bcfa93990eb24345068706) Co-authored-by: Windro.xd <88357206+windro-xdd@users.noreply.github.com> Co-authored-by: Kripa Dev --------- Signed-off-by: dependabot[bot] Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jarek Potiuk Co-authored-by: Justin Pakzad <114518232+justinpakzad@users.noreply.github.com> Co-authored-by: Windro.xd <88357206+windro-xdd@users.noreply.github.com> Co-authored-by: Kripa Dev --- airflow-core/src/airflow/ui/src/components/Graph/graphTypes.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airflow-core/src/airflow/ui/src/components/Graph/graphTypes.ts b/airflow-core/src/airflow/ui/src/components/Graph/graphTypes.ts index 280bc520d7960..c008b5dfc2ea9 100644 --- a/airflow-core/src/airflow/ui/src/components/Graph/graphTypes.ts +++ b/airflow-core/src/airflow/ui/src/components/Graph/graphTypes.ts @@ -33,7 +33,9 @@ export const nodeTypes = { "asset-uri-ref": DefaultNode, dag: DagNode, join: JoinNode, + sensor: DefaultNode, task: TaskNode, + trigger: DefaultNode, }; export const edgeTypes = { custom: Edge }; From 040d74e984c34caca0dd70a83c02ff604ad8080d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 18 Apr 2026 10:13:41 +0200 Subject: [PATCH 016/309] [v3-2-test] Sync local virtualenv before mypy and freeze uv.lock hook (#65326) (#65334) Mypy checks for non-provider projects now synchronize the local virtualenv with uv.lock (uv sync --frozen) before running, so contributors see the same dependency set CI uses and avoid results that drift from CI. The update-uv-lock prek hook now runs with --frozen, so pyproject.toml changes that would touch uv.lock fail the hook and require an explicit uv lock + commit instead of silently rewriting the lock during a commit. (cherry picked from commit 9b08d0586c94d411e3db8d4865fd1a318b35421c) Co-authored-by: Jarek Potiuk --- .pre-commit-config.yaml | 10 ++++--- AGENTS.md | 2 +- contributing-docs/08_static_code_checks.rst | 11 ++++++- scripts/ci/prek/mypy_local_folder.py | 33 +++++++++++++++++++-- 4 files changed, 47 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 748cf045d84fa..ad5d533b2f412 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1006,12 +1006,14 @@ repos: pass_filenames: false files: ^dev/breeze/src/airflow_breeze/global_constants\.py$ require_serial: true - # This is a fast regular hook that runs when any pyproject.toml changes - # It runs locally and usually will not result in modifying the lock unnecessarily - # Unless there is a conflict and uv will determine that the lock needs to be updated to resolve it + # This is a fast regular hook that runs when any pyproject.toml changes. + # It runs locally with `--frozen` so the lock is only verified against pyproject.toml, + # never silently rewritten during a commit. If a pyproject.toml change makes the lock + # stale, the hook fails and the contributor must run `uv lock` explicitly and commit + # the refreshed uv.lock — which keeps lock updates intentional and reviewable. - id: update-uv-lock name: Update uv.lock - entry: uv lock + entry: uv lock --frozen language: system files: > (?x) diff --git a/AGENTS.md b/AGENTS.md index 36af179f50b9d..f38bc2dd33703 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,7 +30,7 @@ - **Run other suites of tests** `breeze testing ` (test groups: `airflow-ctl-tests`, `docker-compose-tests`, `task-sdk-tests`) - **Run scripts tests:** `uv run --project scripts pytest scripts/tests/ -xvs` - **Run Airflow CLI:** `breeze run airflow dags list` -- **Type-check (non-providers):** `uv run --project --with "apache-airflow-devel-common[mypy]" mypy path/to/code` +- **Type-check (non-providers):** first run `uv sync --frozen --project ` to align the local virtualenv with `uv.lock` (the dependency set CI uses), then `uv run --frozen --project --with "apache-airflow-devel-common[mypy]" mypy path/to/code` - **Type-check (providers):** `breeze run mypy path/to/code` - **Lint with ruff only:** `prek run ruff --from-ref ` - **Format with ruff only:** `prek run ruff-format --from-ref ` diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index 23256c6c0e026..4eed1a98a13f9 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -287,7 +287,16 @@ For **non-provider projects** (airflow-core, task-sdk, airflow-ctl, dev, scripts runs locally using the ``uv`` virtualenv — no breeze CI image is needed. These checks run as regular prek hooks in the ``pre-commit`` stage, checking whole directories at once. This means they run both as part of local commits and as part of regular static checks in CI (not as separate mypy CI jobs). -You can also run mypy directly. Use ``--frozen`` to avoid updating ``uv.lock``: + +Before running mypy directly (or via the ``mypy-*`` prek hooks), synchronize your local virtualenv +with ``uv.lock`` so it matches the dependency set CI uses — otherwise mypy may pick up a different +set of installed packages than CI and produce results that diverge from CI: + +.. code-block:: bash + + uv sync --frozen --project + +Then run mypy directly. Use ``--frozen`` so ``uv`` does not update ``uv.lock``: .. code-block:: bash diff --git a/scripts/ci/prek/mypy_local_folder.py b/scripts/ci/prek/mypy_local_folder.py index e578079b11fa8..568fe1c06e49f 100755 --- a/scripts/ci/prek/mypy_local_folder.py +++ b/scripts/ci/prek/mypy_local_folder.py @@ -186,7 +186,33 @@ def get_all_files(folder: str) -> list[str]: }, ) else: - # Locally, run via uv with --frozen to not update the lock file. + # Locally, first synchronize the project's virtualenv with uv.lock so that mypy runs + # against the same dependency set CI uses. Without this, the local .venv can drift from + # uv.lock (e.g. after switching branches or installing extras) and mypy results would + # diverge from CI. --frozen ensures uv.lock itself is not updated. + sync_cmd = ["uv", "sync", "--frozen", "--project", project] + if console: + console.print(f"[magenta]Syncing virtualenv for project {project}: {' '.join(sync_cmd)}[/]") + else: + print(f"Syncing virtualenv for project {project}: {' '.join(sync_cmd)}") + sync_result = subprocess.run( + sync_cmd, + cwd=str(AIRFLOW_ROOT_PATH), + check=False, + env={**os.environ, "TERM": "ansi"}, + ) + if sync_result.returncode != 0: + msg = ( + f"`uv sync --frozen --project {project}` failed. Fix the sync error before running mypy — " + "otherwise the local virtualenv may not match uv.lock and mypy results will diverge from CI.\n" + ) + if console: + console.print(f"[red]{msg}") + else: + print(msg) + sys.exit(sync_result.returncode) + + # Then run mypy via uv with --frozen to not update the lock file. cmd = [ "uv", "run", @@ -211,8 +237,9 @@ def get_all_files(folder: str) -> list[str]: msg = ( "Mypy check failed. You can run mypy locally with:\n" f" prek run mypy-{mypy_folders[0]} --all-files\n" - "Or directly with:\n" - f' uv run --project {project} --with "apache-airflow-devel-common[mypy]" mypy \n' + "Or directly (first sync the virtualenv to match CI's dependency set):\n" + f" uv sync --frozen --project {project}\n" + f' uv run --frozen --project {project} --with "apache-airflow-devel-common[mypy]" mypy \n' "You can also clear the mypy cache with:\n" " breeze down --cleanup-mypy-cache\n" ) From 899bbf59cb69fc9c7227a92fec7c089075f292f9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 18 Apr 2026 23:00:52 +0200 Subject: [PATCH 017/309] [v3-2-test] Avoid false recovery alerts when failed job lookup fails (#64863) (#65473) * CI: Avoid false recovery alerts when failed job lookup fails * Potential fix for pull request finding --------- (cherry picked from commit b41b11d3fc18c23c40703808432f368c321887fe) Co-authored-by: Henry Chen Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- dev/breeze/src/airflow_breeze/utils/workflow_status.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dev/breeze/src/airflow_breeze/utils/workflow_status.py b/dev/breeze/src/airflow_breeze/utils/workflow_status.py index 6ed3bd55390e0..912837e09d63d 100644 --- a/dev/breeze/src/airflow_breeze/utils/workflow_status.py +++ b/dev/breeze/src/airflow_breeze/utils/workflow_status.py @@ -33,6 +33,13 @@ console = Console(width=400, color_system="standard") +def format_failed_jobs_lookup_error(run_id: int) -> str: + return ( + "ERROR: failed to fetch failed jobs for workflow run " + f"{run_id}; inspect it directly with: gh run view {run_id} --json jobs --repo apache/airflow" + ) + + def workflow_status( branch: str, workflow_id: str, @@ -93,7 +100,7 @@ def get_failed_jobs(run_id: int) -> list[str]: ) if result.returncode != 0: console.print(f"[red]Error fetching failed jobs: {result.stderr}[/red]") - return [] + return [format_failed_jobs_lookup_error(run_id)] return [line.strip() for line in result.stdout.strip().splitlines() if line.strip()] From 59eac52b18d9ae86bb5db84dbc69679e21f7a541 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 19 Apr 2026 10:36:54 +0200 Subject: [PATCH 018/309] [v3-2-test] Use monotonic clock for prek command timing (#65481) (#65484) (cherry picked from commit 5709502ecc2e2452ba59449f4a748a7aad8d0d5c) Co-authored-by: Henry Chen --- scripts/ci/prek/common_prek_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/prek/common_prek_utils.py b/scripts/ci/prek/common_prek_utils.py index 3559e523cf09d..d7a4fe9b5d20c 100644 --- a/scripts/ci/prek/common_prek_utils.py +++ b/scripts/ci/prek/common_prek_utils.py @@ -91,9 +91,9 @@ def run_command(*args, **kwargs) -> None: print("#" * min(len(text), 200), file=sys.stderr) print(text, file=sys.stderr) print("#" * min(len(text), 200), file=sys.stderr) - time_start = time.time() + time_start = time.monotonic() subprocess.check_call(*args, **kwargs) - time_end = time.time() + time_end = time.monotonic() if console: console.print(f"[green]After {text}[/]") console.print(f"[green]Command finished in {time_end - time_start:.2f} seconds[/]") From c1449e72a5f8bb63940f5053afa1e3fe442ec3d2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 19 Apr 2026 18:20:02 +0200 Subject: [PATCH 019/309] [v3-2-test] Stop masking quarantined unit test failures (#65500) (#65502) (cherry picked from commit 67e7cc1912c47b6a87657b5c78a02248750b5bc1) Co-authored-by: Henry Chen --- scripts/ci/testing/run_unit_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/testing/run_unit_tests.sh b/scripts/ci/testing/run_unit_tests.sh index 3a4069cdf0761..92a5a29be40d6 100755 --- a/scripts/ci/testing/run_unit_tests.sh +++ b/scripts/ci/testing/run_unit_tests.sh @@ -50,7 +50,7 @@ function core_tests() { set +x elif [[ "${TEST_SCOPE}" == "Quarantined" ]]; then set -x - breeze testing core-tests --test-type "All-Quarantined" || true + breeze testing core-tests --test-type "All-Quarantined" RESULT=$? set +x elif [[ "${TEST_SCOPE}" == "System" ]]; then @@ -93,7 +93,7 @@ function providers_tests() { set +x elif [[ "${TEST_SCOPE}" == "Quarantined" ]]; then set -x - breeze testing providers-tests --test-type "All-Quarantined" || true + breeze testing providers-tests --test-type "All-Quarantined" RESULT=$? set +x else From f7d2dcf748258ac78d038a7a32ee45dc38f48faf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 19 Apr 2026 23:33:09 +0200 Subject: [PATCH 020/309] [v3-2-test] Fix airflow-ctl release verification instructions (#65510) (#65512) (cherry picked from commit 57f54cdd844d2982f45d778050a3870e65e8b300) Co-authored-by: Shahar Epstein <60007259+shahar1@users.noreply.github.com> --- dev/README_RELEASE_AIRFLOWCTL.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/README_RELEASE_AIRFLOWCTL.md b/dev/README_RELEASE_AIRFLOWCTL.md index 2d373e3f7b9f6..c38fbc48c043a 100644 --- a/dev/README_RELEASE_AIRFLOWCTL.md +++ b/dev/README_RELEASE_AIRFLOWCTL.md @@ -494,6 +494,7 @@ cd "${AIRFLOW_REPO_ROOT}" Choose the tag you used for release: ```shell +cd "${AIRFLOW_REPO_ROOT}" git fetch apache --tags --force git checkout airflow-ctl/${VERSION_RC} ``` @@ -559,7 +560,7 @@ tar -xzf /tmp/apache-rat-0.17-bin.tar.gz -C /tmp Unpack the release source archive (the `-source.tar.gz` file) to a folder ```shell script -rm -rf /tmp/apache/airflow-src && mkdir -p /tmp/apache-airflow-src && tar -xzf ${PATH_TO_AIRFLOW_SVN}/${VERSION_RC}/apache_airflow*-source.tar.gz --strip-components 1 -C /tmp/apache-airflow-src +rm -rf /tmp/apache-airflow-src && mkdir -p /tmp/apache-airflow-src && tar -xzf ${PATH_TO_AIRFLOW_SVN}/airflow-ctl/${VERSION_RC}/apache_airflow*-source.tar.gz --strip-components 1 -C /tmp/apache-airflow-src ``` Run the check: From 0c908371dd6a659f0646123ab2f13ac260981d42 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 06:36:22 +0200 Subject: [PATCH 021/309] [v3-2-test] Revert "Stop masking quarantined unit test failures (#65500)" (#65515) (#65516) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Revert "Stop masking quarantined unit test failures (#65500)" This reverts commit 67e7cc1912c47b6a87657b5c78a02248750b5bc1. * Document why quarantined tests use `|| true` Adds a short comment next to both `breeze testing ... --test-type "All-Quarantined" || true` calls explaining that the shell `|| true` is intentional — quarantined tests are known-flaky and must not fail the overall CI run; they are reported separately. (cherry picked from commit b7fcda5136f0600ce74f600129ac800a43177df1) Co-authored-by: Jarek Potiuk --- scripts/ci/testing/run_unit_tests.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/ci/testing/run_unit_tests.sh b/scripts/ci/testing/run_unit_tests.sh index 92a5a29be40d6..1858b09adbae0 100755 --- a/scripts/ci/testing/run_unit_tests.sh +++ b/scripts/ci/testing/run_unit_tests.sh @@ -50,7 +50,9 @@ function core_tests() { set +x elif [[ "${TEST_SCOPE}" == "Quarantined" ]]; then set -x - breeze testing core-tests --test-type "All-Quarantined" + # The `|| true` is deliberate — quarantined tests are known-flaky and we do not want a + # failing run here to fail the whole CI job; they are reported separately. Do not remove. + breeze testing core-tests --test-type "All-Quarantined" || true RESULT=$? set +x elif [[ "${TEST_SCOPE}" == "System" ]]; then @@ -93,7 +95,9 @@ function providers_tests() { set +x elif [[ "${TEST_SCOPE}" == "Quarantined" ]]; then set -x - breeze testing providers-tests --test-type "All-Quarantined" + # The `|| true` is deliberate — quarantined tests are known-flaky and we do not want a + # failing run here to fail the whole CI job; they are reported separately. Do not remove. + breeze testing providers-tests --test-type "All-Quarantined" || true RESULT=$? set +x else From c43bf46a2403b76b1942ad6ee64bea5d29eac1ac Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Mon, 20 Apr 2026 15:57:38 +0200 Subject: [PATCH 022/309] Add cursor based pagination for get_task_instances endpoint (#64845) (#65405) * Add cursor-based pagination to get_task_instances endpoint - Add cursor-based (keyset) pagination as an alternative to offset-based pagination on the get_task_instances endpoint. Offset pagination remains the default and is not deprecated globally. - Response uses a discriminated union: offset responses include total_entries, cursor responses include next_cursor and previous_cursor. - Refactor SortParam to lazily cache column resolution instead of mutating state in to_orm. - Move cursor helpers (encode/decode/apply) to dedicated common/db/cursors.py module. - Cleanly separate cursor vs offset code paths in the endpoint handler. * Simplify cursor token and support first page without sentinel - Remove order_by from cursor token (now just a list of values) - Support empty string cursor for first page (no fake sentinel needed) - Drop order_by consistency check between cursor and query param * Small adjustments * Adjustments * Narrow endpoint return types and encode cursor value types Encode type information directly into cursor tokens as {"type": ..., "value": ...} objects, removing the fragile column-based type guessing during deserialization. Narrow return types for endpoints that only return offset pagination (patch, clear, batch, mapped) so the OpenAPI spec and generated UI client reflect the correct types. Only get_task_instances retains the discriminated union response. Update UI components to use the narrowed types from the spec. * Use msgpack for cursor tokens and nested keyset predicate Switch cursor encoding from typed JSON to msgpack for compactness. Replace flat OR-of-prefix-equalities with nested and/or keyset predicate for better composite index range scans. Always use ascending PK as the final tie-breaker for stable pagination. * Flatten TaskInstanceCollectionRes ponse to avoid oneOf codegen issues Replace the discriminated union (offset | cursor response types) with a single flat model using optional fields. OpenAPI oneOf + discriminator is not handled correctly by hey-api/openapi-ts (#1613, #3270): return types degrade to unknown in generated TypeScript code. * Fix UI * Fix CI * Fix cursor pagination boundary detection and error handling - Fetch limit+1 rows to accurately detect last page, returning next_cursor=null when no more results exist - Return previous_cursor=null on the first page (when no cursor was provided) - Use LimitFilter in apply_filters_to_select for the +1 limit instead of a manual .limit() call - Raise HTTP 400 on invalid UUID in cursor token instead of silently passing the invalid value - Update endpoint docs and add boundary-condition test * Fix backward cursor based pagination (cherry picked from commit e11c6037f3a8d53abb8c03e6010469133621929c) --- .../src/airflow/api_fastapi/common/cursors.py | 169 ++++++++++++++++++ .../airflow/api_fastapi/common/parameters.py | 53 ++++-- .../core_api/datamodels/task_instances.py | 27 ++- .../openapi/v2-rest-api-generated.yaml | 74 +++++++- .../core_api/routes/public/task_instances.py | 127 +++++++++---- .../airflow/ui/openapi-gen/queries/common.ts | 5 +- .../ui/openapi-gen/queries/ensureQueryData.ts | 19 +- .../ui/openapi-gen/queries/prefetch.ts | 19 +- .../airflow/ui/openapi-gen/queries/queries.ts | 19 +- .../ui/openapi-gen/queries/suspense.ts | 19 +- .../ui/openapi-gen/requests/schemas.gen.ts | 47 ++++- .../ui/openapi-gen/requests/services.gen.ts | 15 +- .../ui/openapi-gen/requests/types.gen.ts | 26 ++- .../ActionAccordion/ActionAccordion.tsx | 2 +- .../ClearGroupTaskInstanceDialog.tsx | 2 +- .../ui/src/pages/Dag/Overview/Overview.tsx | 8 +- .../ui/src/pages/Task/Overview/Overview.tsx | 12 +- .../src/pages/TaskInstances/TaskInstances.tsx | 2 +- .../unit/api_fastapi/common/test_cursors.py | 144 +++++++++++++++ .../api_fastapi/common/test_parameters.py | 7 +- .../routes/public/test_task_instances.py | 152 +++++++++++++++- .../airflowctl/api/datamodels/generated.py | 31 +++- 22 files changed, 883 insertions(+), 96 deletions(-) create mode 100644 airflow-core/src/airflow/api_fastapi/common/cursors.py create mode 100644 airflow-core/tests/unit/api_fastapi/common/test_cursors.py diff --git a/airflow-core/src/airflow/api_fastapi/common/cursors.py b/airflow-core/src/airflow/api_fastapi/common/cursors.py new file mode 100644 index 0000000000000..0d06aa10b2ede --- /dev/null +++ b/airflow-core/src/airflow/api_fastapi/common/cursors.py @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Cursor-based (keyset) pagination helpers. + +:meta private: +""" + +from __future__ import annotations + +import base64 +import uuid as uuid_mod +from typing import Any + +import msgspec +from fastapi import HTTPException, status +from sqlalchemy import and_, or_ +from sqlalchemy.sql import Select +from sqlalchemy.sql.elements import ColumnElement +from sqlalchemy.sql.sqltypes import Uuid + +from airflow.api_fastapi.common.parameters import SortParam + + +def _b64url_decode_padded(token: str) -> bytes: + padding = 4 - (len(token) % 4) + if padding != 4: + token = token + ("=" * padding) + return base64.urlsafe_b64decode(token.encode("ascii")) + + +def _nonstrict_bound(col: ColumnElement, value: Any, is_desc: bool) -> ColumnElement[bool]: + """Inclusive range edge on the leading column at each nesting level (``>=`` / ``<=``).""" + return col <= value if is_desc else col >= value + + +def _strict_bound(col: ColumnElement, value: Any, is_desc: bool) -> ColumnElement[bool]: + """Strict inequality for ``or_`` branches (``<`` / ``>``).""" + return col < value if is_desc else col > value + + +def _nested_keyset_predicate( + resolved: list[tuple[str, ColumnElement, bool]], values: list[Any] +) -> ColumnElement[bool]: + """ + Keyset predicate for rows strictly after the cursor in ``ORDER BY`` order. + + Uses nested ``and_(non-strict, or_(strict, ...))`` so leading sort keys use + inclusive range bounds and inner branches use strict inequalities—friendly + for composite index range scans. Logically equivalent to an OR-of-prefix- + equalities formulation. + """ + n = len(resolved) + _, col, is_desc = resolved[n - 1] + inner: ColumnElement[bool] = _strict_bound(col, values[n - 1], is_desc) + for i in range(n - 2, -1, -1): + _, col_i, is_desc_i = resolved[i] + inner = and_( + _nonstrict_bound(col_i, values[i], is_desc_i), + or_(_strict_bound(col_i, values[i], is_desc_i), inner), + ) + return inner + + +def _coerce_value(column: ColumnElement, value: Any) -> Any: + """Normalize decoded values for SQL bind parameters (e.g. UUID columns).""" + if value is None or not isinstance(value, str): + return value + ctype = getattr(column, "type", None) + if isinstance(ctype, Uuid): + try: + return uuid_mod.UUID(value) + except ValueError: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid cursor token") + return value + + +_BACKWARD_PREFIX = "~" + + +def encode_cursor(row: Any, sort_param: SortParam) -> str: + """ + Encode cursor token from the boundary row of a result set. + + The token is a URL-safe base64 encoding of a MessagePack list of sort-key + values (no padding ``=``). + """ + resolved = sort_param.get_resolved_columns() + if not resolved: + raise ValueError("SortParam has no resolved columns.") + + parts = [getattr(row, attr_name, None) for attr_name, _col, _desc in resolved] + payload = msgspec.msgpack.encode(parts) + return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=") + + +def make_backward_cursor(token: str) -> str: + """Prefix a cursor token with the backward direction marker (``~``).""" + return f"{_BACKWARD_PREFIX}{token}" + + +def parse_cursor(cursor: str) -> tuple[str, bool]: + """ + Parse a raw cursor string into ``(token, is_backward)``. + + Strips the ``~`` prefix if present and returns whether the cursor + represents a backward (previous-page) direction. + """ + if cursor.startswith(_BACKWARD_PREFIX): + return cursor[len(_BACKWARD_PREFIX) :], True + return cursor, False + + +def decode_cursor(token: str) -> list[Any]: + """Decode a cursor token to the list of sort-key values.""" + try: + raw = _b64url_decode_padded(token) + except Exception: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid cursor token") + + try: + data: Any = msgspec.msgpack.decode(raw) + except Exception: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid cursor token") + + if not isinstance(data, list): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid cursor token structure") + + return data + + +def apply_cursor_filter( + statement: Select, token: str, sort_param: SortParam, *, is_backward: bool = False +) -> Select: + """ + Apply a keyset pagination WHERE clause from a cursor token. + + For forward cursors the predicate selects rows strictly *after* the cursor + in ORDER BY order. When *is_backward* is True the ``is_desc`` flags are + flipped so the predicate selects rows strictly *before* the cursor in the + original sort order. The caller is responsible for reversing the ORDER BY + and the final result list when using a backward cursor. + """ + raw_values = decode_cursor(token) + + resolved = sort_param.get_resolved_columns() + if len(raw_values) != len(resolved): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Cursor token does not match current query shape") + + parsed_values = [_coerce_value(col, val) for (_, col, _), val in zip(resolved, raw_values, strict=True)] + + if is_backward: + resolved = [(name, col, not is_desc) for name, col, is_desc in resolved] + + return statement.where(_nested_keyset_predicate(resolved, parsed_values)) diff --git a/airflow-core/src/airflow/api_fastapi/common/parameters.py b/airflow-core/src/airflow/api_fastapi/common/parameters.py index 065dff8cda8bb..91250b670cf98 100644 --- a/airflow-core/src/airflow/api_fastapi/common/parameters.py +++ b/airflow-core/src/airflow/api_fastapi/common/parameters.py @@ -119,7 +119,10 @@ def to_orm(self, select: Select) -> Select: return select.offset(self.value) @classmethod - def depends(cls, offset: NonNegativeInt = 0) -> OffsetFilter: + def depends( + cls, + offset: NonNegativeInt = 0, + ) -> OffsetFilter: return cls().set_value(offset) @@ -281,10 +284,16 @@ def __init__( self.allowed_attrs = allowed_attrs self.model = model self.to_replace = to_replace + self._cached_resolution: list[tuple[str, ColumnElement, bool]] | None = None - def to_orm(self, select: Select) -> Select: - if self.skip_none is False: - raise ValueError(f"Cannot set 'skip_none' to False on a {type(self)}") + def set_value(self, value: list[str] | None) -> Self: + self._cached_resolution = None + return super().set_value(value) + + def _resolve(self) -> list[tuple[str, ColumnElement, bool]]: + """Resolve sort columns as (attr_name, column, is_descending) tuples. Cached after first call.""" + if self._cached_resolution is not None: + return self._cached_resolution if self.value is None: self.value = [self.get_primary_key_string()] @@ -296,9 +305,10 @@ def to_orm(self, select: Select) -> Select: f"Ordering with more than {self.MAX_SORT_PARAMS} parameters is not allowed. Provided: {order_by_values}", ) - columns: list[ColumnElement] = [] + resolved: list[tuple[str, ColumnElement, bool]] = [] for order_by_value in order_by_values: lstriped_orderby = order_by_value.lstrip("-") + attr_name = lstriped_orderby column: Column | None = None if self.to_replace: replacement = self.to_replace.get(lstriped_orderby, lstriped_orderby) @@ -316,22 +326,31 @@ def to_orm(self, select: Select) -> Select: if column is None: column = getattr(self.model, lstriped_orderby) - if order_by_value.startswith("-"): - columns.append(column.desc()) - else: - columns.append(column.asc()) - - # Reset default sorting - select = select.order_by(None) + resolved.append((attr_name, column, order_by_value.startswith("-"))) primary_key_column = self.get_primary_key_column() - # Always add a final discriminator to enforce deterministic ordering. - if order_by_values and order_by_values[0].startswith("-"): - columns.append(primary_key_column.desc()) + pk_name = self.get_primary_key_string() + if not any(name == pk_name for name, _, _ in resolved): + pk_desc = bool(order_by_values and order_by_values[0].startswith("-")) + resolved.append((pk_name, primary_key_column, pk_desc)) + + self._cached_resolution = resolved + return self._cached_resolution + + def to_orm(self, select: Select, *, reversed: bool = False) -> Select: + if self.skip_none is False: + raise ValueError(f"Cannot set 'skip_none' to False on a {type(self)}") + + resolved = self._resolve() + if reversed: + columns = [col.asc() if is_desc else col.desc() for _, col, is_desc in resolved] else: - columns.append(primary_key_column.asc()) + columns = [col.desc() if is_desc else col.asc() for _, col, is_desc in resolved] + return select.order_by(None).order_by(*columns) - return select.order_by(*columns) + def get_resolved_columns(self) -> list[tuple[str, ColumnElement, bool]]: + """Return resolved sort columns as (attr_name, column_element, is_descending) tuples.""" + return self._resolve() def get_primary_key_column(self) -> Column: """Get the primary key column of the model of SortParam object.""" diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/task_instances.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/task_instances.py index 397389994a09f..979218f315144 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/task_instances.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/task_instances.py @@ -83,10 +83,33 @@ class TaskInstanceResponse(BaseModel): class TaskInstanceCollectionResponse(BaseModel): - """Task Instance Collection serializer for responses.""" + """ + Task instance collection response supporting both offset and cursor pagination. + + A single flat model is used instead of a discriminated union + (``Annotated[Offset | Cursor, Field(discriminator=...)]``) because + the OpenAPI ``oneOf`` + ``discriminator`` construct is not handled + correctly by ``@hey-api/openapi-ts`` / ``@7nohe/openapi-react-query-codegen``: + return types degrade to ``unknown`` in JSDoc and can produce + incorrect TypeScript types (see hey-api/openapi-ts#1613, #3270). + """ task_instances: Iterable[TaskInstanceResponse] - total_entries: int + total_entries: int | None = Field( + default=None, + description="Total number of matching items. Populated for offset pagination, " + "``null`` when using cursor pagination.", + ) + next_cursor: str | None = Field( + default=None, + description="Token pointing to the next page. Populated for cursor pagination, " + "``null`` when using offset pagination or when there is no next page.", + ) + previous_cursor: str | None = Field( + default=None, + description="Token pointing to the previous page. Populated for cursor pagination, " + "``null`` when using offset pagination or when on the first page.", + ) class TaskDependencyResponse(BaseModel): diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml index 24ab32f7113f7..59c5d1f9295c4 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml @@ -6524,10 +6524,27 @@ paths: description: 'Get list of task instances. - This endpoint allows specifying `~` as the dag_id, dag_run_id to retrieve - Task Instances for all DAGs + This endpoint allows specifying `~` as the dag_id, dag_run_id - and DAG runs.' + to retrieve task instances for all DAGs and DAG runs. + + + Supports two pagination modes: + + + **Offset (default):** use `limit` and `offset` query parameters. Returns `total_entries`. + + + **Cursor:** pass `cursor` (empty string for the first page, then `next_cursor` + from the response). + + When `cursor` is provided, `offset` is ignored and `total_entries` is not + returned. + + ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` + is ``null`` + + on the first page.' operationId: get_task_instances security: - OAuth2PasswordBearer: [] @@ -6545,6 +6562,20 @@ paths: schema: type: string title: Dag Run Id + - name: cursor + in: query + required: false + schema: + anyOf: + - type: string + - type: 'null' + description: Cursor for keyset-based pagination. Pass an empty string for + the first page, then use ``next_cursor`` from the response. When ``cursor`` + is provided, ``offset`` is ignored. + title: Cursor + description: Cursor for keyset-based pagination. Pass an empty string for + the first page, then use ``next_cursor`` from the response. When ``cursor`` + is provided, ``offset`` is ignored. - name: task_id in: query required: false @@ -12569,14 +12600,45 @@ components: type: array title: Task Instances total_entries: - type: integer + anyOf: + - type: integer + - type: 'null' title: Total Entries + description: Total number of matching items. Populated for offset pagination, + ``null`` when using cursor pagination. + next_cursor: + anyOf: + - type: string + - type: 'null' + title: Next Cursor + description: Token pointing to the next page. Populated for cursor pagination, + ``null`` when using offset pagination or when there is no next page. + previous_cursor: + anyOf: + - type: string + - type: 'null' + title: Previous Cursor + description: Token pointing to the previous page. Populated for cursor pagination, + ``null`` when using offset pagination or when on the first page. type: object required: - task_instances - - total_entries title: TaskInstanceCollectionResponse - description: Task Instance Collection serializer for responses. + description: 'Task instance collection response supporting both offset and cursor + pagination. + + + A single flat model is used instead of a discriminated union + + (``Annotated[Offset | Cursor, Field(discriminator=...)]``) because + + the OpenAPI ``oneOf`` + ``discriminator`` construct is not handled + + correctly by ``@hey-api/openapi-ts`` / ``@7nohe/openapi-react-query-codegen``: + + return types degrade to ``unknown`` in JSDoc and can produce + + incorrect TypeScript types (see hey-api/openapi-ts#1613, #3270).' TaskInstanceHistoryCollectionResponse: properties: task_instances: diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/task_instances.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/task_instances.py index 97fa930b1c153..73ba522645f47 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/task_instances.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/task_instances.py @@ -26,13 +26,19 @@ from sqlalchemy.sql.selectable import Select from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity +from airflow.api_fastapi.common.cursors import ( + apply_cursor_filter, + encode_cursor, + make_backward_cursor, + parse_cursor, +) from airflow.api_fastapi.common.dagbag import ( DagBagDep, get_dag_for_run, get_dag_for_run_or_latest_version, get_latest_version_of_dag, ) -from airflow.api_fastapi.common.db.common import SessionDep, paginated_select +from airflow.api_fastapi.common.db.common import SessionDep, apply_filters_to_select, paginated_select from airflow.api_fastapi.common.db.task_instances import eager_load_TI_and_TIH_for_validation from airflow.api_fastapi.common.parameters import ( FilterOptionEnum, @@ -64,6 +70,7 @@ search_param_factory, ) from airflow.api_fastapi.common.router import AirflowRouter +from airflow.api_fastapi.core_api.base import OrmClause from airflow.api_fastapi.core_api.datamodels.common import BulkBody, BulkResponse from airflow.api_fastapi.core_api.datamodels.task_instance_history import ( TaskInstanceHistoryCollectionResponse, @@ -469,13 +476,29 @@ def get_task_instances( ], readable_ti_filter: ReadableTIFilterDep, session: SessionDep, + cursor: str | None = Query( + None, + description="Cursor for keyset-based pagination. " + "Pass an empty string for the first page, then use ``next_cursor`` from the response. " + "When ``cursor`` is provided, ``offset`` is ignored.", + ), ) -> TaskInstanceCollectionResponse: """ Get list of task instances. - This endpoint allows specifying `~` as the dag_id, dag_run_id to retrieve Task Instances for all DAGs - and DAG runs. + This endpoint allows specifying `~` as the dag_id, dag_run_id + to retrieve task instances for all DAGs and DAG runs. + + Supports two pagination modes: + + **Offset (default):** use `limit` and `offset` query parameters. Returns `total_entries`. + + **Cursor:** pass `cursor` (empty string for the first page, then `next_cursor` from the response). + When `cursor` is provided, `offset` is ignored and `total_entries` is not returned. + ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is ``null`` + on the first page. """ + use_cursor = cursor is not None dag_run = None query = eager_load_TI_and_TIH_for_validation(select(TI)) if dag_run_id != "~": @@ -497,40 +520,84 @@ def get_task_instances( if dag: task_group_id.dag = dag + filters: list[OrmClause] = [ + run_after_range, + logical_date_range, + start_date_range, + end_date_range, + update_at_range, + duration_range, + state, + pool, + pool_name_pattern, + queue, + queue_name_pattern, + executor, + task_id, + task_display_name_pattern, + task_group_id, + dag_id_pattern, + run_id_pattern, + version_number, + readable_ti_filter, + try_number, + operator, + operator_name_pattern, + map_index, + ] + + if use_cursor: + # Fetch one extra row so we can detect whether a next page exists. + page_limit = cast( + "int", limit.value + ) # LimitFilter value is guaranteed to be set of the default value of QueryLimit + cursor_limit = LimitFilter().set_value(page_limit + 1) + task_instance_select = apply_filters_to_select( + statement=query, filters=[*filters, order_by, cursor_limit] + ) + + is_backward = False + if cursor: + token, is_backward = parse_cursor(cursor) + if is_backward: + task_instance_select = order_by.to_orm(task_instance_select, reversed=True) + task_instance_select = apply_cursor_filter( + task_instance_select, token, order_by, is_backward=is_backward + ) + + fetched = list(session.scalars(task_instance_select)) + has_more = len(fetched) > page_limit + task_instances = fetched[:page_limit] + + if is_backward: + task_instances.reverse() + has_prev = has_more + has_next = True + else: + has_prev = bool(cursor) + has_next = has_more + + return TaskInstanceCollectionResponse( + task_instances=task_instances, + next_cursor=( + encode_cursor(task_instances[-1], order_by) if has_next and task_instances else None + ), + previous_cursor=( + make_backward_cursor(encode_cursor(task_instances[0], order_by)) + if has_prev and task_instances + else None + ), + ) + task_instance_select, total_entries = paginated_select( statement=query, - filters=[ - run_after_range, - logical_date_range, - start_date_range, - end_date_range, - update_at_range, - duration_range, - state, - pool, - pool_name_pattern, - queue, - queue_name_pattern, - executor, - task_id, - task_display_name_pattern, - task_group_id, - dag_id_pattern, - run_id_pattern, - version_number, - readable_ti_filter, - try_number, - operator, - operator_name_pattern, - map_index, - ], + filters=filters, order_by=order_by, offset=offset, limit=limit, session=session, ) - - task_instances = session.scalars(task_instance_select) + task_instances = list(session.scalars(task_instance_select)) return TaskInstanceCollectionResponse( task_instances=task_instances, total_entries=total_entries, diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts index c02f01452b39e..6c87c143157dc 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts @@ -478,7 +478,8 @@ export const UseTaskInstanceServiceGetMappedTaskInstanceKeyFn = ({ dagId, dagRun export type TaskInstanceServiceGetTaskInstancesDefaultResponse = Awaited>; export type TaskInstanceServiceGetTaskInstancesQueryResult = UseQueryResult; export const useTaskInstanceServiceGetTaskInstancesKey = "TaskInstanceServiceGetTaskInstances"; -export const UseTaskInstanceServiceGetTaskInstancesKeyFn = ({ dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }: { +export const UseTaskInstanceServiceGetTaskInstancesKeyFn = ({ cursor, dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }: { + cursor?: string; dagId: string; dagIdPattern?: string; dagRunId: string; @@ -524,7 +525,7 @@ export const UseTaskInstanceServiceGetTaskInstancesKeyFn = ({ dagId, dagIdPatter updatedAtLt?: string; updatedAtLte?: string; versionNumber?: number[]; -}, queryKey?: Array) => [useTaskInstanceServiceGetTaskInstancesKey, ...(queryKey ?? [{ dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }])]; +}, queryKey?: Array) => [useTaskInstanceServiceGetTaskInstancesKey, ...(queryKey ?? [{ cursor, dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }])]; export type TaskInstanceServiceGetTaskInstanceTryDetailsDefaultResponse = Awaited>; export type TaskInstanceServiceGetTaskInstanceTryDetailsQueryResult = UseQueryResult; export const useTaskInstanceServiceGetTaskInstanceTryDetailsKey = "TaskInstanceServiceGetTaskInstanceTryDetails"; diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts index c0ca5feb6cbd4..23e47e6e21f9a 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts @@ -902,11 +902,21 @@ export const ensureUseTaskInstanceServiceGetMappedTaskInstanceData = (queryClien * Get Task Instances * Get list of task instances. * -* This endpoint allows specifying `~` as the dag_id, dag_run_id to retrieve Task Instances for all DAGs -* and DAG runs. +* This endpoint allows specifying `~` as the dag_id, dag_run_id +* to retrieve task instances for all DAGs and DAG runs. +* +* Supports two pagination modes: +* +* **Offset (default):** use `limit` and `offset` query parameters. Returns `total_entries`. +* +* **Cursor:** pass `cursor` (empty string for the first page, then `next_cursor` from the response). +* When `cursor` is provided, `offset` is ignored and `total_entries` is not returned. +* ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is ``null`` +* on the first page. * @param data The data for the request. * @param data.dagId * @param data.dagRunId +* @param data.cursor Cursor for keyset-based pagination. Pass an empty string for the first page, then use ``next_cursor`` from the response. When ``cursor`` is provided, ``offset`` is ignored. * @param data.taskId * @param data.runAfterGte * @param data.runAfterGt @@ -953,7 +963,8 @@ export const ensureUseTaskInstanceServiceGetMappedTaskInstanceData = (queryClien * @returns TaskInstanceCollectionResponse Successful Response * @throws ApiError */ -export const ensureUseTaskInstanceServiceGetTaskInstancesData = (queryClient: QueryClient, { dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }: { +export const ensureUseTaskInstanceServiceGetTaskInstancesData = (queryClient: QueryClient, { cursor, dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }: { + cursor?: string; dagId: string; dagIdPattern?: string; dagRunId: string; @@ -999,7 +1010,7 @@ export const ensureUseTaskInstanceServiceGetTaskInstancesData = (queryClient: Qu updatedAtLt?: string; updatedAtLte?: string; versionNumber?: number[]; -}) => queryClient.ensureQueryData({ queryKey: Common.UseTaskInstanceServiceGetTaskInstancesKeyFn({ dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }), queryFn: () => TaskInstanceService.getTaskInstances({ dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }) }); +}) => queryClient.ensureQueryData({ queryKey: Common.UseTaskInstanceServiceGetTaskInstancesKeyFn({ cursor, dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }), queryFn: () => TaskInstanceService.getTaskInstances({ cursor, dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }) }); /** * Get Task Instance Try Details * Get task instance details by try number. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts index d7c8731e6a686..f9d107a5b1df2 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts @@ -902,11 +902,21 @@ export const prefetchUseTaskInstanceServiceGetMappedTaskInstance = (queryClient: * Get Task Instances * Get list of task instances. * -* This endpoint allows specifying `~` as the dag_id, dag_run_id to retrieve Task Instances for all DAGs -* and DAG runs. +* This endpoint allows specifying `~` as the dag_id, dag_run_id +* to retrieve task instances for all DAGs and DAG runs. +* +* Supports two pagination modes: +* +* **Offset (default):** use `limit` and `offset` query parameters. Returns `total_entries`. +* +* **Cursor:** pass `cursor` (empty string for the first page, then `next_cursor` from the response). +* When `cursor` is provided, `offset` is ignored and `total_entries` is not returned. +* ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is ``null`` +* on the first page. * @param data The data for the request. * @param data.dagId * @param data.dagRunId +* @param data.cursor Cursor for keyset-based pagination. Pass an empty string for the first page, then use ``next_cursor`` from the response. When ``cursor`` is provided, ``offset`` is ignored. * @param data.taskId * @param data.runAfterGte * @param data.runAfterGt @@ -953,7 +963,8 @@ export const prefetchUseTaskInstanceServiceGetMappedTaskInstance = (queryClient: * @returns TaskInstanceCollectionResponse Successful Response * @throws ApiError */ -export const prefetchUseTaskInstanceServiceGetTaskInstances = (queryClient: QueryClient, { dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }: { +export const prefetchUseTaskInstanceServiceGetTaskInstances = (queryClient: QueryClient, { cursor, dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }: { + cursor?: string; dagId: string; dagIdPattern?: string; dagRunId: string; @@ -999,7 +1010,7 @@ export const prefetchUseTaskInstanceServiceGetTaskInstances = (queryClient: Quer updatedAtLt?: string; updatedAtLte?: string; versionNumber?: number[]; -}) => queryClient.prefetchQuery({ queryKey: Common.UseTaskInstanceServiceGetTaskInstancesKeyFn({ dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }), queryFn: () => TaskInstanceService.getTaskInstances({ dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }) }); +}) => queryClient.prefetchQuery({ queryKey: Common.UseTaskInstanceServiceGetTaskInstancesKeyFn({ cursor, dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }), queryFn: () => TaskInstanceService.getTaskInstances({ cursor, dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }) }); /** * Get Task Instance Try Details * Get task instance details by try number. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts index 1af349e168998..c9dd84d145830 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts @@ -902,11 +902,21 @@ export const useTaskInstanceServiceGetMappedTaskInstance = = unknown[]>({ dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }: { +export const useTaskInstanceServiceGetTaskInstances = = unknown[]>({ cursor, dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }: { + cursor?: string; dagId: string; dagIdPattern?: string; dagRunId: string; @@ -999,7 +1010,7 @@ export const useTaskInstanceServiceGetTaskInstances = , "queryKey" | "queryFn">) => useQuery({ queryKey: Common.UseTaskInstanceServiceGetTaskInstancesKeyFn({ dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }, queryKey), queryFn: () => TaskInstanceService.getTaskInstances({ dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }) as TData, ...options }); +}, queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">) => useQuery({ queryKey: Common.UseTaskInstanceServiceGetTaskInstancesKeyFn({ cursor, dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }, queryKey), queryFn: () => TaskInstanceService.getTaskInstances({ cursor, dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }) as TData, ...options }); /** * Get Task Instance Try Details * Get task instance details by try number. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts index cfc50f2d88308..4727215089c47 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts @@ -902,11 +902,21 @@ export const useTaskInstanceServiceGetMappedTaskInstanceSuspense = = unknown[]>({ dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }: { +export const useTaskInstanceServiceGetTaskInstancesSuspense = = unknown[]>({ cursor, dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }: { + cursor?: string; dagId: string; dagIdPattern?: string; dagRunId: string; @@ -999,7 +1010,7 @@ export const useTaskInstanceServiceGetTaskInstancesSuspense = , "queryKey" | "queryFn">) => useSuspenseQuery({ queryKey: Common.UseTaskInstanceServiceGetTaskInstancesKeyFn({ dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }, queryKey), queryFn: () => TaskInstanceService.getTaskInstances({ dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }) as TData, ...options }); +}, queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">) => useSuspenseQuery({ queryKey: Common.UseTaskInstanceServiceGetTaskInstancesKeyFn({ cursor, dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }, queryKey), queryFn: () => TaskInstanceService.getTaskInstances({ cursor, dagId, dagIdPattern, dagRunId, durationGt, durationGte, durationLt, durationLte, endDateGt, endDateGte, endDateLt, endDateLte, executor, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, offset, operator, operatorNamePattern, orderBy, pool, poolNamePattern, queue, queueNamePattern, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, startDateGt, startDateGte, startDateLt, startDateLte, state, taskDisplayNamePattern, taskGroupId, taskId, tryNumber, updatedAtGt, updatedAtGte, updatedAtLt, updatedAtLte, versionNumber }) as TData, ...options }); /** * Get Task Instance Try Details * Get task instance details by try number. diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts index a129de40e7d15..79959b9ceb123 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -5295,14 +5295,53 @@ export const $TaskInstanceCollectionResponse = { title: 'Task Instances' }, total_entries: { - type: 'integer', - title: 'Total Entries' + anyOf: [ + { + type: 'integer' + }, + { + type: 'null' + } + ], + title: 'Total Entries', + description: 'Total number of matching items. Populated for offset pagination, ``null`` when using cursor pagination.' + }, + next_cursor: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Next Cursor', + description: 'Token pointing to the next page. Populated for cursor pagination, ``null`` when using offset pagination or when there is no next page.' + }, + previous_cursor: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Previous Cursor', + description: 'Token pointing to the previous page. Populated for cursor pagination, ``null`` when using offset pagination or when on the first page.' } }, type: 'object', - required: ['task_instances', 'total_entries'], + required: ['task_instances'], title: 'TaskInstanceCollectionResponse', - description: 'Task Instance Collection serializer for responses.' + description: `Task instance collection response supporting both offset and cursor pagination. + +A single flat model is used instead of a discriminated union +(\`\`Annotated[Offset | Cursor, Field(discriminator=...)]\`\`) because +the OpenAPI \`\`oneOf\`\` + \`\`discriminator\`\` construct is not handled +correctly by \`\`@hey-api/openapi-ts\`\` / \`\`@7nohe/openapi-react-query-codegen\`\`: +return types degrade to \`\`unknown\`\` in JSDoc and can produce +incorrect TypeScript types (see hey-api/openapi-ts#1613, #3270).` } as const; export const $TaskInstanceHistoryCollectionResponse = { diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts index a23f5dbd01247..b27ad6008e4e6 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts @@ -2319,11 +2319,21 @@ export class TaskInstanceService { * Get Task Instances * Get list of task instances. * - * This endpoint allows specifying `~` as the dag_id, dag_run_id to retrieve Task Instances for all DAGs - * and DAG runs. + * This endpoint allows specifying `~` as the dag_id, dag_run_id + * to retrieve task instances for all DAGs and DAG runs. + * + * Supports two pagination modes: + * + * **Offset (default):** use `limit` and `offset` query parameters. Returns `total_entries`. + * + * **Cursor:** pass `cursor` (empty string for the first page, then `next_cursor` from the response). + * When `cursor` is provided, `offset` is ignored and `total_entries` is not returned. + * ``next_cursor`` is ``null`` when there are no more pages; ``previous_cursor`` is ``null`` + * on the first page. * @param data The data for the request. * @param data.dagId * @param data.dagRunId + * @param data.cursor Cursor for keyset-based pagination. Pass an empty string for the first page, then use ``next_cursor`` from the response. When ``cursor`` is provided, ``offset`` is ignored. * @param data.taskId * @param data.runAfterGte * @param data.runAfterGt @@ -2379,6 +2389,7 @@ export class TaskInstanceService { dag_run_id: data.dagRunId }, query: { + cursor: data.cursor, task_id: data.taskId, run_after_gte: data.runAfterGte, run_after_gt: data.runAfterGt, diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts index fdb6e663d4804..c29ee5b71cd19 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts @@ -1374,11 +1374,29 @@ export type TaskInletAssetReference = { }; /** - * Task Instance Collection serializer for responses. + * Task instance collection response supporting both offset and cursor pagination. + * + * A single flat model is used instead of a discriminated union + * (``Annotated[Offset | Cursor, Field(discriminator=...)]``) because + * the OpenAPI ``oneOf`` + ``discriminator`` construct is not handled + * correctly by ``@hey-api/openapi-ts`` / ``@7nohe/openapi-react-query-codegen``: + * return types degrade to ``unknown`` in JSDoc and can produce + * incorrect TypeScript types (see hey-api/openapi-ts#1613, #3270). */ export type TaskInstanceCollectionResponse = { task_instances: Array; - total_entries: number; + /** + * Total number of matching items. Populated for offset pagination, ``null`` when using cursor pagination. + */ + total_entries?: number | null; + /** + * Token pointing to the next page. Populated for cursor pagination, ``null`` when using offset pagination or when there is no next page. + */ + next_cursor?: string | null; + /** + * Token pointing to the previous page. Populated for cursor pagination, ``null`` when using offset pagination or when on the first page. + */ + previous_cursor?: string | null; }; /** @@ -3063,6 +3081,10 @@ export type PatchTaskInstanceByMapIndexData = { export type PatchTaskInstanceByMapIndexResponse = TaskInstanceCollectionResponse; export type GetTaskInstancesData = { + /** + * Cursor for keyset-based pagination. Pass an empty string for the first page, then use ``next_cursor`` from the response. When ``cursor`` is provided, ``offset`` is ignored. + */ + cursor?: string | null; dagId: string; /** * SQL LIKE expression — use `%` / `_` wildcards (e.g. `%customer_%`). or the pipe `|` operator for OR logic (e.g. `dag1 | dag2`). Regular expressions are **not** supported. diff --git a/airflow-core/src/airflow/ui/src/components/ActionAccordion/ActionAccordion.tsx b/airflow-core/src/airflow/ui/src/components/ActionAccordion/ActionAccordion.tsx index e2f5b274a4c10..c04b8d79f730a 100644 --- a/airflow-core/src/airflow/ui/src/components/ActionAccordion/ActionAccordion.tsx +++ b/airflow-core/src/airflow/ui/src/components/ActionAccordion/ActionAccordion.tsx @@ -53,7 +53,7 @@ const ActionAccordion = ({ affectedTasks, note, setNote }: Props) => { {translate("dags:runAndTaskActions.affectedTasks.title", { - count: affectedTasks.total_entries, + count: affectedTasks.total_entries ?? 0, })} diff --git a/airflow-core/src/airflow/ui/src/components/Clear/TaskInstance/ClearGroupTaskInstanceDialog.tsx b/airflow-core/src/airflow/ui/src/components/Clear/TaskInstance/ClearGroupTaskInstanceDialog.tsx index 63e1df78526c4..82fb2ceaf87eb 100644 --- a/airflow-core/src/airflow/ui/src/components/Clear/TaskInstance/ClearGroupTaskInstanceDialog.tsx +++ b/airflow-core/src/airflow/ui/src/components/Clear/TaskInstance/ClearGroupTaskInstanceDialog.tsx @@ -117,7 +117,7 @@ export const ClearGroupTaskInstanceDialog = ({ onClose, open, taskInstance }: Pr {translate("dags:runAndTaskActions.clear.title", { - type: translate("taskInstance", { count: affectedTasks.total_entries }), + type: translate("taskInstance", { count: affectedTasks.total_entries ?? 0 }), })} : {" "} diff --git a/airflow-core/src/airflow/ui/src/pages/Dag/Overview/Overview.tsx b/airflow-core/src/airflow/ui/src/pages/Dag/Overview/Overview.tsx index 7ea155fe84761..b284777b351ac 100644 --- a/airflow-core/src/airflow/ui/src/pages/Dag/Overview/Overview.tsx +++ b/airflow-core/src/airflow/ui/src/pages/Dag/Overview/Overview.tsx @@ -60,6 +60,8 @@ export const Overview = () => { state: ["failed"], }); + const failedTaskCount = failedTasks?.total_entries ?? 0; + const [limit] = useLocalStorage(dagRunsLimitKey(dagId ?? ""), 10); const { data: failedRuns, isLoading: isLoadingFailedRuns } = useDagRunServiceGetDagRuns({ dagId: dagId ?? "", @@ -94,14 +96,14 @@ export const Overview = () => { ({ timestamp: ti.start_date ?? ti.logical_date, }))} isLoading={isLoading} - label={translate("overview.buttons.failedTask", { count: failedTasks?.total_entries ?? 0 })} + label={translate("overview.buttons.failedTask", { count: failedTaskCount })} route={{ pathname: "tasks", search: `${SearchParamsKeys.STATE}=failed`, diff --git a/airflow-core/src/airflow/ui/src/pages/Task/Overview/Overview.tsx b/airflow-core/src/airflow/ui/src/pages/Task/Overview/Overview.tsx index 164b34c02d7ba..e9fc510ff6f93 100644 --- a/airflow-core/src/airflow/ui/src/pages/Task/Overview/Overview.tsx +++ b/airflow-core/src/airflow/ui/src/pages/Task/Overview/Overview.tsx @@ -42,7 +42,7 @@ export const Overview = () => { const refetchInterval = useAutoRefresh({}); - const { data: failedTaskInstances, isLoading: isFailedTaskInstancesLoading } = + const { data: failedTaskInstancesData, isLoading: isFailedTaskInstancesLoading } = useTaskInstanceServiceGetTaskInstances({ dagId, dagRunId: "~", @@ -54,6 +54,8 @@ export const Overview = () => { taskId: Boolean(groupId) ? undefined : taskId, }); + const failedTaskCount = failedTaskInstancesData?.total_entries ?? 0; + const { data: tiData, isLoading: isLoadingTaskInstances } = useTaskInstanceServiceGetTaskInstances( { dagId, @@ -84,15 +86,15 @@ export const Overview = () => { ({ + events={(failedTaskInstancesData?.task_instances ?? []).map((ti) => ({ timestamp: ti.start_date ?? ti.logical_date, }))} isLoading={isFailedTaskInstancesLoading} label={translate("overview.buttons.failedTaskInstance", { - count: failedTaskInstances?.total_entries ?? 0, + count: failedTaskCount, })} route={{ pathname: "task_instances", diff --git a/airflow-core/src/airflow/ui/src/pages/TaskInstances/TaskInstances.tsx b/airflow-core/src/airflow/ui/src/pages/TaskInstances/TaskInstances.tsx index 678f386f11684..f897ad4c28d8b 100644 --- a/airflow-core/src/airflow/ui/src/pages/TaskInstances/TaskInstances.tsx +++ b/airflow-core/src/airflow/ui/src/pages/TaskInstances/TaskInstances.tsx @@ -310,7 +310,7 @@ export const TaskInstances = () => { isLoading={isLoading} modelName="common:taskInstance" onStateChange={setTableURLState} - total={data?.total_entries} + total={data?.total_entries ?? undefined} /> ); diff --git a/airflow-core/tests/unit/api_fastapi/common/test_cursors.py b/airflow-core/tests/unit/api_fastapi/common/test_cursors.py new file mode 100644 index 0000000000000..b863de9eb6f3e --- /dev/null +++ b/airflow-core/tests/unit/api_fastapi/common/test_cursors.py @@ -0,0 +1,144 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import base64 +import uuid +from datetime import datetime, timezone +from unittest.mock import MagicMock + +import msgspec +import pytest +from fastapi import HTTPException +from sqlalchemy import select + +from airflow.api_fastapi.common.cursors import apply_cursor_filter, decode_cursor, encode_cursor +from airflow.api_fastapi.common.parameters import SortParam +from airflow.models.taskinstance import TaskInstance + + +def _msgpack_cursor_token(payload: object) -> str: + """Match production: msgpack + base64url without padding.""" + return base64.urlsafe_b64encode(msgspec.msgpack.encode(payload)).decode("ascii").rstrip("=") + + +class TestCursorPagination: + """Tests for cursor-based pagination helpers.""" + + def _make_sort_param_with_resolved_columns(self, order_by_values=None): + """Build a SortParam for TaskInstance and resolve its columns.""" + sp = SortParam(["id", "start_date", "map_index"], TaskInstance) + sp.set_value(order_by_values or ["map_index"]) + sp.to_orm(select(TaskInstance)) + return sp + + def test_encode_decode_cursor_roundtrip(self): + sp = self._make_sort_param_with_resolved_columns(["start_date"]) + row = MagicMock(spec=["start_date", "id"]) + row.start_date = "2024-01-15T10:00:00+00:00" + row.id = "019462ab-1234-5678-9abc-def012345678" + + token = encode_cursor(row, sp) + decoded = decode_cursor(token) + + assert decoded == [ + "2024-01-15T10:00:00+00:00", + "019462ab-1234-5678-9abc-def012345678", + ] + + def test_decode_cursor_invalid_base64(self): + with pytest.raises(HTTPException, match="Invalid cursor token"): + decode_cursor("not-valid-base64!!!") + + def test_decode_cursor_invalid_msgpack(self): + token = base64.urlsafe_b64encode(b"not-msgpack").decode().rstrip("=") + with pytest.raises(HTTPException, match="Invalid cursor token"): + decode_cursor(token) + + def test_decode_cursor_not_a_list(self): + token = _msgpack_cursor_token({"wrong": "type"}) + with pytest.raises(HTTPException, match="Invalid cursor token structure"): + decode_cursor(token) + + def test_encode_cursor_works_without_prior_to_orm(self): + """get_resolved_columns now lazily resolves, so to_orm is no longer required before encode.""" + sp = SortParam(["id"], TaskInstance) + sp.set_value(["id"]) + row = MagicMock(spec=["id"]) + row.id = "019462ab-1234-5678-9abc-def012345678" + token = encode_cursor(row, sp) + decoded = decode_cursor(token) + assert decoded == ["019462ab-1234-5678-9abc-def012345678"] + + def test_apply_cursor_filter_wrong_value_count(self): + sp = self._make_sort_param_with_resolved_columns(["start_date"]) + token = _msgpack_cursor_token(["only-one-value"]) + + with pytest.raises(HTTPException, match="does not match"): + apply_cursor_filter(select(TaskInstance), token, sp) + + def test_apply_cursor_filter_ascending(self): + sp = self._make_sort_param_with_resolved_columns(["start_date"]) + values = [ + datetime(2024, 1, 15, 10, 0, 0, tzinfo=timezone.utc), + uuid.UUID("019462ab-1234-5678-9abc-def012345678"), + ] + token = _msgpack_cursor_token(values) + + stmt = apply_cursor_filter(select(TaskInstance), token, sp) + sql = str(stmt) + assert ">" in sql + + def test_apply_cursor_filter_descending(self): + sp = self._make_sort_param_with_resolved_columns(["-start_date"]) + values = [ + datetime(2024, 1, 15, 10, 0, 0, tzinfo=timezone.utc), + uuid.UUID("019462ab-1234-5678-9abc-def012345678"), + ] + token = _msgpack_cursor_token(values) + + stmt = apply_cursor_filter(select(TaskInstance), token, sp) + sql = str(stmt) + assert "<" in sql + + def test_sort_param_get_resolved_columns(self): + sp = self._make_sort_param_with_resolved_columns(["start_date"]) + resolved = sp.get_resolved_columns() + + assert len(resolved) == 2 + assert resolved[0][0] == "start_date" + assert resolved[0][2] is False + assert resolved[1][0] == "id" + assert resolved[1][2] is False + + def test_sort_param_get_resolved_columns_descending(self): + sp = self._make_sort_param_with_resolved_columns(["-start_date"]) + resolved = sp.get_resolved_columns() + + assert len(resolved) == 2 + assert resolved[0][0] == "start_date" + assert resolved[0][2] is True + assert resolved[1][0] == "id" + assert resolved[1][2] is True + + def test_sort_param_pk_not_duplicated_when_sorting_by_id(self): + sp = self._make_sort_param_with_resolved_columns(["id"]) + resolved = sp.get_resolved_columns() + + assert len(resolved) == 1 + assert resolved[0][0] == "id" diff --git a/airflow-core/tests/unit/api_fastapi/common/test_parameters.py b/airflow-core/tests/unit/api_fastapi/common/test_parameters.py index 5ad860477d151..40076137467ce 100644 --- a/airflow-core/tests/unit/api_fastapi/common/test_parameters.py +++ b/airflow-core/tests/unit/api_fastapi/common/test_parameters.py @@ -24,7 +24,12 @@ from fastapi import Depends, FastAPI, HTTPException from sqlalchemy import select -from airflow.api_fastapi.common.parameters import FilterParam, SortParam, _SearchParam, filter_param_factory +from airflow.api_fastapi.common.parameters import ( + FilterParam, + SortParam, + _SearchParam, + filter_param_factory, +) from airflow.models import DagModel, DagRun, Log diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_task_instances.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_task_instances.py index 76767a96094b1..8c626615f23c2 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_task_instances.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_task_instances.py @@ -19,6 +19,7 @@ import datetime as dt import itertools +import math import os from datetime import timedelta from typing import TYPE_CHECKING, Any @@ -1712,6 +1713,136 @@ def test_should_respond_200_for_pagination(self, test_client, session): assert (num_entries_batch1 + num_entries_batch2) == ti_count assert response_batch1 != response_batch2 + def test_cursor_pagination_first_page(self, test_client, session): + """First page with cursor='' returns cursor response without needing a real token.""" + dag_id = "example_python_operator" + self.create_task_instances( + session, + task_instances=[ + {"start_date": DEFAULT_DATETIME_1 + dt.timedelta(minutes=(i + 1))} for i in range(5) + ], + dag_id=dag_id, + ) + response = test_client.get( + "/dags/~/dagRuns/~/taskInstances", + params={"limit": 3, "order_by": ["map_index"], "cursor": ""}, + ) + assert response.status_code == 200, response.json() + body = response.json() + assert body["next_cursor"] is not None + assert body["previous_cursor"] is None + assert body["total_entries"] is None + assert len(body["task_instances"]) == 3 + + def test_cursor_pagination_returns_cursor_response(self, test_client, session): + """When cursor param is provided, response has cursor fields and no total_entries.""" + dag_id = "example_python_operator" + self.create_task_instances( + session, + task_instances=[ + {"start_date": DEFAULT_DATETIME_1 + dt.timedelta(minutes=(i + 1))} for i in range(5) + ], + dag_id=dag_id, + ) + # First page in cursor mode (empty cursor) + response1 = test_client.get( + "/dags/~/dagRuns/~/taskInstances", + params={"limit": 3, "order_by": ["map_index"], "cursor": ""}, + ) + assert response1.status_code == 200 + body1 = response1.json() + assert body1["total_entries"] is None + assert len(body1["task_instances"]) == 3 + next_cursor = body1["next_cursor"] + assert next_cursor is not None + + # Second (last) page using next_cursor from first page — only 2 TIs remain + response2 = test_client.get( + "/dags/~/dagRuns/~/taskInstances", + params={"limit": 100, "cursor": next_cursor, "order_by": ["map_index"]}, + ) + assert response2.status_code == 200 + body2 = response2.json() + assert body2["next_cursor"] is None + assert body2["previous_cursor"] is not None + assert body2["total_entries"] is None + + def test_cursor_pagination_forward_and_backward_consistency(self, test_client, session): + """Walk all pages forward via next_cursor, then backward via previous_cursor, and compare.""" + dag_id = "example_python_operator" + total_tis = 13 + page_size = 4 + max_pages = math.ceil(total_tis / page_size) + self.create_task_instances( + session, + task_instances=[ + {"start_date": DEFAULT_DATETIME_1 + dt.timedelta(minutes=(i + 1))} for i in range(total_tis) + ], + dag_id=dag_id, + ) + + # -- Walk forward collecting pages -- + forward_ids: list[str] = [] + forward_pages: list[dict] = [] + cursor_token = "" + for _ in range(max_pages): + response = test_client.get( + "/dags/~/dagRuns/~/taskInstances", + params={"limit": page_size, "order_by": ["map_index"], "cursor": cursor_token}, + ) + assert response.status_code == 200, response.json() + body = response.json() + assert body["total_entries"] is None + forward_pages.append(body) + forward_ids.extend(ti["id"] for ti in body["task_instances"]) + + cursor_token = body.get("next_cursor") + if cursor_token is None: + break + + # Sanity: all TIs collected, no overlaps, multiple pages + assert len(forward_ids) == total_tis + assert len(forward_ids) == len(set(forward_ids)), "Forward pages should not overlap" + assert len(forward_pages) == 4 + + # Boundary cursors + assert forward_pages[0]["previous_cursor"] is None, "First page should have no previous_cursor" + assert forward_pages[-1]["next_cursor"] is None, "Last page should have no next_cursor" + + # -- Walk backward from the last page using previous_cursor -- + backward_ids: list[str] = [] + cursor_token = forward_pages[-1]["previous_cursor"] + assert cursor_token is not None, "Last page should provide a previous_cursor" + + for _ in range(max_pages): + response = test_client.get( + "/dags/~/dagRuns/~/taskInstances", + params={"limit": page_size, "order_by": ["map_index"], "cursor": cursor_token}, + ) + assert response.status_code == 200, response.json() + body = response.json() + backward_ids = [ti["id"] for ti in body["task_instances"]] + backward_ids + + cursor_token = body.get("previous_cursor") + if cursor_token is None: + break + + # Backward walk covers all items except the last page (already collected). + # Order must match exactly — no re-sorting needed if pagination is correct. + all_backward = backward_ids + [ti["id"] for ti in forward_pages[-1]["task_instances"]] + assert all_backward == forward_ids, ( + "Walking backward + last page should produce the same TIs in the same order as walking forward" + ) + + def test_cursor_pagination_invalid_token(self, test_client, session): + """Invalid cursor token returns 400.""" + self.create_task_instances(session) + response = test_client.get( + "/dags/~/dagRuns/~/taskInstances", + params={"cursor": "this-is-not-valid", "order_by": ["map_index"]}, + ) + assert response.status_code == 400 + def test_task_group_filter_uses_run_version_not_latest(self, test_client, dag_maker, session): """ Task group lookup should use the DAG version from the run, not the latest version. @@ -4160,6 +4291,8 @@ def test_should_call_mocked_api(self, mock_set_ti_state, test_client, session): } ], "total_entries": 1, + "next_cursor": None, + "previous_cursor": None, } mock_set_ti_state.assert_called_once_with( @@ -4434,6 +4567,8 @@ def test_should_raise_422_for_invalid_task_instance_state(self, payload, expecte } ], "total_entries": 1, + "next_cursor": None, + "previous_cursor": None, }, 1, ), @@ -4570,6 +4705,8 @@ def test_update_mask_set_note_should_respond_200( } ], "total_entries": 1, + "next_cursor": None, + "previous_cursor": None, } _check_task_instance_note(session, response_data["task_instances"][0]["id"], ti_note_data) @@ -4631,6 +4768,8 @@ def test_set_note_should_respond_200(self, test_client, session): } ], "total_entries": 1, + "next_cursor": None, + "previous_cursor": None, } _check_task_instance_note( @@ -4710,6 +4849,8 @@ def test_set_note_should_respond_200_mapped_task_with_rtif(self, test_client, se } ], "total_entries": 1, + "next_cursor": None, + "previous_cursor": None, } _check_task_instance_note( @@ -4907,6 +5048,8 @@ def test_should_call_mocked_api(self, mock_set_ti_state, test_client, session): } ], "total_entries": 1, + "next_cursor": None, + "previous_cursor": None, } mock_set_ti_state.assert_called_once_with( @@ -5193,6 +5336,8 @@ def test_should_raise_422_for_invalid_task_instance_state(self, payload, expecte } ], "total_entries": 1, + "next_cursor": None, + "previous_cursor": None, }, 1, ), @@ -5271,7 +5416,12 @@ def test_should_return_empty_list_for_updating_same_task_instance_state( }, ) assert response.status_code == 200 - assert response.json() == {"task_instances": [], "total_entries": 0} + assert response.json() == { + "task_instances": [], + "total_entries": 0, + "next_cursor": None, + "previous_cursor": None, + } class TestDeleteTaskInstance(TestTaskInstanceEndpoint): diff --git a/airflow-ctl/src/airflowctl/api/datamodels/generated.py b/airflow-ctl/src/airflowctl/api/datamodels/generated.py index b33543df934d5..986a2a1a6fc6c 100644 --- a/airflow-ctl/src/airflowctl/api/datamodels/generated.py +++ b/airflow-ctl/src/airflowctl/api/datamodels/generated.py @@ -2039,11 +2039,38 @@ class TaskCollectionResponse(BaseModel): class TaskInstanceCollectionResponse(BaseModel): """ - Task Instance Collection serializer for responses. + Task instance collection response supporting both offset and cursor pagination. + + A single flat model is used instead of a discriminated union + (``Annotated[Offset | Cursor, Field(discriminator=...)]``) because + the OpenAPI ``oneOf`` + ``discriminator`` construct is not handled + correctly by ``@hey-api/openapi-ts`` / ``@7nohe/openapi-react-query-codegen``: + return types degrade to ``unknown`` in JSDoc and can produce + incorrect TypeScript types (see hey-api/openapi-ts#1613, #3270). """ task_instances: Annotated[list[TaskInstanceResponse], Field(title="Task Instances")] - total_entries: Annotated[int, Field(title="Total Entries")] + total_entries: Annotated[ + int | None, + Field( + description="Total number of matching items. Populated for offset pagination, ``null`` when using cursor pagination.", + title="Total Entries", + ), + ] = None + next_cursor: Annotated[ + str | None, + Field( + description="Token pointing to the next page. Populated for cursor pagination, ``null`` when using offset pagination or when there is no next page.", + title="Next Cursor", + ), + ] = None + previous_cursor: Annotated[ + str | None, + Field( + description="Token pointing to the previous page. Populated for cursor pagination, ``null`` when using offset pagination or when on the first page.", + title="Previous Cursor", + ), + ] = None class TaskInstanceHistoryCollectionResponse(BaseModel): From 296ae7a5b2bcc2c21fa0ca7dd75a22becc12c2a0 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 20 Apr 2026 17:11:36 +0200 Subject: [PATCH 023/309] [v3-2-test] Work around jpype1 1.7.0 missing macOS arm64 wheels in jdbc provider (#65532) (#65548) jpype1 1.7.0 stopped shipping prebuilt macOS arm64 wheels and tries to build from source against a JDK on Apple Silicon, which breaks `uv sync` out of the box on those machines. Exclude jpype1 1.7.0 as a direct dependency of the jdbc provider, scoped to darwin-arm64 only so every other platform (including macOS x86_64) still resolves to the latest wheels. Upstream plans to restore the arm64 wheels in 1.7.1 (https://github.com/jpype-project/jpype/issues/1357), after which the exclusion can be dropped. (cherry picked from commit 8ed50aebe9d78d54442c84019d9f89fb3018b008) --- providers/jdbc/docs/index.rst | 8 +++++--- providers/jdbc/pyproject.toml | 7 +++++++ uv.lock | 8 ++++++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/providers/jdbc/docs/index.rst b/providers/jdbc/docs/index.rst index 85ffe5c0b1cd8..0a3a9d5f1fb3f 100644 --- a/providers/jdbc/docs/index.rst +++ b/providers/jdbc/docs/index.rst @@ -98,14 +98,16 @@ Requirements The minimum Apache Airflow version supported by this provider distribution is ``2.11.0``. -========================================== ================== +========================================== ============================================================================= PIP package Version required -========================================== ================== +========================================== ============================================================================= ``apache-airflow`` ``>=2.11.0`` ``apache-airflow-providers-common-compat`` ``>=1.14.0`` ``apache-airflow-providers-common-sql`` ``>=1.32.0`` ``jaydebeapi`` ``>=1.1.1`` -========================================== ================== +``jpype1`` ``>=1.4.0,!=1.7.0; sys_platform == "darwin" and platform_machine == "arm64"`` +``jpype1`` ``>=1.4.0; sys_platform != "darwin" or platform_machine != "arm64"`` +========================================== ============================================================================= Cross provider package dependencies ----------------------------------- diff --git a/providers/jdbc/pyproject.toml b/providers/jdbc/pyproject.toml index 52af0df250c72..e64eee863d93a 100644 --- a/providers/jdbc/pyproject.toml +++ b/providers/jdbc/pyproject.toml @@ -63,6 +63,13 @@ dependencies = [ "apache-airflow-providers-common-compat>=1.14.0", "apache-airflow-providers-common-sql>=1.32.0", "jaydebeapi>=1.1.1", + # jpype1 1.7.0 stopped shipping prebuilt macOS arm64 wheels and requires compilation + # against a properly configured JDK on Apple Silicon, which breaks `uv sync` out of + # the box. Exclude only 1.7.0 on darwin-arm64 — upstream is planning to restore the + # wheels in 1.7.1 (see https://github.com/jpype-project/jpype/issues/1357). Other + # platforms (including macOS x86_64) still get all published versions. + "jpype1>=1.4.0,!=1.7.0; sys_platform == 'darwin' and platform_machine == 'arm64'", + "jpype1>=1.4.0; sys_platform != 'darwin' or platform_machine != 'arm64'", ] [dependency-groups] diff --git a/uv.lock b/uv.lock index 0f5b25fbbb071..f54143f15687e 100644 --- a/uv.lock +++ b/uv.lock @@ -22,7 +22,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-04-16T19:13:06.968011Z" +exclude-newer = "2026-04-16T14:57:18.176731926Z" exclude-newer-span = "P4D" [options.exclude-newer-package] @@ -101,9 +101,9 @@ apache-airflow-providers-mongo = false apache-airflow-providers-apprise = false apache-airflow-providers-apache-impala = false apache-airflow-ctl = false +apache-airflow-providers-zendesk = false apache-airflow-providers-github = false apache-airflow-providers-snowflake = false -apache-airflow-providers-zendesk = false apache-airflow-providers-presto = false apache-airflow-providers-airbyte = false apache-airflow-providers-apache-hive = false @@ -5486,6 +5486,8 @@ dependencies = [ { name = "apache-airflow-providers-common-compat" }, { name = "apache-airflow-providers-common-sql" }, { name = "jaydebeapi" }, + { name = "jpype1", version = "1.6.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, + { name = "jpype1", version = "1.7.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, ] [package.dev-dependencies] @@ -5506,6 +5508,8 @@ requires-dist = [ { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, { name = "jaydebeapi", specifier = ">=1.1.1" }, + { name = "jpype1", marker = "platform_machine != 'arm64' or sys_platform != 'darwin'", specifier = ">=1.4.0" }, + { name = "jpype1", marker = "platform_machine == 'arm64' and sys_platform == 'darwin'", specifier = ">=1.4.0,!=1.7.0" }, ] [package.metadata.requires-dev] From 16e8d1c953661231989b04a768a84a0517d35fef Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 18:23:23 +0200 Subject: [PATCH 024/309] [v3-2-test] Add per-DAG authorization to partitioned_dag_runs endpoints (#65344) (#65538) The partitioned_dag_runs endpoints enforced only asset-level access control. Add requires_access_dag and ReadableDagsFilterDep to match the pattern used by the sibling next_run_assets endpoint in assets.py. (cherry picked from commit e36d10a5196d100adb3829219dc91be72d6a7fc9) Generated-by: Claude Opus 4.6 (1M context) following the guidelines at https: //github.com/apache/airflow/blob/main/contributing-docs/05_pull_requests.rst#gen-ai-assisted-contributions Co-authored-by: Jarek Potiuk --- .../routes/ui/partitioned_dag_runs.py | 12 ++++++++-- .../routes/ui/test_partitioned_dag_runs.py | 24 +++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/partitioned_dag_runs.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/partitioned_dag_runs.py index f9090a6ece031..fe4696efbb5a0 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/partitioned_dag_runs.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/partitioned_dag_runs.py @@ -31,7 +31,11 @@ PartitionedDagRunDetailResponse, PartitionedDagRunResponse, ) -from airflow.api_fastapi.core_api.security import requires_access_asset +from airflow.api_fastapi.core_api.security import ( + ReadableDagsFilterDep, + requires_access_asset, + requires_access_dag, +) from airflow.models import DagModel from airflow.models.asset import ( AssetModel, @@ -63,6 +67,7 @@ def _build_response(row, required_count: int) -> PartitionedDagRunResponse: ) def get_partitioned_dag_runs( session: SessionDep, + readable_dags_filter: ReadableDagsFilterDep, dag_id: QueryPartitionedDagRunDagIdFilter, has_created_dag_run_id: QueryPartitionedDagRunHasCreatedDagRunIdFilter, ) -> PartitionedDagRunCollectionResponse: @@ -123,6 +128,9 @@ def get_partitioned_dag_runs( received_subq.label("total_received"), ).outerjoin(DagRun, AssetPartitionDagRun.created_dag_run_id == DagRun.id) query = apply_filters_to_select(statement=query, filters=[dag_id, has_created_dag_run_id]) + readable_dag_ids = readable_dags_filter.value + if readable_dag_ids is not None: + query = query.where(AssetPartitionDagRun.target_dag_id.in_(readable_dag_ids)) query = query.order_by(AssetPartitionDagRun.created_at.desc()) if not (rows := session.execute(query).all()): @@ -162,7 +170,7 @@ def get_partitioned_dag_runs( @partitioned_dag_runs_router.get( "/pending_partitioned_dag_run/{dag_id}/{partition_key}", - dependencies=[Depends(requires_access_asset(method="GET"))], + dependencies=[Depends(requires_access_asset(method="GET")), Depends(requires_access_dag(method="GET"))], ) def get_pending_partitioned_dag_run( dag_id: str, diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_partitioned_dag_runs.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_partitioned_dag_runs.py index 658d23abb214f..a2a86da327b86 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_partitioned_dag_runs.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_partitioned_dag_runs.py @@ -16,6 +16,8 @@ # under the License. from __future__ import annotations +from unittest import mock + import pendulum import pytest from sqlalchemy import select @@ -57,7 +59,7 @@ def test_should_response_200_non_partitioned_dag_returns_empty(self, test_client dag_maker.create_dagrun() dag_maker.sync_dagbag_to_db() - with assert_queries_count(2): + with assert_queries_count(3): resp = test_client.get("/partitioned_dag_runs?dag_id=normal&has_created_dag_run_id=false") assert resp.status_code == 200 assert resp.json() == {"partitioned_dag_runs": [], "total": 0, "asset_expressions": None} @@ -144,7 +146,7 @@ def test_should_response_200( ) session.commit() - with assert_queries_count(2): + with assert_queries_count(3): resp = test_client.get( f"/partitioned_dag_runs?dag_id=list_dag" f"&has_created_dag_run_id={str(has_created_dag_run_id).lower()}" @@ -218,6 +220,24 @@ def _make_schedule(prefix, count): assert pdr_resp["total_required"] == num_target_assets assert pdr_resp["total_received"] == received_count + @mock.patch( + "airflow.api_fastapi.auth.managers.base_auth_manager.BaseAuthManager.get_authorized_dag_ids", + return_value={"other_dag"}, + ) + def test_partitioned_dag_runs_filters_unreadable_dags(self, _, test_client, dag_maker, session): + schedule = PartitionedAssetTimetable(assets=Asset(uri="s3://bucket/a", name="a")) + with dag_maker(dag_id="restricted_dag", schedule=schedule, serialized=True): + EmptyOperator(task_id="t") + dag_maker.sync_dagbag_to_db() + session.add(AssetPartitionDagRun(target_dag_id="restricted_dag", partition_key="2024-06-01")) + session.commit() + + resp = test_client.get("/partitioned_dag_runs?has_created_dag_run_id=false") + assert resp.status_code == 200 + body = resp.json() + dag_ids = {r["dag_id"] for r in body["partitioned_dag_runs"]} + assert "restricted_dag" not in dag_ids + class TestGetPendingPartitionedDagRun: def test_should_response_401(self, unauthenticated_test_client): From e174c11ad2966d9c9ff6244f18dae53bb772a062 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 18:41:56 +0200 Subject: [PATCH 025/309] [v3-2-test] Only fail provider dependency checks on main (#65551) (#65552) The provider_dependency_bump and common_compat_changed_without_next_version checks in selective_checks are meant to guard the main branch from accidental release-manager-only changes. On release branches (v3-X-test), those same changes are expected during cherry-picks, and forcing contributors to set override labels on every backport adds friction without value. Skip both checks when the target branch is not main so release-branch PRs no longer need the override labels. (cherry picked from commit 3dd2ba09ec6766d3996ac9615aaf224d98f7a5cc) Co-authored-by: Jarek Potiuk --- .../airflow_breeze/utils/selective_checks.py | 10 +++ dev/breeze/tests/test_selective_checks.py | 67 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index ca7fbef3a3d56..aae6965dcff12 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -1776,6 +1776,11 @@ def ui_english_translation_changed(self) -> bool: @cached_property def provider_dependency_bump(self) -> bool: """Check for apache-airflow-providers dependency bumps in pyproject.toml files.""" + # Only enforce on PRs targeting main. On release branches (e.g. v3-X-test) + # cherry-picks routinely bump provider dependency lower bounds, and the + # override label is meant for that flow on main. + if self._default_branch != "main": + return False pyproject_files = self._matching_files( FileGroupForCi.ALL_PYPROJECT_TOML_FILES, CI_FILE_GROUP_MATCHES, @@ -2003,6 +2008,11 @@ def common_compat_changed_without_next_version(self) -> bool: """ if self._github_event != GithubEvents.PULL_REQUEST: return False + # Only enforce on PRs targeting main. On release branches (e.g. v3-X-test) + # cherry-picked common.compat changes don't follow the '# use next version' + # convention, and the override label is meant for that flow on main. + if self._default_branch != "main": + return False if not self._has_common_compat_changed(): return False diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 3838f779e4c2d..725e77ee399c8 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -3128,6 +3128,42 @@ def side_effect(*args, **kwargs): ).provider_dependency_bump +@patch("airflow_breeze.utils.selective_checks.run_command") +def test_provider_dependency_bump_check_skipped_on_release_branch(mock_run_command): + """Test that provider dependency bump check is a no-op on release branches (v3-X-test).""" + old_toml = """ +[project] +dependencies = [ + "apache-airflow-providers-common-sql>=1.0.0", +] +""" + new_toml = """ +[project] +dependencies = [ + "apache-airflow-providers-common-sql>=1.1.0", +] +""" + + def side_effect(*args, **kwargs): + result = Mock() + result.returncode = 0 + if "^:" in args[0][2]: + result.stdout = old_toml + else: + result.stdout = new_toml + return result + + mock_run_command.side_effect = side_effect + + assert not SelectiveChecks( + files=("providers/amazon/pyproject.toml",), + commit_ref=NEUTRAL_COMMIT, + pr_labels=(), + github_event=GithubEvents.PULL_REQUEST, + default_branch="v3-2-test", + ).provider_dependency_bump + + @patch("airflow_breeze.utils.selective_checks.run_command") def test_provider_dependency_bump_check_passes_with_label(mock_run_command): """Test that provider dependency bump check passes when label is set.""" @@ -3459,6 +3495,37 @@ def side_effect(*args, **kwargs): assert result is False +@patch("airflow_breeze.utils.selective_checks.run_command") +def test_common_compat_changed_without_next_version_skipped_on_release_branch(mock_run_command): + """Test that common.compat next-version check is a no-op on release branches (v3-X-test).""" + provider_toml = """ +[project] +dependencies = [ + "apache-airflow>=2.11.0", + "apache-airflow-providers-common-compat>=1.8.0", +] +""" + + def side_effect(*args, **kwargs): + result = Mock() + result.returncode = 0 + result.stdout = provider_toml + return result + + mock_run_command.side_effect = side_effect + + assert not SelectiveChecks( + files=( + "providers/common/compat/src/airflow/providers/common/compat/file.py", + "providers/ftp/src/airflow/providers/ftp/hooks/ftp.py", + ), + commit_ref=NEUTRAL_COMMIT, + pr_labels=(), + github_event=GithubEvents.PULL_REQUEST, + default_branch="v3-2-test", + ).common_compat_changed_without_next_version + + @patch("airflow_breeze.utils.selective_checks.run_command") def test_common_compat_changed_without_next_version_bypassed_with_label(mock_run_command): """Test that check can be bypassed with 'skip common compat check' label.""" From 0ecf5b0bd8a63b9c3cdf1c477ffe4ba0ce5cfe94 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 18:05:33 +0100 Subject: [PATCH 026/309] [v3-2-test] Respect dag processor config option to show parsing logs on stdout (#65528) (#65541) This is very useful when running things in a container, where it's much easier to get container logs than files on disk. This config option was added in 2.4.0 but got broken around 3.0 or 3.1. Most of capabilities existed already, we just needed to pass the right value down. Example log line: ``` 2026-04-20T10:44:13.977892Z [info ] Ptint from dag [dag_processor.stdout] bundle_name=main dag_file=dags/example_dag_advanced.py ``` Since we've got structured logging now it includes the bundle name and rel path of the dag file. (cherry picked from commit 1193073f9083ad8e38facbc07eff0476928008a7) Co-authored-by: Ash Berlin-Taylor --- airflow-core/src/airflow/configuration.py | 1 + .../src/airflow/dag_processing/manager.py | 2 + .../src/airflow/dag_processing/processor.py | 28 ++++++++- .../tests/unit/dag_processing/test_manager.py | 32 ++++++++++ .../unit/dag_processing/test_processor.py | 59 +++++++++++++++++++ .../airflow/sdk/execution_time/supervisor.py | 12 ++-- 6 files changed, 129 insertions(+), 5 deletions(-) diff --git a/airflow-core/src/airflow/configuration.py b/airflow-core/src/airflow/configuration.py index be07df7f418c4..7790a252af37a 100644 --- a/airflow-core/src/airflow/configuration.py +++ b/airflow-core/src/airflow/configuration.py @@ -279,6 +279,7 @@ def _update_logging_deprecated_template_to_one_from_defaults(self): ("logging", "gunicorn_logging_level"): _available_logging_levels, ("webserver", "analytical_tool"): ["google_analytics", "metarouter", "segment", "matomo", ""], ("api", "grid_view_sorting_order"): ["topological", "hierarchical_alphabetical"], + ("logging", "dag_processor_log_target"): ["file", "stdout"], } upgraded_values: dict[tuple[str, str], str] diff --git a/airflow-core/src/airflow/dag_processing/manager.py b/airflow-core/src/airflow/dag_processing/manager.py index 2e40230131bde..57ebd56c0e9eb 100644 --- a/airflow-core/src/airflow/dag_processing/manager.py +++ b/airflow-core/src/airflow/dag_processing/manager.py @@ -1034,10 +1034,12 @@ def _create_process(self, dag_file: DagFileInfo) -> DagFileProcessorProcess: path=dag_file.absolute_path, bundle_path=cast("Path", dag_file.bundle_path), bundle_name=dag_file.bundle_name, + dag_file_rel_path=str(dag_file.rel_path), callbacks=callback_to_execute_for_file, selector=self.selector, logger=logger, logger_filehandle=logger_filehandle, + subprocess_logs_to_stdout=conf.get("logging", "dag_processor_log_target") == "stdout", client=self.client, ) diff --git a/airflow-core/src/airflow/dag_processing/processor.py b/airflow-core/src/airflow/dag_processing/processor.py index 758472598d28c..6cbef2fef67bb 100644 --- a/airflow-core/src/airflow/dag_processing/processor.py +++ b/airflow-core/src/airflow/dag_processing/processor.py @@ -18,6 +18,7 @@ import contextlib import importlib +import logging import os import traceback from collections.abc import Callable, Sequence @@ -74,6 +75,8 @@ from airflow.utils.state import TaskInstanceState if TYPE_CHECKING: + from socket import socket + from structlog.typing import FilteringBoundLogger from airflow.api_fastapi.execution_api.app import InProcessExecutionAPI @@ -513,6 +516,9 @@ class DagFileProcessorProcess(WatchedSubprocess): client: Client """The HTTP client to use for communication with the API server.""" + bundle_name: str + dag_file_rel_path: str + @classmethod def start( # type: ignore[override] cls, @@ -520,6 +526,7 @@ def start( # type: ignore[override] path: str | os.PathLike[str], bundle_path: Path, bundle_name: str, + dag_file_rel_path: str, callbacks: list[CallbackRequest], target: Callable[[], None] = _parse_file_entrypoint, client: Client, @@ -529,7 +536,13 @@ def start( # type: ignore[override] _pre_import_airflow_modules(os.fspath(path), logger) - proc: Self = super().start(target=target, client=client, **kwargs) + proc: Self = super().start( + target=target, + client=client, + bundle_name=bundle_name, + dag_file_rel_path=dag_file_rel_path, + **kwargs, + ) proc.had_callbacks = bool(callbacks) # Track if this process had callbacks proc._on_child_started(callbacks, path, bundle_path, bundle_name) return proc @@ -549,6 +562,19 @@ def _on_child_started( ) self.send_msg(msg, request_id=0) + def _get_target_loggers(self) -> tuple[FilteringBoundLogger, ...]: + base = super()._get_target_loggers() + if not self.subprocess_logs_to_stdout: + return base + return tuple( + logger.bind(dag_file=self.dag_file_rel_path, bundle_name=self.bundle_name) for logger in base + ) + + def _create_log_forwarder( + self, loggers: tuple[FilteringBoundLogger, ...], name: str, log_level: int = logging.INFO + ) -> Callable[[socket], bool]: + return super()._create_log_forwarder(loggers, name.replace("task.", "dag_processor.", 1), log_level) + def _handle_request(self, msg: ToManager, log: FilteringBoundLogger, req_id: int) -> None: from airflow.sdk.api.datamodels._generated import ( ConnectionResponse, diff --git a/airflow-core/tests/unit/dag_processing/test_manager.py b/airflow-core/tests/unit/dag_processing/test_manager.py index 3f866dee14d4e..15899a3c7ad1e 100644 --- a/airflow-core/tests/unit/dag_processing/test_manager.py +++ b/airflow-core/tests/unit/dag_processing/test_manager.py @@ -184,6 +184,8 @@ def mock_processor(self, start_time: float | None = None) -> tuple[DagFileProces stdin=write_end, logger_filehandle=logger_filehandle, client=MagicMock(), + bundle_name="testing", + dag_file_rel_path="test_dag.py", ) if start_time: ret.start_time = start_time @@ -796,6 +798,32 @@ def test_cleanup_stale_bundle_versions(self, mock_bundle_manager): manager.cleanup_stale_bundle_versions() mock_bundle_manager.return_value.remove_stale_bundle_versions.assert_called_once_with() + @pytest.mark.parametrize( + ("log_target", "expected_subprocess_logs_to_stdout"), + [ + ("stdout", True), + ("file", False), + ], + ) + @mock.patch.object(DagFileProcessorManager, "_get_logger_for_dag_file") + def test_create_process_subprocess_logs_to_stdout( + self, mock_get_logger, log_target, expected_subprocess_logs_to_stdout + ): + mock_logger = MagicMock() + mock_filehandle = MagicMock() + mock_get_logger.return_value = [mock_logger, mock_filehandle] + + with conf_vars({("logging", "dag_processor_log_target"): log_target}): + manager = DagFileProcessorManager(max_runs=1, processor_timeout=60) + dag_file = DagFileInfo( + bundle_name="testing", rel_path=Path("my_dag.py"), bundle_path=Path("/tmp") + ) + with mock.patch.object(DagFileProcessorProcess, "start") as mock_start: + manager._create_process(dag_file) + + _, kwargs = mock_start.call_args + assert kwargs["subprocess_logs_to_stdout"] is expected_subprocess_logs_to_stdout + def test_kill_timed_out_processors_kill(self): manager = DagFileProcessorManager(max_runs=1, processor_timeout=5) # Set start_time to ensure timeout occurs: start_time = current_time - (timeout + 1) = always (timeout + 1) seconds @@ -1384,10 +1412,12 @@ def test_callback_queue(self, mock_get_logger, configure_testing_dag_bundle): path=Path(dag2_path.bundle_path, dag2_path.rel_path), bundle_path=dag2_path.bundle_path, bundle_name="testing", + dag_file_rel_path=str(dag2_path.rel_path), callbacks=[dag2_req1], selector=mock.ANY, logger=mock_logger, logger_filehandle=mock_filehandle, + subprocess_logs_to_stdout=False, client=mock.ANY, ), mock.call( @@ -1395,10 +1425,12 @@ def test_callback_queue(self, mock_get_logger, configure_testing_dag_bundle): path=Path(dag1_path.bundle_path, dag1_path.rel_path), bundle_path=dag1_path.bundle_path, bundle_name="testing", + dag_file_rel_path=str(dag1_path.rel_path), callbacks=[dag1_req1, dag1_req2], selector=mock.ANY, logger=mock_logger, logger_filehandle=mock_filehandle, + subprocess_logs_to_stdout=False, client=mock.ANY, ), ] diff --git a/airflow-core/tests/unit/dag_processing/test_processor.py b/airflow-core/tests/unit/dag_processing/test_processor.py index c46aceda7949b..667037f9a1e0a 100644 --- a/airflow-core/tests/unit/dag_processing/test_processor.py +++ b/airflow-core/tests/unit/dag_processing/test_processor.py @@ -18,6 +18,7 @@ from __future__ import annotations import inspect +import logging import pathlib import sys import textwrap @@ -164,6 +165,7 @@ def dag_in_a_fn(): path=path, bundle_path=tmp_path, bundle_name="testing", + dag_file_rel_path=str(path.relative_to(tmp_path)), callbacks=[], logger=logger, logger_filehandle=logger_filehandle, @@ -200,6 +202,7 @@ def dag_in_a_fn(): path=path, bundle_path=tmp_path, bundle_name="testing", + dag_file_rel_path=str(path.relative_to(tmp_path)), callbacks=[], logger=logger, logger_filehandle=logger_filehandle, @@ -234,6 +237,7 @@ def dag_in_a_fn(): path=path, bundle_path=tmp_path, bundle_name="testing", + dag_file_rel_path=str(path.relative_to(tmp_path)), callbacks=[], logger=logger, logger_filehandle=logger_filehandle, @@ -278,6 +282,7 @@ def dag_in_a_fn(): path=path, bundle_path=tmp_path, bundle_name="testing", + dag_file_rel_path=str(path.relative_to(tmp_path)), callbacks=[], logger=logger, logger_filehandle=logger_filehandle, @@ -316,6 +321,7 @@ def dag_in_a_fn(): path=path, bundle_path=tmp_path, bundle_name="testing", + dag_file_rel_path=str(path.relative_to(tmp_path)), callbacks=[], logger=logger, logger_filehandle=logger_filehandle, @@ -346,6 +352,7 @@ def dag_in_a_fn(): path=path, bundle_path=tmp_path, bundle_name="testing", + dag_file_rel_path=str(path.relative_to(tmp_path)), callbacks=[], logger=logger, logger_filehandle=logger_filehandle, @@ -380,6 +387,7 @@ def test_import_module_in_bundle_root(self, tmp_path: pathlib.Path, inprocess_cl path=dag1_path, bundle_path=tmp_path, bundle_name="testing", + dag_file_rel_path=str(dag1_path.relative_to(tmp_path)), callbacks=[], logger=MagicMock(spec=FilteringBoundLogger), logger_filehandle=MagicMock(spec=BinaryIO), @@ -1967,3 +1975,54 @@ def get_type_names(union_type): + "\n".join(f" - {t}" for t in sorted(task_diff)) + "\n\nEither handle these types in ToDagProcessor or update in_task_runner_but_not_in_dag_processing_process list." ) + + +class TestDagFileProcessorProcess: + @pytest.fixture + def proc(self): + from socket import socketpair + from unittest.mock import MagicMock + + proc_mock = MagicMock() + proc_mock.create_time.return_value = 0.0 + r, w = socketpair() + instance = DagFileProcessorProcess( + process_log=structlog.get_logger().bind(), + id=uuid.uuid4(), + pid=1234, + process=proc_mock, + stdin=w, + logger_filehandle=MagicMock(spec=BinaryIO), + client=MagicMock(spec=Client), + bundle_name="mybundle", + dag_file_rel_path="dags/my_dag.py", + ) + instance._open_sockets.clear() + r.close() + w.close() + return instance + + def test_get_target_loggers_file_mode_no_context_added(self, proc): + proc.subprocess_logs_to_stdout = False + loggers = proc._get_target_loggers() + assert len(loggers) == 1 + with structlog.testing.capture_logs() as cap: + loggers[0].info("test") + assert "dag_file" not in cap[0] + assert "bundle_name" not in cap[0] + + def test_get_target_loggers_stdout_mode_binds_dag_file_context(self, proc): + proc.subprocess_logs_to_stdout = True + loggers = proc._get_target_loggers() + with structlog.testing.capture_logs() as cap: + for bound_logger in loggers: + bound_logger.info("test") + assert all(e.get("dag_file") == "dags/my_dag.py" for e in cap) + assert all(e.get("bundle_name") == "mybundle" for e in cap) + + def test_create_log_forwarder_rewrites_task_prefix_to_dag_processor(self, proc): + from airflow.sdk.execution_time.supervisor import WatchedSubprocess + + with patch.object(WatchedSubprocess, "_create_log_forwarder") as mock_base: + proc._create_log_forwarder((), "task.stdout") + mock_base.assert_called_once_with((), "dag_processor.stdout", logging.INFO) diff --git a/task-sdk/src/airflow/sdk/execution_time/supervisor.py b/task-sdk/src/airflow/sdk/execution_time/supervisor.py index 0eee4889c2572..a4484199f6297 100644 --- a/task-sdk/src/airflow/sdk/execution_time/supervisor.py +++ b/task-sdk/src/airflow/sdk/execution_time/supervisor.py @@ -566,10 +566,7 @@ def _register_pipe_readers(self, stdout: socket, stderr: socket, requests: socke ) ) - target_loggers: tuple[FilteringBoundLogger, ...] = (self.process_log,) - - if self.subprocess_logs_to_stdout: - target_loggers += (log,) + target_loggers = self._get_target_loggers() self.selector.register( stdout, selectors.EVENT_READ, self._create_log_forwarder(target_loggers, "task.stdout") @@ -592,6 +589,13 @@ def _register_pipe_readers(self, stdout: socket, stderr: socket, requests: socke length_prefixed_frame_reader(self.handle_requests(log), on_close=self._on_socket_closed), ) + def _get_target_loggers(self) -> tuple[FilteringBoundLogger, ...]: + """Return the loggers that child process output should be forwarded to.""" + target_loggers: tuple[FilteringBoundLogger, ...] = (self.process_log,) + if self.subprocess_logs_to_stdout: + target_loggers += (log,) + return target_loggers + def _create_log_forwarder(self, loggers, name, log_level=logging.INFO) -> Callable[[socket], bool]: """Create a socket handler that forwards logs to a logger.""" loggers = tuple( From d999806bae8a0e546fe71be301fcbee1c48667f3 Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Mon, 20 Apr 2026 19:35:03 +0200 Subject: [PATCH 027/309] Feature/cursor pagination task instances UI (#64953) (#65542) * UI: Switch TaskInstances table to cursor-based pagination Replace offset-based pagination with cursor-based pagination for the TaskInstances listing page, leveraging the new cursor API endpoint. Pagination now shows only previous/next buttons without page numbers or total count, which eliminates the expensive COUNT(*) query for large datasets. Add generic cursor pagination support to DataTable via an optional cursorPagination prop so other tables can adopt it. * Small adjustments and cleaning * Address review: cursor in TableState, no useCallback/useEffect * Reuse Pagination.Root for cursor pagination instead of custom buttons * Small updates (cherry picked from commit 44ae2bc5efb5c5fc53b949c90c4e1b24a72cbc8d) --- .../ActionAccordion/ActionAccordion.tsx | 2 +- .../ui/src/components/DataTable/DataTable.tsx | 54 +++++++++++++++++-- .../src/components/DataTable/searchParams.ts | 20 ++++++- .../ui/src/components/DataTable/types.ts | 1 + .../airflow/ui/src/constants/searchParams.ts | 1 + .../src/pages/TaskInstances/TaskInstances.tsx | 11 ++-- .../airflow/ui/src/utils/useFiltersHandler.ts | 2 + 7 files changed, 82 insertions(+), 9 deletions(-) diff --git a/airflow-core/src/airflow/ui/src/components/ActionAccordion/ActionAccordion.tsx b/airflow-core/src/airflow/ui/src/components/ActionAccordion/ActionAccordion.tsx index c04b8d79f730a..d3f190c7ebc39 100644 --- a/airflow-core/src/airflow/ui/src/components/ActionAccordion/ActionAccordion.tsx +++ b/airflow-core/src/airflow/ui/src/components/ActionAccordion/ActionAccordion.tsx @@ -65,7 +65,7 @@ const ActionAccordion = ({ affectedTasks, note, setNote }: Props) => { displayMode="table" modelName="common:taskInstance" noRowsMessage={translate("dags:runAndTaskActions.affectedTasks.noItemsFound")} - total={affectedTasks.total_entries} + total={affectedTasks.total_entries ?? 0} /> diff --git a/airflow-core/src/airflow/ui/src/components/DataTable/DataTable.tsx b/airflow-core/src/airflow/ui/src/components/DataTable/DataTable.tsx index 4ec00ff62f28a..81f5b42d67133 100644 --- a/airflow-core/src/airflow/ui/src/components/DataTable/DataTable.tsx +++ b/airflow-core/src/airflow/ui/src/components/DataTable/DataTable.tsx @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import { Box, Heading, HStack, Text } from "@chakra-ui/react"; +import { Box, Heading, HStack, IconButton, Text } from "@chakra-ui/react"; import { getCoreRowModel, getExpandedRowModel, @@ -31,6 +31,7 @@ import { } from "@tanstack/react-table"; import React, { type ReactNode, useRef, useCallback } from "react"; import { useTranslation } from "react-i18next"; +import { HiChevronLeft, HiChevronRight } from "react-icons/hi2"; import { useLocalStorage } from "usehooks-ts"; import { CardList } from "src/components/DataTable/CardList"; @@ -52,9 +53,11 @@ type DataTableProps = { readonly isFetching?: boolean; readonly isLoading?: boolean; readonly modelName: string; + readonly nextCursor?: string | null; readonly noRowsMessage?: ReactNode; readonly onDisplayToggleChange?: (mode: "card" | "table") => void; readonly onStateChange?: (state: TableState) => void; + readonly previousCursor?: string | null; readonly renderSubComponent?: (props: { row: Row }) => React.ReactElement; readonly showDisplayToggle?: boolean; readonly showRowCountHeading?: boolean; @@ -76,9 +79,11 @@ export const DataTable = ({ isFetching, isLoading, modelName, + nextCursor, noRowsMessage, onDisplayToggleChange, onStateChange, + previousCursor, showDisplayToggle, showRowCountHeading = true, skeletonCount = 10, @@ -148,7 +153,13 @@ export const DataTable = ({ const display = displayMode === "card" && Boolean(cardDef) ? "card" : "table"; const hasRows = rows.length > 0; - const hasPagination = initialState?.pagination !== undefined && (pageIndex !== 0 || rows.length !== total); + const hasNext = nextCursor !== undefined && nextCursor !== null; + const hasPrevious = previousCursor !== undefined && previousCursor !== null; + const hasCursorPagination = hasNext || hasPrevious; + const hasOffsetPagination = + !hasCursorPagination && + initialState?.pagination !== undefined && + (pageIndex !== 0 || rows.length !== total); // Default to show columns filter only if there are actually many columns displayed const showColumnsFilter = allowFiltering ?? columns.length > 5; @@ -158,7 +169,7 @@ export const DataTable = ({ [modelName, translate], ); const showRowCount = Boolean( - showRowCountHeading && !Boolean(isLoading) && !Boolean(isFetching) && total > 0, + showRowCountHeading && !hasCursorPagination && !Boolean(isLoading) && !Boolean(isFetching) && total > 0, ); const noRowsModelName = translateModelName(0); @@ -190,7 +201,7 @@ export const DataTable = ({ )} - {hasPagination ? ( + {hasOffsetPagination ? ( ({ ) : undefined} + {/* Pagination.Root is designed for offset-based pagination and requires count/page/pageSize. + Cursor pagination doesn't have these values, and passing fake values causes + incorrect disabled styling on the triggers. Use plain IconButtons instead. */} + {hasCursorPagination && initialState ? ( + + { + if (onStateChange && previousCursor !== undefined && previousCursor !== null) { + onStateChange({ ...initialState, cursor: previousCursor }); + } + }} + size="sm" + variant="ghost" + > + + + { + if (onStateChange && nextCursor !== undefined && nextCursor !== null) { + onStateChange({ ...initialState, cursor: nextCursor }); + } + }} + size="sm" + variant="ghost" + > + + + + ) : undefined} ); }; diff --git a/airflow-core/src/airflow/ui/src/components/DataTable/searchParams.ts b/airflow-core/src/airflow/ui/src/components/DataTable/searchParams.ts index f1923e90e2244..85ee6cc688f92 100644 --- a/airflow-core/src/airflow/ui/src/components/DataTable/searchParams.ts +++ b/airflow-core/src/airflow/ui/src/components/DataTable/searchParams.ts @@ -22,7 +22,12 @@ import { SearchParamsKeys, type SearchParamsKeysType } from "src/constants/searc import type { TableState } from "./types"; -const { LIMIT: LIMIT_PARAM, OFFSET: OFFSET_PARAM, SORT: SORT_PARAM }: SearchParamsKeysType = SearchParamsKeys; +const { + CURSOR: CURSOR_PARAM, + LIMIT: LIMIT_PARAM, + OFFSET: OFFSET_PARAM, + SORT: SORT_PARAM, +}: SearchParamsKeysType = SearchParamsKeys; export const stateToSearchParams = (state: TableState, defaultTableState?: TableState): URLSearchParams => { const queryParams = new URLSearchParams(globalThis.location.search); @@ -39,6 +44,12 @@ export const stateToSearchParams = (state: TableState, defaultTableState?: Table queryParams.set(OFFSET_PARAM, `${state.pagination.pageIndex}`); } + if (state.cursor !== undefined && state.cursor !== "") { + queryParams.set(CURSOR_PARAM, state.cursor); + } else { + queryParams.delete(CURSOR_PARAM); + } + if (state.sorting.length) { state.sorting.forEach(({ desc, id }) => { if (defaultTableState?.sorting.find((sort) => sort.id === id && sort.desc === desc)) { @@ -68,6 +79,13 @@ export const searchParamsToState = (searchParams: URLSearchParams, defaultState: }, }; } + + const cursorValue = searchParams.get(CURSOR_PARAM); + + if (cursorValue !== null) { + urlState = { ...urlState, cursor: cursorValue }; + } + const sorts = searchParams.getAll(SORT_PARAM); const sorting: SortingState = sorts.map((sort) => ({ desc: sort.startsWith("-"), diff --git a/airflow-core/src/airflow/ui/src/components/DataTable/types.ts b/airflow-core/src/airflow/ui/src/components/DataTable/types.ts index 5e2c047db6f23..e1ce19668a64a 100644 --- a/airflow-core/src/airflow/ui/src/components/DataTable/types.ts +++ b/airflow-core/src/airflow/ui/src/components/DataTable/types.ts @@ -22,6 +22,7 @@ import type { JSX, ReactNode } from "react"; export type TableState = { columnVisibility?: VisibilityState; + cursor?: string; pagination: PaginationState; sorting: SortingState; }; diff --git a/airflow-core/src/airflow/ui/src/constants/searchParams.ts b/airflow-core/src/airflow/ui/src/constants/searchParams.ts index 7194fe69ceeb2..627227a04c1de 100644 --- a/airflow-core/src/airflow/ui/src/constants/searchParams.ts +++ b/airflow-core/src/airflow/ui/src/constants/searchParams.ts @@ -27,6 +27,7 @@ export enum SearchParamsKeys { CREATED_AT_GTE = "created_at_gte", CREATED_AT_LTE = "created_at_lte", CREATED_AT_RANGE = "created_at_range", + CURSOR = "cursor", DAG_DISPLAY_NAME_PATTERN = "dag_display_name_pattern", DAG_ID = "dag_id", DAG_ID_PATTERN = "dag_id_pattern", diff --git a/airflow-core/src/airflow/ui/src/pages/TaskInstances/TaskInstances.tsx b/airflow-core/src/airflow/ui/src/pages/TaskInstances/TaskInstances.tsx index f897ad4c28d8b..7c5f6287649b5 100644 --- a/airflow-core/src/airflow/ui/src/pages/TaskInstances/TaskInstances.tsx +++ b/airflow-core/src/airflow/ui/src/pages/TaskInstances/TaskInstances.tsx @@ -223,6 +223,7 @@ export const TaskInstances = () => { const { t: translate } = useTranslation(); const { dagId, groupId, runId, taskId } = useParams(); const [searchParams] = useSearchParams(); + const { setTableURLState, tableURLState } = useTableURLState({ columnVisibility: { dag_version: false, @@ -233,7 +234,7 @@ export const TaskInstances = () => { queue: false, }, }); - const { pagination, sorting } = tableURLState; + const { cursor, pagination, sorting } = tableURLState; const [sort] = sorting; const orderBy = sort ? [`${sort.desc ? "-" : ""}${sort.id}`] : ["-id"]; @@ -259,6 +260,7 @@ export const TaskInstances = () => { const { data, error, isLoading } = useTaskInstanceServiceGetTaskInstances( { + cursor: cursor ?? "", dagId: dagId ?? "~", dagIdPattern: filteredDagIdPattern ?? undefined, dagRunId: runId ?? "~", @@ -269,7 +271,6 @@ export const TaskInstances = () => { logicalDateGte: logicalDateGte ?? undefined, logicalDateLte: logicalDateLte ?? undefined, mapIndex: mapIndexFilter !== null && mapIndexFilter !== "" ? [Number(mapIndexFilter)] : undefined, - offset: pagination.pageIndex * pagination.pageSize, operatorNamePattern: operatorNamePattern ?? undefined, orderBy, poolNamePattern: poolNamePattern ?? undefined, @@ -292,6 +293,9 @@ export const TaskInstances = () => { }, ); + const nextCursor = data && "next_cursor" in data ? data.next_cursor : undefined; + const previousCursor = data && "previous_cursor" in data ? data.previous_cursor : undefined; + const columns = taskInstanceColumns({ dagId, runId, @@ -309,8 +313,9 @@ export const TaskInstances = () => { initialState={tableURLState} isLoading={isLoading} modelName="common:taskInstance" + nextCursor={nextCursor} onStateChange={setTableURLState} - total={data?.total_entries ?? undefined} + previousCursor={previousCursor} /> ); diff --git a/airflow-core/src/airflow/ui/src/utils/useFiltersHandler.ts b/airflow-core/src/airflow/ui/src/utils/useFiltersHandler.ts index 6df86f6f6ab18..f5dd2b0101a3b 100644 --- a/airflow-core/src/airflow/ui/src/utils/useFiltersHandler.ts +++ b/airflow-core/src/airflow/ui/src/utils/useFiltersHandler.ts @@ -145,6 +145,7 @@ export const useFiltersHandler = (searchParamKeys: Array) => { setTableURLState({ + cursor: undefined, pagination: { ...pagination, pageIndex: 0 }, sorting, }); @@ -167,6 +168,7 @@ export const useFiltersHandler = (searchParamKeys: Array Date: Tue, 21 Apr 2026 01:50:55 +0200 Subject: [PATCH 028/309] [v3-2-test] Isolate non-provider mypy hooks per distribution with dedicated .build/ venvs (#65492) (#65549) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Isolate mypy prek hooks, cover all non-provider dirs, and clean up type errors Each non-provider mypy prek hook now builds and caches its own virtualenv at .build/mypy-venvs// and its own mypy cache at .build/mypy-caches//. UV_PROJECT_ENVIRONMENT redirects uv away from the project's .venv, so running the hook never mutates a contributor's regular development environment while still matching CI's frozen dependency set. Mypy runs with --follow-imports=silent so each hook only reports errors for files it owns; transitive code is covered by its own hook and different venvs no longer produce divergent results on shared code. Adds mypy hooks for the non-provider directories that were previously uncovered: airflow-ctl-tests, helm-tests, airflow-e2e-tests, task-sdk-integration-tests, docker-tests, kubernetes-tests, and shared. The mypy-shared hook iterates every shared/ workspace distribution and builds a separate venv + cache per distribution so each shared library is type-checked against its own dependency set. breeze down --cleanup-mypy-cache additionally removes .build/mypy-venvs/ and .build/mypy-caches/ so all per-hook state is wiped alongside the existing .mypy_cache and mypy-cache-volume. Also fixes pre-existing type errors surfaced by the newly added and cleaned-up checks: platform-specific ignores for Linux-only os.posix_fadvise in the shared logging helper, narrower types and type: ignore where appropriate in shared configuration/observability/ timezones/secrets_backend/secrets_masker, Liskov override markers on the AirflowConfigParser subclass methods, and small correctness fixes in dev/breeze and the docker-tests / kubernetes-tests helpers so the full non-provider mypy suite runs clean on macOS and in CI. * Move mypy prek hooks to their respective distribution configs The new mypy hooks for airflow-ctl-tests, helm-tests, airflow-e2e-tests, task-sdk-integration-tests, docker-tests, and kubernetes-tests now live in each distribution's own .pre-commit-config.yaml, matching the pattern already used by airflow-core, task-sdk, and airflow-ctl. New .pre-commit- config.yaml files are added to distributions that didn't have one. prek auto-discovers nested configs, so the hooks remain part of the default check set. mypy-dev (covers dev + scripts), mypy-devel-common, and mypy-shared stay at the repo root: dev/scripts/devel-common don't have their own configs, and mypy-shared iterates every shared/ distribution so has no single home. * Split mypy-dev and mypy-scripts, each with its own pyproject.toml config Previously the mypy-dev prek hook ran mypy against dev/ and scripts/ in a single invocation under the dev project's virtualenv. The two now get independent hooks — mypy-dev in dev/.pre-commit-config.yaml and mypy-scripts in scripts/.pre-commit-config.yaml — so each can evolve its own dependency set and check its own folder. Copy the full [tool.mypy] section from the root pyproject.toml into both dev/pyproject.toml and scripts/pyproject.toml so each sub-project owns its mypy configuration. Paths inside mypy_path are rewritten from $MYPY_CONFIG_FILE_DIR/ to $MYPY_CONFIG_FILE_DIR/../ so they still resolve to the repo-root siblings from the sub-project location. The decorator/ outputs plugins are scoped to dev only (scripts does not author DAG code). mypy_local_folder.py now passes --config-file /pyproject.toml when the folder maps to one of these sub-project configs, so mypy uses the sub-project's configuration rather than the root one. * Teach selective-checks about the new non-provider mypy hooks Add FileGroupForCi entries and regex patterns for helm-tests, airflow-e2e-tests, docker-tests, kubernetes-tests, scripts, and shared Python files, then wire them into skip_prek_hooks so the corresponding mypy-* prek hook is only kept when its folder changed: - mypy-scripts (split off from the old combined mypy-dev) - mypy-airflow-ctl-tests, mypy-helm-tests, mypy-airflow-e2e-tests, mypy-task-sdk-integration-tests, mypy-docker-tests, mypy-kubernetes-tests - mypy-shared Update test_selective_checks.py skip-list constants and per-case inline skip lists to include the new hooks. Targeted test cases for files under the new-hook directories override skip-prek-hooks to leave the matching hook out of the skip set, confirming it will run when its folder changes. * Trim dev/scripts pyproject mypy_path to just relevant distributions Drop the 200+ provider path entries that were blindly copied from the root pyproject.toml. dev and scripts only import from other non-provider workspace members, so listing every provider src/tests directory under mypy_path just adds noise. The remaining non-provider entries cover everything dev or scripts plausibly import from. * Install mypy into per-hook venvs from uv.lock via a `mypy` dep group Each non-provider distribution with a mypy prek hook now declares a `mypy` dependency group in its pyproject.toml resolving to `apache-airflow-devel-common[mypy]`. mypy_local_folder.py syncs each dedicated virtualenv with `uv sync --frozen --project --group mypy` and runs mypy with `uv run --frozen --project --group mypy` — so mypy and its type stubs come from the workspace uv.lock, not from an ephemeral `--with` overlay whose resolution is independent of the main lockfile. uv.lock is refreshed to include the new group. Covers airflow-core, task-sdk, airflow-ctl, devel-common, dev, scripts, airflow-ctl-tests, helm-tests, airflow-e2e-tests, task-sdk-integration- tests, docker-tests, kubernetes-tests, and every shared/ workspace member. * Drop mypy_path from dev/scripts pyprojects — venv site-packages is enough After the switch to installing mypy (and every transitive workspace dependency) directly into each hook's virtualenv via the `mypy` dep group, workspace packages like airflow, airflow.sdk, airflowctl, airflow_breeze, tests_common are all available via the venv's site-packages. mypy resolves them without needing mypy_path entries, so drop the copied list and leave a short comment explaining why. * Split mypy-shared into per-distribution hooks and enforce the pattern Each shared/ workspace member now owns a mypy-shared- prek hook backed by its own shared//.pre-commit-config.yaml. The single mypy-shared iterator is gone — mypy_local_folder.py accepts shared/ as a first-class folder and the per-hook virtualenv now lives at .build/mypy-venvs/shared-/ (slash in the folder name is replaced with a dash in the venv/cache path). Adds a new check-shared-mypy-hooks prek hook that fails when a shared/ workspace member is missing its dedicated .pre-commit- config.yaml, printing the exact YAML to add. Selective-checks emits one skip entry per dist, enumerated from shared/ at run time. Contributing docs cover the two-step process for adding a new shared library. * Pin minimum_prek_version to 0.3.4 consistently across all configs All .pre-commit-config.yaml files now require prek >= 0.3.4 (the version already declared by the root config). Previously the nested configs pinned a mix of 0.2.0, 0.3.2, and 0.3.4, so a contributor could pass the root's version check and still trip on stale subproject pins as they moved between directories. * Refresh uv.lock after rebase to reflect the `mypy` dep groups The rebase onto main resolved the uv.lock conflict by taking main's version, so `uv sync --group mypy` would fail against uv.lock until the groups added to the per-distribution pyprojects were re-resolved. Regenerates the lockfile to include them. * Add explicit selective-checks test for per-shared-dist mypy hook skipping Verifies that when a file under shared/logging/ changes, only mypy-shared-logging is kept among the thirteen mypy-shared-* hooks; all other shared distributions' hooks land in the skip list. Pins the contract that the runtime enumeration over shared/*/pyproject.toml works as intended. * Refresh mypy docs to match the per-hook venv + --group mypy workflow Fills in the docs that still referenced the pre-split workflow: - AGENTS.md: mentions `mypy-shared-` per shared workspace member and the `uv sync --group mypy` install path for mypy itself. - scripts/ci/prek/AGENTS.md: clarifies that non-provider mypy hooks run locally through mypy_local_folder.py (Breeze image only needed for the providers hook). - dev/breeze/doc/03_developer_tasks.rst: renames stale `mypy-airflow` to `mypy-airflow-core`, and expands the cache note to cover the per-hook virtualenvs and caches under .build/. - dev/breeze/doc/ci/04_selective_checks.md: expands the file-group and skip-reason lists so every new mypy hook (scripts, task-sdk, airflow-ctl, the six test-dir hooks, and mypy-shared- enumerated at runtime) is documented. * Rename mypy_local_folder.py to run_mypy_full_dist_local_venv_or_breeze_in_ci.py Updates every .pre-commit-config.yaml entry and prose references so they point at the new script name. Two shared configs use YAML folded-scalar entries to stay under the 110-char yamllint limit; updates the validation script's expected template to match. (cherry picked from commit 4f3b228be1a785ea5df7b89d8d3291ded78fea4d) --- .pre-commit-config.yaml | 14 +- AGENTS.md | 2 +- airflow-core/.pre-commit-config.yaml | 4 +- airflow-core/pyproject.toml | 4 + airflow-ctl-tests/.pre-commit-config.yaml | 9 +- airflow-ctl-tests/pyproject.toml | 5 + airflow-ctl/.pre-commit-config.yaml | 4 +- airflow-ctl/pyproject.toml | 4 + airflow-e2e-tests/.pre-commit-config.yaml | 31 ++ airflow-e2e-tests/pyproject.toml | 5 + chart/.pre-commit-config.yaml | 2 +- contributing-docs/08_static_code_checks.rst | 46 ++- dev/.pre-commit-config.yaml | 31 ++ dev/breeze/doc/03_developer_tasks.rst | 22 +- dev/breeze/doc/ci/04_selective_checks.md | 24 +- .../commands/developer_commands.py | 5 + .../commands/release_candidate_command.py | 3 + .../commands/testing_commands.py | 2 +- .../src/airflow_breeze/utils/reproducible.py | 2 +- .../airflow_breeze/utils/selective_checks.py | 50 +++ dev/breeze/tests/test_selective_checks.py | 181 ++++++++- dev/pyproject.toml | 70 ++++ devel-common/pyproject.toml | 4 + docker-tests/.pre-commit-config.yaml | 31 ++ docker-tests/pyproject.toml | 5 + .../test_docker_compose_quick_start.py | 2 +- .../tests/docker_tests/test_prod_image.py | 2 +- go-sdk/.pre-commit-config.yaml | 2 +- helm-tests/.pre-commit-config.yaml | 31 ++ helm-tests/pyproject.toml | 5 + kubernetes-tests/.pre-commit-config.yaml | 31 ++ kubernetes-tests/pyproject.toml | 5 + .../tests/kubernetes_tests/test_base.py | 4 +- .../test_kubernetes_pod_operator.py | 2 +- providers/.pre-commit-config.yaml | 2 +- providers/common/ai/.pre-commit-config.yaml | 2 +- .../common/compat/.pre-commit-config.yaml | 2 +- providers/edge3/.pre-commit-config.yaml | 2 +- providers/fab/.pre-commit-config.yaml | 2 +- providers/keycloak/.pre-commit-config.yaml | 2 +- scripts/.pre-commit-config.yaml | 31 ++ scripts/ci/prek/AGENTS.md | 9 +- scripts/ci/prek/check_shared_mypy_hooks.py | 92 +++++ scripts/ci/prek/mypy_local_folder.py | 250 ------------ ...py_full_dist_local_venv_or_breeze_in_ci.py | 325 ++++++++++++++++ scripts/pyproject.toml | 65 ++++ shared/configuration/.pre-commit-config.yaml | 31 ++ shared/configuration/pyproject.toml | 4 + .../airflow_shared/configuration/parser.py | 12 +- shared/dagnode/.pre-commit-config.yaml | 31 ++ shared/dagnode/pyproject.toml | 4 + shared/dagnode/tests/dagnode/test_node.py | 6 +- shared/listeners/.pre-commit-config.yaml | 31 ++ shared/listeners/pyproject.toml | 4 + shared/logging/.pre-commit-config.yaml | 31 ++ shared/logging/pyproject.toml | 4 + .../src/airflow_shared/logging/_noncaching.py | 3 +- shared/module_loading/.pre-commit-config.yaml | 31 ++ shared/module_loading/pyproject.toml | 4 + shared/observability/.pre-commit-config.yaml | 31 ++ shared/observability/pyproject.toml | 4 + .../airflow_shared/observability/common.py | 22 +- .../observability/metrics/otel_logger.py | 2 +- .../observability/metrics/stats.py | 1 + .../plugins_manager/.pre-commit-config.yaml | 31 ++ shared/plugins_manager/pyproject.toml | 4 + .../.pre-commit-config.yaml | 33 ++ shared/providers_discovery/pyproject.toml | 4 + .../secrets_backend/.pre-commit-config.yaml | 31 ++ shared/secrets_backend/pyproject.toml | 4 + .../airflow_shared/secrets_backend/base.py | 2 +- shared/secrets_masker/.pre-commit-config.yaml | 31 ++ shared/secrets_masker/pyproject.toml | 4 + .../secrets_masker/test_secrets_masker.py | 2 +- shared/serialization/.pre-commit-config.yaml | 31 ++ shared/serialization/pyproject.toml | 4 + .../.pre-commit-config.yaml | 33 ++ shared/template_rendering/pyproject.toml | 4 + shared/timezones/.pre-commit-config.yaml | 31 ++ shared/timezones/pyproject.toml | 4 + .../src/airflow_shared/timezones/timezone.py | 8 +- .../.pre-commit-config.yaml | 31 ++ task-sdk-integration-tests/pyproject.toml | 5 + task-sdk/.pre-commit-config.yaml | 4 +- task-sdk/pyproject.toml | 4 + .../src/airflow/sdk/execution_time/comms.py | 4 +- uv.lock | 360 ++++++++++-------- 87 files changed, 1855 insertions(+), 498 deletions(-) create mode 100644 airflow-e2e-tests/.pre-commit-config.yaml create mode 100644 dev/.pre-commit-config.yaml create mode 100644 docker-tests/.pre-commit-config.yaml create mode 100644 helm-tests/.pre-commit-config.yaml create mode 100644 kubernetes-tests/.pre-commit-config.yaml create mode 100644 scripts/.pre-commit-config.yaml create mode 100755 scripts/ci/prek/check_shared_mypy_hooks.py delete mode 100755 scripts/ci/prek/mypy_local_folder.py create mode 100755 scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py create mode 100644 shared/configuration/.pre-commit-config.yaml create mode 100644 shared/dagnode/.pre-commit-config.yaml create mode 100644 shared/listeners/.pre-commit-config.yaml create mode 100644 shared/logging/.pre-commit-config.yaml create mode 100644 shared/module_loading/.pre-commit-config.yaml create mode 100644 shared/observability/.pre-commit-config.yaml create mode 100644 shared/plugins_manager/.pre-commit-config.yaml create mode 100644 shared/providers_discovery/.pre-commit-config.yaml create mode 100644 shared/secrets_backend/.pre-commit-config.yaml create mode 100644 shared/secrets_masker/.pre-commit-config.yaml create mode 100644 shared/serialization/.pre-commit-config.yaml create mode 100644 shared/template_rendering/.pre-commit-config.yaml create mode 100644 shared/timezones/.pre-commit-config.yaml create mode 100644 task-sdk-integration-tests/.pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ad5d533b2f412..3c879b5fc647d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1021,19 +1021,19 @@ repos: ^uv\.lock$ pass_filenames: false require_serial: true - - id: mypy-dev - name: Run mypy for dev + - id: mypy-devel-common + name: Run mypy for devel-common language: python - entry: ./scripts/ci/prek/mypy_local_folder.py dev scripts + entry: ./scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py devel-common pass_filenames: false files: ^.*\.py$ require_serial: true - - id: mypy-devel-common - name: Run mypy for devel-common + - id: check-shared-mypy-hooks + name: Every shared/ has a mypy-shared- prek hook language: python - entry: ./scripts/ci/prek/mypy_local_folder.py devel-common + entry: ./scripts/ci/prek/check_shared_mypy_hooks.py pass_filenames: false - files: ^.*\.py$ + files: ^shared/.*|^scripts/ci/prek/check_shared_mypy_hooks\.py$ require_serial: true ## ADD MOST PREK HOOK ABOVE THAT LINE # The below prek hooks are those requiring CI image to be built diff --git a/AGENTS.md b/AGENTS.md index f38bc2dd33703..7836b9a52dd00 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,7 +30,7 @@ - **Run other suites of tests** `breeze testing ` (test groups: `airflow-ctl-tests`, `docker-compose-tests`, `task-sdk-tests`) - **Run scripts tests:** `uv run --project scripts pytest scripts/tests/ -xvs` - **Run Airflow CLI:** `breeze run airflow dags list` -- **Type-check (non-providers):** first run `uv sync --frozen --project ` to align the local virtualenv with `uv.lock` (the dependency set CI uses), then `uv run --frozen --project --with "apache-airflow-devel-common[mypy]" mypy path/to/code` +- **Type-check (non-providers):** run the prek hook — `prek run mypy- --all-files` (e.g. `mypy-airflow-core`, `mypy-task-sdk`, `mypy-shared-logging`; each `shared/` workspace member has its own `mypy-shared-` hook). The hook uses a dedicated virtualenv and mypy cache under `.build/mypy-venvs//` and `.build/mypy-caches//`; mypy itself is installed from `uv.lock` via the `mypy` dependency group (`uv sync --group mypy`), so it never mutates your project `.venv`. Clear with `breeze down --cleanup-mypy-cache`. - **Type-check (providers):** `breeze run mypy path/to/code` - **Lint with ruff only:** `prek run ruff --from-ref ` - **Format with ruff only:** `prek run ruff-format --from-ref ` diff --git a/airflow-core/.pre-commit-config.yaml b/airflow-core/.pre-commit-config.yaml index 3fb564eceb78d..64ddae194edb3 100644 --- a/airflow-core/.pre-commit-config.yaml +++ b/airflow-core/.pre-commit-config.yaml @@ -16,7 +16,7 @@ # under the License. --- default_stages: [pre-commit, pre-push] -minimum_prek_version: '0.2.0' +minimum_prek_version: '0.3.4' default_language_version: python: python3 node: 22.19.0 @@ -224,7 +224,7 @@ repos: - id: mypy-airflow-core name: Run mypy for airflow-core language: python - entry: ../scripts/ci/prek/mypy_local_folder.py airflow-core + entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py airflow-core pass_filenames: false files: ^.*\.py$ require_serial: true diff --git a/airflow-core/pyproject.toml b/airflow-core/pyproject.toml index bb8b250c159a6..f1a62d84d5b38 100644 --- a/airflow-core/pyproject.toml +++ b/airflow-core/pyproject.toml @@ -311,6 +311,10 @@ dev = [ docs = [ "apache-airflow-devel-common[docs]" ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [tool.uv] diff --git a/airflow-ctl-tests/.pre-commit-config.yaml b/airflow-ctl-tests/.pre-commit-config.yaml index f6213b0f9a83c..1016fa939ebca 100644 --- a/airflow-ctl-tests/.pre-commit-config.yaml +++ b/airflow-ctl-tests/.pre-commit-config.yaml @@ -16,7 +16,7 @@ # under the License. --- default_stages: [pre-commit, pre-push] -minimum_prek_version: '0.2.0' +minimum_prek_version: '0.3.4' default_language_version: python: python3 repos: @@ -30,3 +30,10 @@ repos: files: (?x) ^tests/airflowctl_tests/.*\.py$ + - id: mypy-airflow-ctl-tests + name: Run mypy for airflow-ctl-tests + language: python + entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py airflow-ctl-tests + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/airflow-ctl-tests/pyproject.toml b/airflow-ctl-tests/pyproject.toml index b3b2f8cae8431..70c9eac9005f8 100644 --- a/airflow-ctl-tests/pyproject.toml +++ b/airflow-ctl-tests/pyproject.toml @@ -76,3 +76,8 @@ exclude = ["*"] [tool.hatch.build.targets.wheel] bypass-selection = true + +[dependency-groups] +mypy = [ + "apache-airflow-devel-common[mypy]", +] diff --git a/airflow-ctl/.pre-commit-config.yaml b/airflow-ctl/.pre-commit-config.yaml index a5773e94aabca..4561533c574d0 100644 --- a/airflow-ctl/.pre-commit-config.yaml +++ b/airflow-ctl/.pre-commit-config.yaml @@ -16,7 +16,7 @@ # under the License. --- default_stages: [pre-commit, pre-push] -minimum_prek_version: '0.2.0' +minimum_prek_version: '0.3.4' default_language_version: python: python3 node: 22.19.0 @@ -27,7 +27,7 @@ repos: - id: mypy-airflow-ctl name: Run mypy for airflow-ctl language: python - entry: ../scripts/ci/prek/mypy_local_folder.py airflow-ctl + entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py airflow-ctl pass_filenames: false files: ^.*\.py$ require_serial: true diff --git a/airflow-ctl/pyproject.toml b/airflow-ctl/pyproject.toml index da794913269cf..9bd59aaa7ec90 100644 --- a/airflow-ctl/pyproject.toml +++ b/airflow-ctl/pyproject.toml @@ -151,6 +151,10 @@ codegen = [ ] # uv run --verbose --group codegen --project apache-airflow-ctl --directory airflow-ctl/ datamodel-codegen --url="http://0.0.0.0:28080/auth/openapi.json" --output=src/airflowctl/api/datamodels/auth_generated.py +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [tool.datamodel-codegen] capitalise-enum-members=true # `State.RUNNING` not `State.running` disable-timestamp=true diff --git a/airflow-e2e-tests/.pre-commit-config.yaml b/airflow-e2e-tests/.pre-commit-config.yaml new file mode 100644 index 0000000000000..e3d6413706dfe --- /dev/null +++ b/airflow-e2e-tests/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-airflow-e2e-tests + name: Run mypy for airflow-e2e-tests + language: python + entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py airflow-e2e-tests + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/airflow-e2e-tests/pyproject.toml b/airflow-e2e-tests/pyproject.toml index 394b80b2eee00..3f93c6d07e742 100644 --- a/airflow-e2e-tests/pyproject.toml +++ b/airflow-e2e-tests/pyproject.toml @@ -77,3 +77,8 @@ exclude = ["*"] [tool.hatch.build.targets.wheel] bypass-selection = true + +[dependency-groups] +mypy = [ + "apache-airflow-devel-common[mypy]", +] diff --git a/chart/.pre-commit-config.yaml b/chart/.pre-commit-config.yaml index f95650c53241b..16634adb4e179 100644 --- a/chart/.pre-commit-config.yaml +++ b/chart/.pre-commit-config.yaml @@ -16,7 +16,7 @@ # under the License. --- default_stages: [pre-commit, pre-push] -minimum_prek_version: '0.2.0' +minimum_prek_version: '0.3.4' default_language_version: python: python3 node: 22.19.0 diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index 4eed1a98a13f9..01c5843677c7f 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -281,26 +281,39 @@ Mypy checks ----------- When we run mypy checks locally, the ``mypy-*`` checks run depending on the files you are changing: -``mypy-airflow-core``, ``mypy-dev``, ``mypy-providers``, ``mypy-task-sdk``, ``mypy-airflow-ctl``, etc. +``mypy-airflow-core``, ``mypy-dev``, ``mypy-providers``, ``mypy-scripts``, ``mypy-task-sdk``, +``mypy-airflow-ctl``, ``mypy-devel-common``, ``mypy-airflow-ctl-tests``, ``mypy-helm-tests``, +``mypy-airflow-e2e-tests``, ``mypy-task-sdk-integration-tests``, ``mypy-docker-tests``, +``mypy-kubernetes-tests``, and one ``mypy-shared-`` hook per ``shared/`` workspace +distribution (e.g. ``mypy-shared-configuration``, ``mypy-shared-logging``). -For **non-provider projects** (airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common), mypy -runs locally using the ``uv`` virtualenv — no breeze CI image is needed. These checks run as regular -prek hooks in the ``pre-commit`` stage, checking whole directories at once. This means they run both -as part of local commits and as part of regular static checks in CI (not as separate mypy CI jobs). +For **non-provider projects**, mypy runs locally using ``uv`` — no breeze CI image is needed. These +checks run as regular prek hooks in the ``pre-commit`` stage, checking whole directories at once. This +means they run both as part of local commits and as part of regular static checks in CI (not as +separate mypy CI jobs). -Before running mypy directly (or via the ``mypy-*`` prek hooks), synchronize your local virtualenv -with ``uv.lock`` so it matches the dependency set CI uses — otherwise mypy may pick up a different -set of installed packages than CI and produce results that diverge from CI: +Each non-provider ``mypy-*`` hook uses a **dedicated virtualenv and mypy cache** under ``.build/`` so +running mypy never mutates your regular project ``.venv`` and each hook keeps a stable, CI-aligned +dependency set: -.. code-block:: bash +- virtualenvs: ``.build/mypy-venvs//`` +- mypy caches: ``.build/mypy-caches//`` - uv sync --frozen --project +Adding a new shared library +~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Then run mypy directly. Use ``--frozen`` so ``uv`` does not update ``uv.lock``: +Every ``shared/`` workspace member has its own ``mypy-shared-`` prek hook so it is +type-checked in isolation against its own dependency set. When you add a new shared library under +``shared//``, you also need to: -.. code-block:: bash +1. Add a ``[dependency-groups]`` section with ``mypy = ["apache-airflow-devel-common[mypy]"]`` in + ``shared//pyproject.toml`` (so ``uv sync --group mypy`` installs mypy into the hook's + dedicated virtualenv). +2. Create ``shared//.pre-commit-config.yaml`` with a ``mypy-shared-`` hook + entry that calls ``../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/``. - uv run --frozen --project --with "apache-airflow-devel-common[mypy]" mypy path/to/code +The ``check-shared-mypy-hooks`` prek hook enforces step 2 — it fails and prints the exact config +contents to add when any ``shared/`` is missing its dedicated mypy hook. To run the prek hook for a specific project (example for ``airflow-core`` files): @@ -308,20 +321,19 @@ To run the prek hook for a specific project (example for ``airflow-core`` files) prek mypy-airflow-core --all-files -To show unused mypy ignores for any providers/airflow etc, eg: run below command: +To show unused mypy ignores, run: .. code-block:: bash export SHOW_UNUSED_MYPY_WARNINGS=true prek mypy-airflow-core --all-files -For non-provider projects, the local mypy cache is stored in ``.mypy_cache`` at the repo root. - For **providers**, mypy still runs via breeze (``breeze run mypy``) as a separate CI job and requires ``breeze ci-image build --python 3.10`` to be built locally. Providers use a separate docker-volume (called ``mypy-cache-volume``) that keeps the cache of last MyPy execution. -To clear all mypy caches (both local ``.mypy_cache`` and the Docker volume), run +To clear all mypy caches (the Docker volume used by providers, any legacy repo-root ``.mypy_cache``, +and the per-hook venvs + caches under ``.build/mypy-venvs/`` and ``.build/mypy-caches/``), run ``breeze down --cleanup-mypy-cache``. ----------- diff --git a/dev/.pre-commit-config.yaml b/dev/.pre-commit-config.yaml new file mode 100644 index 0000000000000..f401bd9754594 --- /dev/null +++ b/dev/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-dev + name: Run mypy for dev + language: python + entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py dev + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/dev/breeze/doc/03_developer_tasks.rst b/dev/breeze/doc/03_developer_tasks.rst index 0db3d77fed263..37dff9045ad79 100644 --- a/dev/breeze/doc/03_developer_tasks.rst +++ b/dev/breeze/doc/03_developer_tasks.rst @@ -321,7 +321,7 @@ For example, this following command: .. code-block:: bash - prek mypy-airflow + prek mypy-airflow-core will run mypy check for currently staged files inside ``airflow/`` excluding providers. .. _breeze-dev:running-prek-in-breeze: @@ -349,7 +349,7 @@ re-run latest prek hooks on your changes, but it can take a long time (few minut .. code-block:: bash - prek mypy-airflow --all-files + prek mypy-airflow-core --all-files The above will run mypy check for all files. @@ -358,7 +358,7 @@ specifying (can be multiple times) ``--file`` flag. .. code-block:: bash - prek mypy-airflow --file airflow/utils/code_utils.py --file airflow/utils/timeout.py + prek mypy-airflow-core --file airflow/utils/code_utils.py --file airflow/utils/timeout.py The above will run mypy check for those to files (note: autocomplete should work for the file selection). @@ -370,7 +370,7 @@ of commits you choose. .. code-block:: bash - prek mypy-airflow --last-commit + prek mypy-airflow-core --last-commit The above will run mypy check for all files in the last commit in your branch. @@ -383,9 +383,17 @@ in ``--from-ref`` and ``--to-ref`` flags. .. note:: - When you run static checks, some of the artifacts (mypy_cache) is stored in docker-compose volume - so that it can speed up static checks execution significantly. However, sometimes, the cache might - get broken, in which case you should run ``breeze down`` to clean up the cache. + When you run static checks, some of the artifacts (mypy_cache) is stored to speed up static + checks execution significantly: + + - The providers ``mypy-providers`` hook runs via Breeze and stores its cache in the + ``mypy-cache-volume`` docker-compose volume. + - Each non-provider ``mypy-*`` hook uses its own dedicated virtualenv and mypy cache under + ``.build/mypy-venvs//`` and ``.build/mypy-caches//``; mypy itself is installed + from the workspace ``uv.lock`` via the ``mypy`` dependency group (``uv sync --group mypy``). + + If the cache gets broken, run ``breeze down --cleanup-mypy-cache`` which wipes the docker + volume and every per-hook ``.build/mypy-venvs/`` and ``.build/mypy-caches/`` directory. .. note:: diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index ab20ef96adde2..0290bd9ee530e 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -68,9 +68,20 @@ We have the following Groups of files for CI that determine which tests are run: * `All Python files` - if none of the Python file changed, that indicates that we should not run unit tests * `All source files` - if none of the sources change, that indicates that we should probably not build an image and run any image-based static checks -* `All Airflow Python files` - files that are checked by `mypy-airflow` static checks +* `All Airflow Python files` - files that are checked by `mypy-airflow-core` static checks * `All Providers Python files` - files that are checked by `mypy-providers` static checks * `All Dev Python files` - files that are checked by `mypy-dev` static checks +* `All Scripts Python files` - files that are checked by `mypy-scripts` static checks +* `Task SDK files` - files that are checked by `mypy-task-sdk` static checks +* `All Airflow CTL Python files` - files that are checked by `mypy-airflow-ctl` static checks +* `All Devel Common Python files` - files that are checked by `mypy-devel-common` static checks +* `All Helm Tests Python files` / `All Docker Tests Python files` / + `All Kubernetes Tests Python files` / `All Airflow E2E Tests Python files` / + `Airflow CTL Integration Test files` / `Task SDK Integration Test files` - files that are + checked by the respective `mypy-helm-tests` / `mypy-docker-tests` / `mypy-kubernetes-tests` / + `mypy-airflow-e2e-tests` / `mypy-airflow-ctl-tests` / `mypy-task-sdk-integration-tests` hooks +* `shared//**/*.py` - each `shared/` workspace member has its own `mypy-shared-` + hook (selective-checks enumerates them at runtime) * `All Provider Yaml files` - all provider yaml files We have a number of `TEST_TYPES` that can be selectively disabled/enabled based on the @@ -140,8 +151,17 @@ when some files are not changed. Those are the rules implemented: * If "full tests" mode is detected, no more prek hooks are skipped - we run all of them * The following checks are skipped if those files are not changed: * if no `All Providers Python files` changed - `mypy-providers` check is skipped - * if no `All Airflow Python files` changed - `mypy-airflow` check is skipped + * if no `All Airflow Python files` changed - `mypy-airflow-core` check is skipped * if no `All Dev Python files` changed - `mypy-dev` check is skipped + * if no `All Scripts Python files` changed - `mypy-scripts` check is skipped + * if no `Task SDK files` changed - `mypy-task-sdk` check is skipped + * if no `All Airflow CTL Python files` changed - `mypy-airflow-ctl` check is skipped + * if no `All Devel Common Python files` changed - `mypy-devel-common` check is skipped + * if no files under the matching folder changed, the corresponding per-folder mypy hook + is skipped (`mypy-airflow-ctl-tests`, `mypy-helm-tests`, `mypy-airflow-e2e-tests`, + `mypy-task-sdk-integration-tests`, `mypy-docker-tests`, `mypy-kubernetes-tests`) + * for each `shared/` workspace member, `mypy-shared-` is skipped when no + file under `shared//` changed (enumerated at runtime) * if no `UI files` changed - `ts-compile-format-lint-ui` check is skipped * if no `WWW files` changed - `ts-compile-format-lint-www` check is skipped * if no `All Python files` changed - `flynt` check is skipped diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index f10ed0ec0bd3c..afb22da6355fe 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -936,6 +936,11 @@ def down(preserve_volumes: bool, cleanup_mypy_cache: bool, cleanup_build_cache: if local_mypy_cache.exists(): console_print(f"\n[info]Removing local mypy cache: {local_mypy_cache}\n") shutil.rmtree(local_mypy_cache) + for subdir in ("mypy-venvs", "mypy-caches"): + hook_dir = AIRFLOW_ROOT_PATH / ".build" / subdir + if hook_dir.exists(): + console_print(f"\n[info]Removing dedicated mypy {subdir}: {hook_dir}\n") + shutil.rmtree(hook_dir) if cleanup_build_cache: command_to_execute = ["docker", "volume", "rm", "--force", "airflow-cache-volume"] run_command(command_to_execute) diff --git a/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py b/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py index 651aa803dd6eb..f3401b6074f91 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py +++ b/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py @@ -349,6 +349,9 @@ def create_tarball_release( console_print(f"[error]Unsupported tarball type: {tarball_type}") exit(1) source_date_epoch = get_source_date_epoch(AIRFLOW_ROOT_PATH) + if version is None: + console_print("[error]Version must be set before creating the tarball") + exit(1) # Create the tarball tarball_release( version=version, diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 3ee2e605614a1..48b3a676782bb 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -1454,7 +1454,7 @@ def airflow_e2e_tests( console_print(f"[info]Running Airflow E2E tests with PROD image: {image_name}[/]") # If the image is used from docker hub, test container will pull that part of test. - skip_image_check = True if image_name.startswith("apache/airflow") else False + skip_image_check = bool(image_name and image_name.startswith("apache/airflow")) return_code, info = run_docker_compose_tests( image_name=image_name, python_version=python, diff --git a/dev/breeze/src/airflow_breeze/utils/reproducible.py b/dev/breeze/src/airflow_breeze/utils/reproducible.py index a48f6b048b8be..218598dd66045 100644 --- a/dev/breeze/src/airflow_breeze/utils/reproducible.py +++ b/dev/breeze/src/airflow_breeze/utils/reproducible.py @@ -123,7 +123,7 @@ def reset(tarinfo): temp_file = f"{dest_archive}.temp~" with os.fdopen(os.open(temp_file, os.O_WRONLY | os.O_CREAT, 0o644), "wb") as out_file: with gzip.GzipFile(fileobj=out_file, mtime=0, mode="wb") as gzip_file: - with tarfile.open(fileobj=gzip_file, mode="w:") as tar_file: + with tarfile.open(fileobj=gzip_file, mode="w:") as tar_file: # type: ignore[arg-type] for entry in file_list: entry_path = Path(entry) if not entry_path.is_symlink(): diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index aae6965dcff12..4fd2647f438b1 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -135,6 +135,11 @@ class FileGroupForCi(Enum): ALL_PROVIDERS_DISTRIBUTION_CONFIG_FILES = auto() ALL_DEV_PYTHON_FILES = auto() ALL_DEVEL_COMMON_PYTHON_FILES = auto() + ALL_SCRIPTS_PYTHON_FILES = auto() + ALL_HELM_TESTS_PYTHON_FILES = auto() + ALL_AIRFLOW_E2E_TESTS_PYTHON_FILES = auto() + ALL_DOCKER_TESTS_PYTHON_FILES = auto() + ALL_KUBERNETES_TESTS_PYTHON_FILES = auto() ALL_PROVIDER_YAML_FILES = auto() TESTS_UTILS_FILES = auto() ASSET_FILES = auto() @@ -293,6 +298,21 @@ def __hash__(self): FileGroupForCi.ALL_DEVEL_COMMON_PYTHON_FILES: [ r"^devel-common/.*\.py$", ], + FileGroupForCi.ALL_SCRIPTS_PYTHON_FILES: [ + r"^scripts/.*\.py$", + ], + FileGroupForCi.ALL_HELM_TESTS_PYTHON_FILES: [ + r"^helm-tests/.*\.py$", + ], + FileGroupForCi.ALL_AIRFLOW_E2E_TESTS_PYTHON_FILES: [ + r"^airflow-e2e-tests/.*\.py$", + ], + FileGroupForCi.ALL_DOCKER_TESTS_PYTHON_FILES: [ + r"^docker-tests/.*\.py$", + ], + FileGroupForCi.ALL_KUBERNETES_TESTS_PYTHON_FILES: [ + r"^kubernetes-tests/.*\.py$", + ], FileGroupForCi.ALL_SOURCE_FILES: [ r"^.pre-commit-config.yaml$", r"^airflow-core/src/.*", @@ -1474,12 +1494,42 @@ def skip_prek_hooks(self) -> str: prek_hooks_to_skip.add("mypy-airflow-core") if not self._matching_files(FileGroupForCi.ALL_DEV_PYTHON_FILES, CI_FILE_GROUP_MATCHES): prek_hooks_to_skip.add("mypy-dev") + if not self._matching_files(FileGroupForCi.ALL_SCRIPTS_PYTHON_FILES, CI_FILE_GROUP_MATCHES): + prek_hooks_to_skip.add("mypy-scripts") if not self._matching_files(FileGroupForCi.TASK_SDK_FILES, CI_FILE_GROUP_MATCHES): prek_hooks_to_skip.add("mypy-task-sdk") if not self._matching_files(FileGroupForCi.ALL_DEVEL_COMMON_PYTHON_FILES, CI_FILE_GROUP_MATCHES): prek_hooks_to_skip.add("mypy-devel-common") if not self._matching_files(FileGroupForCi.ALL_AIRFLOW_CTL_PYTHON_FILES, CI_FILE_GROUP_MATCHES): prek_hooks_to_skip.add("mypy-airflow-ctl") + if not self._matching_files( + FileGroupForCi.AIRFLOW_CTL_INTEGRATION_TEST_FILES, CI_FILE_GROUP_MATCHES + ): + prek_hooks_to_skip.add("mypy-airflow-ctl-tests") + if not self._matching_files(FileGroupForCi.ALL_HELM_TESTS_PYTHON_FILES, CI_FILE_GROUP_MATCHES): + prek_hooks_to_skip.add("mypy-helm-tests") + if not self._matching_files( + FileGroupForCi.ALL_AIRFLOW_E2E_TESTS_PYTHON_FILES, CI_FILE_GROUP_MATCHES + ): + prek_hooks_to_skip.add("mypy-airflow-e2e-tests") + if not self._matching_files( + FileGroupForCi.TASK_SDK_INTEGRATION_TEST_FILES, CI_FILE_GROUP_MATCHES + ): + prek_hooks_to_skip.add("mypy-task-sdk-integration-tests") + if not self._matching_files(FileGroupForCi.ALL_DOCKER_TESTS_PYTHON_FILES, CI_FILE_GROUP_MATCHES): + prek_hooks_to_skip.add("mypy-docker-tests") + if not self._matching_files( + FileGroupForCi.ALL_KUBERNETES_TESTS_PYTHON_FILES, CI_FILE_GROUP_MATCHES + ): + prek_hooks_to_skip.add("mypy-kubernetes-tests") + # One mypy-shared- hook per shared/ workspace member; each is only + # kept when that distribution's own files changed. + for dist in sorted( + p.name for p in (AIRFLOW_ROOT_PATH / "shared").iterdir() if (p / "pyproject.toml").exists() + ): + pattern = re.compile(rf"^shared/{re.escape(dist)}/.*\.py$") + if not any(pattern.match(f) for f in self._files): + prek_hooks_to_skip.add(f"mypy-shared-{dist}") return ",".join(sorted(prek_hooks_to_skip)) @cached_property diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 725e77ee399c8..f9443b06a901b 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -101,54 +101,108 @@ ALL_SKIPPED_COMMITS_ON_NO_CI_IMAGE = ( "check-provider-yaml-valid,flynt,identity,lint-helm-chart," - "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED = "identity,update-uv-lock" ALL_SKIPPED_COMMITS_IF_NO_UI = ( - "identity,mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "identity,mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_HELM_TESTS = ( "identity,lint-helm-chart," - "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,update-uv-lock" + "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS = ( "identity,lint-helm-chart," - "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_AND_UI = ( "check-provider-yaml-valid,identity," - "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS = ( "check-provider-yaml-valid,identity,lint-helm-chart," - "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS = ( "check-provider-yaml-valid,identity,lint-helm-chart," - "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_CODE_PROVIDERS_AND_HELM_TESTS = ( "check-provider-yaml-valid,flynt,identity,lint-helm-chart," - "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,update-uv-lock" + "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NOT_IMPORTANT_FILES_CHANGED = ( "check-provider-yaml-valid,flynt,identity,lint-helm-chart," - "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) @@ -345,7 +399,13 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "docs-build": "true", "skip-prek-hooks": ( "check-provider-yaml-valid,identity,lint-helm-chart," - "mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "upgrade-to-newer-dependencies": "false", @@ -612,7 +672,13 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "full-tests-needed": "false", "skip-prek-hooks": ( "check-provider-yaml-valid,identity,lint-helm-chart," - "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common," + "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "skip-providers-tests": "false", @@ -642,7 +708,17 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-task-sdk-integration-tests": "true", "docs-build": "false", "full-tests-needed": "false", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS, + "skip-prek-hooks": ( + "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "skip-providers-tests": "true", "upgrade-to-newer-dependencies": "false", "run-mypy-providers": "false", @@ -671,7 +747,13 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "full-tests-needed": "false", "skip-prek-hooks": ( "check-provider-yaml-valid,identity,lint-helm-chart," - "mypy-airflow-core,mypy-dev,mypy-devel-common,mypy-task-sdk," + "mypy-airflow-core,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "skip-providers-tests": "true", @@ -699,7 +781,17 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-airflow-ctl-integration-tests": "true", "docs-build": "false", "full-tests-needed": "false", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS, + "skip-prek-hooks": ( + "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "skip-providers-tests": "true", "upgrade-to-newer-dependencies": "false", "run-mypy-providers": "false", @@ -1032,7 +1124,13 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-kubernetes-tests": "false", "skip-prek-hooks": ( "identity,lint-helm-chart," - "mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "upgrade-to-newer-dependencies": "false", @@ -1190,7 +1288,13 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "docs-build": "true", "skip-prek-hooks": ( "check-provider-yaml-valid,flynt,identity," - "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "upgrade-to-newer-dependencies": "false", @@ -1307,6 +1411,27 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="Shared library python changes trigger unit tests", ) ), + ( + pytest.param( + ("shared/logging/src/airflow_shared/logging/remote.py",), + { + "skip-prek-hooks": ( + "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests," + "mypy-airflow-e2e-tests,mypy-dev,mypy-devel-common,mypy-docker-tests," + "mypy-helm-tests,mypy-kubernetes-tests,mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners," + "mypy-shared-module_loading,mypy-shared-observability," + "mypy-shared-plugins_manager,mypy-shared-providers_discovery," + "mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering," + "mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), + }, + id=("Shared logging change keeps only mypy-shared-logging among the mypy-shared-* hooks"), + ) + ), ], ) def test_expected_output_pull_request_main( @@ -2030,7 +2155,13 @@ def test_expected_output_push( "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "skip-prek-hooks": ( "check-provider-yaml-valid,identity,lint-helm-chart," - "mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "upgrade-to-newer-dependencies": "false", @@ -2065,7 +2196,13 @@ def test_expected_output_push( "microsoft.mssql mysql openlineage oracle postgres " "presto salesforce samba sftp ssh standard trino", "skip-prek-hooks": ( - "identity,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "identity,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "run-kubernetes-tests": "true", @@ -2105,7 +2242,13 @@ def test_expected_output_push( "docs-list-as-string": "apache-airflow", "skip-prek-hooks": ( "check-provider-yaml-valid,identity,lint-helm-chart," - "mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," + "mypy-scripts," + "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," + "mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager," + "mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker," + "mypy-shared-serialization,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ), "run-kubernetes-tests": "false", diff --git a/dev/pyproject.toml b/dev/pyproject.toml index 82a144f280642..7cb9ae40faf81 100644 --- a/dev/pyproject.toml +++ b/dev/pyproject.toml @@ -66,3 +66,73 @@ exclude = ["*"] [tool.hatch.build.targets.wheel] bypass-selection = true + +[tool.mypy] +ignore_missing_imports = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = false +plugins = [ + "airflow_mypy/plugin/decorators.py", + "airflow_mypy/plugin/outputs.py", +] +pretty = true +show_error_codes = true +disable_error_code = [ + "annotation-unchecked", +] +namespace_packages = true +explicit_package_bases = true +exclude = [ + ".*/node_modules/.*", + # Exclude hidden files and directories + ".*/\\..*" +] +# Workspace packages (airflow_breeze, airflow_perf, airflow_mypy, etc.) are installed +# editable into the hook's virtualenv by `uv sync`, so mypy resolves them via +# site-packages — no mypy_path entries needed. + +[[tool.mypy.overrides]] +module="airflow.config_templates.default_webserver_config" +disable_error_code = [ + "var-annotated", +] + +[[tool.mypy.overrides]] +module="airflow.migrations.*" +ignore_errors = true + +[[tool.mypy.overrides]] +module="airflow.*._vendor.*" +ignore_errors = true + +[[tool.mypy.overrides]] +module= [ + "google.cloud.*", + "azure.*", +] +no_implicit_optional = false + +[[tool.mypy.overrides]] +module = "google.api_core.gapic_v1" +ignore_errors = true + +[[tool.mypy.overrides]] +module=[ + "referencing.*", + # Beam has some old type annotations, and they introduced an error recently with bad signature of + # a function. This is captured in https://github.com/apache/beam/issues/29927 + # and we should remove this exclusion when it is fixed. + "apache_beam.*" +] +ignore_errors = true + +# airflowctl autogenered datamodels +[[tool.mypy.overrides]] +module="airflowctl.api.datamodels.*" +ignore_errors = true + +[dependency-groups] +mypy = [ + "apache-airflow-devel-common[mypy]", +] diff --git a/devel-common/pyproject.toml b/devel-common/pyproject.toml index a1e37d3c54a18..09ee3b008663d 100644 --- a/devel-common/pyproject.toml +++ b/devel-common/pyproject.toml @@ -171,6 +171,10 @@ docs = [ docs-gen = [ "apache-airflow-devel-common[docs-gen]", ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [project.scripts] build-docs = "docs.build_docs:build_docs" diff --git a/docker-tests/.pre-commit-config.yaml b/docker-tests/.pre-commit-config.yaml new file mode 100644 index 0000000000000..27e73ecc067d4 --- /dev/null +++ b/docker-tests/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-docker-tests + name: Run mypy for docker-tests + language: python + entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py docker-tests + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/docker-tests/pyproject.toml b/docker-tests/pyproject.toml index 04d3a062bfd6c..ca471dd20da2a 100644 --- a/docker-tests/pyproject.toml +++ b/docker-tests/pyproject.toml @@ -74,3 +74,8 @@ exclude = ["*"] [tool.hatch.build.targets.wheel] bypass-selection = true + +[dependency-groups] +mypy = [ + "apache-airflow-devel-common[mypy]", +] diff --git a/docker-tests/tests/docker_tests/test_docker_compose_quick_start.py b/docker-tests/tests/docker_tests/test_docker_compose_quick_start.py index bfefd46fc4c47..3cd93078431d8 100644 --- a/docker-tests/tests/docker_tests/test_docker_compose_quick_start.py +++ b/docker-tests/tests/docker_tests/test_docker_compose_quick_start.py @@ -169,7 +169,7 @@ def print_diagnostics(compose: DockerClient, compose_version: str, docker_versio console.print(" Docker Compose Version ".center(72, "=")) console.print(compose_version) console.print(" Compose Config ".center(72, "=")) - console.print(json.dumps(compose.config(return_json=True), indent=4)) + console.print(json.dumps(compose.config(return_json=True), indent=4)) # type: ignore[operator] for service in compose.ps(all=True): console.print(f"Service: {service.name} ".center(72, "=")) console.print(f" Service State {service.name}".center(50, ".")) diff --git a/docker-tests/tests/docker_tests/test_prod_image.py b/docker-tests/tests/docker_tests/test_prod_image.py index 808013538fea1..43b5f3cea9602 100644 --- a/docker-tests/tests/docker_tests/test_prod_image.py +++ b/docker-tests/tests/docker_tests/test_prod_image.py @@ -39,7 +39,7 @@ try: from tomllib import loads as load_tomllib except ImportError: - from tomli import loads as load_tomllib + from tomli import loads as load_tomllib # type: ignore[no-redef] airflow_core_pyproject_toml = load_tomllib(AIRFLOW_CORE_PYPROJECT_TOML.read_text()) diff --git a/go-sdk/.pre-commit-config.yaml b/go-sdk/.pre-commit-config.yaml index 74278d28bee67..72dd3a6cbebcf 100644 --- a/go-sdk/.pre-commit-config.yaml +++ b/go-sdk/.pre-commit-config.yaml @@ -16,7 +16,7 @@ # under the License. --- default_stages: [pre-commit, pre-push] -minimum_prek_version: '0.3.2' +minimum_prek_version: '0.3.4' default_language_version: python: python3 node: 22.19.0 diff --git a/helm-tests/.pre-commit-config.yaml b/helm-tests/.pre-commit-config.yaml new file mode 100644 index 0000000000000..b654d6994cea4 --- /dev/null +++ b/helm-tests/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-helm-tests + name: Run mypy for helm-tests + language: python + entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py helm-tests + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/helm-tests/pyproject.toml b/helm-tests/pyproject.toml index f061aa5950b76..e53dafa6c981e 100644 --- a/helm-tests/pyproject.toml +++ b/helm-tests/pyproject.toml @@ -74,3 +74,8 @@ exclude = ["*"] [tool.hatch.build.targets.wheel] bypass-selection = true + +[dependency-groups] +mypy = [ + "apache-airflow-devel-common[mypy]", +] diff --git a/kubernetes-tests/.pre-commit-config.yaml b/kubernetes-tests/.pre-commit-config.yaml new file mode 100644 index 0000000000000..4bfb8077b9c3b --- /dev/null +++ b/kubernetes-tests/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-kubernetes-tests + name: Run mypy for kubernetes-tests + language: python + entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py kubernetes-tests + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/kubernetes-tests/pyproject.toml b/kubernetes-tests/pyproject.toml index 9afb4e6a2bccf..4930f340df57f 100644 --- a/kubernetes-tests/pyproject.toml +++ b/kubernetes-tests/pyproject.toml @@ -81,3 +81,8 @@ exclude = ["*"] [tool.hatch.build.targets.wheel] bypass-selection = true + +[dependency-groups] +mypy = [ + "apache-airflow-devel-common[mypy]", +] diff --git a/kubernetes-tests/tests/kubernetes_tests/test_base.py b/kubernetes-tests/tests/kubernetes_tests/test_base.py index 326c1191d9c6d..bffc657415380 100644 --- a/kubernetes-tests/tests/kubernetes_tests/test_base.py +++ b/kubernetes-tests/tests/kubernetes_tests/test_base.py @@ -130,8 +130,8 @@ def _describe_resources(self, namespace: str): @staticmethod def _num_pods_in_namespace(namespace: str): air_pod = check_output(["kubectl", "get", "pods", "-n", namespace]).decode() - air_pod = air_pod.splitlines() - names = [re.compile(r"\s+").split(x)[0] for x in air_pod if "airflow" in x] + pod_lines = air_pod.splitlines() + names = [re.compile(r"\s+").split(x)[0] for x in pod_lines if "airflow" in x] return len(names) @staticmethod diff --git a/kubernetes-tests/tests/kubernetes_tests/test_kubernetes_pod_operator.py b/kubernetes-tests/tests/kubernetes_tests/test_kubernetes_pod_operator.py index 64cc3dc075763..40bc4d2fbbb82 100644 --- a/kubernetes-tests/tests/kubernetes_tests/test_kubernetes_pod_operator.py +++ b/kubernetes-tests/tests/kubernetes_tests/test_kubernetes_pod_operator.py @@ -41,7 +41,7 @@ from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction, PodManager from airflow.sdk.definitions.context import Context -from airflow.utils import timezone +from airflow.utils import timezone # type: ignore[attr-defined] from airflow.utils.types import DagRunType from airflow.version import version as airflow_version from kubernetes_tests.test_base import BaseK8STest, StringContainingId diff --git a/providers/.pre-commit-config.yaml b/providers/.pre-commit-config.yaml index 4926188916f9d..db22b1bcec75b 100644 --- a/providers/.pre-commit-config.yaml +++ b/providers/.pre-commit-config.yaml @@ -16,7 +16,7 @@ # under the License. --- default_stages: [pre-commit, pre-push] -minimum_prek_version: '0.2.0' +minimum_prek_version: '0.3.4' default_language_version: python: python3 node: 22.19.0 diff --git a/providers/common/ai/.pre-commit-config.yaml b/providers/common/ai/.pre-commit-config.yaml index 89ffddc5608b3..1a7bc73596fcb 100644 --- a/providers/common/ai/.pre-commit-config.yaml +++ b/providers/common/ai/.pre-commit-config.yaml @@ -16,7 +16,7 @@ # under the License. --- default_stages: [pre-commit, pre-push] -minimum_prek_version: '0.3.2' +minimum_prek_version: '0.3.4' default_language_version: python: python3 node: 22.19.0 diff --git a/providers/common/compat/.pre-commit-config.yaml b/providers/common/compat/.pre-commit-config.yaml index 32a7f0fd76bc1..71cfbd6dad26e 100644 --- a/providers/common/compat/.pre-commit-config.yaml +++ b/providers/common/compat/.pre-commit-config.yaml @@ -16,7 +16,7 @@ # under the License. --- default_stages: [pre-commit, pre-push] -minimum_prek_version: '0.2.0' +minimum_prek_version: '0.3.4' default_language_version: python: python3 node: 22.19.0 diff --git a/providers/edge3/.pre-commit-config.yaml b/providers/edge3/.pre-commit-config.yaml index 133e2c3c4d211..90f5a0344da26 100644 --- a/providers/edge3/.pre-commit-config.yaml +++ b/providers/edge3/.pre-commit-config.yaml @@ -16,7 +16,7 @@ # under the License. --- default_stages: [pre-commit, pre-push] -minimum_prek_version: '0.3.2' +minimum_prek_version: '0.3.4' default_language_version: python: python3 node: 22.19.0 diff --git a/providers/fab/.pre-commit-config.yaml b/providers/fab/.pre-commit-config.yaml index a77a705551c6a..5d59ede01f8c2 100644 --- a/providers/fab/.pre-commit-config.yaml +++ b/providers/fab/.pre-commit-config.yaml @@ -16,7 +16,7 @@ # under the License. --- default_stages: [pre-commit, pre-push] -minimum_prek_version: '0.2.0' +minimum_prek_version: '0.3.4' default_language_version: python: python3 node: 22.19.0 diff --git a/providers/keycloak/.pre-commit-config.yaml b/providers/keycloak/.pre-commit-config.yaml index ff86d492be761..68cdb82835ce6 100644 --- a/providers/keycloak/.pre-commit-config.yaml +++ b/providers/keycloak/.pre-commit-config.yaml @@ -16,7 +16,7 @@ # under the License. --- default_stages: [pre-commit, pre-push] -minimum_prek_version: '0.2.0' +minimum_prek_version: '0.3.4' default_language_version: python: python3 node: 22.19.0 diff --git a/scripts/.pre-commit-config.yaml b/scripts/.pre-commit-config.yaml new file mode 100644 index 0000000000000..2b55cb7cfc9a4 --- /dev/null +++ b/scripts/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-scripts + name: Run mypy for scripts + language: python + entry: ./ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py scripts + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/scripts/ci/prek/AGENTS.md b/scripts/ci/prek/AGENTS.md index 2b36b334f7307..e07c53a256c71 100644 --- a/scripts/ci/prek/AGENTS.md +++ b/scripts/ci/prek/AGENTS.md @@ -10,9 +10,12 @@ This directory contains prek (pre-commit) hook scripts. Shared utilities live in ## Breeze CI image scripts -Some prek scripts require the Breeze CI Docker image to run (e.g. mypy checks, OpenAPI spec -generation, provider validation). These scripts use the `run_command_via_breeze_shell` helper -from `common_prek_utils.py` to execute commands inside the container. +Some prek scripts require the Breeze CI Docker image to run (e.g. `mypy-providers`, OpenAPI +spec generation, provider validation). These scripts use the `run_command_via_breeze_shell` +helper from `common_prek_utils.py` to execute commands inside the container. Non-provider +mypy hooks (`mypy-airflow-core`, `mypy-task-sdk`, `mypy-shared-`, etc.) run locally via +`run_mypy_full_dist_local_venv_or_breeze_in_ci.py`, which builds a dedicated virtualenv per hook under `.build/mypy-venvs/` +using `uv sync --frozen --project --group mypy` — no Breeze image needed. When adding a new breeze-dependent hook: diff --git a/scripts/ci/prek/check_shared_mypy_hooks.py b/scripts/ci/prek/check_shared_mypy_hooks.py new file mode 100755 index 0000000000000..819d81a09f30c --- /dev/null +++ b/scripts/ci/prek/check_shared_mypy_hooks.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.10" +# dependencies = [] +# /// +"""Fail when any shared/ workspace member is missing its mypy prek config. + +Every shared library has its own `mypy-shared-` prek hook backed by a +dedicated `.pre-commit-config.yaml`. When a new shared distribution is added +under `shared/`, the contributor must also add the matching prek config so the +mypy hook runs for that library. This check enforces that rule. +""" + +from __future__ import annotations + +import sys + +from common_prek_utils import AIRFLOW_ROOT_PATH + +SHARED_DIR = AIRFLOW_ROOT_PATH / "shared" + +EXPECTED_TEMPLATE = """\ +# +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-shared-{dist} + name: Run mypy for shared-{dist} + language: python + entry: >- + ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py + shared/{dist} + pass_filenames: false + files: ^.*\\.py$ + require_serial: true +""" + + +def main() -> int: + missing: list[str] = [] + for dist_dir in sorted(SHARED_DIR.iterdir()): + if not (dist_dir / "pyproject.toml").exists(): + continue + dist = dist_dir.name + hook_id = f"mypy-shared-{dist}" + config = dist_dir / ".pre-commit-config.yaml" + if not config.exists() or hook_id not in config.read_text(): + missing.append(dist) + + if missing: + print( + "ERROR: The following shared/ workspace members are missing their " + f"dedicated mypy prek hook: {', '.join(missing)}\n" + ) + print( + "Every shared library must ship its own mypy-shared- hook so it is\n" + "type-checked in isolation (dedicated virtualenv + mypy cache under .build/).\n" + ) + print( + "Create shared//.pre-commit-config.yaml with the following contents\n" + "(add the ASF license header at the top) for each missing distribution:\n" + ) + for dist in missing: + print(f"--- shared/{dist}/.pre-commit-config.yaml ---") + print(EXPECTED_TEMPLATE.format(dist=dist)) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/prek/mypy_local_folder.py b/scripts/ci/prek/mypy_local_folder.py deleted file mode 100755 index 568fe1c06e49f..0000000000000 --- a/scripts/ci/prek/mypy_local_folder.py +++ /dev/null @@ -1,250 +0,0 @@ -#!/usr/bin/env python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# /// script -# requires-python = ">=3.10,<3.11" -# dependencies = [ -# "rich>=13.6.0", -# ] -# /// -"""Run mypy on entire folders using local virtualenv (uv) instead of breeze. - -Used for non-provider projects: airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common. -""" - -from __future__ import annotations - -import os -import re -import shlex -import subprocess -import sys - -from common_prek_utils import ( - AIRFLOW_ROOT_PATH, -) - -CI = os.environ.get("CI") - -try: - from rich.console import Console - - console = Console(width=400, color_system="standard") -except ImportError: - console = None # type: ignore[assignment] - -if __name__ not in ("__main__", "__mp_main__"): - raise SystemExit( - "This file is intended to be executed as an executable program. You cannot use it as a module." - f"To run this script, run the ./{__file__} command" - ) - -ALLOWED_FOLDERS = [ - "airflow-core", - "dev", - "scripts", - "devel-common", - "task-sdk", - "airflow-ctl", -] - -# Map folder(s) to the uv project to use for running mypy. -# When multiple folders are checked together (e.g. dev + scripts), the first folder's project is used. -FOLDER_TO_PROJECT = { - "airflow-core": "airflow-core", - "task-sdk": "task-sdk", - "airflow-ctl": "airflow-ctl", - "devel-common": "devel-common", - "dev": "dev", - "scripts": "scripts", -} - -if len(sys.argv) < 2: - if console: - console.print(f"[yellow]You need to specify the folder to test as parameter: {ALLOWED_FOLDERS}\n") - else: - print(f"You need to specify the folder to test as parameter: {ALLOWED_FOLDERS}") - sys.exit(1) - -mypy_folders = sys.argv[1:] - -show_unused_warnings = os.environ.get("SHOW_UNUSED_MYPY_WARNINGS", "false") -show_unreachable_warnings = os.environ.get("SHOW_UNREACHABLE_MYPY_WARNINGS", "false") - -for mypy_folder in mypy_folders: - if mypy_folder not in ALLOWED_FOLDERS: - if console: - console.print( - f"\n[red]ERROR: Folder `{mypy_folder}` is wrong.[/]\n\n" - f"All folders passed should be one of those: {ALLOWED_FOLDERS}\n" - ) - else: - print( - f"\nERROR: Folder `{mypy_folder}` is wrong.\n\n" - f"All folders passed should be one of those: {ALLOWED_FOLDERS}\n" - ) - sys.exit(1) - -exclude_regexps = [ - re.compile(x) - for x in [ - r"^.*/node_modules/.*", - r"^.*\\..*", - r"^.*/src/airflow/__init__.py$", - ] -] - - -def get_all_files(folder: str) -> list[str]: - files_to_check = [] - python_file_paths = (AIRFLOW_ROOT_PATH / folder).resolve().rglob("*.py") - for file in python_file_paths: - if ( - file.name not in ("conftest.py",) - and not any(x.match(file.as_posix()) for x in exclude_regexps) - and not any(part.startswith(".") for part in file.parts) - ): - files_to_check.append(file.relative_to(AIRFLOW_ROOT_PATH).as_posix()) - return files_to_check - - -all_files_to_check: list[str] = [] -for mypy_folder in mypy_folders: - all_files_to_check.extend(get_all_files(mypy_folder)) - -if not all_files_to_check: - print("No files to test. Quitting") - sys.exit(0) - -# Write file list -mypy_file_list = AIRFLOW_ROOT_PATH / "files" / "mypy_files.txt" -mypy_file_list.parent.mkdir(parents=True, exist_ok=True) -mypy_file_list.write_text("\n".join(all_files_to_check)) - -if console: - console.print(f"[info]You can check the list of files in:[/] {mypy_file_list}") -else: - print(f"You can check the list of files in: {mypy_file_list}") - -file_argument_local = f"@{mypy_file_list}" -file_argument_ci = "@/files/mypy_files.txt" - -project = FOLDER_TO_PROJECT.get(mypy_folders[0], "devel-common") - -mypy_extra_args: list[str] = [] - -if show_unused_warnings == "true": - if console: - console.print("[info]Running mypy with --warn-unused-ignores") - else: - print("Running mypy with --warn-unused-ignores") - mypy_extra_args.append("--warn-unused-ignores") - -if show_unreachable_warnings == "true": - if console: - console.print("[info]Running mypy with --warn-unreachable") - else: - print("Running mypy with --warn-unreachable") - mypy_extra_args.append("--warn-unreachable") - -if console: - console.print(f"[magenta]Running mypy for folders: {mypy_folders}[/]") -else: - print(f"Running mypy for folders: {mypy_folders}") - -if CI: - # In CI, run inside the breeze Docker image to avoid needing a local environment - # and to not change uv.lock or synchronize dependencies. - from common_prek_utils import ( - initialize_breeze_prek, - run_command_via_breeze_shell, - ) - - initialize_breeze_prek(__name__, __file__) - - mypy_cmd = f"TERM=ansi mypy {shlex.quote(file_argument_ci)} {' '.join(mypy_extra_args)}" - result = run_command_via_breeze_shell( - cmd=["bash", "-c", mypy_cmd], - warn_image_upgrade_needed=True, - extra_env={ - "INCLUDE_MYPY_VOLUME": "false", - "MOUNT_SOURCES": "selected", - }, - ) -else: - # Locally, first synchronize the project's virtualenv with uv.lock so that mypy runs - # against the same dependency set CI uses. Without this, the local .venv can drift from - # uv.lock (e.g. after switching branches or installing extras) and mypy results would - # diverge from CI. --frozen ensures uv.lock itself is not updated. - sync_cmd = ["uv", "sync", "--frozen", "--project", project] - if console: - console.print(f"[magenta]Syncing virtualenv for project {project}: {' '.join(sync_cmd)}[/]") - else: - print(f"Syncing virtualenv for project {project}: {' '.join(sync_cmd)}") - sync_result = subprocess.run( - sync_cmd, - cwd=str(AIRFLOW_ROOT_PATH), - check=False, - env={**os.environ, "TERM": "ansi"}, - ) - if sync_result.returncode != 0: - msg = ( - f"`uv sync --frozen --project {project}` failed. Fix the sync error before running mypy — " - "otherwise the local virtualenv may not match uv.lock and mypy results will diverge from CI.\n" - ) - if console: - console.print(f"[red]{msg}") - else: - print(msg) - sys.exit(sync_result.returncode) - - # Then run mypy via uv with --frozen to not update the lock file. - cmd = [ - "uv", - "run", - "--frozen", - "--project", - project, - "--with", - "apache-airflow-devel-common[mypy]", - "mypy", - file_argument_local, - *mypy_extra_args, - ] - - result = subprocess.run( - cmd, - cwd=str(AIRFLOW_ROOT_PATH), - check=False, - env={**os.environ, "TERM": "ansi"}, - ) - -if result.returncode != 0: - msg = ( - "Mypy check failed. You can run mypy locally with:\n" - f" prek run mypy-{mypy_folders[0]} --all-files\n" - "Or directly (first sync the virtualenv to match CI's dependency set):\n" - f" uv sync --frozen --project {project}\n" - f' uv run --frozen --project {project} --with "apache-airflow-devel-common[mypy]" mypy \n' - "You can also clear the mypy cache with:\n" - " breeze down --cleanup-mypy-cache\n" - ) - if console: - console.print(f"[yellow]{msg}") - else: - print(msg) -sys.exit(result.returncode) diff --git a/scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py b/scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py new file mode 100755 index 0000000000000..44664d37c270b --- /dev/null +++ b/scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.10,<3.11" +# dependencies = [ +# "rich>=13.6.0", +# ] +# /// +"""Run mypy on entire folders using local virtualenv (uv) instead of breeze. + +Used for non-provider projects: airflow-core, task-sdk, airflow-ctl, dev, scripts, +devel-common, airflow-ctl-tests, helm-tests, airflow-e2e-tests, +task-sdk-integration-tests, docker-tests, kubernetes-tests, shared. +""" + +from __future__ import annotations + +import os +import re +import shlex +import subprocess +import sys +from pathlib import Path + +from common_prek_utils import ( + AIRFLOW_ROOT_PATH, +) + +CI = os.environ.get("CI") + +try: + from rich.console import Console + + console = Console(width=400, color_system="standard") +except ImportError: + console = None # type: ignore[assignment] + +if __name__ not in ("__main__", "__mp_main__"): + raise SystemExit( + "This file is intended to be executed as an executable program. You cannot use it as a module." + f"To run this script, run the ./{__file__} command" + ) + +_TOP_LEVEL_ALLOWED_FOLDERS = [ + "airflow-core", + "dev", + "scripts", + "devel-common", + "task-sdk", + "airflow-ctl", + "airflow-ctl-tests", + "helm-tests", + "airflow-e2e-tests", + "task-sdk-integration-tests", + "docker-tests", + "kubernetes-tests", +] + +# Each shared/ workspace member is an allowed folder in its own right, giving +# every shared library its own dedicated mypy hook / virtualenv / cache. +_SHARED_DISTS = sorted( + f"shared/{p.name}" for p in (AIRFLOW_ROOT_PATH / "shared").iterdir() if (p / "pyproject.toml").exists() +) + +ALLOWED_FOLDERS = _TOP_LEVEL_ALLOWED_FOLDERS + _SHARED_DISTS + +# Map folder(s) to the uv project to use for running mypy. +# When multiple folders are checked together, the first folder's project is used. +FOLDER_TO_PROJECT = {f: f for f in ALLOWED_FOLDERS} + +# Projects that ship their own [tool.mypy] configuration in their pyproject.toml; +# mypy must be invoked with --config-file pointing at that file so those sections +# take precedence over the root pyproject.toml. +FOLDER_TO_MYPY_CONFIG = { + "dev": "dev/pyproject.toml", + "scripts": "scripts/pyproject.toml", +} + +if len(sys.argv) < 2: + if console: + console.print(f"[yellow]You need to specify the folder to test as parameter: {ALLOWED_FOLDERS}\n") + else: + print(f"You need to specify the folder to test as parameter: {ALLOWED_FOLDERS}") + sys.exit(1) + +mypy_folders = sys.argv[1:] + +show_unused_warnings = os.environ.get("SHOW_UNUSED_MYPY_WARNINGS", "false") +show_unreachable_warnings = os.environ.get("SHOW_UNREACHABLE_MYPY_WARNINGS", "false") + +for mypy_folder in mypy_folders: + if mypy_folder not in ALLOWED_FOLDERS: + if console: + console.print( + f"\n[red]ERROR: Folder `{mypy_folder}` is wrong.[/]\n\n" + f"All folders passed should be one of those: {ALLOWED_FOLDERS}\n" + ) + else: + print( + f"\nERROR: Folder `{mypy_folder}` is wrong.\n\n" + f"All folders passed should be one of those: {ALLOWED_FOLDERS}\n" + ) + sys.exit(1) + +exclude_regexps = [ + re.compile(x) + for x in [ + r"^.*/node_modules/.*", + r"^.*\\..*", + r"^.*/src/airflow/__init__.py$", + ] +] + + +def get_all_files(folder: str) -> list[str]: + files_to_check = [] + python_file_paths = (AIRFLOW_ROOT_PATH / folder).resolve().rglob("*.py") + for file in python_file_paths: + if ( + file.name not in ("conftest.py",) + and not any(x.match(file.as_posix()) for x in exclude_regexps) + and not any(part.startswith(".") for part in file.parts) + ): + files_to_check.append(file.relative_to(AIRFLOW_ROOT_PATH).as_posix()) + return files_to_check + + +mypy_extra_args: list[str] = [] + +if show_unused_warnings == "true": + if console: + console.print("[info]Running mypy with --warn-unused-ignores") + else: + print("Running mypy with --warn-unused-ignores") + mypy_extra_args.append("--warn-unused-ignores") + +if show_unreachable_warnings == "true": + if console: + console.print("[info]Running mypy with --warn-unreachable") + else: + print("Running mypy with --warn-unreachable") + mypy_extra_args.append("--warn-unreachable") + + +def write_file_list(files: list[str], suffix: str = "") -> Path: + name = "mypy_files.txt" if not suffix else f"mypy_files_{suffix}.txt" + mypy_file_list = AIRFLOW_ROOT_PATH / "files" / name + mypy_file_list.parent.mkdir(parents=True, exist_ok=True) + mypy_file_list.write_text("\n".join(files)) + if console: + console.print(f"[info]You can check the list of files in:[/] {mypy_file_list}") + else: + print(f"You can check the list of files in: {mypy_file_list}") + return mypy_file_list + + +def run_local_mypy(project: str, hook_name: str, files: list[str], config_file: str | None = None) -> int: + """Sync a dedicated mypy venv under .build/ and run mypy on the given files. + + Each hook gets its own virtualenv and mypy cache so running mypy never mutates the + contributor's regular project .venv and each hook keeps a stable, CI-aligned + dependency set. UV_PROJECT_ENVIRONMENT redirects uv away from the default + /.venv to our cached location. + """ + mypy_venv_dir = AIRFLOW_ROOT_PATH / ".build" / "mypy-venvs" / hook_name + mypy_cache_dir = AIRFLOW_ROOT_PATH / ".build" / "mypy-caches" / hook_name + mypy_venv_dir.parent.mkdir(parents=True, exist_ok=True) + mypy_cache_dir.parent.mkdir(parents=True, exist_ok=True) + + run_env = { + **os.environ, + "TERM": "ansi", + "UV_PROJECT_ENVIRONMENT": str(mypy_venv_dir), + } + + sync_cmd = ["uv", "sync", "--frozen", "--project", project, "--group", "mypy"] + if console: + console.print( + f"[magenta]Syncing dedicated mypy virtualenv ({mypy_venv_dir}) " + f"for project {project}: {' '.join(sync_cmd)}[/]" + ) + else: + print( + f"Syncing dedicated mypy virtualenv ({mypy_venv_dir}) for project {project}: {' '.join(sync_cmd)}" + ) + sync_result = subprocess.run( + sync_cmd, + cwd=str(AIRFLOW_ROOT_PATH), + check=False, + env=run_env, + ) + if sync_result.returncode != 0: + msg = ( + f"`uv sync --frozen --project {project}` failed for the mypy virtualenv at " + f"{mypy_venv_dir}. Fix the sync error before running mypy — otherwise the " + "dedicated mypy virtualenv will not match uv.lock and results will diverge " + "from CI. You can remove the cached virtualenv with:\n" + " breeze down --cleanup-mypy-cache\n" + ) + if console: + console.print(f"[red]{msg}") + else: + print(msg) + return sync_result.returncode + + mypy_file_list = write_file_list(files, suffix=hook_name.replace("/", "_")) + + # --follow-imports=silent: each hook only reports errors for files it owns. Transitive + # errors from imports into other workspace projects are not reported here — those files + # are covered by their own hook. Without this, mypy can produce different results for the + # same file across hooks because each hook's virtualenv installs a different dependency + # set that influences type inference. + cmd = [ + "uv", + "run", + "--frozen", + "--project", + project, + "--group", + "mypy", + "mypy", + "--cache-dir", + str(mypy_cache_dir), + "--follow-imports=silent", + ] + if config_file is not None: + cmd += ["--config-file", config_file] + cmd += [f"@{mypy_file_list}", *mypy_extra_args] + + result = subprocess.run( + cmd, + cwd=str(AIRFLOW_ROOT_PATH), + check=False, + env=run_env, + ) + return result.returncode + + +if console: + console.print(f"[magenta]Running mypy for folders: {mypy_folders}[/]") +else: + print(f"Running mypy for folders: {mypy_folders}") + +if CI: + # In CI, run inside the breeze Docker image to avoid needing a local environment + # and to not change uv.lock or synchronize dependencies. + from common_prek_utils import ( + initialize_breeze_prek, + run_command_via_breeze_shell, + ) + + initialize_breeze_prek(__name__, __file__) + + all_files_to_check = [] + for mypy_folder in mypy_folders: + all_files_to_check.extend(get_all_files(mypy_folder)) + + if not all_files_to_check: + print("No files to test. Quitting") + sys.exit(0) + + write_file_list(all_files_to_check) + file_argument_ci = "@/files/mypy_files.txt" + + mypy_cmd = ( + f"TERM=ansi mypy --follow-imports=silent {shlex.quote(file_argument_ci)} {' '.join(mypy_extra_args)}" + ) + result = run_command_via_breeze_shell( + cmd=["bash", "-c", mypy_cmd], + warn_image_upgrade_needed=True, + extra_env={ + "INCLUDE_MYPY_VOLUME": "false", + "MOUNT_SOURCES": "selected", + }, + ) + returncode = result.returncode +else: + all_files_to_check = [] + for mypy_folder in mypy_folders: + all_files_to_check.extend(get_all_files(mypy_folder)) + + if not all_files_to_check: + print("No files to test. Quitting") + sys.exit(0) + + project = FOLDER_TO_PROJECT.get(mypy_folders[0], "devel-common") + # hook_name derives venv/cache paths under .build/; replace "/" so "shared/dist" + # becomes ".build/mypy-venvs/shared-dist" rather than a nested path. + hook_name = mypy_folders[0].replace("/", "-") + returncode = run_local_mypy( + project=project, + hook_name=hook_name, + files=all_files_to_check, + config_file=FOLDER_TO_MYPY_CONFIG.get(mypy_folders[0]), + ) + +if returncode != 0: + hook_label = mypy_folders[0].replace("/", "-") + msg = ( + "Mypy check failed. You can run mypy locally with:\n" + f" prek run mypy-{hook_label} --all-files\n" + "The hook uses dedicated virtualenv(s) and mypy cache(s) under .build/ so it does\n" + "not touch your regular project .venv. You can clear both with:\n" + " breeze down --cleanup-mypy-cache\n" + ) + if console: + console.print(f"[yellow]{msg}") + else: + print(msg) +sys.exit(returncode) diff --git a/scripts/pyproject.toml b/scripts/pyproject.toml index d2c3e06c96e13..952969a15df76 100644 --- a/scripts/pyproject.toml +++ b/scripts/pyproject.toml @@ -61,6 +61,10 @@ dependencies = [ dev = [ "apache-airflow-devel-common", ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [tool.uv.sources] apache-airflow-devel-common = {workspace = true} @@ -76,3 +80,64 @@ packages = ["ci", "cov", "docker", "in_container", "tools"] # "from common_prek_utils import ..." inside those modules resolve correctly # (mirroring what Python does automatically when scripts are run directly). pythonpath = [".", "ci/prek", "in_container"] + +[tool.mypy] +ignore_missing_imports = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = false +pretty = true +show_error_codes = true +disable_error_code = [ + "annotation-unchecked", +] +namespace_packages = true +explicit_package_bases = true +exclude = [ + ".*/node_modules/.*", + # Exclude hidden files and directories + ".*/\\..*" +] +# Workspace packages (airflow, airflow.sdk, airflowctl, airflow_breeze, tests_common) +# are installed editable into the hook's virtualenv by `uv sync`, so mypy resolves them +# via site-packages — no mypy_path entries needed. + +[[tool.mypy.overrides]] +module="airflow.config_templates.default_webserver_config" +disable_error_code = [ + "var-annotated", +] + +[[tool.mypy.overrides]] +module="airflow.migrations.*" +ignore_errors = true + +[[tool.mypy.overrides]] +module="airflow.*._vendor.*" +ignore_errors = true + +[[tool.mypy.overrides]] +module= [ + "google.cloud.*", + "azure.*", +] +no_implicit_optional = false + +[[tool.mypy.overrides]] +module = "google.api_core.gapic_v1" +ignore_errors = true + +[[tool.mypy.overrides]] +module=[ + "referencing.*", + # Beam has some old type annotations, and they introduced an error recently with bad signature of + # a function. This is captured in https://github.com/apache/beam/issues/29927 + # and we should remove this exclusion when it is fixed. + "apache_beam.*" +] +ignore_errors = true + +# airflowctl autogenered datamodels +[[tool.mypy.overrides]] +module="airflowctl.api.datamodels.*" +ignore_errors = true diff --git a/shared/configuration/.pre-commit-config.yaml b/shared/configuration/.pre-commit-config.yaml new file mode 100644 index 0000000000000..9706d8602d6f3 --- /dev/null +++ b/shared/configuration/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-shared-configuration + name: Run mypy for shared-configuration + language: python + entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/configuration + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/shared/configuration/pyproject.toml b/shared/configuration/pyproject.toml index 939df7d0dadb4..bef10b4ce0858 100644 --- a/shared/configuration/pyproject.toml +++ b/shared/configuration/pyproject.toml @@ -35,6 +35,10 @@ dev = [ "apache-airflow-devel-common", "apache-airflow-shared-module-loading", ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [build-system] requires = [ diff --git a/shared/configuration/src/airflow_shared/configuration/parser.py b/shared/configuration/src/airflow_shared/configuration/parser.py index c34911039a484..9ffd5a7b441b5 100644 --- a/shared/configuration/src/airflow_shared/configuration/parser.py +++ b/shared/configuration/src/airflow_shared/configuration/parser.py @@ -622,7 +622,7 @@ def get_default_value(self, section: str, key: str, fallback: Any = None, raw=Fa ) if value is VALUE_NOT_FOUND_SENTINEL: value = fallback - if raw and value is not None: + if raw and isinstance(value, str): return value.replace("%", "%%") return value @@ -844,7 +844,7 @@ def _filter_by_source( # when display_source = true, we know that the config_sources contains tuple opt, source = config_sources[section][key] # type: ignore else: - opt = config_sources[section][key] + opt = config_sources[section][key] # type: ignore[assignment] if opt == self.get_default_value(section, key): del config_sources[section][key] @@ -1631,7 +1631,9 @@ def get_options_including_defaults(self, section: str) -> list[str]: ) return list(dict.fromkeys(itertools.chain(all_options_from_defaults, my_own_options))) - def has_option(self, section: str, option: str, lookup_from_deprecated: bool = True, **kwargs) -> bool: + def has_option( # type: ignore[override] + self, section: str, option: str, lookup_from_deprecated: bool = True, **kwargs + ) -> bool: """ Check if option is defined. @@ -1660,7 +1662,7 @@ def has_option(self, section: str, option: str, lookup_from_deprecated: bool = T except (NoOptionError, NoSectionError, AirflowConfigException): return False - def set(self, section: str, option: str, value: str | None = None) -> None: + def set(self, section: str, option: str, value: str | None = None) -> None: # type: ignore[override] """ Set an option to the given value. @@ -1675,7 +1677,7 @@ def set(self, section: str, option: str, value: str | None = None) -> None: self.add_section(section) super().set(section, option, value) - def remove_option(self, section: str, option: str, remove_default: bool = True): + def remove_option(self, section: str, option: str, remove_default: bool = True): # type: ignore[override] """ Remove an option if it exists in config from a file or default config. diff --git a/shared/dagnode/.pre-commit-config.yaml b/shared/dagnode/.pre-commit-config.yaml new file mode 100644 index 0000000000000..34f7fe42212b1 --- /dev/null +++ b/shared/dagnode/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-shared-dagnode + name: Run mypy for shared-dagnode + language: python + entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/dagnode + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/shared/dagnode/pyproject.toml b/shared/dagnode/pyproject.toml index 55c9e84552ebd..a4de72c566c1b 100644 --- a/shared/dagnode/pyproject.toml +++ b/shared/dagnode/pyproject.toml @@ -31,6 +31,10 @@ dependencies = [ dev = [ "apache-airflow-devel-common", ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [build-system] requires = [ diff --git a/shared/dagnode/tests/dagnode/test_node.py b/shared/dagnode/tests/dagnode/test_node.py index 4259ca7555ffe..fa6600b880464 100644 --- a/shared/dagnode/tests/dagnode/test_node.py +++ b/shared/dagnode/tests/dagnode/test_node.py @@ -34,7 +34,7 @@ class TaskGroup: """Task group type for tests.""" node_id: str = attrs.field(init=False, default="test_group_id") - prefix_group_id: str + prefix_group_id: bool class Dag: @@ -46,8 +46,8 @@ class Dag: class ConcreteDAGNode(GenericDAGNode[Dag, Task, TaskGroup]): """Concrete DAGNode variant for tests.""" - dag = None - task_group = None + dag: Dag | None = None + task_group: TaskGroup | None = None @property def node_id(self) -> str: diff --git a/shared/listeners/.pre-commit-config.yaml b/shared/listeners/.pre-commit-config.yaml new file mode 100644 index 0000000000000..62f226fab7407 --- /dev/null +++ b/shared/listeners/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-shared-listeners + name: Run mypy for shared-listeners + language: python + entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/listeners + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/shared/listeners/pyproject.toml b/shared/listeners/pyproject.toml index 6ab004f642fa0..90f0a1860aaf5 100644 --- a/shared/listeners/pyproject.toml +++ b/shared/listeners/pyproject.toml @@ -32,6 +32,10 @@ dependencies = [ dev = [ "apache-airflow-devel-common", ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [build-system] requires = [ diff --git a/shared/logging/.pre-commit-config.yaml b/shared/logging/.pre-commit-config.yaml new file mode 100644 index 0000000000000..7440bc4add9a1 --- /dev/null +++ b/shared/logging/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-shared-logging + name: Run mypy for shared-logging + language: python + entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/logging + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/shared/logging/pyproject.toml b/shared/logging/pyproject.toml index 6ff377300cf10..1cf0e140b30f8 100644 --- a/shared/logging/pyproject.toml +++ b/shared/logging/pyproject.toml @@ -33,6 +33,10 @@ dependencies = [ dev = [ "apache-airflow-devel-common", ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [build-system] requires = [ diff --git a/shared/logging/src/airflow_shared/logging/_noncaching.py b/shared/logging/src/airflow_shared/logging/_noncaching.py index 7a8e21fbe3811..e6509e14a409c 100644 --- a/shared/logging/src/airflow_shared/logging/_noncaching.py +++ b/shared/logging/src/airflow_shared/logging/_noncaching.py @@ -29,7 +29,8 @@ def make_file_io_non_caching(io: _IO) -> _IO: try: fd = io.fileno() - os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) + # posix_fadvise / POSIX_FADV_DONTNEED are Linux-only; ignored on other platforms at runtime. + os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) # type: ignore[attr-defined,unused-ignore] except Exception: # in case either file descriptor cannot be retrieved or fadvise is not available # we should simply return the wrapper retrieved by FileHandler's open method diff --git a/shared/module_loading/.pre-commit-config.yaml b/shared/module_loading/.pre-commit-config.yaml new file mode 100644 index 0000000000000..fd00e315b0e53 --- /dev/null +++ b/shared/module_loading/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-shared-module_loading + name: Run mypy for shared-module_loading + language: python + entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/module_loading + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/shared/module_loading/pyproject.toml b/shared/module_loading/pyproject.toml index a26fa76db88e1..330ea06ace88c 100644 --- a/shared/module_loading/pyproject.toml +++ b/shared/module_loading/pyproject.toml @@ -32,6 +32,10 @@ dependencies = [ dev = [ "apache-airflow-devel-common", ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [build-system] requires = [ diff --git a/shared/observability/.pre-commit-config.yaml b/shared/observability/.pre-commit-config.yaml new file mode 100644 index 0000000000000..eab5206f13303 --- /dev/null +++ b/shared/observability/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-shared-observability + name: Run mypy for shared-observability + language: python + entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/observability + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/shared/observability/pyproject.toml b/shared/observability/pyproject.toml index 85b5cd6bcaeff..7efee710eef1d 100644 --- a/shared/observability/pyproject.toml +++ b/shared/observability/pyproject.toml @@ -49,6 +49,10 @@ dev = [ "apache-airflow-devel-common", "apache-airflow-shared-observability[all]", ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [build-system] requires = [ diff --git a/shared/observability/src/airflow_shared/observability/common.py b/shared/observability/src/airflow_shared/observability/common.py index a92a26543d3d8..c611782daa087 100644 --- a/shared/observability/src/airflow_shared/observability/common.py +++ b/shared/observability/src/airflow_shared/observability/common.py @@ -48,13 +48,25 @@ def get_otel_data_exporter( # If the protocol env var isn't set, then it will be None, # and it will default to an http/protobuf exporter. + # The grpc and http variants are incompatible types to mypy but functionally interchangeable here. + OTLPMetricExporter: type + OTLPSpanExporter: type if env_endpoint and env_exporter_protocol == "grpc": - from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, # type: ignore[no-redef] + ) + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, # type: ignore[no-redef] + ) else: - from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter - from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter - + from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, # type: ignore[no-redef] + ) + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, # type: ignore[no-redef] + ) + + exporter: SpanExporter | MetricExporter if env_endpoint: if host is not None and port is not None: log.warning( diff --git a/shared/observability/src/airflow_shared/observability/metrics/otel_logger.py b/shared/observability/src/airflow_shared/observability/metrics/otel_logger.py index 14726e3ecc064..c8db3ee02f989 100644 --- a/shared/observability/src/airflow_shared/observability/metrics/otel_logger.py +++ b/shared/observability/src/airflow_shared/observability/metrics/otel_logger.py @@ -418,7 +418,7 @@ def get_otel_logger( readers = [ PeriodicExportingMetricReader( - exporter=metric_exporter, + exporter=metric_exporter, # type: ignore[arg-type] export_interval_millis=interval, # type: ignore[arg-type] ) ] diff --git a/shared/observability/src/airflow_shared/observability/metrics/stats.py b/shared/observability/src/airflow_shared/observability/metrics/stats.py index be153c298d6d1..5cd3c098d81e6 100644 --- a/shared/observability/src/airflow_shared/observability/metrics/stats.py +++ b/shared/observability/src/airflow_shared/observability/metrics/stats.py @@ -65,6 +65,7 @@ def normalize_name_for_stats(name: str, log_warning: bool = True) -> str: class _Stats(type): factory: Callable[[], StatsLogger | NoStatsLogger] | None = None instance: StatsLogger | NoStatsLogger | None = None + _instance_pid: int | None = None def __getattr__(cls, name: str) -> str: factory = type.__getattribute__(cls, "factory") diff --git a/shared/plugins_manager/.pre-commit-config.yaml b/shared/plugins_manager/.pre-commit-config.yaml new file mode 100644 index 0000000000000..9180e2b10358e --- /dev/null +++ b/shared/plugins_manager/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-shared-plugins_manager + name: Run mypy for shared-plugins_manager + language: python + entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/plugins_manager + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/shared/plugins_manager/pyproject.toml b/shared/plugins_manager/pyproject.toml index b80b85268e586..1fae534b6b3ae 100644 --- a/shared/plugins_manager/pyproject.toml +++ b/shared/plugins_manager/pyproject.toml @@ -36,6 +36,10 @@ dev = [ "apache-airflow-shared-module-loading", "apache-airflow-shared-listeners", ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [build-system] requires = [ diff --git a/shared/providers_discovery/.pre-commit-config.yaml b/shared/providers_discovery/.pre-commit-config.yaml new file mode 100644 index 0000000000000..2c26b25a954ff --- /dev/null +++ b/shared/providers_discovery/.pre-commit-config.yaml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-shared-providers_discovery + name: Run mypy for shared-providers_discovery + language: python + entry: >- + ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py + shared/providers_discovery + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/shared/providers_discovery/pyproject.toml b/shared/providers_discovery/pyproject.toml index 7517d6d13d2d0..ef86a049db4cb 100644 --- a/shared/providers_discovery/pyproject.toml +++ b/shared/providers_discovery/pyproject.toml @@ -39,6 +39,10 @@ dev = [ "apache-airflow-devel-common", "apache-airflow-shared-module-loading", ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [build-system] requires = [ diff --git a/shared/secrets_backend/.pre-commit-config.yaml b/shared/secrets_backend/.pre-commit-config.yaml new file mode 100644 index 0000000000000..93175830f9f9b --- /dev/null +++ b/shared/secrets_backend/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-shared-secrets_backend + name: Run mypy for shared-secrets_backend + language: python + entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/secrets_backend + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/shared/secrets_backend/pyproject.toml b/shared/secrets_backend/pyproject.toml index 59aa0af2915e6..1d3d8df067a96 100644 --- a/shared/secrets_backend/pyproject.toml +++ b/shared/secrets_backend/pyproject.toml @@ -29,6 +29,10 @@ dependencies = [] dev = [ "apache-airflow-devel-common", ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [build-system] requires = [ diff --git a/shared/secrets_backend/src/airflow_shared/secrets_backend/base.py b/shared/secrets_backend/src/airflow_shared/secrets_backend/base.py index 566ed379c8501..a2715b93ca993 100644 --- a/shared/secrets_backend/src/airflow_shared/secrets_backend/base.py +++ b/shared/secrets_backend/src/airflow_shared/secrets_backend/base.py @@ -84,7 +84,7 @@ def _get_connection_class(self) -> type: def _deserialize_connection_value(conn_class: type, conn_id: str, value: str): value = value.strip() if value[0] == "{": - return conn_class.from_json(value=value, conn_id=conn_id) + return conn_class.from_json(value=value, conn_id=conn_id) # type: ignore[attr-defined] # TODO: Only sdk has from_uri defined on it. Is it worthwhile developing the core path or not? if hasattr(conn_class, "from_uri"): diff --git a/shared/secrets_masker/.pre-commit-config.yaml b/shared/secrets_masker/.pre-commit-config.yaml new file mode 100644 index 0000000000000..8025b4429c216 --- /dev/null +++ b/shared/secrets_masker/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-shared-secrets_masker + name: Run mypy for shared-secrets_masker + language: python + entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/secrets_masker + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/shared/secrets_masker/pyproject.toml b/shared/secrets_masker/pyproject.toml index a971925888ebd..9be22a40edf78 100644 --- a/shared/secrets_masker/pyproject.toml +++ b/shared/secrets_masker/pyproject.toml @@ -34,6 +34,10 @@ dependencies = [ dev = [ "apache-airflow-devel-common", ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [build-system] requires = [ diff --git a/shared/secrets_masker/tests/secrets_masker/test_secrets_masker.py b/shared/secrets_masker/tests/secrets_masker/test_secrets_masker.py index 274e611d57c99..e23cb1c274438 100644 --- a/shared/secrets_masker/tests/secrets_masker/test_secrets_masker.py +++ b/shared/secrets_masker/tests/secrets_masker/test_secrets_masker.py @@ -46,7 +46,7 @@ def configure_secrets_masker_for_test( - masker: SecretsMasker, min_length: int = 5, sensitive_fields: list[str] = None + masker: SecretsMasker, min_length: int = 5, sensitive_fields: list[str] | None = None ): """Helper function to configure a SecretsMasker instance for testing.""" masker.min_length_to_mask = min_length diff --git a/shared/serialization/.pre-commit-config.yaml b/shared/serialization/.pre-commit-config.yaml new file mode 100644 index 0000000000000..e0619c11dfd61 --- /dev/null +++ b/shared/serialization/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-shared-serialization + name: Run mypy for shared-serialization + language: python + entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/serialization + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/shared/serialization/pyproject.toml b/shared/serialization/pyproject.toml index 48c2cb91edbd6..c547da3ee5ce4 100644 --- a/shared/serialization/pyproject.toml +++ b/shared/serialization/pyproject.toml @@ -29,6 +29,10 @@ dependencies = [] dev = [ "apache-airflow-devel-common", ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [build-system] requires = [ diff --git a/shared/template_rendering/.pre-commit-config.yaml b/shared/template_rendering/.pre-commit-config.yaml new file mode 100644 index 0000000000000..cbe27c38eba39 --- /dev/null +++ b/shared/template_rendering/.pre-commit-config.yaml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-shared-template_rendering + name: Run mypy for shared-template_rendering + language: python + entry: >- + ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py + shared/template_rendering + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/shared/template_rendering/pyproject.toml b/shared/template_rendering/pyproject.toml index 03e56b2d1b11e..83ac811ce2761 100644 --- a/shared/template_rendering/pyproject.toml +++ b/shared/template_rendering/pyproject.toml @@ -29,6 +29,10 @@ dependencies = [] dev = [ "apache-airflow-devel-common", ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [build-system] requires = [ diff --git a/shared/timezones/.pre-commit-config.yaml b/shared/timezones/.pre-commit-config.yaml new file mode 100644 index 0000000000000..25c6cf6d8a599 --- /dev/null +++ b/shared/timezones/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-shared-timezones + name: Run mypy for shared-timezones + language: python + entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/timezones + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/shared/timezones/pyproject.toml b/shared/timezones/pyproject.toml index 7d62b3ff99b3f..a405c57fac6c8 100644 --- a/shared/timezones/pyproject.toml +++ b/shared/timezones/pyproject.toml @@ -32,6 +32,10 @@ dependencies = [ dev = [ "apache-airflow-devel-common", ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [build-system] requires = [ diff --git a/shared/timezones/src/airflow_shared/timezones/timezone.py b/shared/timezones/src/airflow_shared/timezones/timezone.py index 69898b24d0bab..45b538a52a8fb 100644 --- a/shared/timezones/src/airflow_shared/timezones/timezone.py +++ b/shared/timezones/src/airflow_shared/timezones/timezone.py @@ -107,7 +107,9 @@ def make_aware(value: DateTime, timezone: dt.tzinfo | None = None) -> DateTime: @overload -def make_aware(value: dt.datetime, timezone: dt.tzinfo | None = None) -> dt.datetime: ... +def make_aware( # type: ignore[overload-cannot-match] + value: dt.datetime, timezone: dt.tzinfo | None = None +) -> dt.datetime: ... def make_aware(value: dt.datetime | None, timezone: dt.tzinfo | None = None) -> dt.datetime | None: @@ -200,7 +202,9 @@ def coerce_datetime(v: DateTime, tz: dt.tzinfo | None = None) -> DateTime: ... @overload -def coerce_datetime(v: dt.datetime, tz: dt.tzinfo | None = None) -> DateTime: ... +def coerce_datetime( # type: ignore[overload-cannot-match] + v: dt.datetime, tz: dt.tzinfo | None = None +) -> DateTime: ... def coerce_datetime(v: dt.datetime | None, tz: dt.tzinfo | None = None) -> DateTime | None: diff --git a/task-sdk-integration-tests/.pre-commit-config.yaml b/task-sdk-integration-tests/.pre-commit-config.yaml new file mode 100644 index 0000000000000..f09e0e1f6fe6f --- /dev/null +++ b/task-sdk-integration-tests/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 +repos: + - repo: local + hooks: + - id: mypy-task-sdk-integration-tests + name: Run mypy for task-sdk-integration-tests + language: python + entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py task-sdk-integration-tests + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/task-sdk-integration-tests/pyproject.toml b/task-sdk-integration-tests/pyproject.toml index f4364a751305d..898c71dfc588a 100644 --- a/task-sdk-integration-tests/pyproject.toml +++ b/task-sdk-integration-tests/pyproject.toml @@ -75,3 +75,8 @@ exclude = ["*"] [tool.hatch.build.targets.wheel] bypass-selection = true + +[dependency-groups] +mypy = [ + "apache-airflow-devel-common[mypy]", +] diff --git a/task-sdk/.pre-commit-config.yaml b/task-sdk/.pre-commit-config.yaml index df1be38500f71..aa0cefc74f151 100644 --- a/task-sdk/.pre-commit-config.yaml +++ b/task-sdk/.pre-commit-config.yaml @@ -16,7 +16,7 @@ # under the License. --- default_stages: [pre-commit, pre-push] -minimum_prek_version: '0.2.0' +minimum_prek_version: '0.3.4' default_language_version: python: python3 node: 22.19.0 @@ -58,7 +58,7 @@ repos: - id: mypy-task-sdk name: Run mypy for task-sdk language: python - entry: ../scripts/ci/prek/mypy_local_folder.py task-sdk + entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py task-sdk pass_filenames: false files: ^.*\.py$ require_serial: true diff --git a/task-sdk/pyproject.toml b/task-sdk/pyproject.toml index 90dba8508d969..3b2cba7ebbf44 100644 --- a/task-sdk/pyproject.toml +++ b/task-sdk/pyproject.toml @@ -226,6 +226,10 @@ dev = [ docs = [ "apache-airflow-devel-common[docs]", ] +mypy = [ + "apache-airflow-devel-common[mypy]", +] + [tool.uv.sources] # These names must match the names as defined in the pyproject.toml of the workspace items, # *not* the workspace folder paths diff --git a/task-sdk/src/airflow/sdk/execution_time/comms.py b/task-sdk/src/airflow/sdk/execution_time/comms.py index ab59365946201..c8f22b9d5a484 100644 --- a/task-sdk/src/airflow/sdk/execution_time/comms.py +++ b/task-sdk/src/airflow/sdk/execution_time/comms.py @@ -133,7 +133,7 @@ def _new_encoder() -> msgspec.msgpack.Encoder: return msgspec.msgpack.Encoder(enc_hook=_msgpack_enc_hook) -class _RequestFrame(msgspec.Struct, array_like=True, frozen=True, omit_defaults=True): +class _RequestFrame(msgspec.Struct, array_like=True, frozen=True, omit_defaults=True): # type: ignore[call-arg] id: int """ The request id, set by the sender. @@ -159,7 +159,7 @@ def as_bytes(self) -> bytearray: return buffer -class _ResponseFrame(_RequestFrame, frozen=True): +class _ResponseFrame(_RequestFrame, frozen=True): # type: ignore[call-arg] id: int """ The id of the request this is a response to diff --git a/uv.lock b/uv.lock index f54143f15687e..8c87e66ecd652 100644 --- a/uv.lock +++ b/uv.lock @@ -5,24 +5,19 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", "python_full_version == '3.13.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and sys_platform != 'darwin'", + "(python_full_version == '3.13.*' and platform_machine != 'arm64') or (python_full_version == '3.13.*' and sys_platform != 'darwin')", "python_full_version == '3.12.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", "python_full_version == '3.11.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", "python_full_version < '3.11' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", + "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] [options] -exclude-newer = "2026-04-16T14:57:18.176731926Z" +exclude-newer = "2026-04-16T15:21:28.729569834Z" exclude-newer-span = "P4D" [options.exclude-newer-package] @@ -1849,6 +1844,9 @@ dev = [ docs = [ { name = "apache-airflow-devel-common", extra = ["docs"] }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] requires-dist = [ @@ -1949,6 +1947,7 @@ dev = [ { name = "apache-airflow-providers-git", editable = "providers/git" }, ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-ctl" @@ -1985,6 +1984,9 @@ dev = [ docs = [ { name = "apache-airflow-devel-common", extra = ["docs"] }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] requires-dist = [ @@ -2014,6 +2016,7 @@ dev = [ { name = "apache-airflow-devel-common", editable = "devel-common" }, ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-ctl-tests" @@ -2024,12 +2027,20 @@ dependencies = [ { name = "apache-airflow-devel-common" }, ] +[package.dev-dependencies] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] + [package.metadata] requires-dist = [ { name = "apache-airflow-ctl", editable = "airflow-ctl" }, { name = "apache-airflow-devel-common", editable = "devel-common" }, ] +[package.metadata.requires-dev] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] + [[package]] name = "apache-airflow-dev" version = "0.0.1" @@ -2053,6 +2064,11 @@ dependencies = [ { name = "tabulate" }, ] +[package.dev-dependencies] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] + [package.metadata] requires-dist = [ { name = "beautifulsoup4", specifier = ">=4.12.0" }, @@ -2073,6 +2089,9 @@ requires-dist = [ { name = "tabulate", specifier = ">=0.9.0" }, ] +[package.metadata.requires-dev] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] + [[package]] name = "apache-airflow-devel-common" version = "0.1.0" @@ -2384,6 +2403,9 @@ docs = [ docs-gen = [ { name = "apache-airflow-devel-common", extra = ["docs-gen"] }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] requires-dist = [ @@ -2489,6 +2511,7 @@ provides-extras = ["all", "basic", "coverage", "duckdb", "debuggers", "devscript dev = [{ name = "apache-airflow-core", editable = "airflow-core" }] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] docs-gen = [{ name = "apache-airflow-devel-common", extras = ["docs-gen"], editable = "devel-common" }] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-docker-tests" @@ -2498,9 +2521,17 @@ dependencies = [ { name = "apache-airflow-devel-common" }, ] +[package.dev-dependencies] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] + [package.metadata] requires-dist = [{ name = "apache-airflow-devel-common", editable = "devel-common" }] +[package.metadata.requires-dev] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] + [[package]] name = "apache-airflow-e2e-tests" version = "0.0.1" @@ -2512,6 +2543,11 @@ dependencies = [ { name = "testcontainers" }, ] +[package.dev-dependencies] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] + [package.metadata] requires-dist = [ { name = "apache-airflow-core", editable = "airflow-core" }, @@ -2520,6 +2556,9 @@ requires-dist = [ { name = "testcontainers", specifier = ">=4.12.0" }, ] +[package.metadata.requires-dev] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] + [[package]] name = "apache-airflow-helm-tests" version = "0.0.1" @@ -2529,12 +2568,20 @@ dependencies = [ { name = "apache-airflow-providers-cncf-kubernetes" }, ] +[package.dev-dependencies] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] + [package.metadata] requires-dist = [ { name = "apache-airflow-devel-common", editable = "devel-common" }, { name = "apache-airflow-providers-cncf-kubernetes", editable = "providers/cncf/kubernetes" }, ] +[package.metadata.requires-dev] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] + [[package]] name = "apache-airflow-kubernetes-tests" version = "0.0.1" @@ -2549,6 +2596,11 @@ dependencies = [ { name = "urllib3" }, ] +[package.dev-dependencies] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] + [package.metadata] requires-dist = [ { name = "apache-airflow-core", editable = "airflow-core" }, @@ -2560,6 +2612,9 @@ requires-dist = [ { name = "urllib3", specifier = ">=2.1.0,!=2.6.0" }, ] +[package.metadata.requires-dev] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] + [[package]] name = "apache-airflow-providers" version = "0.0.1" @@ -5486,8 +5541,8 @@ dependencies = [ { name = "apache-airflow-providers-common-compat" }, { name = "apache-airflow-providers-common-sql" }, { name = "jaydebeapi" }, - { name = "jpype1", version = "1.6.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, - { name = "jpype1", version = "1.7.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, + { name = "jpype1", version = "1.6.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'arm64' and sys_platform == 'darwin'" }, + { name = "jpype1", version = "1.7.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'arm64' or sys_platform != 'darwin'" }, ] [package.dev-dependencies] @@ -7732,6 +7787,9 @@ dependencies = [ dev = [ { name = "apache-airflow-devel-common" }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] requires-dist = [ @@ -7751,6 +7809,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [{ name = "apache-airflow-devel-common", editable = "devel-common" }] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-shared-configuration" @@ -7768,6 +7827,9 @@ dev = [ { name = "apache-airflow-devel-common" }, { name = "apache-airflow-shared-module-loading" }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] requires-dist = [ @@ -7782,6 +7844,7 @@ dev = [ { name = "apache-airflow-devel-common", editable = "devel-common" }, { name = "apache-airflow-shared-module-loading", editable = "shared/module_loading" }, ] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-shared-dagnode" @@ -7795,12 +7858,16 @@ dependencies = [ dev = [ { name = "apache-airflow-devel-common" }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] requires-dist = [{ name = "structlog", specifier = ">=25.4.0" }] [package.metadata.requires-dev] dev = [{ name = "apache-airflow-devel-common", editable = "devel-common" }] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-shared-listeners" @@ -7815,6 +7882,9 @@ dependencies = [ dev = [ { name = "apache-airflow-devel-common" }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] requires-dist = [ @@ -7824,6 +7894,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [{ name = "apache-airflow-devel-common", editable = "devel-common" }] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-shared-logging" @@ -7839,6 +7910,9 @@ dependencies = [ dev = [ { name = "apache-airflow-devel-common" }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] requires-dist = [ @@ -7849,6 +7923,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [{ name = "apache-airflow-devel-common", editable = "devel-common" }] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-shared-module-loading" @@ -7863,6 +7938,9 @@ dependencies = [ dev = [ { name = "apache-airflow-devel-common" }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] requires-dist = [ @@ -7872,6 +7950,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [{ name = "apache-airflow-devel-common", editable = "devel-common" }] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-shared-observability" @@ -7909,6 +7988,9 @@ dev = [ { name = "apache-airflow-devel-common" }, { name = "apache-airflow-shared-observability", extra = ["all"] }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] requires-dist = [ @@ -7930,6 +8012,7 @@ dev = [ { name = "apache-airflow-devel-common", editable = "devel-common" }, { name = "apache-airflow-shared-observability", extras = ["all"], editable = "shared/observability" }, ] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-shared-plugins-manager" @@ -7947,6 +8030,9 @@ dev = [ { name = "apache-airflow-shared-listeners" }, { name = "apache-airflow-shared-module-loading" }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] requires-dist = [ @@ -7962,6 +8048,7 @@ dev = [ { name = "apache-airflow-shared-listeners", editable = "shared/listeners" }, { name = "apache-airflow-shared-module-loading", editable = "shared/module_loading" }, ] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-shared-providers-discovery" @@ -7982,6 +8069,9 @@ dev = [ { name = "apache-airflow-devel-common" }, { name = "apache-airflow-shared-module-loading" }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] requires-dist = [ @@ -8000,6 +8090,7 @@ dev = [ { name = "apache-airflow-devel-common", editable = "devel-common" }, { name = "apache-airflow-shared-module-loading", editable = "shared/module_loading" }, ] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-shared-secrets-backend" @@ -8010,11 +8101,15 @@ source = { editable = "shared/secrets_backend" } dev = [ { name = "apache-airflow-devel-common" }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] [package.metadata.requires-dev] dev = [{ name = "apache-airflow-devel-common", editable = "devel-common" }] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-shared-secrets-masker" @@ -8031,6 +8126,9 @@ dependencies = [ dev = [ { name = "apache-airflow-devel-common" }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] requires-dist = [ @@ -8042,6 +8140,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [{ name = "apache-airflow-devel-common", editable = "devel-common" }] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-shared-serialization" @@ -8052,11 +8151,15 @@ source = { editable = "shared/serialization" } dev = [ { name = "apache-airflow-devel-common" }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] [package.metadata.requires-dev] dev = [{ name = "apache-airflow-devel-common", editable = "devel-common" }] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-shared-template-rendering" @@ -8067,11 +8170,15 @@ source = { editable = "shared/template_rendering" } dev = [ { name = "apache-airflow-devel-common" }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] [package.metadata.requires-dev] dev = [{ name = "apache-airflow-devel-common", editable = "devel-common" }] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-shared-timezones" @@ -8086,6 +8193,9 @@ dependencies = [ dev = [ { name = "apache-airflow-devel-common" }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] requires-dist = [ @@ -8095,6 +8205,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [{ name = "apache-airflow-devel-common", editable = "devel-common" }] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-task-sdk" @@ -8166,6 +8277,9 @@ dev = [ docs = [ { name = "apache-airflow-devel-common", extra = ["docs"] }, ] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] [package.metadata] requires-dist = [ @@ -8225,6 +8339,7 @@ dev = [ { name = "pandas", marker = "python_full_version >= '3.14'", specifier = ">=2.3.3" }, ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] [[package]] name = "apache-airflow-task-sdk-integration-tests" @@ -8235,12 +8350,20 @@ dependencies = [ { name = "apache-airflow-devel-common" }, ] +[package.dev-dependencies] +mypy = [ + { name = "apache-airflow-devel-common", extra = ["mypy"] }, +] + [package.metadata] requires-dist = [ { name = "apache-airflow-core", editable = "airflow-core" }, { name = "apache-airflow-devel-common", editable = "devel-common" }, ] +[package.metadata.requires-dev] +mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] + [[package]] name = "apispec" version = "6.10.0" @@ -9847,17 +9970,13 @@ version = "3.1.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version == '3.13.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and sys_platform != 'darwin'", + "(python_full_version == '3.13.*' and platform_machine != 'arm64') or (python_full_version == '3.13.*' and sys_platform != 'darwin')", "python_full_version == '3.12.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", "python_full_version == '3.11.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", "python_full_version < '3.11' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", + "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] sdist = { url = "https://files.pythonhosted.org/packages/52/39/069100b84d7418bc358d81669d5748efb14b9cceacd2f9c75f550424132f/cloudpickle-3.1.1.tar.gz", hash = "sha256:b216fa8ae4019d5482a8ac3c95d8f6346115d8835911fd4aefd1a445e4242c64", size = 22113, upload-time = "2025-01-14T17:02:05.085Z" } wheels = [ @@ -9872,8 +9991,7 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", ] sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } wheels = [ @@ -10539,8 +10657,7 @@ version = "0.21.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", + "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] sdist = { url = "https://files.pythonhosted.org/packages/ae/ed/aefcc8cd0ba62a0560c3c18c33925362d46c6075480bfa4df87b28e169a9/docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f", size = 2204444, upload-time = "2024-04-23T18:57:18.24Z" } wheels = [ @@ -10555,17 +10672,13 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", "python_full_version == '3.13.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and sys_platform != 'darwin'", + "(python_full_version == '3.13.*' and platform_machine != 'arm64') or (python_full_version == '3.13.*' and sys_platform != 'darwin')", "python_full_version == '3.12.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", "python_full_version == '3.11.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] sdist = { url = "https://files.pythonhosted.org/packages/ae/b6/03bb70946330e88ffec97aefd3ea75ba575cb2e762061e0e62a213befee8/docutils-0.22.4.tar.gz", hash = "sha256:4db53b1fde9abecbb74d91230d32ab626d94f6badfc575d6db9194a49df29968", size = 2291750, upload-time = "2025-12-18T19:00:26.443Z" } wheels = [ @@ -13406,8 +13519,7 @@ version = "8.39.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", + "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, @@ -13433,8 +13545,7 @@ version = "9.10.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version == '3.11.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ { name = "colorama", marker = "python_full_version == '3.11.*' and sys_platform == 'win32'" }, @@ -13462,14 +13573,11 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", "python_full_version == '3.13.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and sys_platform != 'darwin'", + "(python_full_version == '3.13.*' and platform_machine != 'arm64') or (python_full_version == '3.13.*' and sys_platform != 'darwin')", "python_full_version == '3.12.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", ] dependencies = [ { name = "colorama", marker = "python_full_version >= '3.12' and sys_platform == 'win32'" }, @@ -13568,8 +13676,8 @@ name = "jaydebeapi" version = "1.2.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jpype1", version = "1.6.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation != 'Jython' and sys_platform == 'darwin'" }, - { name = "jpype1", version = "1.7.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation != 'Jython' and sys_platform != 'darwin'" }, + { name = "jpype1", version = "1.6.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'arm64' and platform_python_implementation != 'Jython' and sys_platform == 'darwin'" }, + { name = "jpype1", version = "1.7.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'arm64' and platform_python_implementation != 'Jython') or (platform_python_implementation != 'Jython' and sys_platform != 'darwin')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5c/8c/f27750106bf1fba33f92d83fb866af164179f7046495bc5a61fae26d9475/JayDeBeApi-1.2.3.tar.gz", hash = "sha256:f25e9307fbb5960cb035394c26e37731b64cc465b197c4344cee85ec450ab92f", size = 32929, upload-time = "2020-06-12T07:03:27.204Z" } wheels = [ @@ -13748,18 +13856,13 @@ version = "1.6.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin'", "python_full_version == '3.13.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", "python_full_version == '3.12.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", "python_full_version == '3.11.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine != 'arm64' and sys_platform == 'darwin'", ] dependencies = [ - { name = "packaging", marker = "sys_platform == 'darwin'" }, + { name = "packaging", marker = "platform_machine == 'arm64' and sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/34/49/6090a131d84b22c6aae13b1853092028b060fd17da1af87c0e42ad69d50f/jpype1-1.6.0.tar.gz", hash = "sha256:2d46b2a14f8f0e6f17d8aa22b4fc3a64b2790851ebf1409ad79a37c698fd6e9a", size = 1057888, upload-time = "2025-07-07T14:04:11.244Z" } wheels = [ @@ -13776,29 +13879,33 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.13.*' and sys_platform != 'darwin'", - "python_full_version == '3.12.*' and sys_platform != 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", + "(python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", + "(python_full_version == '3.13.*' and platform_machine != 'arm64') or (python_full_version == '3.13.*' and sys_platform != 'darwin')", + "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", + "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", + "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ - { name = "packaging", marker = "sys_platform != 'darwin'" }, + { name = "packaging", marker = "platform_machine != 'arm64' or sys_platform != 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ff/b3/a95951c2d967ca5e61f50d96549f528193315c2e2f38817bfbe214cc162d/jpype1-1.7.0.tar.gz", hash = "sha256:2109138b7264f6360c717b887b6a4d20b29e997c516e8d7fa8756e40595bb537", size = 782128, upload-time = "2026-04-09T22:18:53.873Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/01/4c/0ac784fe2ef4dce88cda18969d08dd584709279861367b79d0738a9bda9c/jpype1-1.7.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:fffa7228d25efd00ff75df5192d69ec8674d2aaa51b48e60871bc07bb151afb0", size = 375246, upload-time = "2026-04-09T22:17:57.324Z" }, { url = "https://files.pythonhosted.org/packages/20/55/a06cb7fdc9b025b203fdd9ade1f5b649e743f02dce0806cd93aa7bbda32b/jpype1-1.7.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b109abef2dc94f159828f15e161a83ef2243ca45a0c23c311de7679e62bd664", size = 407739, upload-time = "2026-04-09T22:17:59.713Z" }, { url = "https://files.pythonhosted.org/packages/79/93/4101a487aece26cac567b23daca9627876d459e46486b79649254f8d47c8/jpype1-1.7.0-cp310-cp310-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:51275f9f25cd431bec65aaa752806521e3abd19ad3ac29e1de35ec216697e615", size = 454431, upload-time = "2026-04-09T22:18:01.561Z" }, { url = "https://files.pythonhosted.org/packages/32/f5/5005e0ca4b222bfdd7e24814dbfbb6b6a0b93b65a21506e7ed79a3134c2f/jpype1-1.7.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1baea397292698579d69207df1b23f4261c6814c90f91081359f4aff958b804", size = 438987, upload-time = "2026-04-09T22:18:03.425Z" }, { url = "https://files.pythonhosted.org/packages/df/0c/51bd6e0ec919bcadca0074e6e90c5b725e6989f3d00d809c5175d160261f/jpype1-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:280fe6aecf086c1d311dff008c1b0129d7197940a6871ea845dc6a5f8b3319a0", size = 357687, upload-time = "2026-04-09T22:18:05.035Z" }, + { url = "https://files.pythonhosted.org/packages/78/c8/9613746ac28b4a1c5ec3871b9e56297493ac586d56c61b93724fc1a0cd36/jpype1-1.7.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:83486bd4629f40386ae6557311e7baca89bb9f2b70c2210d16bcdfafe1ac812c", size = 375265, upload-time = "2026-04-09T22:18:07.084Z" }, { url = "https://files.pythonhosted.org/packages/46/05/4052f15ce9d5d8a82a9a8537454119acfaf6dfcc42595f75ca6a366bf227/jpype1-1.7.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b248d080a3a95249a2aad3ce53f9b56f84916d1bdc01df18b06c152deb12b8d", size = 407735, upload-time = "2026-04-09T22:18:09.641Z" }, { url = "https://files.pythonhosted.org/packages/66/bd/a75f8309d98efebd4dfea57c4009d0f86834dac4eeaa8bd6d2c9db37b468/jpype1-1.7.0-cp311-cp311-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:9198e7a87f6afd7c6953df23ea0eab424b0a30ec103ca91104082b075e0e11c5", size = 454398, upload-time = "2026-04-09T22:18:11.871Z" }, { url = "https://files.pythonhosted.org/packages/13/21/63fea3b7b18c929c85ab29b6b5cafc9909f6fca8361b4442b26da177f87f/jpype1-1.7.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b042813690e68cec1611e10982bf3e496cf47754e681abd8c3342be0aa97a802", size = 438963, upload-time = "2026-04-09T22:18:13.764Z" }, { url = "https://files.pythonhosted.org/packages/cf/5d/d6331ad2350f10451cb9082ea3cf0dee63be2e7246c74998a9de57e3fdc6/jpype1-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:1d1f5b67eade4ef21e0e9f184034e47aee1e349eebb55ab97078219a50cf1e9c", size = 357721, upload-time = "2026-04-09T22:18:15.499Z" }, + { url = "https://files.pythonhosted.org/packages/39/87/7512ac8d3d2499f8fb23611d6d870189da609683239e50fa961ea82edf0a/jpype1-1.7.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:ce9a4addeb0494da774ae5ef7284f1f2eb52e05842d6700aa21988e2b50b2b88", size = 374171, upload-time = "2026-04-09T22:18:17.731Z" }, { url = "https://files.pythonhosted.org/packages/9e/9e/2544fabe521f969258bb532bc82d54cf2d799c5d93fd97f7453496ff3936/jpype1-1.7.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10d81fe60d4c37b3dc175e3ca2a08b7f655f8854d117a74d190e66f10833adcc", size = 406974, upload-time = "2026-04-09T22:18:19.896Z" }, { url = "https://files.pythonhosted.org/packages/0f/f2/03763b0e7ff5038307a919c4bcd629fba398f48472876b974db7d7f5a31e/jpype1-1.7.0-cp312-cp312-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:e3c4befd919f05e23e0a4245af2bc188771bc8daff5041d544ae29f467976a29", size = 453801, upload-time = "2026-04-09T22:18:21.706Z" }, { url = "https://files.pythonhosted.org/packages/37/d5/56214f4a93943d6786e7b024b8c07cfbc36df20c79ed520bf1db0053c780/jpype1-1.7.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ee5d72fee3b39e2e339adf3c4344d01d4473e10fbc4a3bb7d79ac06a8426da0", size = 437982, upload-time = "2026-04-09T22:18:23.58Z" }, { url = "https://files.pythonhosted.org/packages/8e/d1/64671f546467ba3733f7824d9728d03a563024a83ee6037a8b0a4c255710/jpype1-1.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:22ddcf4b8616b3ef54ac666a484f8238a921ab3f435d066de724a9705c2eeb84", size = 356788, upload-time = "2026-04-09T22:18:25.606Z" }, + { url = "https://files.pythonhosted.org/packages/43/8d/1c15575ae2d3a5e7da2515886b4c411eef3a4f079003e1fdbb5b5a8052c4/jpype1-1.7.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:74d6a88a84f0ad44f5962119a3e9c7876bfed2b5385804ff1f755620849b5aaf", size = 374362, upload-time = "2026-04-09T22:18:27.174Z" }, { url = "https://files.pythonhosted.org/packages/4b/e7/c1259ad95eaa29385d1407d8dc8ccff063de5cdb8054445ee7c4b2f77321/jpype1-1.7.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:783251ea09387efb490405ffeeccde167b30f12375f94ffc5db0ccb79f276a25", size = 407130, upload-time = "2026-04-09T22:18:29.42Z" }, { url = "https://files.pythonhosted.org/packages/5a/4e/65220f2e27df18c1f8696343436a9e3645beaee073f80c4a17fa8fd542ac/jpype1-1.7.0-cp313-cp313-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:28bbcf8532cfe71507d2ec4416f7387db7d7ee631ddf820e8c6ad559167d9abb", size = 453893, upload-time = "2026-04-09T22:18:31.257Z" }, { url = "https://files.pythonhosted.org/packages/a0/00/5991cbdf56898459f79817c0d91f1957c094d0805c0815e26c38f8fc39b3/jpype1-1.7.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9b22b58becd92cc899a88298a5b06b001a1988147c37c4fee503415a7cb5539", size = 438095, upload-time = "2026-04-09T22:18:33.393Z" }, @@ -15658,8 +15765,7 @@ version = "3.4.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", + "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368, upload-time = "2024-10-21T12:39:38.695Z" } wheels = [ @@ -15674,17 +15780,13 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", "python_full_version == '3.13.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and sys_platform != 'darwin'", + "(python_full_version == '3.13.*' and platform_machine != 'arm64') or (python_full_version == '3.13.*' and sys_platform != 'darwin')", "python_full_version == '3.12.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", "python_full_version == '3.11.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } wheels = [ @@ -15740,8 +15842,7 @@ version = "2.2.6" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", + "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" } wheels = [ @@ -15809,17 +15910,13 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", "python_full_version == '3.13.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and sys_platform != 'darwin'", + "(python_full_version == '3.13.*' and platform_machine != 'arm64') or (python_full_version == '3.13.*' and sys_platform != 'darwin')", "python_full_version == '3.12.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", "python_full_version == '3.11.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] sdist = { url = "https://files.pythonhosted.org/packages/d7/9f/b8cef5bffa569759033adda9481211426f12f53299629b410340795c2514/numpy-2.4.4.tar.gz", hash = "sha256:2d390634c5182175533585cc89f3608a4682ccb173cc9bb940b2881c8d6f8fa0", size = 20731587, upload-time = "2026-03-29T13:22:01.298Z" } wheels = [ @@ -16526,8 +16623,7 @@ version = "2.3.3.260113" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", + "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -16546,17 +16642,13 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", "python_full_version == '3.13.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and sys_platform != 'darwin'", + "(python_full_version == '3.13.*' and platform_machine != 'arm64') or (python_full_version == '3.13.*' and sys_platform != 'darwin')", "python_full_version == '3.12.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", "python_full_version == '3.11.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, @@ -19650,8 +19742,7 @@ version = "1.5.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", + "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ { name = "joblib", marker = "python_full_version < '3.11'" }, @@ -19691,17 +19782,13 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", "python_full_version == '3.13.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and sys_platform != 'darwin'", + "(python_full_version == '3.13.*' and platform_machine != 'arm64') or (python_full_version == '3.13.*' and sys_platform != 'darwin')", "python_full_version == '3.12.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", "python_full_version == '3.11.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ { name = "joblib", marker = "python_full_version >= '3.11'" }, @@ -19755,8 +19842,7 @@ version = "1.15.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", + "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -19818,17 +19904,13 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", "python_full_version == '3.13.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and sys_platform != 'darwin'", + "(python_full_version == '3.13.*' and platform_machine != 'arm64') or (python_full_version == '3.13.*' and sys_platform != 'darwin')", "python_full_version == '3.12.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", "python_full_version == '3.11.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, @@ -19937,8 +20019,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "jeepney", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "cryptography", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (platform_machine != 'arm64' and sys_platform == 'darwin') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "jeepney", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (platform_machine != 'arm64' and sys_platform == 'darwin') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -19977,14 +20059,11 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", "python_full_version == '3.13.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and sys_platform != 'darwin'", + "(python_full_version == '3.13.*' and platform_machine != 'arm64') or (python_full_version == '3.13.*' and sys_platform != 'darwin')", "python_full_version == '3.12.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", ] dependencies = [ { name = "ecdsa", marker = "python_full_version >= '3.12'" }, @@ -20002,11 +20081,9 @@ version = "6.12.5" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version == '3.11.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", "python_full_version < '3.11' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", + "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ { name = "cryptography", marker = "python_full_version < '3.12'" }, @@ -20323,8 +20400,7 @@ version = "8.1.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", + "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ { name = "alabaster", marker = "python_full_version < '3.11'" }, @@ -20356,8 +20432,7 @@ version = "9.0.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version == '3.11.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ { name = "alabaster", marker = "python_full_version == '3.11.*'" }, @@ -20391,14 +20466,11 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", "python_full_version == '3.13.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and sys_platform != 'darwin'", + "(python_full_version == '3.13.*' and platform_machine != 'arm64') or (python_full_version == '3.13.*' and sys_platform != 'darwin')", "python_full_version == '3.12.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", ] dependencies = [ { name = "alabaster", marker = "python_full_version >= '3.12'" }, @@ -20480,8 +20552,7 @@ version = "2024.10.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", + "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ { name = "colorama", marker = "python_full_version < '3.11'" }, @@ -20504,17 +20575,13 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", "python_full_version == '3.13.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and sys_platform != 'darwin'", + "(python_full_version == '3.13.*' and platform_machine != 'arm64') or (python_full_version == '3.13.*' and sys_platform != 'darwin')", "python_full_version == '3.12.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", "python_full_version == '3.11.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ { name = "colorama", marker = "python_full_version >= '3.11'" }, @@ -20550,8 +20617,7 @@ version = "0.6.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", + "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -20569,17 +20635,13 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", "python_full_version == '3.13.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and sys_platform != 'darwin'", + "(python_full_version == '3.13.*' and platform_machine != 'arm64') or (python_full_version == '3.13.*' and sys_platform != 'darwin')", "python_full_version == '3.12.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", "python_full_version == '3.11.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine != 'arm64' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, From 38986e614769bbb0aa760c59951f904e1cce8538 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 11:53:22 +0200 Subject: [PATCH 029/309] [v3-2-test] Filter external dependency nodes by readable DAGs in structure_data endpoint (#65342) (#65534) The structure_data endpoint returned external dependency nodes for linked DAGs without checking whether the caller had read permission on those DAGs. Add the ReadableDagsFilterDep and skip dependency entries that reference DAGs outside the caller's readable set. (cherry picked from commit 01888df3ae9b437e9086d296d29d2fee147d6312) Generated-by: Claude Opus 4.6 (1M context) following the guidelines at https: //github.com/apache/airflow/blob/main/contributing-docs/05_pull_requests.rst#gen-ai-assisted-contributions Co-authored-by: Jarek Potiuk --- .../core_api/routes/ui/structure.py | 14 ++++++++- .../core_api/routes/ui/test_structure.py | 29 +++++++++++++++---- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/structure.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/structure.py index 8fdb19f34a86c..bc7efc0a75afb 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/structure.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/structure.py @@ -26,7 +26,7 @@ from airflow.api_fastapi.common.router import AirflowRouter from airflow.api_fastapi.core_api.datamodels.ui.structure import StructureDataResponse from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc -from airflow.api_fastapi.core_api.security import requires_access_dag +from airflow.api_fastapi.core_api.security import ReadableDagsFilterDep, requires_access_dag from airflow.api_fastapi.core_api.services.ui.structure import ( bind_output_assets_to_tasks, get_upstream_assets, @@ -52,6 +52,7 @@ def structure_data( session: SessionDep, dag_id: str, + readable_dags_filter: ReadableDagsFilterDep, include_upstream: QueryIncludeUpstream = False, include_downstream: QueryIncludeDownstream = False, depth: int | None = None, @@ -105,11 +106,22 @@ def structure_data( start_edges: list[dict] = [] end_edges: list[dict] = [] + readable_dag_ids = readable_dags_filter.value for dependency_dag_id, dependencies in sorted(SerializedDagModel.get_dag_dependencies().items()): + if readable_dag_ids is not None and dependency_dag_id not in readable_dag_ids: + continue for dependency in dependencies: # Dependencies not related to `dag_id` are ignored if dependency_dag_id != dag_id and dependency.target != dag_id: continue + # When target is a real DAG ID (not a type label), hide it + # if the caller cannot read that DAG. + if ( + readable_dag_ids is not None + and dependency.target != dependency.dependency_type + and dependency.target not in readable_dag_ids + ): + continue # upstream assets are handled by the `get_upstream_assets` function. if dependency.target != dependency.dependency_type and dependency.dependency_type in [ diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_structure.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_structure.py index f4a56db93e4a4..db436ca3cf93a 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_structure.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_structure.py @@ -18,6 +18,7 @@ from __future__ import annotations import copy +from unittest import mock import pendulum import pytest @@ -303,7 +304,7 @@ class TestStructureDataEndpoint: }, ], }, - 6, + 7, ), ( { @@ -311,7 +312,7 @@ class TestStructureDataEndpoint: "root": "unknown_task", }, {"edges": [], "nodes": []}, - 6, + 7, ), ( { @@ -336,7 +337,7 @@ class TestStructureDataEndpoint: }, ], }, - 6, + 7, ), ( {"dag_id": DAG_ID_EXTERNAL_TRIGGER, "external_dependencies": True}, @@ -375,7 +376,7 @@ class TestStructureDataEndpoint: }, ], }, - 13, + 14, ), ], ) @@ -572,7 +573,7 @@ def test_should_return_200_with_asset(self, test_client, asset1_id, asset2_id, a ], } - with assert_queries_count(13): + with assert_queries_count(14): response = test_client.get("/structure/structure_data", params=params) assert response.status_code == 200 assert response.json() == expected @@ -685,6 +686,24 @@ def test_delete_dag_should_response_403(self, unauthorized_test_client): response = unauthorized_test_client.get("/structure/structure_data", params={"dag_id": DAG_ID}) assert response.status_code == 403 + @mock.patch( + "airflow.api_fastapi.auth.managers.base_auth_manager.BaseAuthManager.get_authorized_dag_ids", + return_value={DAG_ID_EXTERNAL_TRIGGER}, + ) + @pytest.mark.usefixtures("make_dags") + def test_external_deps_filters_unreadable_dags(self, _, test_client): + response = test_client.get( + "/structure/structure_data", + params={"dag_id": DAG_ID_EXTERNAL_TRIGGER, "external_dependencies": True}, + ) + assert response.status_code == 200 + result = response.json() + node_ids = {node["id"] for node in result["nodes"]} + assert "trigger_dag_run_operator" in node_ids + assert not any(DAG_ID in nid for nid in node_ids if nid != "trigger_dag_run_operator") + edge_targets = {edge["target_id"] for edge in result["edges"]} + assert not any(DAG_ID in tid for tid in edge_targets) + def test_should_return_404(self, test_client): response = test_client.get("/structure/structure_data", params={"dag_id": "not_existing"}) assert response.status_code == 404 From 7f144fdfc8aec5c6ee8aa7f1847255c6b7389172 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 12:13:54 +0200 Subject: [PATCH 030/309] [v3-2-test] unmock Graph component in Graph.test.tsx so hook assertions are reached (#65555) (#65556) (cherry picked from commit 7d13c2cd0253539a8b4df62b4b661b277a688427) Co-authored-by: Bugra Ozturk --- .../src/airflow/ui/src/layouts/Details/Graph/Graph.test.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airflow-core/src/airflow/ui/src/layouts/Details/Graph/Graph.test.tsx b/airflow-core/src/airflow/ui/src/layouts/Details/Graph/Graph.test.tsx index 8fa32dcc2c835..da463360e9683 100644 --- a/airflow-core/src/airflow/ui/src/layouts/Details/Graph/Graph.test.tsx +++ b/airflow-core/src/airflow/ui/src/layouts/Details/Graph/Graph.test.tsx @@ -25,6 +25,10 @@ import { Wrapper } from "src/utils/Wrapper"; import { Graph } from "./Graph"; +// testsSetup.ts globally mocks Graph to null so full-page tests don't need to +// stub ELK layout data. Unmock it here so we test the real component. +vi.unmock("src/layouts/Details/Graph/Graph"); + let mockParams: Record = { dagId: "test_dag" }; vi.mock("react-router-dom", async () => { From 4e00fc5e7d7d287b21400406980b51ac40d3f73f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 17:04:43 +0200 Subject: [PATCH 031/309] [v3-2-test] Fix run_id_pattern pipe OR operator dropping single-term edge cases (#65190) (#65565) When a pattern like "term|" or "|term" was passed, the OR-split guard `if len(search_terms) > 1` evaluated False for the single remaining term, falling through to `ilike("%term|%")` / `ilike("%|term%")` which matched nothing. Changing the guard to `if search_terms:` ensures a single valid term is used correctly. (cherry picked from commit 43556cc36081f7ce7f9004503ee0cb81e02641e1) closes: #65129 Co-authored-by: Cole Heflin <75401093+coleheflin@users.noreply.github.com> --- .../airflow/api_fastapi/common/parameters.py | 2 +- .../api_fastapi/common/test_parameters.py | 20 +++++++++++++++++++ .../core_api/routes/public/test_dag_run.py | 5 +++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/airflow-core/src/airflow/api_fastapi/common/parameters.py b/airflow-core/src/airflow/api_fastapi/common/parameters.py index 91250b670cf98..0206c9d0ac635 100644 --- a/airflow-core/src/airflow/api_fastapi/common/parameters.py +++ b/airflow-core/src/airflow/api_fastapi/common/parameters.py @@ -185,7 +185,7 @@ def to_orm(self, select: Select) -> Select: val_str = str(self.value) if "|" in val_str: search_terms = [term.strip() for term in val_str.split("|") if term.strip()] - if len(search_terms) > 1: + if search_terms: return select.where(or_(*(self.attribute.ilike(f"%{term}%") for term in search_terms))) return select.where(self.attribute.ilike(f"%{self.value}%")) diff --git a/airflow-core/tests/unit/api_fastapi/common/test_parameters.py b/airflow-core/tests/unit/api_fastapi/common/test_parameters.py index 40076137467ce..99fe13b3c6078 100644 --- a/airflow-core/tests/unit/api_fastapi/common/test_parameters.py +++ b/airflow-core/tests/unit/api_fastapi/common/test_parameters.py @@ -128,3 +128,23 @@ def test_to_orm_multiple_values_or(self): assert "OR" in sql assert "example_bash" in sql assert "example_python" in sql + + def test_to_orm_pipe_with_trailing_pipe(self): + """Test that a trailing pipe is ignored and only the valid term is searched.""" + param = _SearchParam(DagModel.dag_id).set_value("example_bash|") + statement = select(DagModel) + statement = param.to_orm(statement) + + sql = str(statement.compile(compile_kwargs={"literal_binds": True})) + assert "example_bash" in sql + assert "|" not in sql + + def test_to_orm_pipe_with_leading_pipe(self): + """Test that a leading pipe is ignored and only the valid term is searched.""" + param = _SearchParam(DagModel.dag_id).set_value("|example_bash") + statement = select(DagModel) + statement = param.to_orm(statement) + + sql = str(statement.compile(compile_kwargs={"literal_binds": True})) + assert "example_bash" in sql + assert "|" not in sql diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py index 9d27392de9251..5c13f09ef5a4a 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py @@ -662,6 +662,11 @@ def test_bad_limit_and_offset(self, test_client, query_params, expected_detail): (DAG1_ID, {"run_id_pattern": "run_1"}, [DAG1_RUN1_ID]), (DAG1_ID, {"run_id_pattern": "dag_%_1"}, [DAG1_RUN1_ID]), ("~", {"run_id_pattern": "dag_run_"}, [DAG1_RUN1_ID, DAG1_RUN2_ID, DAG2_RUN1_ID, DAG2_RUN2_ID]), + # Pipe (OR) operator returns results matching either term + ("~", {"run_id_pattern": f"{DAG1_RUN1_ID}|{DAG1_RUN2_ID}"}, [DAG1_RUN1_ID, DAG1_RUN2_ID]), + # Trailing/leading pipe should not leak into the LIKE pattern + ("~", {"run_id_pattern": f"{DAG1_RUN1_ID}|"}, [DAG1_RUN1_ID]), + ("~", {"run_id_pattern": f"|{DAG1_RUN1_ID}"}, [DAG1_RUN1_ID]), ( DAG1_ID, { From a232851d9757f311fb6a9a9b15e32a5fd9bec955 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 17:28:41 +0200 Subject: [PATCH 032/309] [v3-2-test] Define translation agent skill guidelines for Hebrew (he) locale (#65122) (#65596) (cherry picked from commit 7e01004596a98bfc3bd7a45328d68ad0ef06923a) Co-authored-by: Ali Asghar <98263017+alliasgher@users.noreply.github.com> --- .../skills/airflow-translations/locales/he.md | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 .github/skills/airflow-translations/locales/he.md diff --git a/.github/skills/airflow-translations/locales/he.md b/.github/skills/airflow-translations/locales/he.md new file mode 100644 index 0000000000000..385d0f5c0dea3 --- /dev/null +++ b/.github/skills/airflow-translations/locales/he.md @@ -0,0 +1,206 @@ + + +# Hebrew (he) Translation Agent Skill + +**Locale code:** `he` +**Preferred variant:** Modern Hebrew (he), consistent with existing translations in `airflow-core/src/airflow/ui/public/i18n/locales/he/` + +This file contains locale-specific guidelines so AI translation agents produce +new Hebrew strings that stay fully consistent with the existing translations. + +## 1. Core Airflow Terminology + +### Global Airflow terms (never translate) + +These terms are defined as untranslatable across **all** Airflow locales. +Do not translate them regardless of language: + +- `Airflow` — Product name +- `Dag` / `Dags` — Airflow concept; never write "DAG" +- `XCom` / `XComs` — Airflow cross-communication mechanism +- `UTC` — Time standard +- `JSON` — Standard technical format name +- `REST API` — Standard technical term +- `Unix` — Operating system name +- Log levels: `CRITICAL`, `ERROR`, `WARNING`, `INFO`, `DEBUG` + +### Translated by convention (Hebrew-specific) + +The existing Hebrew translations translate most Airflow terms into native Hebrew. +These established translations **must be used consistently**: + +- `Asset` / `Assets` → `נכס` / `נכסים` +- `Backfill` → `השלמה למפרע` / `השלמות למפרע` +- `Plugin` / `Plugins` → `תוסף` / `תוספים` +- `Pool` / `Pools` → `מאגר משאבים` +- `Provider` / `Providers` → `חבילות עזר` +- `Trigger` / `Triggerer` → `מפעיל` (component noun) +- `Executor` → `מבצע` +- `Heartbeat` → `עדכון חיים` (e.g., "עדכון חיים אחרון" for "Latest Heartbeat") + +## 2. Standard Translations + +| English Term | Hebrew Translation | Notes | +| --------------------- | ----------------------------- | ---------------------------------------------- | +| Task | משימה | | +| Task Instance | מופע משימה | | +| Task Group | קבוצת משימות | | +| Dag Run | הרצת Dag | | +| Trigger (verb) | הפעלה | "מופעל על-ידי" for "Triggered by" | +| Trigger Rule | כלל הפעלה | | +| Scheduler | מתזמן | | +| Schedule (noun) | תזמון | | +| Operator | אופרטור | | +| Connection | חיבור | | +| Variable | משתנה | | +| Configuration | הגדרות | | +| Audit Log | יומן ביקורת | | +| State | מצב | | +| Queue (noun) | בתור | "תור" for "queued" | +| Duration | משך זמן | | +| Owner | בעלים | | +| Tags | תגיות | | +| Description | תיאור | | +| Documentation | תיעוד | | +| Timezone | אזור זמן | | +| Dark Mode | מצב כהה | | +| Light Mode | מצב בהיר | | +| Asset Event | אירוע נכס | | +| Dag Processor | מעבד Dag | | +| Try Number | מספר נסיון | | + +## 3. Task/Run States + +| English State | Hebrew Translation | +| ------------------- | ----------------------------- | +| running | בריצה | +| failed | נכשלו | +| success | הצליחו | +| queued | בתור | +| scheduled | בתזמון | +| skipped | דולגו | +| deferred | בהשהייה | +| removed | הוסרו | +| restarting | בהפעלה מחדש | +| up_for_retry | בהמתנה לניסיון חוזר | +| up_for_reschedule | בהמתנה לתזמון מחדש | +| upstream_failed | משימות קודמות נכשלו | +| no_status / none | ללא סטטוס | +| planned | בתכנון | + +## 4. Hebrew-Specific Guidelines + +### Tone and Register + +- Use a **neutral, professional Hebrew** tone suitable for technical software UIs. +- The existing translations use masculine forms for imperatives and general references. Follow this established convention for consistency. +- Keep UI strings concise — they appear in buttons, labels, and tooltips. + +### Right-to-Left (RTL) Considerations + +- Hebrew is an RTL language. UI layout should flip accordingly. +- When mixing Hebrew and English (e.g., "הרצת Dag"), the LTR English term will naturally appear in the correct reading order within an RTL context. +- Preserve all i18next placeholders exactly as-is: `{{count}}`, `{{dagName}}`, etc. + +### Plural Forms + +- Hebrew uses i18next plural suffixes `_one`, `_two`, and `_other`. For most Airflow UI strings `_two` will be identical to `_other`, but check existing translations and keep the `_two` key when it is present. +- Note: colloquial Hebrew has a true dual form for things that come in pairs (e.g. one sock = גרב, two socks = גרביים, not "2 גרבים"). This rarely applies to Airflow UI terminology but is worth being aware of. + + ```json + "task_one": "משימה", + "task_other": "משימות" + ``` + + ```json + "dagRun_one": "הרצת Dag", + "dagRun_other": "הרצת Dags" + ``` + +### Capitalization of English terms + +- For English terms embedded in Hebrew strings, preserve their original casing (e.g., "Dag", "XCom", "Dags"). + +## 5. Examples from Existing Translations + +**Terms translated to Hebrew:** + +``` +Asset → "נכס" +Backfill → "השלמה למפרע" +Pool → "מאגר משאבים" +Plugin → "תוסף" +Provider → "חבילות עזר" +Executor → "מבצע" +Trigger → "מפעיל" +Heartbeat → "עדכון חיים" +``` + +**Common translation patterns:** + +``` +task_one → "משימה" +task_other → "משימות" +dagRun_one → "הרצת Dag" +dagRun_other → "הרצת Dags" +backfill_one → "השלמה למפרע" +backfill_other → "השלמות למפרע" +taskInstance_one → "מופע משימה" +taskInstance_other → "מופעי משימות" +running → "בריצה" +failed → "נכשלו" +success → "הצליחו" +queued → "בתור" +scheduled → "בתזמון" +``` + +**Action verbs (buttons):** + +``` +Add → "הוסף" +Delete → "מחק" +Save → "שמור" +Reset → "אתחל" +Cancel → "בטל" +Confirm → "אשר" +Download → "הורד" +Expand → "הרחב" +Collapse → "צמצם" +Filter → "סנן" +``` + +**Triggerer compound nouns:** + +``` +triggerer.class → "סוג מפעיל" +triggerer.id → "מזהה מפעיל" +triggerer.createdAt → "זמן יצירת מפעיל" +triggerer.assigned → "מפעיל מוקצה" +triggerer.latestHeartbeat → "עדכון חיים אחרון" +triggerer.title → "פרטי מפעיל" +``` + +## 6. Agent Instructions (DO / DON'T) + +**DO:** + +- Match tone, style, and terminology from existing `he/*.json` files +- Use professional, neutral Hebrew +- Preserve all i18next placeholders: `{{count}}`, `{{dagName}}`, `{{hotkey}}`, etc. +- Use construct state (סמיכות) for compound nouns as established +- Provide all needed plural suffixes (`_one`, `_other`) for each plural key +- Check existing translations before adding new ones to maintain consistency + +**DON'T:** + +- Write "DAG" — always write "Dag" +- Use colloquial or slang Hebrew +- Invent new vocabulary when an equivalent already exists in the current translations +- Change hotkey values (e.g., `"hotkey": "e"` must stay `"e"`) +- Translate variable names or placeholders inside `{{...}}` +- Add Hebrew prefixed prepositions to English terms (e.g., don't write "ב-Dag", use "ב-Dag" only if established) + +--- + +**Version:** 1.0 — derived from existing `he/*.json` locale files (April 2026) From 3fb1963b410a920be8c0fd6e24b7e81086636ff2 Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Tue, 21 Apr 2026 18:06:11 +0200 Subject: [PATCH 033/309] Fix backfill params not overriding existing DAG run conf (#64939) (#65599) * Fix backfill params not overriding existing DAG run conf When reprocessing an existing DAG run during backfill, the dag_run_conf was not being applied to the cleared run. This adds the dag_run_conf parameter to _handle_clear_run() and conditionally updates conf in the DagRun UPDATE statement. Also adds conf validation in _create_backfill() using dag.params.deep_merge().validate() to match the validation done in create_dagrun() for new runs, ensuring invalid conf is rejected before any runs are created or cleared. closes: #59043 * UI: Add override params checkbox to backfill form Add an "Override parameters on existing runs" checkbox to the backfill form, unchecked by default. When unchecked, dag_run_conf is sent as null, preserving existing run params during reprocessing. When checked, the form values are sent and override existing conf. Also changes BackfillPostBody.dag_run_conf from dict={} to dict|None=None so the API can distinguish between "no conf provided" (null, preserve existing) and "empty conf" ({}, override). related: #59043 * Address code review: pass null through and regenerate openapi artifacts * Address review: InvalidBackfillConf exception and partitioned path comment * UI: Disable max-lines in RunBackfillForm with justification * Apply suggestion from @uranusjr * Apply suggestion from @uranusjr: chained .values() --------- (cherry picked from commit 8b401f28f03ce99dc304d924d67a8b04befb2e7b) Co-authored-by: Shivam Rastogi <6463385+shivaam@users.noreply.github.com> Co-authored-by: Tzu-ping Chung --- .../core_api/datamodels/backfills.py | 2 +- .../openapi/v2-rest-api-generated.yaml | 7 +- .../core_api/routes/public/backfills.py | 3 + airflow-core/src/airflow/models/backfill.py | 29 +++- .../ui/openapi-gen/requests/schemas.gen.ts | 14 +- .../ui/openapi-gen/requests/types.gen.ts | 4 +- .../ui/public/i18n/locales/en/components.json | 1 + .../components/DagActions/RunBackfillForm.tsx | 13 +- .../ui/src/queries/useCreateBackfill.ts | 2 +- .../tests/unit/models/test_backfill.py | 150 ++++++++++++++++++ .../airflowctl/api/datamodels/generated.py | 2 +- 11 files changed, 211 insertions(+), 16 deletions(-) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/backfills.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/backfills.py index 87d2677028eab..64ed8bb05e91f 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/backfills.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/backfills.py @@ -33,7 +33,7 @@ class BackfillPostBody(StrictBaseModel): from_date: datetime to_date: datetime run_backwards: bool = False - dag_run_conf: dict = {} + dag_run_conf: dict | None = None reprocess_behavior: ReprocessBehavior = ReprocessBehavior.NONE max_active_runs: int = 10 run_on_latest_version: bool = True diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml index 59c5d1f9295c4..d8579e9844918 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml @@ -9281,10 +9281,11 @@ components: title: Run Backwards default: false dag_run_conf: - additionalProperties: true - type: object + anyOf: + - additionalProperties: true + type: object + - type: 'null' title: Dag Run Conf - default: {} reprocess_behavior: $ref: '#/components/schemas/ReprocessBehavior' default: none diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/backfills.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/backfills.py index c1e90ab0fa62b..e920c85f3c16d 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/backfills.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/backfills.py @@ -50,6 +50,7 @@ Backfill, BackfillDagRun, DagNoScheduleException, + InvalidBackfillConf, InvalidBackfillDate, InvalidBackfillDirection, InvalidReprocessBehavior, @@ -264,6 +265,7 @@ def create_backfill( InvalidBackfillDirection, DagNoScheduleException, InvalidBackfillDate, + InvalidBackfillConf, ) as e: raise RequestValidationError(str(e)) @@ -311,5 +313,6 @@ def create_backfill_dry_run( InvalidBackfillDirection, DagNoScheduleException, InvalidBackfillDate, + InvalidBackfillConf, ) as e: raise RequestValidationError(str(e)) diff --git a/airflow-core/src/airflow/models/backfill.py b/airflow-core/src/airflow/models/backfill.py index bb54faec615fe..bd48f8daa563e 100644 --- a/airflow-core/src/airflow/models/backfill.py +++ b/airflow-core/src/airflow/models/backfill.py @@ -100,6 +100,14 @@ class InvalidBackfillDate(AirflowException): """ +class InvalidBackfillConf(AirflowException): + """ + Raised when the provided ``dag_run_conf`` fails validation against the DAG's params. + + :meta private: + """ + + class UnknownActiveBackfills(AirflowException): """ Raised when the quantity of active backfills cannot be determined. @@ -249,6 +257,7 @@ def _validate_backfill_params( from_date: datetime, to_date: datetime, reprocess_behavior: ReprocessBehavior | None, + dag_run_conf: dict | None = None, ) -> None: depends_on_past = any(x.depends_on_past for x in dag.tasks) if depends_on_past: @@ -264,6 +273,11 @@ def _validate_backfill_params( current_time = timezone.utcnow() if from_date >= current_time and to_date >= current_time: raise InvalidBackfillDate("Backfill cannot be executed for future dates.") + if dag_run_conf is not None: + try: + dag.params.deep_merge(dag_run_conf).validate() + except ValueError as e: + raise InvalidBackfillConf(str(e)) from e def _do_dry_run( @@ -359,6 +373,7 @@ def _create_backfill_dag_run_non_partitioned( backfill_id=backfill_id, sort_ordinal=backfill_sort_ordinal, run_on_latest=run_on_latest_version, + dag_run_conf=dag_run_conf, ) else: session.add( @@ -432,6 +447,10 @@ def _create_backfill_dag_run_partitioned( triggering_user_name: str | None, session: Session, ) -> None: + # Partitioned backfills don't currently reprocess existing runs — if a run exists + # for this partition, it's recorded as skipped via exception_reason rather than + # cleared and re-queued. As a result, this function never calls ``_handle_clear_run`` + # and therefore doesn't need to forward ``dag_run_conf`` for the reprocess path. stmt = _get_latest_dag_run_row_query(dag_id=dag.dag_id, info=info) dr = session.scalar(stmt) if dr: @@ -508,6 +527,7 @@ def _handle_clear_run( backfill_id: int, sort_ordinal: int, run_on_latest: bool = False, + dag_run_conf: dict | None = None, ) -> None: """Clear the existing Dag run and update backfill metadata.""" from sqlalchemy.sql import update @@ -524,8 +544,8 @@ def _handle_clear_run( run_on_latest_version=run_on_latest, ) - # Update backfill_id and run_type in DagRun table - session.execute( + # Update backfill_id, run_type, and optionally conf in DagRun table + stmt = ( update(DagRun) .where(DagRun.logical_date == info.logical_date, DagRun.dag_id == dag.dag_id) .values( @@ -534,6 +554,9 @@ def _handle_clear_run( triggered_by=DagRunTriggeredByType.BACKFILL, ) ) + if dag_run_conf is not None: + stmt = stmt.values(conf=dag_run_conf) + session.execute(stmt) session.add( BackfillDagRun( backfill_id=backfill_id, @@ -589,7 +612,7 @@ def _create_backfill( ) dag = serdag.dag - _validate_backfill_params(dag, reverse, from_date, to_date, reprocess_behavior) + _validate_backfill_params(dag, reverse, from_date, to_date, reprocess_behavior, dag_run_conf) br = Backfill( dag_id=dag_id, diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts index 79959b9ceb123..0e9e8339c31b0 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -445,10 +445,16 @@ export const $BackfillPostBody = { default: false }, dag_run_conf: { - additionalProperties: true, - type: 'object', - title: 'Dag Run Conf', - default: {} + anyOf: [ + { + additionalProperties: true, + type: 'object' + }, + { + type: 'null' + } + ], + title: 'Dag Run Conf' }, reprocess_behavior: { '$ref': '#/components/schemas/ReprocessBehavior', diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts index c29ee5b71cd19..0f161e60213f6 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts @@ -122,8 +122,8 @@ export type BackfillPostBody = { to_date: string; run_backwards?: boolean; dag_run_conf?: { - [key: string]: unknown; - }; + [key: string]: unknown; +} | null; reprocess_behavior?: ReprocessBehavior; max_active_runs?: number; run_on_latest_version?: boolean; diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/en/components.json b/airflow-core/src/airflow/ui/public/i18n/locales/en/components.json index 67e348ca11e35..299f3852b1a65 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/en/components.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/en/components.json @@ -10,6 +10,7 @@ "maxRuns": "Max Active Runs", "missingAndErroredRuns": "Missing and Errored Runs", "missingRuns": "Missing Runs", + "overrideExistingParams": "Override parameters on existing runs", "permissionDenied": "Dry Run Failed: User does not have permission to create backfills.", "reprocessBehavior": "Reprocess Behavior", "run": "Run Backfill", diff --git a/airflow-core/src/airflow/ui/src/components/DagActions/RunBackfillForm.tsx b/airflow-core/src/airflow/ui/src/components/DagActions/RunBackfillForm.tsx index 089c914707d86..c1873b9d352dd 100644 --- a/airflow-core/src/airflow/ui/src/components/DagActions/RunBackfillForm.tsx +++ b/airflow-core/src/airflow/ui/src/components/DagActions/RunBackfillForm.tsx @@ -1,3 +1,5 @@ +/* eslint-disable max-lines -- form aggregates date range, reprocess behavior, and param controls in a single UX flow */ + /*! * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -50,6 +52,7 @@ const RunBackfillForm = ({ dag, onClose }: RunBackfillFormProps) => { const { t: translate } = useTranslation(["components", "common"]); const [errors, setErrors] = useState<{ conf?: string; date?: unknown }>({}); const [unpause, setUnpause] = useState(true); + const [overrideParams, setOverrideParams] = useState(false); const [formError, setFormError] = useState(false); const initialParamsDict = useDagParams(dag.dag_id, true); const { conf } = useParamStore(); @@ -119,7 +122,7 @@ const RunBackfillForm = ({ dag, onClose }: RunBackfillFormProps) => { createBackfill({ requestBody: { ...fdata, - dag_run_conf: JSON.parse(fdata.conf) as Record, + dag_run_conf: overrideParams ? (JSON.parse(fdata.conf) as Record) : null, }, }); }; @@ -250,6 +253,14 @@ const RunBackfillForm = ({ dag, onClose }: RunBackfillFormProps) => { ) : undefined} + setOverrideParams(!overrideParams)} + > + {translate("backfill.overrideExistingParams")} + + Date: Tue, 21 Apr 2026 13:05:24 -0400 Subject: [PATCH 034/309] [v3-2-test] Support ordering XCom entries in the REST API and UI (#65418) (#65600) * Support ordering XCom entries in the REST API and UI * Fix CI error in XComs e2e spec * Replace networkidle wait in XComs sort test (cherry picked from commit 1fb2d0e36487a4fd8daefa7b6a33f0010622a160) Co-authored-by: Yuseok Jo --- .../airflow/api_fastapi/common/parameters.py | 13 +++++++--- .../openapi/v2-rest-api-generated.yaml | 20 ++++++++++++++ .../core_api/routes/public/xcom.py | 15 ++++++++--- .../airflow/ui/openapi-gen/queries/common.ts | 5 ++-- .../ui/openapi-gen/queries/ensureQueryData.ts | 6 +++-- .../ui/openapi-gen/queries/prefetch.ts | 6 +++-- .../airflow/ui/openapi-gen/queries/queries.ts | 6 +++-- .../ui/openapi-gen/queries/suspense.ts | 6 +++-- .../ui/openapi-gen/requests/services.gen.ts | 4 ++- .../ui/openapi-gen/requests/types.gen.ts | 4 +++ .../src/airflow/ui/src/pages/XCom/XCom.tsx | 14 +++++----- .../airflow/ui/tests/e2e/pages/XComsPage.ts | 26 +++++++++++++++++++ .../airflow/ui/tests/e2e/specs/xcoms.spec.ts | 8 ++++++ 13 files changed, 108 insertions(+), 25 deletions(-) diff --git a/airflow-core/src/airflow/api_fastapi/common/parameters.py b/airflow-core/src/airflow/api_fastapi/common/parameters.py index 0206c9d0ac635..28180b4d81fb8 100644 --- a/airflow-core/src/airflow/api_fastapi/common/parameters.py +++ b/airflow-core/src/airflow/api_fastapi/common/parameters.py @@ -18,7 +18,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Sequence from datetime import datetime from enum import Enum from typing import ( @@ -364,14 +364,21 @@ def get_primary_key_string(self) -> str: def depends(cls, *args: Any, **kwargs: Any) -> Self: raise NotImplementedError("Use dynamic_depends, depends not implemented.") - def dynamic_depends(self, default: str | None = None) -> Callable: + def dynamic_depends(self, default: str | Sequence[str] | None = None) -> Callable: to_replace_attrs = list(self.to_replace.keys()) if self.to_replace else [] all_attrs = self.allowed_attrs + to_replace_attrs + if default is None: + default_list = [self.get_primary_key_string()] + elif isinstance(default, str): + default_list = [default] + else: + default_list = list(default) + def inner( order_by: list[str] = Query( - default=[default] if default is not None else [self.get_primary_key_string()], + default=default_list, description=f"Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. " f"Supported attributes: `{', '.join(all_attrs) if all_attrs else self.get_primary_key_string()}`", ), diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml index d8579e9844918..b69b98e6a127a 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml @@ -5348,6 +5348,26 @@ paths: format: date-time - type: 'null' title: Run After Lt + - name: order_by + in: query + required: false + schema: + type: array + items: + type: string + description: 'Attributes to order by, multi criteria sort is supported. + Prefix with `-` for descending order. Supported attributes: `key, dag_id, + run_id, task_id, map_index, timestamp, run_after`' + default: + - dag_id + - task_id + - run_id + - map_index + - key + title: Order By + description: 'Attributes to order by, multi criteria sort is supported. Prefix + with `-` for descending order. Supported attributes: `key, dag_id, run_id, + task_id, map_index, timestamp, run_after`' responses: '200': description: Successful Response diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/xcom.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/xcom.py index 7bf64592aa640..7f5b49ca2e2e8 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/xcom.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/xcom.py @@ -36,6 +36,7 @@ QueryXComRunIdPatternSearch, QueryXComTaskIdPatternSearch, RangeFilter, + SortParam, datetime_range_filter_factory, filter_param_factory, ) @@ -162,6 +163,16 @@ def get_xcom_entries( ], logical_date_range: Annotated[RangeFilter, Depends(datetime_range_filter_factory("logical_date", DR))], run_after_range: Annotated[RangeFilter, Depends(datetime_range_filter_factory("run_after", DR))], + order_by: Annotated[ + SortParam, + Depends( + SortParam( + ["key", "dag_id", "run_id", "task_id", "map_index", "timestamp"], + XComModel, + to_replace={"run_after": DR.run_after}, + ).dynamic_depends(default=("dag_id", "task_id", "run_id", "map_index", "key")) + ), + ], xcom_key: Annotated[str | None, Query()] = None, map_index: Annotated[int | None, Query(ge=-1)] = None, ) -> XComCollectionResponse: @@ -200,13 +211,11 @@ def get_xcom_entries( logical_date_range, run_after_range, ], + order_by=order_by, offset=offset, limit=limit, session=session, ) - query = query.order_by( - XComModel.dag_id, XComModel.task_id, XComModel.run_id, XComModel.map_index, XComModel.key - ) return XComCollectionResponse(xcom_entries=session.scalars(query), total_entries=total_entries) diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts index 6c87c143157dc..25b8985c4e96d 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts @@ -696,7 +696,7 @@ export const UseXcomServiceGetXcomEntryKeyFn = ({ dagId, dagRunId, deserialize, export type XcomServiceGetXcomEntriesDefaultResponse = Awaited>; export type XcomServiceGetXcomEntriesQueryResult = UseQueryResult; export const useXcomServiceGetXcomEntriesKey = "XcomServiceGetXcomEntries"; -export const UseXcomServiceGetXcomEntriesKeyFn = ({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }: { +export const UseXcomServiceGetXcomEntriesKeyFn = ({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, orderBy, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }: { dagDisplayNamePattern?: string; dagId: string; dagRunId: string; @@ -708,6 +708,7 @@ export const UseXcomServiceGetXcomEntriesKeyFn = ({ dagDisplayNamePattern, dagId mapIndex?: number; mapIndexFilter?: number; offset?: number; + orderBy?: string[]; runAfterGt?: string; runAfterGte?: string; runAfterLt?: string; @@ -717,7 +718,7 @@ export const UseXcomServiceGetXcomEntriesKeyFn = ({ dagDisplayNamePattern, dagId taskIdPattern?: string; xcomKey?: string; xcomKeyPattern?: string; -}, queryKey?: Array) => [useXcomServiceGetXcomEntriesKey, ...(queryKey ?? [{ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }])]; +}, queryKey?: Array) => [useXcomServiceGetXcomEntriesKey, ...(queryKey ?? [{ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, orderBy, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }])]; export type TaskServiceGetTasksDefaultResponse = Awaited>; export type TaskServiceGetTasksQueryResult = UseQueryResult; export const useTaskServiceGetTasksKey = "TaskServiceGetTasks"; diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts index 23e47e6e21f9a..6fbc3416bb9be 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts @@ -1352,10 +1352,11 @@ export const ensureUseXcomServiceGetXcomEntryData = (queryClient: QueryClient, { * @param data.runAfterGt * @param data.runAfterLte * @param data.runAfterLt +* @param data.orderBy Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `key, dag_id, run_id, task_id, map_index, timestamp, run_after` * @returns XComCollectionResponse Successful Response * @throws ApiError */ -export const ensureUseXcomServiceGetXcomEntriesData = (queryClient: QueryClient, { dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }: { +export const ensureUseXcomServiceGetXcomEntriesData = (queryClient: QueryClient, { dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, orderBy, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }: { dagDisplayNamePattern?: string; dagId: string; dagRunId: string; @@ -1367,6 +1368,7 @@ export const ensureUseXcomServiceGetXcomEntriesData = (queryClient: QueryClient, mapIndex?: number; mapIndexFilter?: number; offset?: number; + orderBy?: string[]; runAfterGt?: string; runAfterGte?: string; runAfterLt?: string; @@ -1376,7 +1378,7 @@ export const ensureUseXcomServiceGetXcomEntriesData = (queryClient: QueryClient, taskIdPattern?: string; xcomKey?: string; xcomKeyPattern?: string; -}) => queryClient.ensureQueryData({ queryKey: Common.UseXcomServiceGetXcomEntriesKeyFn({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }), queryFn: () => XcomService.getXcomEntries({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }) }); +}) => queryClient.ensureQueryData({ queryKey: Common.UseXcomServiceGetXcomEntriesKeyFn({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, orderBy, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }), queryFn: () => XcomService.getXcomEntries({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, orderBy, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }) }); /** * Get Tasks * Get tasks for DAG. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts index f9d107a5b1df2..840cd9508d5ba 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts @@ -1352,10 +1352,11 @@ export const prefetchUseXcomServiceGetXcomEntry = (queryClient: QueryClient, { d * @param data.runAfterGt * @param data.runAfterLte * @param data.runAfterLt +* @param data.orderBy Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `key, dag_id, run_id, task_id, map_index, timestamp, run_after` * @returns XComCollectionResponse Successful Response * @throws ApiError */ -export const prefetchUseXcomServiceGetXcomEntries = (queryClient: QueryClient, { dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }: { +export const prefetchUseXcomServiceGetXcomEntries = (queryClient: QueryClient, { dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, orderBy, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }: { dagDisplayNamePattern?: string; dagId: string; dagRunId: string; @@ -1367,6 +1368,7 @@ export const prefetchUseXcomServiceGetXcomEntries = (queryClient: QueryClient, { mapIndex?: number; mapIndexFilter?: number; offset?: number; + orderBy?: string[]; runAfterGt?: string; runAfterGte?: string; runAfterLt?: string; @@ -1376,7 +1378,7 @@ export const prefetchUseXcomServiceGetXcomEntries = (queryClient: QueryClient, { taskIdPattern?: string; xcomKey?: string; xcomKeyPattern?: string; -}) => queryClient.prefetchQuery({ queryKey: Common.UseXcomServiceGetXcomEntriesKeyFn({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }), queryFn: () => XcomService.getXcomEntries({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }) }); +}) => queryClient.prefetchQuery({ queryKey: Common.UseXcomServiceGetXcomEntriesKeyFn({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, orderBy, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }), queryFn: () => XcomService.getXcomEntries({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, orderBy, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }) }); /** * Get Tasks * Get tasks for DAG. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts index c9dd84d145830..af30805d4e1a7 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts @@ -1352,10 +1352,11 @@ export const useXcomServiceGetXcomEntry = = unknown[]>({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }: { +export const useXcomServiceGetXcomEntries = = unknown[]>({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, orderBy, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }: { dagDisplayNamePattern?: string; dagId: string; dagRunId: string; @@ -1367,6 +1368,7 @@ export const useXcomServiceGetXcomEntries = , "queryKey" | "queryFn">) => useQuery({ queryKey: Common.UseXcomServiceGetXcomEntriesKeyFn({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }, queryKey), queryFn: () => XcomService.getXcomEntries({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }) as TData, ...options }); +}, queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">) => useQuery({ queryKey: Common.UseXcomServiceGetXcomEntriesKeyFn({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, orderBy, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }, queryKey), queryFn: () => XcomService.getXcomEntries({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, orderBy, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }) as TData, ...options }); /** * Get Tasks * Get tasks for DAG. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts index 4727215089c47..e1453e19e4290 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts @@ -1352,10 +1352,11 @@ export const useXcomServiceGetXcomEntrySuspense = = unknown[]>({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }: { +export const useXcomServiceGetXcomEntriesSuspense = = unknown[]>({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, orderBy, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }: { dagDisplayNamePattern?: string; dagId: string; dagRunId: string; @@ -1367,6 +1368,7 @@ export const useXcomServiceGetXcomEntriesSuspense = , "queryKey" | "queryFn">) => useSuspenseQuery({ queryKey: Common.UseXcomServiceGetXcomEntriesKeyFn({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }, queryKey), queryFn: () => XcomService.getXcomEntries({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }) as TData, ...options }); +}, queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">) => useSuspenseQuery({ queryKey: Common.UseXcomServiceGetXcomEntriesKeyFn({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, orderBy, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }, queryKey), queryFn: () => XcomService.getXcomEntries({ dagDisplayNamePattern, dagId, dagRunId, limit, logicalDateGt, logicalDateGte, logicalDateLt, logicalDateLte, mapIndex, mapIndexFilter, offset, orderBy, runAfterGt, runAfterGte, runAfterLt, runAfterLte, runIdPattern, taskId, taskIdPattern, xcomKey, xcomKeyPattern }) as TData, ...options }); /** * Get Tasks * Get tasks for DAG. diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts index b27ad6008e4e6..fff137401c721 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts @@ -3384,6 +3384,7 @@ export class XcomService { * @param data.runAfterGt * @param data.runAfterLte * @param data.runAfterLt + * @param data.orderBy Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `key, dag_id, run_id, task_id, map_index, timestamp, run_after` * @returns XComCollectionResponse Successful Response * @throws ApiError */ @@ -3413,7 +3414,8 @@ export class XcomService { run_after_gte: data.runAfterGte, run_after_gt: data.runAfterGt, run_after_lte: data.runAfterLte, - run_after_lt: data.runAfterLt + run_after_lt: data.runAfterLt, + order_by: data.orderBy }, errors: { 400: 'Bad Request', diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts index 0f161e60213f6..0a0a8d6b9c871 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts @@ -3469,6 +3469,10 @@ export type GetXcomEntriesData = { mapIndex?: number | null; mapIndexFilter?: number | null; offset?: number; + /** + * Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `key, dag_id, run_id, task_id, map_index, timestamp, run_after` + */ + orderBy?: Array<(string)>; runAfterGt?: string | null; runAfterGte?: string | null; runAfterLt?: string | null; diff --git a/airflow-core/src/airflow/ui/src/pages/XCom/XCom.tsx b/airflow-core/src/airflow/ui/src/pages/XCom/XCom.tsx index 66895c76cc6c6..4bf1a5a8229e8 100644 --- a/airflow-core/src/airflow/ui/src/pages/XCom/XCom.tsx +++ b/airflow-core/src/airflow/ui/src/pages/XCom/XCom.tsx @@ -54,7 +54,6 @@ type ColumnsProps = { const getColumns = ({ open, translate }: ColumnsProps): Array> => [ { accessorKey: "key", - enableSorting: false, header: translate("xcom.columns.key"), }, { @@ -64,7 +63,6 @@ const getColumns = ({ open, translate }: ColumnsProps): Array{original.dag_display_name} ), - enableSorting: false, header: translate("xcom.columns.dag"), }, { @@ -76,7 +74,6 @@ const getColumns = ({ open, translate }: ColumnsProps): Array ), - enableSorting: false, header: translate("common:dagRunId"), }, { @@ -88,7 +85,6 @@ const getColumns = ({ open, translate }: ColumnsProps): Array ), - enableSorting: false, header: translate("common:dagRun.runAfter"), }, { @@ -107,18 +103,15 @@ const getColumns = ({ open, translate }: ColumnsProps): Array ), - enableSorting: false, header: translate("common:task_one"), }, { accessorKey: "map_index", - enableSorting: false, header: translate("common:mapIndex"), }, { accessorKey: "timestamp", cell: ({ row: { original } }) =>