diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 96b4108ab7703..0de707561c774 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -99,7 +99,7 @@ jobs: }' --jq '.data.node.labels.nodes[]' | jq --slurp -c '[.[].name]' >> ${GITHUB_OUTPUT} if: github.event_name == 'pull_request_target' # Retrieve it to be able to determine which files has changed in the incoming commit of the PR - # we checkout the target commit and it's parent to be able to compare them + # we checkout the target commit and its parent to be able to compare them - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - uses: actions/checkout@v3 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2c022c308cfe..a716ca2f78d33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -886,7 +886,7 @@ jobs: DB_RESET: "false" PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" JOB_ID: "helm-tests" - COVERAGE: "${{needs.build-info.outputs.run-coverage}}" + COVERAGE: "false" # We do not need to run coverage on Helm tests if: > needs.build-info.outputs.needs-helm-tests == 'true' && (github.repository == 'apache/airflow' || github.event_name != 'schedule') && @@ -1089,6 +1089,7 @@ jobs: MYSQL_VERSION: "${{matrix.mysql-version}}" BACKEND_VERSION: "${{matrix.mysql-version}}" JOB_ID: "mysql-${{matrix.mysql-version}}-${{matrix.python-version}}" + COVERAGE: "${{needs.build-info.outputs.run-coverage}}" if: needs.build-info.outputs.run-tests == 'true' steps: - name: Cleanup repo diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cb0545eddffa6..778b4d6db0026 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -412,7 +412,7 @@ repos: language: python entry: ./scripts/ci/pre_commit/pre_commit_version_heads_map.py pass_filenames: false - additional_dependencies: ['packaging'] + additional_dependencies: ['packaging','google-re2'] - id: update-version name: Update version to the latest version in the documentation entry: ./scripts/ci/pre_commit/pre_commit_update_versions.py diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index ef358cbdc107c..5379f31160be9 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -679,7 +679,7 @@ ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm mysql, neo4j, odbc, openfaas, openlineage, opsgenie, oracle, otel, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, spark, sqlite, ssh, statsd, -tableau, tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, zendesk +tableau, tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk .. END EXTRAS HERE Provider packages diff --git a/CONTRIBUTORS_QUICK_START.rst b/CONTRIBUTORS_QUICK_START.rst index 75168b4efccf8..7ccfac30d09fb 100644 --- a/CONTRIBUTORS_QUICK_START.rst +++ b/CONTRIBUTORS_QUICK_START.rst @@ -104,7 +104,7 @@ Colima ------ If you use Colima as your container runtimes engine, please follow the next steps: -1. `Install buildx manually `_ and follow it's instructions +1. `Install buildx manually `_ and follow its instructions 2. Link the Colima socket to the default socket path. Note that this may break other Docker servers. @@ -252,7 +252,7 @@ Typical development tasks ######################### For many of the development tasks you will need ``Breeze`` to be configured. ``Breeze`` is a development -environment which uses docker and docker-compose and it's main purpose is to provide a consistent +environment which uses docker and docker-compose and its main purpose is to provide a consistent and repeatable environment for all the contributors and CI. When using ``Breeze`` you avoid the "works for me" syndrome - because not only others can reproduce easily what you do, but also the CI of Airflow uses the same environment to run all tests - so you should be able to easily reproduce the same failures you diff --git a/Dockerfile b/Dockerfile index 5023e3ef9a3c0..43f085ce6afc2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1336,7 +1336,7 @@ RUN if [[ -f /docker-context-files/requirements.txt ]]; then \ ############################################################################################## # This is the actual Airflow image - much smaller than the build one. We copy -# installed Airflow and all it's dependencies from the build image to make it smaller. +# installed Airflow and all its dependencies from the build image to make it smaller. ############################################################################################## FROM ${PYTHON_BASE_IMAGE} as main diff --git a/Dockerfile.ci b/Dockerfile.ci index a4503f2d7cd59..0761021304197 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1020,10 +1020,11 @@ else fi if [[ ${ENABLE_TEST_COVERAGE:="false"} == "true" ]]; then + _suffix="$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8)" EXTRA_PYTEST_ARGS+=( "--cov=airflow" "--cov-config=pyproject.toml" - "--cov-report=xml:/files/coverage-${TEST_TYPE/\[*\]/}-${BACKEND}.xml" + "--cov-report=xml:/files/coverage-${TEST_TYPE/\[*\]/}-${BACKEND}-${_suffix}.xml" ) fi @@ -1395,7 +1396,7 @@ RUN echo "Airflow version: ${AIRFLOW_VERSION}" # force them on the main Airflow package. Currently we need no extra limits as PIP 23.1+ has much better # dependency resolution and we do not need to limit the versions of the dependencies # aiobotocore is limited temporarily until it stops backtracking pip -ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="aiobotocore<2.6.0" +ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="" ARG UPGRADE_TO_NEWER_DEPENDENCIES="false" ARG VERSION_SUFFIX_FOR_PYPI="" diff --git a/INSTALL b/INSTALL index b30f0dbd5f10f..4e017cc9a8065 100644 --- a/INSTALL +++ b/INSTALL @@ -106,7 +106,7 @@ ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm mysql, neo4j, odbc, openfaas, openlineage, opsgenie, oracle, otel, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, spark, sqlite, ssh, statsd, -tableau, tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, zendesk +tableau, tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk # END EXTRAS HERE # For installing Airflow in development environments - see CONTRIBUTING.rst diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index fbd2ba786ef24..66794268e2ac9 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -5439,7 +5439,7 @@ It has been removed. ``airflow.settings.CONTEXT_MANAGER_DAG`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -CONTEXT_MANAGER_DAG was removed from settings. It's role has been taken by ``DagContext`` in +CONTEXT_MANAGER_DAG was removed from settings. Its role has been taken by ``DagContext`` in 'airflow.models.dag'. One of the reasons was that settings should be rather static than store dynamic context from the DAG, but the main one is that moving the context out of settings allowed to untangle cyclic imports between DAG, BaseOperator, SerializedDAG, SerializedBaseOperator which was diff --git a/TESTING.rst b/TESTING.rst index c9e1525c2f98c..d5657124684e7 100644 --- a/TESTING.rst +++ b/TESTING.rst @@ -451,9 +451,9 @@ Running Tests with provider packages Airflow 2.0 introduced the concept of splitting the monolithic Airflow package into separate providers packages. The main "apache-airflow" package contains the bare Airflow implementation, -and additionally we have 70+ providers that we can install additionally to get integrations with -external services. Those providers live in the same monorepo as Airflow, but we build separate -packages for them and the main "apache-airflow" package does not contain the providers. +and there are more than 70 additional providers that can be installed to integrate with +external services. Such providers live in the same monorepo as Airflow but are built as +separate packages, distinct from the main "apache-airflow" package. Most of the development in Breeze happens by iterating on sources and when you run your tests during development, you usually do not want to build packages and install them separately. diff --git a/airflow/auth/managers/base_auth_manager.py b/airflow/auth/managers/base_auth_manager.py index bcc5e13892e2b..a512804b4ca94 100644 --- a/airflow/auth/managers/base_auth_manager.py +++ b/airflow/auth/managers/base_auth_manager.py @@ -20,11 +20,12 @@ from abc import abstractmethod from typing import TYPE_CHECKING -from airflow.auth.managers.models.base_user import BaseUser from airflow.exceptions import AirflowException from airflow.utils.log.logging_mixin import LoggingMixin if TYPE_CHECKING: + from airflow.auth.managers.models.base_user import BaseUser + from airflow.cli.cli_config import CLICommand from airflow.www.security import AirflowSecurityManager @@ -38,6 +39,14 @@ class BaseAuthManager(LoggingMixin): def __init__(self): self._security_manager: AirflowSecurityManager | None = None + @staticmethod + def get_cli_commands() -> list[CLICommand]: + """Vends CLI commands to be included in Airflow CLI. + + Override this method to expose commands via Airflow CLI to manage this auth manager. + """ + return [] + @abstractmethod def get_user_name(self) -> str: """Return the username associated to the user in session.""" diff --git a/airflow/auth/managers/fab/cli_commands/__init__.py b/airflow/auth/managers/fab/cli_commands/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/airflow/auth/managers/fab/cli_commands/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/auth/managers/fab/cli_commands/definition.py b/airflow/auth/managers/fab/cli_commands/definition.py new file mode 100644 index 0000000000000..478f6d8d309e0 --- /dev/null +++ b/airflow/auth/managers/fab/cli_commands/definition.py @@ -0,0 +1,220 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import textwrap + +from airflow.cli.cli_config import ( + ARG_OUTPUT, + ARG_VERBOSE, + ActionCommand, + Arg, + lazy_load_command, +) + +############ +# # ARGS # # +############ + +# users +ARG_USERNAME = Arg(("-u", "--username"), help="Username of the user", required=True, type=str) +ARG_USERNAME_OPTIONAL = Arg(("-u", "--username"), help="Username of the user", type=str) +ARG_FIRSTNAME = Arg(("-f", "--firstname"), help="First name of the user", required=True, type=str) +ARG_LASTNAME = Arg(("-l", "--lastname"), help="Last name of the user", required=True, type=str) +ARG_ROLE = Arg( + ("-r", "--role"), + help="Role of the user. Existing roles include Admin, User, Op, Viewer, and Public", + required=True, + type=str, +) +ARG_EMAIL = Arg(("-e", "--email"), help="Email of the user", required=True, type=str) +ARG_EMAIL_OPTIONAL = Arg(("-e", "--email"), help="Email of the user", type=str) +ARG_PASSWORD = Arg( + ("-p", "--password"), + help="Password of the user, required to create a user without --use-random-password", + type=str, +) +ARG_USE_RANDOM_PASSWORD = Arg( + ("--use-random-password",), + help="Do not prompt for password. Use random string instead." + " Required to create a user without --password ", + default=False, + action="store_true", +) +ARG_USER_IMPORT = Arg( + ("import",), + metavar="FILEPATH", + help="Import users from JSON file. Example format::\n" + + textwrap.indent( + textwrap.dedent( + """ + [ + { + "email": "foo@bar.org", + "firstname": "Jon", + "lastname": "Doe", + "roles": ["Public"], + "username": "jondoe" + } + ]""" + ), + " " * 4, + ), +) +ARG_USER_EXPORT = Arg(("export",), metavar="FILEPATH", help="Export all users to JSON file") + +# roles +ARG_CREATE_ROLE = Arg(("-c", "--create"), help="Create a new role", action="store_true") +ARG_LIST_ROLES = Arg(("-l", "--list"), help="List roles", action="store_true") +ARG_ROLES = Arg(("role",), help="The name of a role", nargs="*") +ARG_PERMISSIONS = Arg(("-p", "--permission"), help="Show role permissions", action="store_true") +ARG_ROLE_RESOURCE = Arg(("-r", "--resource"), help="The name of permissions", nargs="*", required=True) +ARG_ROLE_ACTION = Arg(("-a", "--action"), help="The action of permissions", nargs="*") +ARG_ROLE_ACTION_REQUIRED = Arg(("-a", "--action"), help="The action of permissions", nargs="*", required=True) + +ARG_ROLE_IMPORT = Arg(("file",), help="Import roles from JSON file", nargs=None) +ARG_ROLE_EXPORT = Arg(("file",), help="Export all roles to JSON file", nargs=None) +ARG_ROLE_EXPORT_FMT = Arg( + ("-p", "--pretty"), + help="Format output JSON file by sorting role names and indenting by 4 spaces", + action="store_true", +) + +# sync-perm +ARG_INCLUDE_DAGS = Arg( + ("--include-dags",), help="If passed, DAG specific permissions will also be synced.", action="store_true" +) + +################ +# # COMMANDS # # +################ + +USERS_COMMANDS = ( + ActionCommand( + name="list", + help="List users", + func=lazy_load_command("airflow.auth.managers.fab.cli_commands.user_command.users_list"), + args=(ARG_OUTPUT, ARG_VERBOSE), + ), + ActionCommand( + name="create", + help="Create a user", + func=lazy_load_command("airflow.auth.managers.fab.cli_commands.user_command.users_create"), + args=( + ARG_ROLE, + ARG_USERNAME, + ARG_EMAIL, + ARG_FIRSTNAME, + ARG_LASTNAME, + ARG_PASSWORD, + ARG_USE_RANDOM_PASSWORD, + ARG_VERBOSE, + ), + epilog=( + "examples:\n" + 'To create an user with "Admin" role and username equals to "admin", run:\n' + "\n" + " $ airflow users create \\\n" + " --username admin \\\n" + " --firstname FIRST_NAME \\\n" + " --lastname LAST_NAME \\\n" + " --role Admin \\\n" + " --email admin@example.org" + ), + ), + ActionCommand( + name="delete", + help="Delete a user", + func=lazy_load_command("airflow.auth.managers.fab.cli_commands.user_command.users_delete"), + args=(ARG_USERNAME_OPTIONAL, ARG_EMAIL_OPTIONAL, ARG_VERBOSE), + ), + ActionCommand( + name="add-role", + help="Add role to a user", + func=lazy_load_command("airflow.auth.managers.fab.cli_commands.user_command.add_role"), + args=(ARG_USERNAME_OPTIONAL, ARG_EMAIL_OPTIONAL, ARG_ROLE, ARG_VERBOSE), + ), + ActionCommand( + name="remove-role", + help="Remove role from a user", + func=lazy_load_command("airflow.auth.managers.fab.cli_commands.user_command.remove_role"), + args=(ARG_USERNAME_OPTIONAL, ARG_EMAIL_OPTIONAL, ARG_ROLE, ARG_VERBOSE), + ), + ActionCommand( + name="import", + help="Import users", + func=lazy_load_command("airflow.auth.managers.fab.cli_commands.user_command.users_import"), + args=(ARG_USER_IMPORT, ARG_VERBOSE), + ), + ActionCommand( + name="export", + help="Export all users", + func=lazy_load_command("airflow.auth.managers.fab.cli_commands.user_command.users_export"), + args=(ARG_USER_EXPORT, ARG_VERBOSE), + ), +) +ROLES_COMMANDS = ( + ActionCommand( + name="list", + help="List roles", + func=lazy_load_command("airflow.auth.managers.fab.cli_commands.role_command.roles_list"), + args=(ARG_PERMISSIONS, ARG_OUTPUT, ARG_VERBOSE), + ), + ActionCommand( + name="create", + help="Create role", + func=lazy_load_command("airflow.auth.managers.fab.cli_commands.role_command.roles_create"), + args=(ARG_ROLES, ARG_VERBOSE), + ), + ActionCommand( + name="delete", + help="Delete role", + func=lazy_load_command("airflow.auth.managers.fab.cli_commands.role_command.roles_delete"), + args=(ARG_ROLES, ARG_VERBOSE), + ), + ActionCommand( + name="add-perms", + help="Add roles permissions", + func=lazy_load_command("airflow.auth.managers.fab.cli_commands.role_command.roles_add_perms"), + args=(ARG_ROLES, ARG_ROLE_RESOURCE, ARG_ROLE_ACTION_REQUIRED, ARG_VERBOSE), + ), + ActionCommand( + name="del-perms", + help="Delete roles permissions", + func=lazy_load_command("airflow.auth.managers.fab.cli_commands.role_command.roles_del_perms"), + args=(ARG_ROLES, ARG_ROLE_RESOURCE, ARG_ROLE_ACTION, ARG_VERBOSE), + ), + ActionCommand( + name="export", + help="Export roles (without permissions) from db to JSON file", + func=lazy_load_command("airflow.auth.managers.fab.cli_commands.role_command.roles_export"), + args=(ARG_ROLE_EXPORT, ARG_ROLE_EXPORT_FMT, ARG_VERBOSE), + ), + ActionCommand( + name="import", + help="Import roles (without permissions) from JSON file to db", + func=lazy_load_command("airflow.auth.managers.fab.cli_commands.role_command.roles_import"), + args=(ARG_ROLE_IMPORT, ARG_VERBOSE), + ), +) + +SYNC_PERM_COMMAND = ActionCommand( + name="sync-perm", + help="Update permissions for existing roles and optionally DAGs", + func=lazy_load_command("airflow.auth.managers.fab.cli_commands.sync_perm_command.sync_perm"), + args=(ARG_INCLUDE_DAGS, ARG_VERBOSE), +) diff --git a/airflow/cli/commands/role_command.py b/airflow/auth/managers/fab/cli_commands/role_command.py similarity index 94% rename from airflow/cli/commands/role_command.py rename to airflow/auth/managers/fab/cli_commands/role_command.py index a582b33195320..34ea8fb9d30e0 100644 --- a/airflow/cli/commands/role_command.py +++ b/airflow/auth/managers/fab/cli_commands/role_command.py @@ -23,6 +23,7 @@ import json import os +from airflow.auth.managers.fab.cli_commands.utils import get_application_builder from airflow.auth.managers.fab.models import Action, Permission, Resource, Role from airflow.cli.simple_table import AirflowConsole from airflow.utils import cli as cli_utils @@ -35,8 +36,6 @@ @providers_configuration_loaded def roles_list(args): """List all existing roles.""" - from airflow.utils.cli_app_builder import get_application_builder - with get_application_builder() as appbuilder: roles = appbuilder.sm.get_all_roles() @@ -63,8 +62,6 @@ def roles_list(args): @providers_configuration_loaded def roles_create(args): """Create new empty role in DB.""" - from airflow.utils.cli_app_builder import get_application_builder - with get_application_builder() as appbuilder: for role_name in args.role: appbuilder.sm.add_role(role_name) @@ -76,8 +73,6 @@ def roles_create(args): @providers_configuration_loaded def roles_delete(args): """Delete role in DB.""" - from airflow.utils.cli_app_builder import get_application_builder - with get_application_builder() as appbuilder: for role_name in args.role: role = appbuilder.sm.find_role(role_name) @@ -90,8 +85,6 @@ def roles_delete(args): def __roles_add_or_remove_permissions(args): - from airflow.utils.cli_app_builder import get_application_builder - with get_application_builder() as appbuilder: is_add: bool = args.subcommand.startswith("add") @@ -165,8 +158,6 @@ def roles_export(args): Note, this function does not export the permissions associated for each role. Strictly, it exports the role names into the passed role json file. """ - from airflow.utils.cli_app_builder import get_application_builder - with get_application_builder() as appbuilder: roles = appbuilder.sm.get_all_roles() exporting_roles = [role.name for role in roles if role.name not in EXISTING_ROLES] @@ -196,7 +187,6 @@ def roles_import(args): except ValueError as e: print(f"File '{json_file}' is not a valid JSON file. Error: {e}") exit(1) - from airflow.utils.cli_app_builder import get_application_builder with get_application_builder() as appbuilder: existing_roles = [role.name for role in appbuilder.sm.get_all_roles()] diff --git a/airflow/cli/commands/sync_perm_command.py b/airflow/auth/managers/fab/cli_commands/sync_perm_command.py similarity index 94% rename from airflow/cli/commands/sync_perm_command.py rename to airflow/auth/managers/fab/cli_commands/sync_perm_command.py index 4d4e280637f9c..14b6e58bbb08a 100644 --- a/airflow/cli/commands/sync_perm_command.py +++ b/airflow/auth/managers/fab/cli_commands/sync_perm_command.py @@ -26,7 +26,7 @@ @providers_configuration_loaded def sync_perm(args): """Update permissions for existing roles and DAGs.""" - from airflow.utils.cli_app_builder import get_application_builder + from airflow.auth.managers.fab.cli_commands.utils import get_application_builder with get_application_builder() as appbuilder: print("Updating actions and resources for all existing roles") diff --git a/airflow/cli/commands/user_command.py b/airflow/auth/managers/fab/cli_commands/user_command.py similarity index 95% rename from airflow/cli/commands/user_command.py rename to airflow/auth/managers/fab/cli_commands/user_command.py index bc982719c94df..84e6318e40537 100644 --- a/airflow/cli/commands/user_command.py +++ b/airflow/auth/managers/fab/cli_commands/user_command.py @@ -29,6 +29,7 @@ from marshmallow import Schema, fields, validate from marshmallow.exceptions import ValidationError +from airflow.auth.managers.fab.cli_commands.utils import get_application_builder from airflow.cli.simple_table import AirflowConsole from airflow.utils import cli as cli_utils from airflow.utils.cli import suppress_logs_and_warning @@ -50,8 +51,6 @@ class UserSchema(Schema): @providers_configuration_loaded def users_list(args): """List users at the command line.""" - from airflow.utils.cli_app_builder import get_application_builder - with get_application_builder() as appbuilder: users = appbuilder.sm.get_all_users() fields = ["id", "username", "email", "first_name", "last_name", "roles"] @@ -65,8 +64,6 @@ def users_list(args): @providers_configuration_loaded def users_create(args): """Create new user in the DB.""" - from airflow.utils.cli_app_builder import get_application_builder - with get_application_builder() as appbuilder: role = appbuilder.sm.find_role(args.role) if not role: @@ -101,8 +98,6 @@ def _find_user(args): if args.username and args.email: raise SystemExit("Conflicting args: must supply either --username or --email, but not both") - from airflow.utils.cli_app_builder import get_application_builder - with get_application_builder() as appbuilder: user = appbuilder.sm.find_user(username=args.username, email=args.email) if not user: @@ -119,8 +114,6 @@ def users_delete(args): # Clear the associated user roles first. user.roles.clear() - from airflow.utils.cli_app_builder import get_application_builder - with get_application_builder() as appbuilder: if appbuilder.sm.del_register_user(user): print(f'User "{user.username}" deleted') @@ -134,8 +127,6 @@ def users_manage_role(args, remove=False): """Delete or appends user roles.""" user = _find_user(args) - from airflow.utils.cli_app_builder import get_application_builder - with get_application_builder() as appbuilder: role = appbuilder.sm.find_role(args.role) if not role: @@ -161,8 +152,6 @@ def users_manage_role(args, remove=False): @providers_configuration_loaded def users_export(args): """Export all users to the json file.""" - from airflow.utils.cli_app_builder import get_application_builder - with get_application_builder() as appbuilder: users = appbuilder.sm.get_all_users() fields = ["id", "username", "email", "first_name", "last_name", "roles"] @@ -211,8 +200,6 @@ def users_import(args): def _import_users(users_list: list[dict[str, Any]]): - from airflow.utils.cli_app_builder import get_application_builder - with get_application_builder() as appbuilder: users_created = [] users_updated = [] diff --git a/airflow/utils/cli_app_builder.py b/airflow/auth/managers/fab/cli_commands/utils.py similarity index 100% rename from airflow/utils/cli_app_builder.py rename to airflow/auth/managers/fab/cli_commands/utils.py diff --git a/airflow/auth/managers/fab/fab_auth_manager.py b/airflow/auth/managers/fab/fab_auth_manager.py index 848f5ff188d6c..def0590b1f5c9 100644 --- a/airflow/auth/managers/fab/fab_auth_manager.py +++ b/airflow/auth/managers/fab/fab_auth_manager.py @@ -17,13 +17,22 @@ # under the License. from __future__ import annotations -from flask import url_for -from flask_login import current_user +from typing import TYPE_CHECKING from airflow import AirflowException from airflow.auth.managers.base_auth_manager import BaseAuthManager -from airflow.auth.managers.fab.models import User -from airflow.auth.managers.fab.security_manager.override import FabAirflowSecurityManagerOverride +from airflow.auth.managers.fab.cli_commands.definition import ( + ROLES_COMMANDS, + SYNC_PERM_COMMAND, + USERS_COMMANDS, +) +from airflow.cli.cli_config import ( + CLICommand, + GroupCommand, +) + +if TYPE_CHECKING: + from airflow.auth.managers.fab.models import User class FabAuthManager(BaseAuthManager): @@ -33,6 +42,23 @@ class FabAuthManager(BaseAuthManager): This auth manager is responsible for providing a backward compatible user management experience to users. """ + @staticmethod + def get_cli_commands() -> list[CLICommand]: + """Vends CLI commands to be included in Airflow CLI.""" + return [ + GroupCommand( + name="users", + help="Manage users", + subcommands=USERS_COMMANDS, + ), + GroupCommand( + name="roles", + help="Manage roles", + subcommands=ROLES_COMMANDS, + ), + SYNC_PERM_COMMAND, # not in a command group + ] + def get_user_name(self) -> str: """ Return the username associated to the user in session. @@ -47,6 +73,8 @@ def get_user_name(self) -> str: def get_user(self) -> User: """Return the user associated to the user in session.""" + from flask_login import current_user + return current_user def get_user_id(self) -> str: @@ -59,25 +87,33 @@ def is_logged_in(self) -> bool: def get_security_manager_override_class(self) -> type: """Return the security manager override.""" + from airflow.auth.managers.fab.security_manager.override import FabAirflowSecurityManagerOverride + return FabAirflowSecurityManagerOverride + def url_for(self, *args, **kwargs): + """Wrapper to allow mocking without having to import at the top of the file.""" + from flask import url_for + + return url_for(*args, **kwargs) + def get_url_login(self, **kwargs) -> str: """Return the login page url.""" if not self.security_manager.auth_view: raise AirflowException("`auth_view` not defined in the security manager.") if "next_url" in kwargs and kwargs["next_url"]: - return url_for(f"{self.security_manager.auth_view.endpoint}.login", next=kwargs["next_url"]) + return self.url_for(f"{self.security_manager.auth_view.endpoint}.login", next=kwargs["next_url"]) else: - return url_for(f"{self.security_manager.auth_view.endpoint}.login") + return self.url_for(f"{self.security_manager.auth_view.endpoint}.login") def get_url_logout(self): """Return the logout page url.""" if not self.security_manager.auth_view: raise AirflowException("`auth_view` not defined in the security manager.") - return url_for(f"{self.security_manager.auth_view.endpoint}.logout") + return self.url_for(f"{self.security_manager.auth_view.endpoint}.logout") def get_url_user_profile(self) -> str | None: """Return the url to a page displaying info about the current user.""" if not self.security_manager.user_view: return None - return url_for(f"{self.security_manager.user_view.endpoint}.userinfo") + return self.url_for(f"{self.security_manager.user_view.endpoint}.userinfo") diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index 2ec55ff79498f..c06bd32d96cbb 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -229,6 +229,12 @@ def string_lower_type(val): ), default=None, ) +ARG_SKIP_SERVE_LOGS = Arg( + ("-s", "--skip-serve-logs"), + default=False, + help="Don't start the serve logs process along with the workers", + action="store_true", +) # list_dag_runs ARG_DAG_ID_REQ_FLAG = Arg( @@ -593,7 +599,7 @@ def string_lower_type(val): ARG_DEPENDS_ON_PAST = Arg( ("-d", "--depends-on-past"), help="Determine how Airflow should deal with past dependencies. The default action is `check`, Airflow " - "will check if the the past dependencies are met for the tasks having `depends_on_past=True` before run " + "will check if the past dependencies are met for the tasks having `depends_on_past=True` before run " "them, if `ignore` is provided, the past dependencies will be ignored, if `wait` is provided and " "`depends_on_past=True`, Airflow will wait the past dependencies until they are met before running or " "skipping the task", @@ -878,76 +884,6 @@ def string_lower_type(val): action="store_true", ) -# users -ARG_USERNAME = Arg(("-u", "--username"), help="Username of the user", required=True, type=str) -ARG_USERNAME_OPTIONAL = Arg(("-u", "--username"), help="Username of the user", type=str) -ARG_FIRSTNAME = Arg(("-f", "--firstname"), help="First name of the user", required=True, type=str) -ARG_LASTNAME = Arg(("-l", "--lastname"), help="Last name of the user", required=True, type=str) -ARG_ROLE = Arg( - ("-r", "--role"), - help="Role of the user. Existing roles include Admin, User, Op, Viewer, and Public", - required=True, - type=str, -) -ARG_EMAIL = Arg(("-e", "--email"), help="Email of the user", required=True, type=str) -ARG_EMAIL_OPTIONAL = Arg(("-e", "--email"), help="Email of the user", type=str) -ARG_PASSWORD = Arg( - ("-p", "--password"), - help="Password of the user, required to create a user without --use-random-password", - type=str, -) -ARG_USE_RANDOM_PASSWORD = Arg( - ("--use-random-password",), - help="Do not prompt for password. Use random string instead." - " Required to create a user without --password ", - default=False, - action="store_true", -) -ARG_USER_IMPORT = Arg( - ("import",), - metavar="FILEPATH", - help="Import users from JSON file. Example format::\n" - + textwrap.indent( - textwrap.dedent( - """ - [ - { - "email": "foo@bar.org", - "firstname": "Jon", - "lastname": "Doe", - "roles": ["Public"], - "username": "jondoe" - } - ]""" - ), - " " * 4, - ), -) -ARG_USER_EXPORT = Arg(("export",), metavar="FILEPATH", help="Export all users to JSON file") - -# roles -ARG_CREATE_ROLE = Arg(("-c", "--create"), help="Create a new role", action="store_true") -ARG_LIST_ROLES = Arg(("-l", "--list"), help="List roles", action="store_true") -ARG_ROLES = Arg(("role",), help="The name of a role", nargs="*") -ARG_PERMISSIONS = Arg(("-p", "--permission"), help="Show role permissions", action="store_true") -ARG_ROLE_RESOURCE = Arg(("-r", "--resource"), help="The name of permissions", nargs="*", required=True) -ARG_ROLE_ACTION = Arg(("-a", "--action"), help="The action of permissions", nargs="*") -ARG_ROLE_ACTION_REQUIRED = Arg(("-a", "--action"), help="The action of permissions", nargs="*", required=True) -ARG_AUTOSCALE = Arg(("-a", "--autoscale"), help="Minimum and Maximum number of worker to autoscale") -ARG_SKIP_SERVE_LOGS = Arg( - ("-s", "--skip-serve-logs"), - default=False, - help="Don't start the serve logs process along with the workers", - action="store_true", -) -ARG_ROLE_IMPORT = Arg(("file",), help="Import roles from JSON file", nargs=None) -ARG_ROLE_EXPORT = Arg(("file",), help="Export all roles to JSON file", nargs=None) -ARG_ROLE_EXPORT_FMT = Arg( - ("-p", "--pretty"), - help="Format output JSON file by sorting role names and indenting by 4 spaces", - action="store_true", -) - # info ARG_ANONYMIZE = Arg( ("--anonymize",), @@ -1024,11 +960,6 @@ def string_lower_type(val): help="If passed, this command will be successful even if multiple matching alive jobs are found.", ) -# sync-perm -ARG_INCLUDE_DAGS = Arg( - ("--include-dags",), help="If passed, DAG specific permissions will also be synced.", action="store_true" -) - # triggerer ARG_CAPACITY = Arg( ("--capacity",), @@ -1839,115 +1770,6 @@ class GroupCommand(NamedTuple): ) -USERS_COMMANDS = ( - ActionCommand( - name="list", - help="List users", - func=lazy_load_command("airflow.cli.commands.user_command.users_list"), - args=(ARG_OUTPUT, ARG_VERBOSE), - ), - ActionCommand( - name="create", - help="Create a user", - func=lazy_load_command("airflow.cli.commands.user_command.users_create"), - args=( - ARG_ROLE, - ARG_USERNAME, - ARG_EMAIL, - ARG_FIRSTNAME, - ARG_LASTNAME, - ARG_PASSWORD, - ARG_USE_RANDOM_PASSWORD, - ARG_VERBOSE, - ), - epilog=( - "examples:\n" - 'To create an user with "Admin" role and username equals to "admin", run:\n' - "\n" - " $ airflow users create \\\n" - " --username admin \\\n" - " --firstname FIRST_NAME \\\n" - " --lastname LAST_NAME \\\n" - " --role Admin \\\n" - " --email admin@example.org" - ), - ), - ActionCommand( - name="delete", - help="Delete a user", - func=lazy_load_command("airflow.cli.commands.user_command.users_delete"), - args=(ARG_USERNAME_OPTIONAL, ARG_EMAIL_OPTIONAL, ARG_VERBOSE), - ), - ActionCommand( - name="add-role", - help="Add role to a user", - func=lazy_load_command("airflow.cli.commands.user_command.add_role"), - args=(ARG_USERNAME_OPTIONAL, ARG_EMAIL_OPTIONAL, ARG_ROLE, ARG_VERBOSE), - ), - ActionCommand( - name="remove-role", - help="Remove role from a user", - func=lazy_load_command("airflow.cli.commands.user_command.remove_role"), - args=(ARG_USERNAME_OPTIONAL, ARG_EMAIL_OPTIONAL, ARG_ROLE, ARG_VERBOSE), - ), - ActionCommand( - name="import", - help="Import users", - func=lazy_load_command("airflow.cli.commands.user_command.users_import"), - args=(ARG_USER_IMPORT, ARG_VERBOSE), - ), - ActionCommand( - name="export", - help="Export all users", - func=lazy_load_command("airflow.cli.commands.user_command.users_export"), - args=(ARG_USER_EXPORT, ARG_VERBOSE), - ), -) -ROLES_COMMANDS = ( - ActionCommand( - name="list", - help="List roles", - func=lazy_load_command("airflow.cli.commands.role_command.roles_list"), - args=(ARG_PERMISSIONS, ARG_OUTPUT, ARG_VERBOSE), - ), - ActionCommand( - name="create", - help="Create role", - func=lazy_load_command("airflow.cli.commands.role_command.roles_create"), - args=(ARG_ROLES, ARG_VERBOSE), - ), - ActionCommand( - name="delete", - help="Delete role", - func=lazy_load_command("airflow.cli.commands.role_command.roles_delete"), - args=(ARG_ROLES, ARG_VERBOSE), - ), - ActionCommand( - name="add-perms", - help="Add roles permissions", - func=lazy_load_command("airflow.cli.commands.role_command.roles_add_perms"), - args=(ARG_ROLES, ARG_ROLE_RESOURCE, ARG_ROLE_ACTION_REQUIRED, ARG_VERBOSE), - ), - ActionCommand( - name="del-perms", - help="Delete roles permissions", - func=lazy_load_command("airflow.cli.commands.role_command.roles_del_perms"), - args=(ARG_ROLES, ARG_ROLE_RESOURCE, ARG_ROLE_ACTION, ARG_VERBOSE), - ), - ActionCommand( - name="export", - help="Export roles (without permissions) from db to JSON file", - func=lazy_load_command("airflow.cli.commands.role_command.roles_export"), - args=(ARG_ROLE_EXPORT, ARG_ROLE_EXPORT_FMT, ARG_VERBOSE), - ), - ActionCommand( - name="import", - help="Import roles (without permissions) from JSON file to db", - func=lazy_load_command("airflow.cli.commands.role_command.roles_import"), - args=(ARG_ROLE_IMPORT, ARG_VERBOSE), - ), -) - CONFIG_COMMANDS = ( ActionCommand( name="get-value", @@ -2171,22 +1993,6 @@ class GroupCommand(NamedTuple): help="Display providers", subcommands=PROVIDERS_COMMANDS, ), - GroupCommand( - name="users", - help="Manage users", - subcommands=USERS_COMMANDS, - ), - GroupCommand( - name="roles", - help="Manage roles", - subcommands=ROLES_COMMANDS, - ), - ActionCommand( - name="sync-perm", - help="Update permissions for existing roles and optionally DAGs", - func=lazy_load_command("airflow.cli.commands.sync_perm_command.sync_perm"), - args=(ARG_INCLUDE_DAGS, ARG_VERBOSE), - ), ActionCommand( name="rotate-fernet-key", func=lazy_load_command("airflow.cli.commands.rotate_fernet_key_command.rotate_fernet_key"), diff --git a/airflow/cli/cli_parser.py b/airflow/cli/cli_parser.py index ee1e60f1b210a..07c7695f2dddb 100644 --- a/airflow/cli/cli_parser.py +++ b/airflow/cli/cli_parser.py @@ -46,6 +46,7 @@ from airflow.exceptions import AirflowException from airflow.executors.executor_loader import ExecutorLoader from airflow.utils.helpers import partition +from airflow.www.extensions.init_auth_manager import get_auth_manager_cls airflow_commands = core_commands.copy() # make a copy to prevent bad interactions in tests @@ -64,6 +65,13 @@ # Do not re-raise the exception since we want the CLI to still function for # other commands. +try: + auth_mgr = get_auth_manager_cls() + airflow_commands.extend(auth_mgr.get_cli_commands()) +except Exception: + log.exception("cannot load CLI commands from auth manager") + # do not re-raise for the same reason as above + ALL_COMMANDS_DICT: dict[str, CLICommand] = {sp.name: sp for sp in airflow_commands} diff --git a/airflow/cli/commands/dag_command.py b/airflow/cli/commands/dag_command.py index ea06619917f8f..4b54ac2f9f675 100644 --- a/airflow/cli/commands/dag_command.py +++ b/airflow/cli/commands/dag_command.py @@ -240,7 +240,7 @@ def dag_dependencies_show(args) -> None: @providers_configuration_loaded def dag_show(args) -> None: - """Display DAG or saves it's graphic representation to the file.""" + """Display DAG or saves its graphic representation to the file.""" dag = get_dag(args.subdir, args.dag_id) dot = render_dag(dag) filename = args.save diff --git a/airflow/cli/commands/db_command.py b/airflow/cli/commands/db_command.py index be7dff62d74a3..449ba9c26c224 100644 --- a/airflow/cli/commands/db_command.py +++ b/airflow/cli/commands/db_command.py @@ -29,7 +29,7 @@ from airflow import settings from airflow.exceptions import AirflowException from airflow.utils import cli as cli_utils, db -from airflow.utils.db import REVISION_HEADS_MAP +from airflow.utils.db import _REVISION_HEADS_MAP from airflow.utils.db_cleanup import config_dict, drop_archived_tables, export_archived_records, run_cleanup from airflow.utils.process_utils import execute_interactive from airflow.utils.providers_configuration_loader import providers_configuration_loaded @@ -65,6 +65,27 @@ def upgradedb(args): migratedb(args) +def get_version_revision(version: str, recursion_limit=10) -> str | None: + """ + Recursively search for the revision of the given version. + + This searches REVISION_HEADS_MAP for the revision of the given version, recursively + searching for the previous version if the given version is not found. + """ + if version in _REVISION_HEADS_MAP: + return _REVISION_HEADS_MAP[version] + try: + major, minor, patch = map(int, version.split(".")) + except ValueError: + return None + new_version = f"{major}.{minor}.{patch - 1}" + recursion_limit -= 1 + if recursion_limit <= 0: + # Prevent infinite recursion as I can't imagine 10 successive versions without migration + return None + return get_version_revision(new_version, recursion_limit) + + @cli_utils.action_cli(check_db=False) @providers_configuration_loaded def migratedb(args): @@ -85,12 +106,12 @@ def migratedb(args): elif args.from_version: if parse_version(args.from_version) < parse_version("2.0.0"): raise SystemExit("--from-version must be greater or equal to than 2.0.0") - from_revision = REVISION_HEADS_MAP.get(args.from_version) + from_revision = get_version_revision(args.from_version) if not from_revision: raise SystemExit(f"Unknown version {args.from_version!r} supplied as `--from-version`.") if args.to_version: - to_revision = REVISION_HEADS_MAP.get(args.to_version) + to_revision = get_version_revision(args.to_version) if not to_revision: raise SystemExit(f"Upgrading to version {args.to_version} is not supported.") elif args.to_revision: @@ -129,11 +150,11 @@ def downgrade(args): if args.from_revision: from_revision = args.from_revision elif args.from_version: - from_revision = REVISION_HEADS_MAP.get(args.from_version) + from_revision = get_version_revision(args.from_version) if not from_revision: raise SystemExit(f"Unknown version {args.from_version!r} supplied as `--from-version`.") if args.to_version: - to_revision = REVISION_HEADS_MAP.get(args.to_version) + to_revision = get_version_revision(args.to_version) if not to_revision: raise SystemExit(f"Downgrading to version {args.to_version} is not supported.") elif args.to_revision: diff --git a/airflow/cli/commands/pool_command.py b/airflow/cli/commands/pool_command.py index b26f2032a85bc..13386d6fbe8b6 100644 --- a/airflow/cli/commands/pool_command.py +++ b/airflow/cli/commands/pool_command.py @@ -97,7 +97,7 @@ def pool_import(args): if not os.path.exists(args.file): raise SystemExit(f"Missing pools file {args.file}") pools, failed = pool_import_helper(args.file) - if len(failed) > 0: + if failed: raise SystemExit(f"Failed to update pool(s): {', '.join(failed)}") print(f"Uploaded {len(pools)} pool(s)") diff --git a/airflow/cli/commands/standalone_command.py b/airflow/cli/commands/standalone_command.py index ceae1c6dcaec6..0beacb71d159d 100644 --- a/airflow/cli/commands/standalone_command.py +++ b/airflow/cli/commands/standalone_command.py @@ -182,7 +182,7 @@ def initialize_database(self): # server. Thus, we make a random password and store it in AIRFLOW_HOME, # with the reasoning that if you can read that directory, you can see # the database credentials anyway. - from airflow.utils.cli_app_builder import get_application_builder + from airflow.auth.managers.fab.cli_commands.utils import get_application_builder with get_application_builder() as appbuilder: user_exists = appbuilder.sm.find_user("admin") diff --git a/airflow/cli/commands/webserver_command.py b/airflow/cli/commands/webserver_command.py index 00ac66372465c..1714e2bd55031 100644 --- a/airflow/cli/commands/webserver_command.py +++ b/airflow/cli/commands/webserver_command.py @@ -138,7 +138,7 @@ def _get_num_ready_workers_running(self) -> int: def ready_prefix_on_cmdline(proc): try: cmdline = proc.cmdline() - if len(cmdline) > 0: + if cmdline: return settings.GUNICORN_WORKER_READY_PREFIX in cmdline[0] except psutil.NoSuchProcess: pass diff --git a/airflow/dag_processing/processor.py b/airflow/dag_processing/processor.py index 64858be94ed9e..e38bae653c540 100644 --- a/airflow/dag_processing/processor.py +++ b/airflow/dag_processing/processor.py @@ -826,7 +826,7 @@ def process_file( Stats.incr("dag_file_refresh_error", 1, 1, tags={"file_path": file_path}) return 0, 0 - if len(dagbag.dags) > 0: + if dagbag.dags: self.log.info("DAG(s) %s retrieved from %s", dagbag.dags.keys(), file_path) else: self.log.warning("No viable dags retrieved from %s", file_path) diff --git a/airflow/example_dags/example_setup_teardown_taskflow.py b/airflow/example_dags/example_setup_teardown_taskflow.py index 4dcdbf253f6ca..b1bdbccf7fadc 100644 --- a/airflow/example_dags/example_setup_teardown_taskflow.py +++ b/airflow/example_dags/example_setup_teardown_taskflow.py @@ -50,7 +50,7 @@ def my_third_task(): # The method `as_teardown` will mark task_3 as teardown, task_1 as setup, and # arrow task_1 >> task_3. - # Now if you clear task_2, then it's setup task, task_1, will be cleared in + # Now if you clear task_2, then its setup task, task_1, will be cleared in # addition to its teardown task, task_3 # it's also possible to use a decorator to mark a task as setup or diff --git a/airflow/jobs/local_task_job_runner.py b/airflow/jobs/local_task_job_runner.py index c142504639328..d04af55dcedc3 100644 --- a/airflow/jobs/local_task_job_runner.py +++ b/airflow/jobs/local_task_job_runner.py @@ -43,7 +43,7 @@ an attempt by a program/library to write or read outside its allocated memory. In Python environment usually this signal refers to libraries which use low level C API. -Make sure that you use use right libraries/Docker Images +Make sure that you use right libraries/Docker Images for your architecture (Intel/ARM) and/or Operational System (Linux/macOS). Suggested way to debug diff --git a/airflow/jobs/scheduler_job_runner.py b/airflow/jobs/scheduler_job_runner.py index f128c7857351e..50f78a48a9680 100644 --- a/airflow/jobs/scheduler_job_runner.py +++ b/airflow/jobs/scheduler_job_runner.py @@ -1752,7 +1752,7 @@ def _cleanup_stale_dags(self, session: Session = NEW_SESSION) -> None: Find all dags that were not updated by Dag Processor recently and mark them as inactive. In case one of DagProcessors is stopped (in case there are multiple of them - for different dag folders), it's dags are never marked as inactive. + for different dag folders), its dags are never marked as inactive. Also remove dags from SerializedDag table. Executed on schedule only if [scheduler]standalone_dag_processor is True. """ diff --git a/airflow/jobs/triggerer_job_runner.py b/airflow/jobs/triggerer_job_runner.py index 81ac8c178f776..298b6cf38843d 100644 --- a/airflow/jobs/triggerer_job_runner.py +++ b/airflow/jobs/triggerer_job_runner.py @@ -508,7 +508,7 @@ async def cancel_triggers(self): """ Drain the to_cancel queue and ensure all triggers that are not in the DB are cancelled. - This allows the the cleanup job to delete them. + This allows the cleanup job to delete them. """ while self.to_cancel: trigger_id = self.to_cancel.popleft() diff --git a/airflow/listeners/listener.py b/airflow/listeners/listener.py index dd5481cba1b34..eb738c3e918bf 100644 --- a/airflow/listeners/listener.py +++ b/airflow/listeners/listener.py @@ -46,7 +46,7 @@ def __init__(self): @property def has_listeners(self) -> bool: - return len(self.pm.get_plugins()) > 0 + return bool(self.pm.get_plugins()) @property def hook(self) -> _HookRelay: diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py index b3167fda06706..0c8c3d3149672 100644 --- a/airflow/models/baseoperator.py +++ b/airflow/models/baseoperator.py @@ -391,7 +391,7 @@ def apply_defaults(self: BaseOperator, *args: Any, **kwargs: Any) -> Any: from airflow.models.dag import DagContext from airflow.utils.task_group import TaskGroupContext - if len(args) > 0: + if args: raise AirflowException("Use keyword arguments when initializing operators") instantiated_from_mapped = kwargs.pop( diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 80b29f8e4ce55..cc2626278f2ee 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -1583,7 +1583,7 @@ def get_task_instances_before( .limit(num) ).all() - if len(execution_dates) == 0: + if not execution_dates: return self.get_task_instances(start_date=base_date, end_date=base_date, session=session) min_date: datetime | None = execution_dates[-1]._mapping.get( @@ -3146,7 +3146,7 @@ def deactivate_unknown_dags(active_dag_ids, session=NEW_SESSION): :param active_dag_ids: list of DAG IDs that are active :return: None """ - if len(active_dag_ids) == 0: + if not active_dag_ids: return for dag in session.scalars(select(DagModel).where(~DagModel.dag_id.in_(active_dag_ids))).all(): dag.is_active = False diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py index 99e517656a129..6b3f561ffc8d3 100644 --- a/airflow/models/dagrun.py +++ b/airflow/models/dagrun.py @@ -1320,7 +1320,7 @@ def schedule_tis( """ Set the given task instances in to the scheduled state. - Each element of ``schedulable_tis`` should have it's ``task`` attribute already set. + Each element of ``schedulable_tis`` should have its ``task`` attribute already set. Any EmptyOperator without callbacks or outlets is instead set straight to the success state. diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index 975e615d95414..0be2f005bf5fd 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -2139,7 +2139,7 @@ def get_prev_ds() -> str | None: execution_date = get_prev_execution_date() if execution_date is None: return None - return execution_date.strftime(r"%Y-%m-%d") + return execution_date.strftime("%Y-%m-%d") def get_prev_ds_nodash() -> str | None: prev_ds = get_prev_ds() diff --git a/airflow/provider_info.schema.json b/airflow/provider_info.schema.json index 3a37810f7a04b..b97bd44fe4daf 100644 --- a/airflow/provider_info.schema.json +++ b/airflow/provider_info.schema.json @@ -25,6 +25,60 @@ "deprecatedVersion": "2.2.0" } }, + "transfers": { + "type": "array", + "items": { + "type": "object", + "properties": { + "how-to-guide": { + "description": "Path to how-to-guide for the transfer. The path must start with '/docs/'", + "type": "string" + }, + "source-integration-name": { + "type": "string", + "description": "Integration name. It must have a matching item in the 'integration' section of any provider." + }, + "target-integration-name": { + "type": "string", + "description": "Target integration name. It must have a matching item in the 'integration' section of any provider." + }, + "python-module": { + "type": "string", + "description": "List of python modules containing the transfers." + } + }, + "additionalProperties": false, + "required": [ + "source-integration-name", + "target-integration-name", + "python-module" + ] + } + }, + "triggers": { + "type": "array", + "items": { + "type": "object", + "properties": { + "integration-name": { + "type": "string", + "description": "Integration name. It must have a matching item in the 'integration' section of any provider." + }, + "python-modules": { + "description": "List of Python modules containing the triggers.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "integration-name", + "python-modules" + ] + } + }, "connection-types": { "type": "array", "description": "Map of connection types mapped to hook class names.", diff --git a/airflow/providers/alibaba/cloud/hooks/analyticdb_spark.py b/airflow/providers/alibaba/cloud/hooks/analyticdb_spark.py index 9881ca38ae4cd..e06ee912281bb 100644 --- a/airflow/providers/alibaba/cloud/hooks/analyticdb_spark.py +++ b/airflow/providers/alibaba/cloud/hooks/analyticdb_spark.py @@ -321,7 +321,7 @@ def _validate_extra_conf(conf: dict[Any, Any]) -> bool: if conf: if not isinstance(conf, dict): raise ValueError("'conf' argument must be a dict") - if not all((v and isinstance(v, str)) or isinstance(v, int) for v in conf.values()): + if not all(isinstance(v, (str, int)) and v != "" for v in conf.values()): raise ValueError("'conf' values must be either strings or ints") return True diff --git a/airflow/providers/amazon/aws/hooks/appflow.py b/airflow/providers/amazon/aws/hooks/appflow.py index 71e7ddd1c7f46..c9741278c00a7 100644 --- a/airflow/providers/amazon/aws/hooks/appflow.py +++ b/airflow/providers/amazon/aws/hooks/appflow.py @@ -20,6 +20,7 @@ from typing import TYPE_CHECKING from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook +from airflow.providers.amazon.aws.utils.waiter_with_logging import wait if TYPE_CHECKING: from mypy_boto3_appflow.client import AppflowClient @@ -49,13 +50,20 @@ def conn(self) -> AppflowClient: """Get the underlying boto3 Appflow client (cached).""" return super().conn - def run_flow(self, flow_name: str, poll_interval: int = 20, wait_for_completion: bool = True) -> str: + def run_flow( + self, + flow_name: str, + poll_interval: int = 20, + wait_for_completion: bool = True, + max_attempts: int = 60, + ) -> str: """ Execute an AppFlow run. :param flow_name: The flow name :param poll_interval: Time (seconds) to wait between two consecutive calls to check the run status :param wait_for_completion: whether to wait for the run to end to return + :param max_attempts: the number of polls to do before timing out/returning a failure. :return: The run execution ID """ response_start = self.conn.start_flow(flowName=flow_name) @@ -63,9 +71,17 @@ def run_flow(self, flow_name: str, poll_interval: int = 20, wait_for_completion: self.log.info("executionId: %s", execution_id) if wait_for_completion: - self.get_waiter("run_complete", {"EXECUTION_ID": execution_id}).wait( - flowName=flow_name, - WaiterConfig={"Delay": poll_interval}, + wait( + waiter=self.get_waiter("run_complete", {"EXECUTION_ID": execution_id}), + waiter_delay=poll_interval, + waiter_max_attempts=max_attempts, + args={"flowName": flow_name}, + failure_message="error while waiting for flow to complete", + status_message="waiting for flow completion, status", + status_args=[ + f"flowExecutions[?executionId=='{execution_id}'].executionStatus", + f"flowExecutions[?executionId=='{execution_id}'].executionResult.errorInfo", + ], ) self._log_execution_description(flow_name, execution_id) diff --git a/airflow/providers/amazon/aws/hooks/quicksight.py b/airflow/providers/amazon/aws/hooks/quicksight.py index 02ea582631a69..586a73b97b750 100644 --- a/airflow/providers/amazon/aws/hooks/quicksight.py +++ b/airflow/providers/amazon/aws/hooks/quicksight.py @@ -152,7 +152,7 @@ def wait_for_state( :param target_state: Describes the QuickSight Job's Target State :param check_interval: the time interval in seconds which the operator will check the status of QuickSight Ingestion - :return: response of describe_ingestion call after Ingestion is is done + :return: response of describe_ingestion call after Ingestion is done """ while True: status = self.get_status(aws_account_id, data_set_id, ingestion_id) diff --git a/airflow/providers/amazon/aws/hooks/sagemaker.py b/airflow/providers/amazon/aws/hooks/sagemaker.py index b8247d9de97bb..364e073b1a8d3 100644 --- a/airflow/providers/amazon/aws/hooks/sagemaker.py +++ b/airflow/providers/amazon/aws/hooks/sagemaker.py @@ -128,10 +128,8 @@ def secondary_training_status_message( status_strs = [] for transition in transitions_to_print: message = transition["StatusMessage"] - time_str = timezone.convert_to_utc(cast(datetime, job_description["LastModifiedTime"])).strftime( - "%Y-%m-%d %H:%M:%S" - ) - status_strs.append(f"{time_str} {transition['Status']} - {message}") + time_utc = timezone.convert_to_utc(cast(datetime, job_description["LastModifiedTime"])) + status_strs.append(f"{time_utc:%Y-%m-%d %H:%M:%S} {transition['Status']} - {message}") return "\n".join(status_strs) diff --git a/airflow/providers/amazon/aws/operators/appflow.py b/airflow/providers/amazon/aws/operators/appflow.py index f2fe75f39501e..7b7402acde887 100644 --- a/airflow/providers/amazon/aws/operators/appflow.py +++ b/airflow/providers/amazon/aws/operators/appflow.py @@ -16,12 +16,13 @@ # under the License. from __future__ import annotations +import warnings from datetime import datetime, timedelta from functools import cached_property from time import sleep from typing import TYPE_CHECKING, cast -from airflow.exceptions import AirflowException +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import BaseOperator from airflow.operators.python import ShortCircuitOperator from airflow.providers.amazon.aws.hooks.appflow import AppflowHook @@ -51,6 +52,7 @@ class AppflowBaseOperator(BaseOperator): :param source_field: The field name to apply filters :param filter_date: The date value (or template) to be used in filters. :param poll_interval: how often in seconds to check the query status + :param max_attempts: how many times to check for status before timing out :param aws_conn_id: aws connection to use :param region: aws region to use :param wait_for_completion: whether to wait for the run to end to return @@ -58,29 +60,33 @@ class AppflowBaseOperator(BaseOperator): ui_color = "#2bccbd" + template_fields = ("flow_name", "source", "source_field", "filter_date") + UPDATE_PROPAGATION_TIME: int = 15 def __init__( self, - source: str, flow_name: str, flow_update: bool, + source: str | None = None, source_field: str | None = None, filter_date: str | None = None, poll_interval: int = 20, + max_attempts: int = 60, aws_conn_id: str = "aws_default", region: str | None = None, wait_for_completion: bool = True, **kwargs, ) -> None: super().__init__(**kwargs) - if source not in SUPPORTED_SOURCES: + if source is not None and source not in SUPPORTED_SOURCES: raise ValueError(f"{source} is not a supported source (options: {SUPPORTED_SOURCES})!") self.filter_date = filter_date self.flow_name = flow_name self.source = source self.source_field = source_field self.poll_interval = poll_interval + self.max_attempts = max_attempts self.aws_conn_id = aws_conn_id self.region = region self.flow_update = flow_update @@ -95,7 +101,8 @@ def execute(self, context: Context) -> None: self.filter_date_parsed: datetime | None = ( datetime.fromisoformat(self.filter_date) if self.filter_date else None ) - self.connector_type = self._get_connector_type() + if self.source is not None: + self.connector_type = self._get_connector_type() if self.flow_update: self._update_flow() # while schedule flows will pick up the update right away, on-demand flows might use out of date @@ -118,6 +125,7 @@ def _run_flow(self, context) -> str: execution_id = self.hook.run_flow( flow_name=self.flow_name, poll_interval=self.poll_interval, + max_attempts=self.max_attempts, wait_for_completion=self.wait_for_completion, ) task_instance = context["task_instance"] @@ -127,13 +135,13 @@ def _run_flow(self, context) -> str: class AppflowRunOperator(AppflowBaseOperator): """ - Execute a Appflow run with filters as is. + Execute a Appflow run as is. .. seealso:: For more information on how to use this operator, take a look at the guide: :ref:`howto/operator:AppflowRunOperator` - :param source: The source name (Supported: salesforce, zendesk) + :param source: Obsolete, unnecessary for this operator :param flow_name: The flow name :param poll_interval: how often in seconds to check the query status :param aws_conn_id: aws connection to use @@ -143,18 +151,21 @@ class AppflowRunOperator(AppflowBaseOperator): def __init__( self, - source: str, flow_name: str, + source: str | None = None, poll_interval: int = 20, aws_conn_id: str = "aws_default", region: str | None = None, wait_for_completion: bool = True, **kwargs, ) -> None: - if source not in {"salesforce", "zendesk"}: - raise ValueError(NOT_SUPPORTED_SOURCE_MSG.format(source=source, entity="AppflowRunOperator")) + if source is not None: + warnings.warn( + "The `source` parameter is unused when simply running a flow, please remove it.", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) super().__init__( - source=source, flow_name=flow_name, flow_update=False, source_field=None, @@ -227,8 +238,6 @@ class AppflowRunBeforeOperator(AppflowBaseOperator): :param wait_for_completion: whether to wait for the run to end to return """ - template_fields = ("filter_date",) - def __init__( self, source: str, @@ -297,8 +306,6 @@ class AppflowRunAfterOperator(AppflowBaseOperator): :param wait_for_completion: whether to wait for the run to end to return """ - template_fields = ("filter_date",) - def __init__( self, source: str, @@ -365,8 +372,6 @@ class AppflowRunDailyOperator(AppflowBaseOperator): :param wait_for_completion: whether to wait for the run to end to return """ - template_fields = ("filter_date",) - def __init__( self, source: str, diff --git a/airflow/providers/amazon/aws/operators/eks.py b/airflow/providers/amazon/aws/operators/eks.py index 557bf29f5feed..98fbc66da43b4 100644 --- a/airflow/providers/amazon/aws/operators/eks.py +++ b/airflow/providers/amazon/aws/operators/eks.py @@ -987,7 +987,7 @@ class EksPodOperator(KubernetesPodOperator): empty, then the default boto3 configuration would be used (and must be maintained on each worker node). :param on_finish_action: What to do when the pod reaches its final state, or the execution is interrupted. - If "delete_pod", the pod will be deleted regardless it's state; if "delete_succeeded_pod", + If "delete_pod", the pod will be deleted regardless its state; if "delete_succeeded_pod", only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod. Current default is `keep_pod`, but this will be changed in the next major release of this provider. :param is_delete_operator_pod: What to do when the pod reaches its final diff --git a/airflow/providers/amazon/aws/triggers/README.md b/airflow/providers/amazon/aws/triggers/README.md index cd0c0baae5d53..bd998a5a294ff 100644 --- a/airflow/providers/amazon/aws/triggers/README.md +++ b/airflow/providers/amazon/aws/triggers/README.md @@ -38,7 +38,7 @@ The first step to making an existing operator deferrable is to add `deferrable` The next step is to determine where the operator should be deferred. This will be dependent on what the operator does, and how it is written. Although every operator is different, there are a few guidelines to determine the best place to defer an operator. 1. If the operator has a `wait_for_completion` parameter, the `self.defer` method should be called right before the check for wait_for_completion . -2. If there is no `wait_for_completion` , look for the "main" task that the operator does. Often, operators will make various describe calls to to the boto3 API to verify certain conditions, or look up some information before performing its "main" task. Often, right after the "main" call to the boto3 API is made is a good place to call `self.defer`. +2. If there is no `wait_for_completion` , look for the "main" task that the operator does. Often, operators will make various describe calls to the boto3 API to verify certain conditions, or look up some information before performing its "main" task. Often, right after the "main" call to the boto3 API is made is a good place to call `self.defer`. Once the location to defer is decided in the operator, call the `self.defer` method if the `deferrable` flag is `True`. The `self.defer` method takes in several parameters, listed below: diff --git a/airflow/providers/amazon/aws/waiters/appflow.json b/airflow/providers/amazon/aws/waiters/appflow.json index f45c427467901..ed239a4048320 100644 --- a/airflow/providers/amazon/aws/waiters/appflow.json +++ b/airflow/providers/amazon/aws/waiters/appflow.json @@ -10,13 +10,13 @@ "expected": "Successful", "matcher": "path", "state": "success", - "argument": "flowExecutions[?executionId=='{{EXECUTION_ID}}'].executionStatus" + "argument": "flowExecutions[?executionId=='{{EXECUTION_ID}}'].executionStatus | [0]" }, { "expected": "Error", "matcher": "path", "state": "failure", - "argument": "flowExecutions[?executionId=='{{EXECUTION_ID}}'].executionStatus" + "argument": "flowExecutions[?executionId=='{{EXECUTION_ID}}'].executionStatus | [0]" }, { "expected": true, diff --git a/airflow/providers/amazon/aws/waiters/batch.json b/airflow/providers/amazon/aws/waiters/batch.json index 3fbdd433771c8..a8cd489ea3809 100644 --- a/airflow/providers/amazon/aws/waiters/batch.json +++ b/airflow/providers/amazon/aws/waiters/batch.json @@ -17,7 +17,7 @@ "argument": "jobs[].status", "expected": "FAILED", "matcher": "pathAll", - "state": "failed" + "state": "failure" } ] }, @@ -37,13 +37,13 @@ "argument": "computeEnvironments[].status", "expected": "INVALID", "matcher": "pathAny", - "state": "failed" + "state": "failure" }, { "argument": "computeEnvironments[].status", "expected": "DELETED", "matcher": "pathAny", - "state": "failed" + "state": "failure" } ] } diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 6aa32825105bf..2959764e00237 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -67,7 +67,19 @@ dependencies: - apache-airflow>=2.4.0 - apache-airflow-providers-common-sql>=1.3.1 - apache-airflow-providers-http - - boto3>=1.24.0 + # We should update minimum version of boto3 and here regularly to avoid `pip` backtracking with the number + # of candidates to consider. We should also make sure that all the below related packages have also the + # same minimum version specified. Boto3 1.28.0 has been released on July 6 2023. We should also make sure we + # set it to the version that `aiobotocore` supports (see `aiobotocore` optional dependency at the end + # of this file). Currently we set aiobotocore as minimum 2.5.3 - as this is was the first version + # that supported boto3 1.28. NOTE!!! BOTOCORE VERSIONS ARE SHIFTED BY 3 MINOR VERSIONS + - boto3>=1.28.0 + - mypy-boto3-rds>=1.28.0 + - mypy-boto3-redshift-data>=1.28.0 + - mypy-boto3-s3>=1.28.0 + - mypy-boto3-appflow>=1.28.0 + # NOTE!!! BOTOCORE VERSIONS ARE SHIFTED BY 3 MINOR VERSIONS + - botocore>=1.31.0 - asgiref # watchtower 3 has been released end Jan and introduced breaking change across the board that might # change logging behaviour: @@ -77,11 +89,7 @@ dependencies: - jsonpath_ng>=1.5.3 - redshift_connector>=2.0.888 - sqlalchemy_redshift>=0.8.6 - - mypy-boto3-rds>=1.24.0 - - mypy-boto3-redshift-data>=1.24.0 - - mypy-boto3-appflow>=1.28.16 - asgiref - - mypy-boto3-s3>=1.24.0 integrations: - integration-name: Amazon Athena @@ -699,7 +707,7 @@ additional-extras: # boto3 have native async support and we move away from aio aiobotocore - name: aiobotocore dependencies: - - aiobotocore[boto3]>=2.2.0 + - aiobotocore[boto3]>=2.5.3 - name: cncf.kubernetes dependencies: - apache-airflow-providers-cncf-kubernetes>=7.2.0 diff --git a/airflow/providers/apache/beam/hooks/beam.py b/airflow/providers/apache/beam/hooks/beam.py index 762dd2f07b676..72dc224626bba 100644 --- a/airflow/providers/apache/beam/hooks/beam.py +++ b/airflow/providers/apache/beam/hooks/beam.py @@ -104,10 +104,8 @@ def process_fd( fd_to_log = {proc.stderr: log.warning, proc.stdout: log.info} func_log = fd_to_log[fd] - while True: - line = fd.readline().decode() - if not line: - return + for line in iter(fd.readline, b""): + line = line.decode() if process_line_callback: process_line_callback(line) func_log(line.rstrip("\n")) diff --git a/airflow/providers/apache/beam/triggers/beam.py b/airflow/providers/apache/beam/triggers/beam.py index 9c29a9fbe2728..0d201cd8c9a4d 100644 --- a/airflow/providers/apache/beam/triggers/beam.py +++ b/airflow/providers/apache/beam/triggers/beam.py @@ -85,32 +85,29 @@ def serialize(self) -> tuple[str, dict[str, Any]]: async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override] """Get current pipeline status and yields a TriggerEvent.""" hook = self._get_async_hook() - while True: - try: - return_code = await hook.start_python_pipeline_async( - variables=self.variables, - py_file=self.py_file, - py_options=self.py_options, - py_interpreter=self.py_interpreter, - py_requirements=self.py_requirements, - py_system_site_packages=self.py_system_site_packages, + try: + return_code = await hook.start_python_pipeline_async( + variables=self.variables, + py_file=self.py_file, + py_options=self.py_options, + py_interpreter=self.py_interpreter, + py_requirements=self.py_requirements, + py_system_site_packages=self.py_system_site_packages, + ) + except Exception as e: + self.log.exception("Exception occurred while checking for pipeline state") + yield TriggerEvent({"status": "error", "message": str(e)}) + else: + if return_code == 0: + yield TriggerEvent( + { + "status": "success", + "message": "Pipeline has finished SUCCESSFULLY", + } ) - if return_code == 0: - yield TriggerEvent( - { - "status": "success", - "message": "Pipeline has finished SUCCESSFULLY", - } - ) - return - else: - yield TriggerEvent({"status": "error", "message": "Operation failed"}) - return - - except Exception as e: - self.log.exception("Exception occurred while checking for pipeline state") - yield TriggerEvent({"status": "error", "message": str(e)}) - return + else: + yield TriggerEvent({"status": "error", "message": "Operation failed"}) + return def _get_async_hook(self) -> BeamAsyncHook: return BeamAsyncHook(runner=self.runner) diff --git a/airflow/providers/apache/hdfs/CHANGELOG.rst b/airflow/providers/apache/hdfs/CHANGELOG.rst index 37de3a05cc49f..8a17a7f4a6924 100644 --- a/airflow/providers/apache/hdfs/CHANGELOG.rst +++ b/airflow/providers/apache/hdfs/CHANGELOG.rst @@ -75,7 +75,7 @@ you can use 3.* version of the provider, but the recommendation is to switch to Protobuf 3 required by the snakebite-py3 library has ended its life in June 2023 and Airflow and it's providers stopped supporting it. If you would like to continue using HDFS hooks and sensors based on snakebite-py3 library when you have protobuf library 4.+ you can install the 3.* version - of the provider but due to Protobuf incompatibility, you need to do one of the the two things: + of the provider but due to Protobuf incompatibility, you need to do one of the two things: * set ``PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python`` variable in your environment. * downgrade protobuf to latest 3.* version (3.20.3 at this time) diff --git a/airflow/providers/apache/hive/hooks/hive.py b/airflow/providers/apache/hive/hooks/hive.py index ea004860b4944..7f02619024fdd 100644 --- a/airflow/providers/apache/hive/hooks/hive.py +++ b/airflow/providers/apache/hive/hooks/hive.py @@ -277,13 +277,11 @@ def run_cli( ) self.sub_process = sub_process stdout = "" - while True: - line = sub_process.stdout.readline() - if not line: - break - stdout += line.decode("UTF-8") + for line in iter(sub_process.stdout.readline, b""): + line = line.decode() + stdout += line if verbose: - self.log.info(line.decode("UTF-8").strip()) + self.log.info(line.strip()) sub_process.wait() if sub_process.returncode: @@ -704,25 +702,20 @@ def _get_max_partition_from_part_specs( # Assuming all specs have the same keys. if partition_key not in part_specs[0].keys(): raise AirflowException(f"Provided partition_key {partition_key} is not in part_specs.") - is_subset = None - if filter_map: - is_subset = set(filter_map.keys()).issubset(set(part_specs[0].keys())) - if filter_map and not is_subset: + if filter_map and not set(filter_map).issubset(part_specs[0]): raise AirflowException( f"Keys in provided filter_map {', '.join(filter_map.keys())} " f"are not subset of part_spec keys: {', '.join(part_specs[0].keys())}" ) - candidates = [ - p_dict[partition_key] - for p_dict in part_specs - if filter_map is None or all(item in p_dict.items() for item in filter_map.items()) - ] - - if not candidates: - return None - else: - return max(candidates) + return max( + ( + p_dict[partition_key] + for p_dict in part_specs + if filter_map is None or all(item in p_dict.items() for item in filter_map.items()) + ), + default=None, + ) def max_partition( self, diff --git a/airflow/providers/apache/kafka/triggers/await_message.py b/airflow/providers/apache/kafka/triggers/await_message.py index f2c1bb81ca6b6..8f89d2ba1f180 100644 --- a/airflow/providers/apache/kafka/triggers/await_message.py +++ b/airflow/providers/apache/kafka/triggers/await_message.py @@ -49,7 +49,7 @@ class AwaitMessageTrigger(BaseTrigger): defaults to None :param poll_timeout: How long the Kafka client should wait before returning from a poll request to Kafka (seconds), defaults to 1 - :param poll_interval: How long the the trigger should sleep after reaching the end of the Kafka log + :param poll_interval: How long the trigger should sleep after reaching the end of the Kafka log (seconds), defaults to 5 """ diff --git a/airflow/providers/apache/livy/hooks/livy.py b/airflow/providers/apache/livy/hooks/livy.py index ede3d2eb985e5..ba2ff1bb1318f 100644 --- a/airflow/providers/apache/livy/hooks/livy.py +++ b/airflow/providers/apache/livy/hooks/livy.py @@ -432,7 +432,7 @@ def _validate_list_of_stringables(vals: Sequence[str | int | float]) -> bool: if ( vals is None or not isinstance(vals, (tuple, list)) - or any(1 for val in vals if not isinstance(val, (str, int, float))) + or not all(isinstance(val, (str, int, float)) for val in vals) ): raise ValueError("List of strings expected") return True @@ -448,7 +448,7 @@ def _validate_extra_conf(conf: dict[Any, Any]) -> bool: if conf: if not isinstance(conf, dict): raise ValueError("'conf' argument must be a dict") - if not all((v and isinstance(v, str)) or isinstance(v, int) for v in conf.values()): + if not all(isinstance(v, (str, int)) and v != "" for v in conf.values()): raise ValueError("'conf' values must be either strings or ints") return True @@ -542,8 +542,7 @@ async def _do_api_call_async( else: return {"Response": f"Unexpected HTTP Method: {self.method}", "status": "error"} - attempt_num = 1 - while True: + for attempt_num in range(1, 1 + self.retry_limit): response = await request_func( url, json=data if self.method in ("POST", "PATCH") else None, @@ -568,7 +567,6 @@ async def _do_api_call_async( # Don't retry. return {"Response": {e.message}, "Status Code": {e.status}, "status": "error"} - attempt_num += 1 await asyncio.sleep(self.retry_delay) def _generate_base_url(self, conn: Connection) -> str: @@ -815,7 +813,7 @@ def _validate_list_of_stringables(vals: Sequence[str | int | float]) -> bool: if ( vals is None or not isinstance(vals, (tuple, list)) - or any(1 for val in vals if not isinstance(val, (str, int, float))) + or not all(isinstance(val, (str, int, float)) for val in vals) ): raise ValueError("List of strings expected") return True @@ -831,6 +829,6 @@ def _validate_extra_conf(conf: dict[Any, Any]) -> bool: if conf: if not isinstance(conf, dict): raise ValueError("'conf' argument must be a dict") - if not all((v and isinstance(v, str)) or isinstance(v, int) for v in conf.values()): + if not all(isinstance(v, (str, int)) and v != "" for v in conf.values()): raise ValueError("'conf' values must be either strings or ints") return True diff --git a/airflow/providers/apache/pinot/hooks/pinot.py b/airflow/providers/apache/pinot/hooks/pinot.py index 0246a5d3a0fee..0fe11a04e4b82 100644 --- a/airflow/providers/apache/pinot/hooks/pinot.py +++ b/airflow/providers/apache/pinot/hooks/pinot.py @@ -56,6 +56,11 @@ class PinotAdminHook(BaseHook): "Exception" is in the output message. """ + conn_name_attr = "conn_id" + default_conn_name = "pinot_admin_default" + conn_type = "pinot_admin" + hook_name = "Pinot Admin" + def __init__( self, conn_id: str = "pinot_admin_default", @@ -258,6 +263,8 @@ class PinotDbApiHook(DbApiHook): conn_name_attr = "pinot_broker_conn_id" default_conn_name = "pinot_broker_default" + conn_type = "pinot" + hook_name = "Pinot Broker" supports_autocommit = False def get_conn(self) -> Any: diff --git a/airflow/providers/apache/pinot/provider.yaml b/airflow/providers/apache/pinot/provider.yaml index 51146a1e6aa69..3c10b9c05f818 100644 --- a/airflow/providers/apache/pinot/provider.yaml +++ b/airflow/providers/apache/pinot/provider.yaml @@ -59,5 +59,7 @@ hooks: - airflow.providers.apache.pinot.hooks.pinot connection-types: - - hook-class-name: airflow.providers.apache.pinot.hooks.pinot.PinotHook + - hook-class-name: airflow.providers.apache.pinot.hooks.pinot.PinotDbApiHook connection-type: pinot + - hook-class-name: airflow.providers.apache.pinot.hooks.pinot.PinotAdminHook + connection-type: pinot_admin diff --git a/airflow/providers/apache/spark/hooks/spark_sql.py b/airflow/providers/apache/spark/hooks/spark_sql.py index 6864aa52fe0c8..41dc741ccdd39 100644 --- a/airflow/providers/apache/spark/hooks/spark_sql.py +++ b/airflow/providers/apache/spark/hooks/spark_sql.py @@ -134,7 +134,7 @@ def _prepare_command(self, cmd: str | list[str]) -> list[str]: connection_cmd += ["--num-executors", str(self._num_executors)] if self._sql: sql = self._sql.strip() - if sql.endswith(".sql") or sql.endswith(".hql"): + if sql.endswith((".sql", ".hql")): connection_cmd += ["-f", sql] else: connection_cmd += ["-e", sql] diff --git a/airflow/providers/celery/executors/celery_executor.py b/airflow/providers/celery/executors/celery_executor.py index 8bdff2a25e727..cc1b6e8122744 100644 --- a/airflow/providers/celery/executors/celery_executor.py +++ b/airflow/providers/celery/executors/celery_executor.py @@ -37,7 +37,6 @@ try: from airflow.cli.cli_config import ( - ARG_AUTOSCALE, ARG_DAEMON, ARG_LOG_FILE, ARG_PID, @@ -143,6 +142,7 @@ def __getattr__(name): ) # worker cli args +ARG_AUTOSCALE = Arg(("-a", "--autoscale"), help="Minimum and Maximum number of worker to autoscale") ARG_QUEUES = Arg( ("-q", "--queues"), help="Comma delimited list of queues to serve", diff --git a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py index 8e95f7a6c9387..5bc39625da97c 100644 --- a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py +++ b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py @@ -37,7 +37,7 @@ from sqlalchemy import select, update from sqlalchemy.orm import Session -from airflow import AirflowException +from airflow.exceptions import AirflowException try: from airflow.cli.cli_config import ( diff --git a/airflow/providers/cncf/kubernetes/operators/pod.py b/airflow/providers/cncf/kubernetes/operators/pod.py index 7a1cd975e6931..e32e8b4c1f2f7 100644 --- a/airflow/providers/cncf/kubernetes/operators/pod.py +++ b/airflow/providers/cncf/kubernetes/operators/pod.py @@ -22,6 +22,7 @@ import logging import re import secrets +import shlex import string import warnings from collections.abc import Container @@ -29,7 +30,8 @@ from functools import cached_property from typing import TYPE_CHECKING, Any, Iterable, Sequence -from kubernetes.client import CoreV1Api, models as k8s +from kubernetes.client import CoreV1Api, V1Pod, models as k8s +from kubernetes.stream import stream from slugify import slugify from urllib3.exceptions import HTTPError @@ -59,6 +61,7 @@ PodManager, PodOperatorHookProtocol, PodPhase, + container_is_succeeded, get_container_termination_message, ) from airflow.settings import pod_mutation_hook @@ -173,7 +176,7 @@ class KubernetesPodOperator(BaseOperator): They can be exposed as environment vars or files in a volume. :param in_cluster: run kubernetes client with in_cluster configuration. :param cluster_context: context that points to kubernetes cluster. - Ignored when in_cluster is True. If None, current-context is used. + Ignored when in_cluster is True. If None, current-context is used. (templated) :param reattach_on_restart: if the worker dies while the pod is running, reattach and monitor during the next try. If False, always create a new pod for each try. :param labels: labels to apply to the Pod. (templated) @@ -232,7 +235,7 @@ class KubernetesPodOperator(BaseOperator): :param poll_interval: Polling period in seconds to check for the status. Used only in deferrable mode. :param log_pod_spec_on_failure: Log the pod's specification if a failure occurs :param on_finish_action: What to do when the pod reaches its final state, or the execution is interrupted. - If "delete_pod", the pod will be deleted regardless it's state; if "delete_succeeded_pod", + If "delete_pod", the pod will be deleted regardless its state; if "delete_succeeded_pod", only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod. :param is_delete_operator_pod: What to do when the pod reaches its final state, or the execution is interrupted. If True (default), delete the @@ -246,7 +249,7 @@ class KubernetesPodOperator(BaseOperator): # This field can be overloaded at the instance level via base_container_name BASE_CONTAINER_NAME = "base" - + ISTIO_CONTAINER_NAME = "istio-proxy" POD_CHECKED_KEY = "already_checked" POST_TERMINATION_TIMEOUT = 120 @@ -262,6 +265,7 @@ class KubernetesPodOperator(BaseOperator): "container_resources", "volumes", "volume_mounts", + "cluster_context", ) template_fields_renderers = {"env_vars": "py"} @@ -604,7 +608,10 @@ def execute_sync(self, context: Context): if self.do_xcom_push: self.pod_manager.await_xcom_sidecar_container_start(pod=self.pod) result = self.extract_xcom(pod=self.pod) - self.remote_pod = self.pod_manager.await_pod_completion(self.pod) + istio_enabled = self.is_istio_enabled(self.pod) + self.remote_pod = self.pod_manager.await_pod_completion( + self.pod, istio_enabled, self.base_container_name + ) finally: self.cleanup( pod=self.pod or self.pod_request_obj, @@ -667,7 +674,8 @@ def execute_complete(self, context: Context, event: dict, **kwargs): xcom_sidecar_output = self.extract_xcom(pod=pod) return xcom_sidecar_output finally: - pod = self.pod_manager.await_pod_completion(pod) + istio_enabled = self.is_istio_enabled(pod) + pod = self.pod_manager.await_pod_completion(pod, istio_enabled, self.base_container_name) if pod is not None: self.post_complete_action( pod=pod, @@ -699,13 +707,16 @@ def post_complete_action(self, *, pod, remote_pod, **kwargs): ) def cleanup(self, pod: k8s.V1Pod, remote_pod: k8s.V1Pod): + istio_enabled = self.is_istio_enabled(remote_pod) pod_phase = remote_pod.status.phase if hasattr(remote_pod, "status") else None # if the pod fails or success, but we don't want to delete it if pod_phase != PodPhase.SUCCEEDED or self.on_finish_action == OnFinishAction.KEEP_POD: self.patch_already_checked(remote_pod, reraise=False) - if pod_phase != PodPhase.SUCCEEDED: + if (pod_phase != PodPhase.SUCCEEDED and not istio_enabled) or ( + istio_enabled and not container_is_succeeded(remote_pod, self.base_container_name) + ): if self.log_events_on_failure: self._read_pod_events(pod, reraise=False) @@ -752,16 +763,61 @@ def _read_pod_events(self, pod, *, reraise=True): for event in self.pod_manager.read_pod_events(pod).items: self.log.error("Pod Event: %s - %s", event.reason, event.message) + def is_istio_enabled(self, pod: V1Pod) -> bool: + """Checks if istio is enabled for the namespace of the pod by inspecting the namespace labels.""" + if not pod: + return False + + remote_pod = self.pod_manager.read_pod(pod) + + for container in remote_pod.spec.containers: + if container.name == self.ISTIO_CONTAINER_NAME: + return True + + return False + + def kill_istio_sidecar(self, pod: V1Pod) -> None: + command = "/bin/sh -c curl -fsI -X POST http://localhost:15020/quitquitquit && exit 0" + command_to_container = shlex.split(command) + try: + resp = stream( + self.client.connect_get_namespaced_pod_exec, + name=pod.metadata.name, + namespace=pod.metadata.namespace, + container=self.ISTIO_CONTAINER_NAME, + command=command_to_container, + stderr=True, + stdin=True, + stdout=True, + tty=False, + _preload_content=False, + ) + resp.close() + except Exception as e: + self.log.error("Error while deleting istio-proxy sidecar: %s", e) + raise e + def process_pod_deletion(self, pod: k8s.V1Pod, *, reraise=True): + istio_enabled = self.is_istio_enabled(pod) with _optionally_suppress(reraise=reraise): if pod is not None: - should_delete_pod = (self.on_finish_action == OnFinishAction.DELETE_POD) or ( - self.on_finish_action == OnFinishAction.DELETE_SUCCEEDED_POD - and pod.status.phase == PodPhase.SUCCEEDED + should_delete_pod = ( + (self.on_finish_action == OnFinishAction.DELETE_POD) + or ( + self.on_finish_action == OnFinishAction.DELETE_SUCCEEDED_POD + and pod.status.phase == PodPhase.SUCCEEDED + ) + or ( + self.on_finish_action == OnFinishAction.DELETE_SUCCEEDED_POD + and container_is_succeeded(pod, self.base_container_name) + ) ) - if should_delete_pod: + if should_delete_pod and not istio_enabled: self.log.info("Deleting pod: %s", pod.metadata.name) self.pod_manager.delete_pod(pod) + elif should_delete_pod and istio_enabled: + self.log.info("Deleting istio-proxy sidecar inside %s: ", pod.metadata.name) + self.kill_istio_sidecar(pod) else: self.log.info("Skipping deleting pod: %s", pod.metadata.name) diff --git a/airflow/providers/cncf/kubernetes/provider.yaml b/airflow/providers/cncf/kubernetes/provider.yaml index 11a661e63ecae..491deb6e99dd8 100644 --- a/airflow/providers/cncf/kubernetes/provider.yaml +++ b/airflow/providers/cncf/kubernetes/provider.yaml @@ -100,7 +100,6 @@ integrations: operators: - integration-name: Kubernetes python-modules: - - airflow.providers.cncf.kubernetes.operators.kubernetes_pod - airflow.providers.cncf.kubernetes.operators.pod - airflow.providers.cncf.kubernetes.operators.spark_kubernetes - airflow.providers.cncf.kubernetes.operators.resource diff --git a/airflow/providers/cncf/kubernetes/triggers/pod.py b/airflow/providers/cncf/kubernetes/triggers/pod.py index 7b7a9afe59455..f3f18a660ab97 100644 --- a/airflow/providers/cncf/kubernetes/triggers/pod.py +++ b/airflow/providers/cncf/kubernetes/triggers/pod.py @@ -61,7 +61,7 @@ class KubernetesPodTrigger(BaseTrigger): :param get_logs: get the stdout of the container as logs of the tasks. :param startup_timeout: timeout in seconds to start up the pod. :param on_finish_action: What to do when the pod reaches its final state, or the execution is interrupted. - If "delete_pod", the pod will be deleted regardless it's state; if "delete_succeeded_pod", + If "delete_pod", the pod will be deleted regardless its state; if "delete_succeeded_pod", only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod. :param should_delete_pod: What to do when the pod reaches its final state, or the execution is interrupted. If True (default), delete the diff --git a/airflow/providers/cncf/kubernetes/utils/pod_manager.py b/airflow/providers/cncf/kubernetes/utils/pod_manager.py index 1c2e0ab59719d..77600b4fcf195 100644 --- a/airflow/providers/cncf/kubernetes/utils/pod_manager.py +++ b/airflow/providers/cncf/kubernetes/utils/pod_manager.py @@ -145,6 +145,21 @@ def container_is_completed(pod: V1Pod, container_name: str) -> bool: return container_status.state.terminated is not None +def container_is_succeeded(pod: V1Pod, container_name: str) -> bool: + """ + Examines V1Pod ``pod`` to determine whether ``container_name`` is completed and succeeded. + + If that container is present and completed and succeeded, returns True. Returns False otherwise. + """ + if not container_is_completed(pod, container_name): + return False + + container_status = get_container_status(pod, container_name) + if not container_status: + return False + return container_status.state.terminated.exit_code == 0 + + def container_is_terminated(pod: V1Pod, container_name: str) -> bool: """ Examines V1Pod ``pod`` to determine whether ``container_name`` is terminated. @@ -508,17 +523,23 @@ def await_container_completion(self, pod: V1Pod, container_name: str) -> None: self.log.info("Waiting for container '%s' state to be completed", container_name) time.sleep(1) - def await_pod_completion(self, pod: V1Pod) -> V1Pod: + def await_pod_completion( + self, pod: V1Pod, istio_enabled: bool = False, container_name: str = "base" + ) -> V1Pod: """ Monitors a pod and returns the final state. + :param istio_enabled: whether istio is enabled in the namespace :param pod: pod spec that will be monitored + :param container_name: name of the container within the pod :return: tuple[State, str | None] """ while True: remote_pod = self.read_pod(pod) if remote_pod.status.phase in PodPhase.terminal_states: break + if istio_enabled and container_is_completed(remote_pod, container_name): + break self.log.info("Pod %s has phase %s", pod.metadata.name, remote_pod.status.phase) time.sleep(2) return remote_pod diff --git a/airflow/providers/common/sql/hooks/sql.pyi b/airflow/providers/common/sql/hooks/sql.pyi index 30d8eef488d9b..dedac037dfcb2 100644 --- a/airflow/providers/common/sql/hooks/sql.pyi +++ b/airflow/providers/common/sql/hooks/sql.pyi @@ -18,7 +18,7 @@ # This is automatically generated stub for the `common.sql` provider # # This file is generated automatically by the `update-common-sql-api stubs` pre-commit -# and the .pyi file represents part of the the "public" API that the +# and the .pyi file represents part of the "public" API that the # `common.sql` provider exposes to other providers. # # Any, potentially breaking change in the stubs will require deliberate manual action from the contributor diff --git a/airflow/providers/common/sql/operators/sql.py b/airflow/providers/common/sql/operators/sql.py index cf42fa4f57363..979f815c5bbc0 100644 --- a/airflow/providers/common/sql/operators/sql.py +++ b/airflow/providers/common/sql/operators/sql.py @@ -837,14 +837,13 @@ def execute(self, context: Context): pass_value_conv = _convert_to_float_if_possible(self.pass_value) is_numeric_value_check = isinstance(pass_value_conv, float) - tolerance_pct_str = str(self.tol * 100) + "%" if self.tol is not None else None error_msg = ( "Test failed.\nPass value:{pass_value_conv}\n" "Tolerance:{tolerance_pct_str}\n" "Query:\n{sql}\nResults:\n{records!s}" ).format( pass_value_conv=pass_value_conv, - tolerance_pct_str=tolerance_pct_str, + tolerance_pct_str=f"{self.tol:.1%}" if self.tol is not None else None, sql=self.sql, records=records, ) diff --git a/airflow/providers/common/sql/operators/sql.pyi b/airflow/providers/common/sql/operators/sql.pyi index f2735b4a252ab..d9d099f948941 100644 --- a/airflow/providers/common/sql/operators/sql.pyi +++ b/airflow/providers/common/sql/operators/sql.pyi @@ -18,7 +18,7 @@ # This is automatically generated stub for the `common.sql` provider # # This file is generated automatically by the `update-common-sql-api stubs` pre-commit -# and the .pyi file represents part of the the "public" API that the +# and the .pyi file represents part of the "public" API that the # `common.sql` provider exposes to other providers. # # Any, potentially breaking change in the stubs will require deliberate manual action from the contributor diff --git a/airflow/providers/common/sql/sensors/sql.pyi b/airflow/providers/common/sql/sensors/sql.pyi index ed88c4b8109fe..0343d04871e50 100644 --- a/airflow/providers/common/sql/sensors/sql.pyi +++ b/airflow/providers/common/sql/sensors/sql.pyi @@ -18,7 +18,7 @@ # This is automatically generated stub for the `common.sql` provider # # This file is generated automatically by the `update-common-sql-api stubs` pre-commit -# and the .pyi file represents part of the the "public" API that the +# and the .pyi file represents part of the "public" API that the # `common.sql` provider exposes to other providers. # # Any, potentially breaking change in the stubs will require deliberate manual action from the contributor diff --git a/airflow/providers/elasticsearch/log/es_json_formatter.py b/airflow/providers/elasticsearch/log/es_json_formatter.py index 7ac543c0a3a0d..cf77896a9218e 100644 --- a/airflow/providers/elasticsearch/log/es_json_formatter.py +++ b/airflow/providers/elasticsearch/log/es_json_formatter.py @@ -31,11 +31,7 @@ class ElasticsearchJSONFormatter(JSONFormatter): def formatTime(self, record, datefmt=None): """Return the creation time of the LogRecord in ISO 8601 date/time format in the local time zone.""" dt = pendulum.from_timestamp(record.created, tz=pendulum.local_timezone()) - if datefmt: - s = dt.strftime(datefmt) - else: - s = dt.strftime(self.default_time_format) - + s = dt.strftime(datefmt or self.default_time_format) if self.default_msec_format: s = self.default_msec_format % (s, record.msecs) if self.default_tz_format: diff --git a/airflow/providers/google/cloud/hooks/bigquery.py b/airflow/providers/google/cloud/hooks/bigquery.py index 96340c0d50002..9f4be0d4f8f87 100644 --- a/airflow/providers/google/cloud/hooks/bigquery.py +++ b/airflow/providers/google/cloud/hooks/bigquery.py @@ -3182,7 +3182,6 @@ def value_check( raise AirflowException("The query returned None") pass_value_conv = self._convert_to_float_if_possible(pass_value) is_numeric_value_check = isinstance(pass_value_conv, float) - tolerance_pct_str = str(tolerance * 100) + "%" if tolerance else None error_msg = ( "Test failed.\nPass value:{pass_value_conv}\n" @@ -3190,7 +3189,7 @@ def value_check( "Query:\n{sql}\nResults:\n{records!s}" ).format( pass_value_conv=pass_value_conv, - tolerance_pct_str=tolerance_pct_str, + tolerance_pct_str=f"{tolerance:.1%}" if tolerance else None, sql=sql, records=records, ) diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py index 09e76ae2069fe..eaa65ba7c15db 100644 --- a/airflow/providers/google/cloud/operators/dataproc.py +++ b/airflow/providers/google/cloud/operators/dataproc.py @@ -1578,8 +1578,7 @@ class DataprocSubmitPySparkJobOperator(DataprocJobBaseOperator): @staticmethod def _generate_temp_filename(filename): - date = time.strftime("%Y%m%d%H%M%S") - return f"{date}_{str(uuid.uuid4())[:8]}_{ntpath.basename(filename)}" + return f"{time:%Y%m%d%H%M%S}_{str(uuid.uuid4())[:8]}_{ntpath.basename(filename)}" def _upload_file_temp(self, bucket, local_file): """Upload a local file to a Google Cloud Storage bucket.""" diff --git a/airflow/providers/google/cloud/operators/kubernetes_engine.py b/airflow/providers/google/cloud/operators/kubernetes_engine.py index 3f730d5bda343..642c4bf279555 100644 --- a/airflow/providers/google/cloud/operators/kubernetes_engine.py +++ b/airflow/providers/google/cloud/operators/kubernetes_engine.py @@ -429,7 +429,7 @@ class GKEStartPodOperator(KubernetesPodOperator): :param regional: The location param is region name. :param deferrable: Run operator in the deferrable mode. :param on_finish_action: What to do when the pod reaches its final state, or the execution is interrupted. - If "delete_pod", the pod will be deleted regardless it's state; if "delete_succeeded_pod", + If "delete_pod", the pod will be deleted regardless its state; if "delete_succeeded_pod", only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod. Current default is `keep_pod`, but this will be changed in the next major release of this provider. :param is_delete_operator_pod: What to do when the pod reaches its final diff --git a/airflow/providers/google/cloud/operators/pubsub.py b/airflow/providers/google/cloud/operators/pubsub.py index 8e0f7a12d99a3..f91e9ea8c18dd 100644 --- a/airflow/providers/google/cloud/operators/pubsub.py +++ b/airflow/providers/google/cloud/operators/pubsub.py @@ -709,7 +709,7 @@ class PubSubPullOperator(GoogleCloudBaseOperator): :param gcp_conn_id: The connection ID to use connecting to Google Cloud. :param messages_callback: (Optional) Callback to process received messages. - It's return value will be saved to XCom. + Its return value will be saved to XCom. If you are pulling large messages, you probably want to provide a custom callback. If not provided, the default implementation will convert `ReceivedMessage` objects into JSON-serializable dicts using `google.protobuf.json_format.MessageToDict` function. diff --git a/airflow/providers/google/cloud/sensors/pubsub.py b/airflow/providers/google/cloud/sensors/pubsub.py index c818f7168e761..7bd07a08e5059 100644 --- a/airflow/providers/google/cloud/sensors/pubsub.py +++ b/airflow/providers/google/cloud/sensors/pubsub.py @@ -73,7 +73,7 @@ class PubSubPullSensor(BaseSensorOperator): :param gcp_conn_id: The connection ID to use connecting to Google Cloud. :param messages_callback: (Optional) Callback to process received messages. - It's return value will be saved to XCom. + Its return value will be saved to XCom. If you are pulling large messages, you probably want to provide a custom callback. If not provided, the default implementation will convert `ReceivedMessage` objects into JSON-serializable dicts using `google.protobuf.json_format.MessageToDict` function. diff --git a/airflow/providers/google/cloud/triggers/bigquery_dts.py b/airflow/providers/google/cloud/triggers/bigquery_dts.py index def8b90b66a30..3a5ab2267f97a 100644 --- a/airflow/providers/google/cloud/triggers/bigquery_dts.py +++ b/airflow/providers/google/cloud/triggers/bigquery_dts.py @@ -95,7 +95,7 @@ async def run(self) -> AsyncIterator[TriggerEvent]: self.log.info("Current state is %s", state) if state == TransferState.SUCCEEDED: - self.log.info("Job has completed it's work.") + self.log.info("Job has completed its work.") yield TriggerEvent( { "status": "success", diff --git a/airflow/providers/google/cloud/triggers/kubernetes_engine.py b/airflow/providers/google/cloud/triggers/kubernetes_engine.py index ba0df0fc153c2..1e0780fbc66c9 100644 --- a/airflow/providers/google/cloud/triggers/kubernetes_engine.py +++ b/airflow/providers/google/cloud/triggers/kubernetes_engine.py @@ -55,7 +55,7 @@ class GKEStartPodTrigger(KubernetesPodTrigger): will consult the class variable BASE_CONTAINER_NAME (which defaults to "base") for the base container name to use. :param on_finish_action: What to do when the pod reaches its final state, or the execution is interrupted. - If "delete_pod", the pod will be deleted regardless it's state; if "delete_succeeded_pod", + If "delete_pod", the pod will be deleted regardless its state; if "delete_succeeded_pod", only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod. :param should_delete_pod: What to do when the pod reaches its final state, or the execution is interrupted. If True (default), delete the diff --git a/airflow/providers/google/cloud/triggers/pubsub.py b/airflow/providers/google/cloud/triggers/pubsub.py index 40c43f7cb79dd..27dbd8285cac6 100644 --- a/airflow/providers/google/cloud/triggers/pubsub.py +++ b/airflow/providers/google/cloud/triggers/pubsub.py @@ -41,7 +41,7 @@ class PubsubPullTrigger(BaseTrigger): immediately rather than by any downstream tasks :param gcp_conn_id: Reference to google cloud connection id :param messages_callback: (Optional) Callback to process received messages. - It's return value will be saved to XCom. + Its return value will be saved to XCom. If you are pulling large messages, you probably want to provide a custom callback. If not provided, the default implementation will convert `ReceivedMessage` objects into JSON-serializable dicts using `google.protobuf.json_format.MessageToDict` function. diff --git a/airflow/providers/hashicorp/hooks/vault.py b/airflow/providers/hashicorp/hooks/vault.py index 31867c938397f..0fe8e21031129 100644 --- a/airflow/providers/hashicorp/hooks/vault.py +++ b/airflow/providers/hashicorp/hooks/vault.py @@ -49,7 +49,7 @@ class VaultHook(BaseHook): The mount point should be placed as a path in the URL - similarly to Vault's URL schema: This indicates the "path" the secret engine is mounted on. Default id not specified is "secret". Note that this ``mount_point`` is not used for authentication if authentication is done via a - different engines. Each engine uses it's own engine-specific authentication mount_point. + different engines. Each engine uses its own engine-specific authentication mount_point. The extras in the connection are named the same as the parameters ('kv_engine_version', 'auth_type', ...). diff --git a/airflow/providers/microsoft/azure/CHANGELOG.rst b/airflow/providers/microsoft/azure/CHANGELOG.rst index c5dac31b9d86f..deec92ca60bc8 100644 --- a/airflow/providers/microsoft/azure/CHANGELOG.rst +++ b/airflow/providers/microsoft/azure/CHANGELOG.rst @@ -718,11 +718,11 @@ Breaking changes This change removes ``azure_container_instance_default`` connection type and replaces it with the ``azure_default``. The problem was that AzureContainerInstance was not needed as it was exactly the -same as the plain "azure" connection, however it's presence caused duplication in the field names +same as the plain "azure" connection, however its presence caused duplication in the field names used in the UI editor for connections and unnecessary warnings generated. This version uses plain Azure Hook and connection also for Azure Container Instance. If you already have ``azure_container_instance_default`` connection created in your DB, it will continue to work, but -the first time you edit it with the UI you will have to change it's type to ``azure_default``. +the first time you edit it with the UI you will have to change its type to ``azure_default``. Features ~~~~~~~~ diff --git a/airflow/providers/microsoft/azure/hooks/adx.py b/airflow/providers/microsoft/azure/hooks/adx.py index 705db4b5b39ae..83f955753dbf8 100644 --- a/airflow/providers/microsoft/azure/hooks/adx.py +++ b/airflow/providers/microsoft/azure/hooks/adx.py @@ -28,8 +28,8 @@ import warnings from typing import Any +from azure.kusto.data import ClientRequestProperties, KustoClient, KustoConnectionStringBuilder from azure.kusto.data.exceptions import KustoServiceError -from azure.kusto.data.request import ClientRequestProperties, KustoClient, KustoConnectionStringBuilder from azure.kusto.data.response import KustoResponseDataSetV2 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning diff --git a/airflow/providers/microsoft/azure/hooks/data_lake.py b/airflow/providers/microsoft/azure/hooks/data_lake.py index ef84ec51c2681..95ef4c6cc2789 100644 --- a/airflow/providers/microsoft/azure/hooks/data_lake.py +++ b/airflow/providers/microsoft/azure/hooks/data_lake.py @@ -241,7 +241,7 @@ class AzureDataLakeStorageV2Hook(BaseHook): accounts that have a hierarchical namespace. Using Adls_v2 connection details create DataLakeServiceClient object. - Due to Wasb is marked as legacy and and retirement of the (ADLS1), it would + Due to Wasb is marked as legacy and retirement of the (ADLS1), it would be nice to implement ADLS gen2 hook for interacting with the storage account. .. seealso:: diff --git a/airflow/providers/microsoft/azure/operators/data_factory.py b/airflow/providers/microsoft/azure/operators/data_factory.py index 1823212473f6b..d6b4592e35b6e 100644 --- a/airflow/providers/microsoft/azure/operators/data_factory.py +++ b/airflow/providers/microsoft/azure/operators/data_factory.py @@ -92,7 +92,7 @@ class AzureDataFactoryRunPipelineOperator(BaseOperator): ``AzureDataFactoryHook`` will attempt to use the resource group name provided in the corresponding connection. :param factory_name: The data factory name. If a value is not passed in to the operator, the - ``AzureDataFactoryHook`` will attempt to use the factory name name provided in the corresponding + ``AzureDataFactoryHook`` will attempt to use the factory name provided in the corresponding connection. :param reference_pipeline_run_id: The pipeline run identifier. If this run ID is specified the parameters of the specified run will be used to create a new run. diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index 24045abcc955c..9719b03d8900f 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -78,11 +78,11 @@ dependencies: - azure-synapse-spark - adal>=1.2.7 - azure-storage-file-datalake>=12.9.1 + - azure-kusto-data>=4.1.0 # TODO: upgrade to newer versions of all the below libraries. # See issue https://github.com/apache/airflow/issues/30199 - azure-mgmt-containerinstance>=1.5.0,<2.0 - azure-mgmt-datafactory>=1.0.0,<2.0 - - azure-kusto-data>=0.0.43,<0.1 integrations: - integration-name: Microsoft Azure Batch diff --git a/airflow/providers/oracle/hooks/oracle.py b/airflow/providers/oracle/hooks/oracle.py index 225a8ca6ccd60..f207df72d9af5 100644 --- a/airflow/providers/oracle/hooks/oracle.py +++ b/airflow/providers/oracle/hooks/oracle.py @@ -302,9 +302,7 @@ def insert_rows( elif numpy and isinstance(cell, numpy.datetime64): lst.append("'" + str(cell) + "'") elif isinstance(cell, datetime): - lst.append( - "to_date('" + cell.strftime("%Y-%m-%d %H:%M:%S") + "','YYYY-MM-DD HH24:MI:SS')" - ) + lst.append(f"to_date('{cell:%Y-%m-%d %H:%M:%S}','YYYY-MM-DD HH24:MI:SS')") else: lst.append(str(cell)) values = tuple(lst) diff --git a/airflow/providers/slack/provider.yaml b/airflow/providers/slack/provider.yaml index b6195dc04a8d4..9f46d284d40cb 100644 --- a/airflow/providers/slack/provider.yaml +++ b/airflow/providers/slack/provider.yaml @@ -82,4 +82,4 @@ connection-types: connection-type: slackwebhook notifications: - - airflow.providers.slack.notifications.slack_notifier.SlackNotifier + - airflow.providers.slack.notifications.slack.SlackNotifier diff --git a/airflow/providers/snowflake/provider.yaml b/airflow/providers/snowflake/provider.yaml index 824551981c0a4..f688ca0e894c5 100644 --- a/airflow/providers/snowflake/provider.yaml +++ b/airflow/providers/snowflake/provider.yaml @@ -23,6 +23,7 @@ description: | suspended: false versions: + - 5.0.0 - 4.4.2 - 4.4.1 - 4.4.0 diff --git a/airflow/providers/yandex/operators/yandexcloud_dataproc.py b/airflow/providers/yandex/operators/yandexcloud_dataproc.py index 625827d109963..dfd3a07fe4f94 100644 --- a/airflow/providers/yandex/operators/yandexcloud_dataproc.py +++ b/airflow/providers/yandex/operators/yandexcloud_dataproc.py @@ -95,6 +95,8 @@ class DataprocCreateClusterOperator(BaseOperator): Docs: https://cloud.yandex.com/docs/data-proc/concepts/logs :param initialization_actions: Set of init-actions to run when cluster starts. Docs: https://cloud.yandex.com/docs/data-proc/concepts/init-action + :param labels: Cluster labels as key:value pairs. No more than 64 per resource. + Docs: https://cloud.yandex.ru/docs/resource-manager/concepts/labels """ def __init__( @@ -135,6 +137,7 @@ def __init__( security_group_ids: Iterable[str] | None = None, log_group_id: str | None = None, initialization_actions: Iterable[InitializationAction] | None = None, + labels: dict[str, str] | None = None, **kwargs, ) -> None: super().__init__(**kwargs) @@ -173,6 +176,7 @@ def __init__( self.security_group_ids = security_group_ids self.log_group_id = log_group_id self.initialization_actions = initialization_actions + self.labels = labels self.hook: DataprocHook | None = None @@ -214,6 +218,7 @@ def execute(self, context: Context) -> dict: host_group_ids=self.host_group_ids, security_group_ids=self.security_group_ids, log_group_id=self.log_group_id, + labels=self.labels, initialization_actions=self.initialization_actions and [ self.hook.sdk.wrappers.InitializationAction( diff --git a/airflow/providers/yandex/provider.yaml b/airflow/providers/yandex/provider.yaml index ea7a4fb4a3cba..76a3459e6435e 100644 --- a/airflow/providers/yandex/provider.yaml +++ b/airflow/providers/yandex/provider.yaml @@ -20,7 +20,7 @@ package-name: apache-airflow-providers-yandex name: Yandex description: | Yandex including `Yandex.Cloud `__ -suspended: true # see https://github.com/apache/airflow/pull/30667/ +suspended: false versions: - 3.3.0 - 3.2.0 @@ -37,7 +37,7 @@ versions: dependencies: - apache-airflow>=2.4.0 - - yandexcloud>=0.173.0 + - yandexcloud>=0.228.0 integrations: - integration-name: Yandex.Cloud diff --git a/airflow/sensors/base.py b/airflow/sensors/base.py index 3f8b6bf2e6bce..77094269a1951 100644 --- a/airflow/sensors/base.py +++ b/airflow/sensors/base.py @@ -339,12 +339,6 @@ def reschedule(self): def get_serialized_fields(cls): return super().get_serialized_fields() | {"reschedule"} - def raise_failed_or_skiping_exception(self, *, failed_message: str, skipping_message: str = "") -> None: - """Raise AirflowSkipException if self.soft_fail is set to True. Otherwise raise AirflowException.""" - if self.soft_fail: - raise AirflowSkipException(skipping_message or failed_message) - raise AirflowException(failed_message) - def poke_mode_only(cls): """ diff --git a/airflow/sensors/external_task.py b/airflow/sensors/external_task.py index ffc2cc1313f0a..92562589b212b 100644 --- a/airflow/sensors/external_task.py +++ b/airflow/sensors/external_task.py @@ -222,8 +222,6 @@ def __init__( self.deferrable = deferrable self.poll_interval = poll_interval - self._skipping_message_postfix = " Skipping due to soft_fail." - def _get_dttm_filter(self, context): if self.execution_delta: dttm = context["logical_date"] - self.execution_delta @@ -276,28 +274,32 @@ def poke(self, context: Context, session: Session = NEW_SESSION) -> bool: # Fail if anything in the list has failed. if count_failed > 0: if self.external_task_ids: - failed_message = ( + if self.soft_fail: + raise AirflowSkipException( + f"Some of the external tasks {self.external_task_ids} " + f"in DAG {self.external_dag_id} failed. Skipping due to soft_fail." + ) + raise AirflowException( f"Some of the external tasks {self.external_task_ids} " f"in DAG {self.external_dag_id} failed." ) - - self.raise_failed_or_skiping_exception( - failed_message=failed_message, - skipping_message=f"{failed_message}{self._skipping_message_postfix}", - ) elif self.external_task_group_id: - self.raise_failed_or_skiping_exception( - failed_message=( + if self.soft_fail: + raise AirflowSkipException( f"The external task_group '{self.external_task_group_id}' " - f"in DAG '{self.external_dag_id}' failed." + f"in DAG '{self.external_dag_id}' failed. Skipping due to soft_fail." ) + raise AirflowException( + f"The external task_group '{self.external_task_group_id}' " + f"in DAG '{self.external_dag_id}' failed." ) + else: - failed_message = f"The external DAG {self.external_dag_id} failed." - self.raise_failed_or_skiping_exception( - failed_message=failed_message, - skipping_message=f"{failed_message}{self._skipping_message_postfix}", - ) + if self.soft_fail: + raise AirflowSkipException( + f"The external DAG {self.external_dag_id} failed. Skipping due to soft_fail." + ) + raise AirflowException(f"The external DAG {self.external_dag_id} failed.") count_skipped = -1 if self.skipped_states: @@ -349,20 +351,12 @@ def execute_complete(self, context, event=None): self.log.info("External task %s has executed successfully.", self.external_task_id) return None elif event["status"] == "timeout": - failed_message = "Dag was not started within 1 minute, assuming fail." - self.raise_failed_or_skiping_exception( - failed_message=failed_message, - skipping_message=f"{failed_message}{self._skipping_message_postfix}", - ) + raise AirflowException("Dag was not started within 1 minute, assuming fail.") else: - failed_message = ( + raise AirflowException( "Error occurred while trying to retrieve task status. Please, check the " "name of executed task and Dag." ) - self.raise_failed_or_skiping_exception( - failed_message=failed_message, - skipping_message=f"{failed_message}{self._skipping_message_postfix}", - ) def _check_for_existence(self, session) -> None: dag_to_wait = DagModel.get_current(self.external_dag_id, session) diff --git a/airflow/sensors/filesystem.py b/airflow/sensors/filesystem.py index dea1643d9ae21..3eba10380f740 100644 --- a/airflow/sensors/filesystem.py +++ b/airflow/sensors/filesystem.py @@ -70,6 +70,6 @@ def poke(self, context: Context): return True for _, _, files in os.walk(path): - if len(files) > 0: + if files: return True return False diff --git a/airflow/sentry.py b/airflow/sentry.py index 34133228ca13f..da46f924786df 100644 --- a/airflow/sentry.py +++ b/airflow/sentry.py @@ -87,7 +87,7 @@ def __init__(self): # LoggingIntegration is set by default. integrations = [sentry_flask] - executor_class, _ = ExecutorLoader.import_default_executor_cls() + executor_class, _ = ExecutorLoader.import_default_executor_cls(validate=False) if executor_class.supports_sentry: from sentry_sdk.integrations.celery import CeleryIntegration diff --git a/airflow/task/task_runner/cgroup_task_runner.py b/airflow/task/task_runner/cgroup_task_runner.py index 2ab011471377a..14354453bc62a 100644 --- a/airflow/task/task_runner/cgroup_task_runner.py +++ b/airflow/task/task_runner/cgroup_task_runner.py @@ -134,7 +134,7 @@ def start(self): return # Create a unique cgroup name - cgroup_name = f"airflow/{datetime.datetime.utcnow().strftime('%Y-%m-%d')}/{str(uuid.uuid4())}" + cgroup_name = f"airflow/{datetime.datetime.utcnow():%Y-%m-%d}/{uuid.uuid4()}" self.mem_cgroup_name = f"memory/{cgroup_name}" self.cpu_cgroup_name = f"cpu/{cgroup_name}" diff --git a/airflow/ti_deps/deps/mapped_task_expanded.py b/airflow/ti_deps/deps/mapped_task_expanded.py index 87a804006be45..8138de9f9e476 100644 --- a/airflow/ti_deps/deps/mapped_task_expanded.py +++ b/airflow/ti_deps/deps/mapped_task_expanded.py @@ -21,7 +21,7 @@ class MappedTaskIsExpanded(BaseTIDep): - """Checks that a mapped task has been expanded before it's TaskInstance can run.""" + """Checks that a mapped task has been expanded before its TaskInstance can run.""" NAME = "Task has been mapped" IGNORABLE = False diff --git a/airflow/ti_deps/deps/not_previously_skipped_dep.py b/airflow/ti_deps/deps/not_previously_skipped_dep.py index 855f04af533d1..92dd2b373acdb 100644 --- a/airflow/ti_deps/deps/not_previously_skipped_dep.py +++ b/airflow/ti_deps/deps/not_previously_skipped_dep.py @@ -84,7 +84,7 @@ def _get_dep_statuses(self, ti, session, dep_context): ) if not past_depends_met: yield self._failing_status( - reason=("Task should be skipped but the the past depends are not met") + reason=("Task should be skipped but the past depends are not met") ) return ti.set_state(TaskInstanceState.SKIPPED, session) diff --git a/airflow/ti_deps/deps/trigger_rule_dep.py b/airflow/ti_deps/deps/trigger_rule_dep.py index c7e2982fffdd7..dbdf692e769ec 100644 --- a/airflow/ti_deps/deps/trigger_rule_dep.py +++ b/airflow/ti_deps/deps/trigger_rule_dep.py @@ -294,7 +294,7 @@ def _iter_upstream_conditions() -> Iterator[ColumnOperators]: ) if not past_depends_met: yield self._failing_status( - reason=("Task should be skipped but the the past depends are not met") + reason=("Task should be skipped but the past depends are not met") ) return changed = ti.set_state(new_state, session) diff --git a/airflow/triggers/file.py b/airflow/triggers/file.py index 12b2d0e8272e5..85a5a373baecf 100644 --- a/airflow/triggers/file.py +++ b/airflow/triggers/file.py @@ -68,6 +68,6 @@ async def run(self) -> typing.AsyncIterator[TriggerEvent]: self.log.info("Found File %s last modified: %s", str(path), str(mod_time)) yield TriggerEvent(True) for _, _, files in os.walk(self.filepath): - if len(files) > 0: + if files: yield TriggerEvent(True) await asyncio.sleep(self.poll_interval) diff --git a/airflow/utils/db.py b/airflow/utils/db.py index 9d82114c562b3..4429cb8dfe893 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -67,39 +67,27 @@ log = logging.getLogger(__name__) -REVISION_HEADS_MAP = { +_REVISION_HEADS_MAP = { "2.0.0": "e959f08ac86c", "2.0.1": "82b7c48c147f", "2.0.2": "2e42bb497a22", "2.1.0": "a13f7613ad25", - "2.1.1": "a13f7613ad25", - "2.1.2": "a13f7613ad25", "2.1.3": "97cdd93827b8", "2.1.4": "ccde3e26fe78", "2.2.0": "7b2661a43ba3", - "2.2.1": "7b2661a43ba3", - "2.2.2": "7b2661a43ba3", "2.2.3": "be2bfac3da23", "2.2.4": "587bdf053233", - "2.2.5": "587bdf053233", "2.3.0": "b1b348e02d07", "2.3.1": "1de7bc13c950", "2.3.2": "3c94c427fdf6", "2.3.3": "f5fcbda3e651", - "2.3.4": "f5fcbda3e651", "2.4.0": "ecb43d2a1842", - "2.4.1": "ecb43d2a1842", "2.4.2": "b0d31815b5a6", "2.4.3": "e07f49787c9d", "2.5.0": "290244fb8b83", - "2.5.1": "290244fb8b83", - "2.5.2": "290244fb8b83", - "2.5.3": "290244fb8b83", "2.6.0": "98ae134e6fff", - "2.6.1": "98ae134e6fff", "2.6.2": "c804e5c76e3e", - "2.6.3": "c804e5c76e3e", - "2.7.0": "788397e78828", + "2.7.0": "405de8318b3a", } diff --git a/airflow/utils/db_cleanup.py b/airflow/utils/db_cleanup.py index 79a804a9e5011..4c82f90ab3814 100644 --- a/airflow/utils/db_cleanup.py +++ b/airflow/utils/db_cleanup.py @@ -194,6 +194,7 @@ def _do_delete(*, query, orm_model, skip_archive, session): session.execute(delete) session.commit() if skip_archive: + metadata.bind = session.get_bind() target_table.drop() session.commit() print("Finished Performing Delete") diff --git a/airflow/utils/log/logging_mixin.py b/airflow/utils/log/logging_mixin.py index 59c1d7980c1e3..a3d3b4b00c71d 100644 --- a/airflow/utils/log/logging_mixin.py +++ b/airflow/utils/log/logging_mixin.py @@ -38,7 +38,7 @@ class SetContextPropagate(enum.Enum): :meta private: """ - # If a `set_context` function wants to _keep_ propagation set on it's logger it needs to return this + # If a `set_context` function wants to _keep_ propagation set on its logger it needs to return this # special value. MAINTAIN_PROPAGATE = object() # Don't use this one anymore! diff --git a/airflow/utils/log/timezone_aware.py b/airflow/utils/log/timezone_aware.py index 999ccda5a722a..ae96a11116a1f 100644 --- a/airflow/utils/log/timezone_aware.py +++ b/airflow/utils/log/timezone_aware.py @@ -40,11 +40,7 @@ def formatTime(self, record, datefmt=None): date and time format in the local time zone. """ dt = pendulum.from_timestamp(record.created, tz=pendulum.local_timezone()) - if datefmt: - s = dt.strftime(datefmt) - else: - s = dt.strftime(self.default_time_format) - + s = dt.strftime(datefmt or self.default_time_format) if self.default_msec_format: s = self.default_msec_format % (s, record.msecs) if self.default_tz_format: diff --git a/airflow/utils/process_utils.py b/airflow/utils/process_utils.py index f3104df918cdf..1f7c4771e8c02 100644 --- a/airflow/utils/process_utils.py +++ b/airflow/utils/process_utils.py @@ -94,7 +94,7 @@ def signal_procs(sig): + [str(p.pid) for p in all_processes_in_the_group] ) elif err_killpg.errno == errno.ESRCH: - # There is a rare condition that the process has not managed yet to change it's process + # There is a rare condition that the process has not managed yet to change its process # group. In this case os.killpg fails with ESRCH error # So we additionally send a kill signal to the process itself. logger.info( @@ -119,7 +119,7 @@ def signal_procs(sig): all_processes_in_the_group = parent.children(recursive=True) all_processes_in_the_group.append(parent) except psutil.NoSuchProcess: - # The process already exited, but maybe it's children haven't. + # The process already exited, but maybe its children haven't. all_processes_in_the_group = [] for proc in psutil.process_iter(): try: diff --git a/airflow/utils/task_group.py b/airflow/utils/task_group.py index 167eb53b71eaa..841abc710678a 100644 --- a/airflow/utils/task_group.py +++ b/airflow/utils/task_group.py @@ -482,7 +482,7 @@ def topological_sort(self, _include_subdag_tasks: bool = False): while graph_unsorted: # Go through each of the node/edges pairs in the unsorted graph. If a set of edges doesn't contain # any nodes that haven't been resolved, that is, that are still in the unsorted graph, remove the - # pair from the unsorted graph, and append it to the sorted graph. Note here that by using using + # pair from the unsorted graph, and append it to the sorted graph. Note here that by using # the values() method for iterating, a copy of the unsorted graph is used, allowing us to modify # the unsorted graph as we move through it. # diff --git a/airflow/www/extensions/init_appbuilder.py b/airflow/www/extensions/init_appbuilder.py index 9c2948e32490f..bbc6343fcf05a 100644 --- a/airflow/www/extensions/init_appbuilder.py +++ b/airflow/www/extensions/init_appbuilder.py @@ -586,6 +586,10 @@ def security_converge(self, dry=False) -> dict: def get_url_for_login_with(self, next_url: str | None = None) -> str: return get_auth_manager().get_url_login(next_url=next_url) + @property + def get_url_for_login(self): + return get_auth_manager().get_url_login() + @property def get_url_for_index(self): return url_for(f"{self.indexview.endpoint}.{self.indexview.default_view}") diff --git a/airflow/www/extensions/init_auth_manager.py b/airflow/www/extensions/init_auth_manager.py index a53fdf304befc..24ae020862dc9 100644 --- a/airflow/www/extensions/init_auth_manager.py +++ b/airflow/www/extensions/init_auth_manager.py @@ -26,12 +26,10 @@ from airflow.auth.managers.base_auth_manager import BaseAuthManager -@cache -def get_auth_manager() -> BaseAuthManager: - """ - Initialize auth manager. +def get_auth_manager_cls() -> type[BaseAuthManager]: + """Returns just the auth manager class without initializing it. - Import the user manager class, instantiate it and return it. + Useful to save execution time if only static methods need to be called. """ auth_manager_cls = conf.getimport(section="core", key="auth_manager") @@ -41,4 +39,16 @@ def get_auth_manager() -> BaseAuthManager: "Please specify one using section/key [core/auth_manager]." ) + return auth_manager_cls + + +@cache +def get_auth_manager() -> BaseAuthManager: + """ + Initialize auth manager. + + Import the user manager class, instantiate it and return it. + """ + auth_manager_cls = get_auth_manager_cls() + return auth_manager_cls() diff --git a/airflow/www/extensions/init_jinja_globals.py b/airflow/www/extensions/init_jinja_globals.py index 13baeea7bc67e..ff5481dd468f6 100644 --- a/airflow/www/extensions/init_jinja_globals.py +++ b/airflow/www/extensions/init_jinja_globals.py @@ -74,7 +74,7 @@ def prepare_jinja_globals(): } backends = conf.get("api", "auth_backends") - if len(backends) > 0 and backends[0] != "airflow.api.auth.backend.deny_all": + if backends and backends[0] != "airflow.api.auth.backend.deny_all": extra_globals["rest_api_enabled"] = True if "analytics_tool" in conf.getsection("webserver"): diff --git a/airflow/www/fab_security/manager.py b/airflow/www/fab_security/manager.py index 0e08c93d913a6..145e099d081b9 100644 --- a/airflow/www/fab_security/manager.py +++ b/airflow/www/fab_security/manager.py @@ -374,7 +374,7 @@ def oauth_user_info_getter(self, f): Decorator function to be the OAuth user info getter for all the providers. Receives provider and response return a dict with the information returned from the provider. - The returned user info dict should have it's keys with the same name as the User Model. + The returned user info dict should have its keys with the same name as the User Model. Use it like this an example for GitHub :: @@ -521,7 +521,7 @@ def _search_ldap(self, ldap, con, username): self.auth_ldap_lastname_field, self.auth_ldap_email_field, ] - if len(self.auth_roles_mapping) > 0: + if self.auth_roles_mapping: request_fields.append(self.auth_ldap_group_field) # perform the LDAP search @@ -561,7 +561,7 @@ def _ldap_calculate_user_roles(self, user_attributes: dict[str, list[bytes]]) -> user_role_objects = set() # apply AUTH_ROLES_MAPPING - if len(self.auth_roles_mapping) > 0: + if self.auth_roles_mapping: user_role_keys = self.ldap_extract_list(user_attributes, self.auth_ldap_group_field) user_role_objects.update(self.get_roles_from_keys(user_role_keys)) @@ -852,7 +852,7 @@ def _oauth_calculate_user_roles(self, userinfo) -> list[str]: user_role_objects = set() # apply AUTH_ROLES_MAPPING - if len(self.auth_roles_mapping) > 0: + if self.auth_roles_mapping: user_role_keys = userinfo.get("role_keys", []) user_role_objects.update(self.get_roles_from_keys(user_role_keys)) @@ -1096,7 +1096,7 @@ def security_cleanup(self, baseviews, menus): self.delete_resource(resource.name) def find_user(self, username=None, email=None): - """Generic function find a user by it's username or email.""" + """Generic function find a user by its username or email.""" raise NotImplementedError def get_role_permissions_from_db(self, role_id: int) -> list[Permission]: diff --git a/airflow/www/templates/appbuilder/navbar_right.html b/airflow/www/templates/appbuilder/navbar_right.html index 8eec9f9fcf1e3..1ccf28db418f1 100644 --- a/airflow/www/templates/appbuilder/navbar_right.html +++ b/airflow/www/templates/appbuilder/navbar_right.html @@ -67,8 +67,13 @@
  • - {% set user_names = auth_manager.get_user_name().split(" ", 1) %} - {% for name in user_names %}{{ name[0].upper() }}{% endfor %} + {% set user_name = auth_manager.get_user_name() %} + {% if user_name %} + {% set user_names = user_name.split(" ", 1) %} + {% for name in user_names %}{{ name[0].upper() }}{% endfor %} + {% else %} + person + {% endif %} diff --git a/airflow/www/views.py b/airflow/www/views.py index 951ed496da646..018b720b4f287 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -3857,7 +3857,7 @@ def dataset_dependencies(self): for dag, dependencies in SerializedDagModel.get_dag_dependencies().items(): dag_node_id = f"dag:{dag}" - if dag_node_id not in nodes_dict and len(dependencies) > 0: + if dag_node_id not in nodes_dict: for dep in dependencies: if dep.dependency_type == "dag" or dep.dependency_type == "dataset": nodes_dict[dag_node_id] = node_dict(dag_node_id, dag, "dag") diff --git a/codecov.yml b/codecov.yml index 3caf236b58be9..e78b83b58206b 100644 --- a/codecov.yml +++ b/codecov.yml @@ -25,7 +25,7 @@ codecov: coverage: precision: 2 round: down - range: 55..80 + range: 60..85 status: project: default: diff --git a/dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md b/dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md index d8f8d4c2c654c..8cd229066134d 100644 --- a/dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md +++ b/dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md @@ -136,7 +136,7 @@ candidates that you got from the `find-backtracking-candidates` command. This ** the next step is to narrow down the list of candidates to the one that is causing the backtracking. We narrow-down the list by "bisecting" the list. We remove half of the dependency limits and see if it -still works or not. It it works - we continue. If it does not work, we restore the removed half and remove +still works or not. It works - we continue. If it does not work, we restore the removed half and remove the other half. Rinse and repeat until there is only one dependency left - hopefully (sometimes you will need to leave few of them). diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 7326a5eec8c29..faff07a55e206 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -280,7 +280,6 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag ./dev/airflow-github changelog v2-3-stable v2-3-test ``` -- Update the `REVISION_HEADS_MAP` at airflow/utils/db.py to include the revision head of the release even if there are no migrations. - Commit the version change. - PR from the 'test' branch to the 'stable' branch @@ -828,7 +827,8 @@ Documentation for providers can be found in the ``/docs/apache-airflow`` directo ```shell script breeze release-management publish-docs --package-filter apache-airflow --package-filter docker-stack - breeze release-management add-back-references apache-airflow + breeze release-management add-back-references apache-airflow --airflow-site-directory "${AIRFLOW_SITE_DIRECTORY}" + cd "${AIRFLOW_SITE_DIRECTORY}" git add . git commit -m "Add documentation for Apache Airflow ${VERSION}" git push @@ -972,8 +972,7 @@ EOF This includes: - Modify `./scripts/ci/pre_commit/pre_commit_supported_versions.py` and let pre-commit do the job. -- For major/minor release, update version in `airflow/__main__.py`, `docs/docker-stack/` and `airflow/api_connexion/openapi/v1.yaml` to the next likely minor version release. -- Update the `REVISION_HEADS_MAP` at airflow/utils/db.py to include the revision head of the release even if there are no migrations. +- For major/minor release, update version in `airflow/__init__.py`, `docs/docker-stack/` and `airflow/api_connexion/openapi/v1.yaml` to the next likely minor version release. - Sync `RELEASE_NOTES.rst` (including deleting relevant `newsfragments`) and `README.md` changes. - Updating `Dockerfile` with the new version. - Updating `airflow_bug_report.yml` issue template in `.github/ISSUE_TEMPLATE/` with the new version. diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index 92e848c38094a..928cc29196625 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -97,7 +97,7 @@ Details about maintaining the SEMVER version are going to be discussed and imple First thing that release manager has to do is to change version of the provider to a target version. Each provider has a `provider.yaml` file that, among others, stores information about provider versions. When you attempt to release a provider you should update that -information based on the changes for the provider, and it's `CHANGELOG.rst`. It might be that +information based on the changes for the provider, and its `CHANGELOG.rst`. It might be that `CHANGELOG.rst` already contains the right target version. This will be especially true if some changes in the provider add new features (then minor version is increased) or when the changes introduce backwards-incompatible, breaking change in the provider (then major version is diff --git a/dev/breeze/SELECTIVE_CHECKS.md b/dev/breeze/SELECTIVE_CHECKS.md index f2d2ab72b6725..d6e525485a97a 100644 --- a/dev/breeze/SELECTIVE_CHECKS.md +++ b/dev/breeze/SELECTIVE_CHECKS.md @@ -125,10 +125,10 @@ Github Actions to pass the list of parameters to a command to execute | all-python-versions | List of all python versions there are available in the form of JSON array | ['3.8', '3.9', '3.10'] | | | all-python-versions-list-as-string | List of all python versions there are available in the form of space separated string | 3.8 3.9 3.10 | * | | basic-checks-only | Whether to run all static checks ("false") or only basic set of static checks ("true") | false | | -| cache-directive | Which cache should be be used for images ("registry", "local" , "disabled") | registry | | +| cache-directive | Which cache should be used for images ("registry", "local" , "disabled") | registry | | | debug-resources | Whether resources usage should be printed during parallel job execution ("true"/ "false") | false | | -| default-branch | Which branch is default for the the build ("main" for main branch, "v2-4-test" for 2.4 line etc.) | main | | -| default-constraints-branch | Which branch is default for the the build ("constraints-main" for main branch, "constraints-2-4" etc.) | constraints-main | | +| default-branch | Which branch is default for the build ("main" for main branch, "v2-4-test" for 2.4 line etc.) | main | | +| default-constraints-branch | Which branch is default for the build ("constraints-main" for main branch, "constraints-2-4" etc.) | constraints-main | | | default-helm-version | Which Helm version to use as default | v3.9.4 | | | default-kind-version | Which Kind version to use as default | v0.16.0 | | | default-kubernetes-version | Which Kubernetes version to use as default | v1.25.2 | | diff --git a/dev/breeze/doc/adr/0003-bootstrapping-virtual-environment.md b/dev/breeze/doc/adr/0003-bootstrapping-virtual-environment.md index 5ade92a9c82b4..eb750460d60dc 100644 --- a/dev/breeze/doc/adr/0003-bootstrapping-virtual-environment.md +++ b/dev/breeze/doc/adr/0003-bootstrapping-virtual-environment.md @@ -87,7 +87,7 @@ Integration we run. It's been established practice of the CI is that the logic of the CI is stored in the same repository as the source code of the application it tests and part of the Breeze functions are shared with CI. -In the future when breeze stabilizes and it's update cadence will be +In the future when breeze stabilizes and its update cadence will be much slower (which is likele as it happened with the Breeze predecessor) there could be an option that Breeze is installed as separate package and same released Breeze version could be ued to manage multiple Airflow @@ -150,7 +150,7 @@ of Airflow) performs the following tasks: re-installation if new dependencies are used - which is not as seamlessly integrate in the regular development environment, and it might create some confusions for the - users who would have to learn `pipx` and it's commands. + users who would have to learn `pipx` and its commands. Another drawback of `pipx` is that installs one global version of breeze for all projects, where it is quite possible that someone has two different versions of @@ -192,7 +192,7 @@ The alternatives considered were: mostly "used" but not deliberately activated is a better choice - especially that most users will simply "use" breeze as an app rather than activate the environment deliberately. - Also choosing `pyenv` and it's virtualenv plugin would + Also choosing `pyenv` and its virtualenv plugin would add extra, unnecessary steps and prerequisites for Breeze. diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 124d6ca31809b..2eecbfff3ca81 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -455,7 +455,10 @@ def should_we_run_the_build(build_ci_params: BuildCiParams) -> bool: # We import those locally so that click autocomplete works from inputimeout import TimeoutOccurred - if not md5sum_check_if_build_is_needed(md5sum_cache_dir=build_ci_params.md5sum_cache_dir): + if not md5sum_check_if_build_is_needed( + md5sum_cache_dir=build_ci_params.md5sum_cache_dir, + skip_provider_dependencies_check=build_ci_params.skip_provider_dependencies_check, + ): return False try: answer = user_confirm( @@ -631,6 +634,7 @@ def rebuild_or_pull_ci_image_if_needed(command_params: ShellParams | BuildCiPara image_tag=command_params.image_tag, platform=command_params.platform, force_build=command_params.force_build, + skip_provider_dependencies_check=command_params.skip_provider_dependencies_check, ) if command_params.image_tag is not None and command_params.image_tag != "latest": return_code, message = run_pull_image( diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 40fc0954780e9..671ced7d5a649 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -515,6 +515,9 @@ def static_checks( force_build=force_build, image_tag=image_tag, github_repository=github_repository, + # for static checks we do not want to regenerate dependencies before pre-commits are run + # we want the pre-commit to do it for us (and detect the case the dependencies are updated) + skip_provider_dependencies_check=True, ) if not skip_image_check: rebuild_or_pull_ci_image_if_needed(command_params=build_params) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index dc01f1aff5606..aa4f7c3a8573f 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -1203,7 +1203,7 @@ class ProviderPRInfo(NamedTuple): get_console().print() get_console().print( "Issue title: [yellow]Status of testing Providers that were " - f"prepared on {datetime.now().strftime('%B %d, %Y')}[/]" + f"prepared on {datetime.now():%B %d, %Y}[/]" ) get_console().print() syntax = Syntax(issue_content, "markdown", theme="ansi_dark") diff --git a/dev/breeze/src/airflow_breeze/commands/setup_commands.py b/dev/breeze/src/airflow_breeze/commands/setup_commands.py index 2668b1c319281..61e1f5ac7b57c 100644 --- a/dev/breeze/src/airflow_breeze/commands/setup_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/setup_commands.py @@ -297,7 +297,7 @@ def get_command_hash_export() -> str: f"[error]The `breeze {command} {subcommand}` is missing in rich-click options[/]" ) get_console().print( - "[info]Please add add it to rich_click.OPTION_GROUPS " + "[info]Please add it to rich_click.OPTION_GROUPS " "via one of the `*_commands_config.py` " "files in `dev/breeze/src/airflow_breeze/commands`[/]" ) diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index c0740801d59c0..6e07343aecc77 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -323,13 +323,13 @@ def get_airflow_extras(): "setup.cfg", "Dockerfile.ci", ".dockerignore", + "generated/provider_dependencies.json", "scripts/docker/common.sh", "scripts/docker/install_additional_dependencies.sh", "scripts/docker/install_airflow.sh", "scripts/docker/install_airflow_dependencies_from_branch_tip.sh", "scripts/docker/install_from_docker_context_files.sh", "scripts/docker/install_mysql.sh", - *ALL_PROVIDER_YAML_FILES, ] ENABLED_SYSTEMS = "" diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py b/dev/breeze/src/airflow_breeze/params/build_ci_params.py index 1f0663ec9f95c..1f49f0597b53c 100644 --- a/dev/breeze/src/airflow_breeze/params/build_ci_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py @@ -37,6 +37,7 @@ class BuildCiParams(CommonBuildParams): airflow_pre_cached_pip_packages: bool = True force_build: bool = False eager_upgrade_additional_requirements: str = "" + skip_provider_dependencies_check: bool = False @property def airflow_version(self): diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index a70a404d0ed64..99bc8c4d3cfd5 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -125,6 +125,7 @@ class ShellParams: celery_flower: bool = False only_min_version_update: bool = False regenerate_missing_docs: bool = False + skip_provider_dependencies_check: bool = False def clone_with_test(self, test_type: str) -> ShellParams: new_params = deepcopy(self) diff --git a/dev/breeze/src/airflow_breeze/utils/cdxgen.py b/dev/breeze/src/airflow_breeze/utils/cdxgen.py index b03d7b8bb8587..b1a64efac8f97 100644 --- a/dev/breeze/src/airflow_breeze/utils/cdxgen.py +++ b/dev/breeze/src/airflow_breeze/utils/cdxgen.py @@ -201,7 +201,7 @@ def get_requirements_for_provider( ) get_console().print( f"[info]Provider {provider_id} has {len(provider_packages)} transitively " - f"dependent packages (excluding airflow and it's dependencies)" + f"dependent packages (excluding airflow and its dependencies)" ) get_console().print(provider_packages) provider_file = target_dir / provider_file_name diff --git a/dev/breeze/src/airflow_breeze/utils/coertions.py b/dev/breeze/src/airflow_breeze/utils/coertions.py index 8f5daa6898e06..415b9472c23d1 100644 --- a/dev/breeze/src/airflow_breeze/utils/coertions.py +++ b/dev/breeze/src/airflow_breeze/utils/coertions.py @@ -30,4 +30,4 @@ def coerce_bool_value(value: str | bool) -> bool: def one_or_none_set(iterable: Iterable[bool]) -> bool: - return sum(1 for i in iterable if i) in (0, 1) + return 0 <= sum(1 for i in iterable if i) <= 1 diff --git a/dev/breeze/src/airflow_breeze/utils/md5_build_check.py b/dev/breeze/src/airflow_breeze/utils/md5_build_check.py index 235fbf22495b4..d1ec943785d42 100644 --- a/dev/breeze/src/airflow_breeze/utils/md5_build_check.py +++ b/dev/breeze/src/airflow_breeze/utils/md5_build_check.py @@ -20,16 +20,19 @@ from __future__ import annotations import hashlib +import os +import sys from pathlib import Path -from airflow_breeze.global_constants import FILES_FOR_REBUILD_CHECK +from airflow_breeze.global_constants import ALL_PROVIDER_YAML_FILES, FILES_FOR_REBUILD_CHECK from airflow_breeze.utils.console import get_console from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT +from airflow_breeze.utils.run_utils import run_command def check_md5checksum_in_cache_modified(file_hash: str, cache_path: Path, update: bool) -> bool: """ - Check if the file hash is present in cache and it's content has been modified. Optionally updates + Check if the file hash is present in cache and its content has been modified. Optionally updates the hash. :param file_hash: hash of the current version of the file @@ -59,8 +62,19 @@ def generate_md5(filename, file_size: int = 65536): return hash_md5.hexdigest() +def check_md5_sum_for_file(file_to_check: str, md5sum_cache_dir: Path, update: bool): + file_to_get_md5 = AIRFLOW_SOURCES_ROOT / file_to_check + md5_checksum = generate_md5(file_to_get_md5) + sub_dir_name = file_to_get_md5.parts[-2] + actual_file_name = file_to_get_md5.parts[-1] + cache_file_name = Path(md5sum_cache_dir, sub_dir_name + "-" + actual_file_name + ".md5sum") + file_content = md5_checksum + " " + str(file_to_get_md5) + "\n" + is_modified = check_md5checksum_in_cache_modified(file_content, cache_file_name, update=update) + return is_modified + + def calculate_md5_checksum_for_files( - md5sum_cache_dir: Path, update: bool = False + md5sum_cache_dir: Path, update: bool = False, skip_provider_dependencies_check: bool = False ) -> tuple[list[str], list[str]]: """ Calculates checksums for all interesting files and stores the hashes in the md5sum_cache_dir. @@ -68,35 +82,63 @@ def calculate_md5_checksum_for_files( :param md5sum_cache_dir: directory where to store cached information :param update: whether to update the hashes + :param skip_provider_dependencies_check: whether to skip regeneration of the provider dependencies :return: Tuple of two lists: modified and not-modified files """ not_modified_files = [] modified_files = [] - for calculate_md5_file in FILES_FOR_REBUILD_CHECK: - file_to_get_md5 = AIRFLOW_SOURCES_ROOT / calculate_md5_file - md5_checksum = generate_md5(file_to_get_md5) - sub_dir_name = file_to_get_md5.parts[-2] - actual_file_name = file_to_get_md5.parts[-1] - cache_file_name = Path(md5sum_cache_dir, sub_dir_name + "-" + actual_file_name + ".md5sum") - file_content = md5_checksum + " " + str(file_to_get_md5) + "\n" - is_modified = check_md5checksum_in_cache_modified(file_content, cache_file_name, update=update) + if not skip_provider_dependencies_check: + modified_provider_yaml_files = [] + for file in ALL_PROVIDER_YAML_FILES: + # Only check provider yaml files once and save the result immediately. + # If we need to regenerate the dependencies and they are not modified then + # all is fine and we can save checksums for the new files + if check_md5_sum_for_file(file, md5sum_cache_dir, True): + modified_provider_yaml_files.append(file) + if modified_provider_yaml_files: + get_console().print( + "[info]Attempting to generate provider dependencies. " + "Provider yaml files changed since last check:[/]" + ) + get_console().print( + [os.fspath(file.relative_to(AIRFLOW_SOURCES_ROOT)) for file in modified_provider_yaml_files] + ) + # Regenerate provider_dependencies.json + run_command( + [ + sys.executable, + os.fspath( + AIRFLOW_SOURCES_ROOT + / "scripts" + / "ci" + / "pre_commit" + / "pre_commit_update_providers_dependencies.py" + ), + ], + cwd=AIRFLOW_SOURCES_ROOT, + ) + for file in FILES_FOR_REBUILD_CHECK: + is_modified = check_md5_sum_for_file(file, md5sum_cache_dir, update) if is_modified: - modified_files.append(calculate_md5_file) + modified_files.append(file) else: - not_modified_files.append(calculate_md5_file) + not_modified_files.append(file) return modified_files, not_modified_files -def md5sum_check_if_build_is_needed(md5sum_cache_dir: Path) -> bool: +def md5sum_check_if_build_is_needed(md5sum_cache_dir: Path, skip_provider_dependencies_check: bool) -> bool: """ Checks if build is needed based on whether important files were modified. :param md5sum_cache_dir: directory where cached md5 sums are stored + :param skip_provider_dependencies_check: whether to skip regeneration of the provider dependencies :return: True if build is needed. """ build_needed = False - modified_files, not_modified_files = calculate_md5_checksum_for_files(md5sum_cache_dir, update=False) + modified_files, not_modified_files = calculate_md5_checksum_for_files( + md5sum_cache_dir, update=False, skip_provider_dependencies_check=skip_provider_dependencies_check + ) if modified_files: get_console().print( f"[warning]The following important files are modified in {AIRFLOW_SOURCES_ROOT} " diff --git a/dev/breeze/src/airflow_breeze/utils/parallel.py b/dev/breeze/src/airflow_breeze/utils/parallel.py index 70f5ec737eecb..9e2ffb64950d4 100644 --- a/dev/breeze/src/airflow_breeze/utils/parallel.py +++ b/dev/breeze/src/airflow_breeze/utils/parallel.py @@ -402,7 +402,7 @@ def check_async_run_results( completed_number = current_completed_number get_console().print( f"\n[info]Completed {completed_number} out of {total_number_of_results} " - f"({int(100*completed_number/total_number_of_results)}%).[/]\n" + f"({completed_number / total_number_of_results:.0%}).[/]\n" ) print_async_summary(completed_list) time.sleep(poll_time_seconds) @@ -410,7 +410,7 @@ def check_async_run_results( completed_number = len(completed_list) get_console().print( f"\n[info]Completed {completed_number} out of {total_number_of_results} " - f"({int(100*completed_number/total_number_of_results)}%).[/]\n" + f"({completed_number / total_number_of_results:.0%}).[/]\n" ) print_async_summary(completed_list) errors = False diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 4e447d24c14ba..413fe8be71634 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -347,7 +347,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Always Providers[airbyte,http]", }, - id="Providers tests tests run without amazon tests if no amazon file changed", + id="Providers tests run without amazon tests if no amazon file changed", ), pytest.param( ("airflow/providers/amazon/file.py",), @@ -1134,53 +1134,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d assert_outputs_are_printed(expected_outputs, str(stderr)) -@pytest.mark.parametrize( - "files, labels, expected_outputs, should_fail", - [ - pytest.param( - ("airflow/providers/yandex/test.py",), - (), - None, - True, - id="Suspended provider changes should fail", - ), - pytest.param( - ("airflow/providers/yandex/test.py",), - ("allow suspended provider changes",), - {"affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED}, - False, - id="Suspended provider changes should not fail if appropriate label is set", - ), - pytest.param( - ("airflow/providers/yandex/test.py", "airflow/providers/airbyte/test.py"), - ("allow suspended provider changes",), - {"affected-providers-list-as-string": "airbyte http"}, - False, - id="Only non-suspended provider changes should be listed", - ), - ], -) -def test_suspended_providers( - files: tuple[str, ...], labels: tuple[str], expected_outputs: dict[str, str], should_fail: bool -): - failed = False - try: - stderr = str( - SelectiveChecks( - files=files, - commit_ref="HEAD", - github_event=GithubEvents.PULL_REQUEST, - pr_labels=labels, - default_branch="main", - ) - ) - except SystemExit: - failed = True - assert failed == should_fail - if not failed: - assert_outputs_are_printed(expected_outputs, str(stderr)) - - @pytest.mark.parametrize( "github_event, github_actor, github_repository, pr_labels, github_context_dict, runs_on", [ diff --git a/dev/prepare_release_issue.py b/dev/prepare_release_issue.py index 4db602fce55b9..de50e910ebfb2 100755 --- a/dev/prepare_release_issue.py +++ b/dev/prepare_release_issue.py @@ -179,7 +179,7 @@ def render_template( keep_trailing_newline: bool = False, ) -> str: """ - Renders template based on it's name. Reads the template from _TEMPLATE.md.jinja2 in current dir. + Renders template based on its name. Reads the template from _TEMPLATE.md.jinja2 in current dir. :param template_name: name of the template to use :param context: Jinja2 context :param autoescape: Whether to autoescape HTML diff --git a/dev/system_tests/update_issue_status.py b/dev/system_tests/update_issue_status.py index 34a0262170273..83ba3a73e8621 100755 --- a/dev/system_tests/update_issue_status.py +++ b/dev/system_tests/update_issue_status.py @@ -171,7 +171,7 @@ def update_issue_status( console.print(f"[blue]Summary of performed actions: for {issue.title}[/]") console.print(f" Re-added file number (still there): {count_re_added}") console.print(f" Completed file number: {count_completed}") - console.print(f" Done {count_done}/{count_all} = {(count_done * 100/ count_all):.2f}%") + console.print(f" Done {count_done}/{count_all} = {count_done / count_all:.2%}") console.print() total_re_added += count_re_added total_completed += count_completed @@ -199,11 +199,11 @@ def update_issue_status( console.print(f"[green]Summary of ALL issues: for {num_issues} issues[/]") console.print( f" Completed and closed issues: {len(completed_closed_issues)}/{num_issues}: " - f"{len(completed_closed_issues) * 100/num_issues:.2f}%" + f"{len(completed_closed_issues) / num_issues:.2%}" ) console.print( f" Completed files {total_count_done}/{total_count_all} = " - f"{(total_count_done * 100/ total_count_all):.2f}%" + f"{total_count_done / total_count_all:.2%}" ) console.print() if not_completed_closed_issues: @@ -211,10 +211,7 @@ def update_issue_status( for issue in not_completed_closed_issues: all = per_issue_num_all[issue.id] done = per_issue_num_done[issue.id] - console.print( - rf" * [[yellow]{issue.title}[/]]({issue.html_url}): " - f"{done}/{all} : {done * 100 / all:.2f}%" - ) + console.print(f" * [[yellow]{issue.title}[/]]({issue.html_url}): {done}/{all} : {done / all:.2%}") console.print() if completed_open_issues: console.print("[yellow] Issues that are completed and should be closed:[/]\n") @@ -226,10 +223,7 @@ def update_issue_status( for issue in not_completed_opened_issues: all = per_issue_num_all[issue.id] done = per_issue_num_done[issue.id] - console.print( - rf" * [[yellow]{issue.title}[/]]({issue.html_url}): " - f"{done}/{all} : {done * 100 / all:.2f}%" - ) + console.print(f" * [[yellow]{issue.title}[/]]({issue.html_url}): {done}/{all} : {done / all:.2%}") console.print() if completed_closed_issues: console.print("[green] Issues that are completed and are already closed:[/]\n") diff --git a/docs/apache-airflow-providers-amazon/connections/aws.rst b/docs/apache-airflow-providers-amazon/connections/aws.rst index b3b222913fcd8..c12e24e76541e 100644 --- a/docs/apache-airflow-providers-amazon/connections/aws.rst +++ b/docs/apache-airflow-providers-amazon/connections/aws.rst @@ -60,10 +60,10 @@ automatically the credentials from there. .. caution:: If you do not run "airflow connections create-default-connections" command, most probably you do not have ``aws_default``. For historical reasons, the Amazon Provider components (Hooks, Operators, Sensors, etc.) fallback to the default boto3 credentials strategy - in case of a missing Connection ID. This behaviour is deprecated and will be removed in a future releases. + in case of a missing Connection ID. If you need to use the default boto3 credential strategy (credentials in environment variables, IAM Profile, etc.) - please provide ``None`` instead of a connection ID. + please provide ``None``, instead of a missing connection ID, to avoid getting a warning in your logs. .. _howto/connection:aws:configuring-the-connection: @@ -746,7 +746,7 @@ Using IAM Roles for Service Accounts (IRSA) on EKS If you are running Airflow on `Amazon EKS `_, you can grant AWS related permission (such as S3 Read/Write for remote logging) to the Airflow service -by granting the IAM role to it's service account. +by granting the IAM role to its service account. IRSA provides fine-grained permission management for apps(e.g., pods) that run on EKS and use other AWS services. These could be apps that use S3, any other AWS services like Secrets Manager, CloudWatch, DynamoDB etc. @@ -779,7 +779,7 @@ Create IAM Role for Service Account(IRSA) using eksctl eksctl utils associate-iam-oidc-provider --cluster="" --approve -4. Replace ``EKS_CLUSTER_ID``, ``SERVICE_ACCOUNT_NAME`` and ``NAMESPACE`` and execute the the following command. +4. Replace ``EKS_CLUSTER_ID``, ``SERVICE_ACCOUNT_NAME`` and ``NAMESPACE`` and execute the following command. This command will use an existing EKS Cluster ID and create an IAM role, service account and namespace. .. code-block:: bash diff --git a/docs/apache-airflow-providers-amazon/operators/appflow.rst b/docs/apache-airflow-providers-amazon/operators/appflow.rst index 8ef7e9c288fc6..14d3a5e5cad3d 100644 --- a/docs/apache-airflow-providers-amazon/operators/appflow.rst +++ b/docs/apache-airflow-providers-amazon/operators/appflow.rst @@ -41,10 +41,10 @@ Operators Run Flow ======== -To run an AppFlow flow keeping all filters as is, use: +To run an AppFlow flow keeping as is, use: :class:`~airflow.providers.amazon.aws.operators.appflow.AppflowRunOperator`. -.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_appflow.py +.. exampleinclude:: /../../tests/system/providers/amazon/aws/example_appflow_run.py :language: python :dedent: 4 :start-after: [START howto_operator_appflow_run] diff --git a/docs/apache-airflow-providers-apache-spark/connections/spark.rst b/docs/apache-airflow-providers-apache-spark/connections/spark.rst index 97a432c887af3..28249f3af9270 100644 --- a/docs/apache-airflow-providers-apache-spark/connections/spark.rst +++ b/docs/apache-airflow-providers-apache-spark/connections/spark.rst @@ -48,7 +48,7 @@ Extra (optional) When specifying the connection in environment variable you should specify it using URI syntax. -Note that all components of the URI should be URL-encoded. The URI and and the mongo +Note that all components of the URI should be URL-encoded. The URI and the mongo connection string are not the same. For example: diff --git a/docs/apache-airflow-providers-google/operators/cloud/bigquery_dts.rst b/docs/apache-airflow-providers-google/operators/cloud/bigquery_dts.rst index 46dbeaf14c273..0d3663419baad 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/bigquery_dts.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/bigquery_dts.rst @@ -44,7 +44,7 @@ In the case of Airflow, the customer needs to create a transfer config with the and then trigger a transfer run using a specialized Airflow operator that will call StartManualTransferRuns API for example :class:`~airflow.providers.google.cloud.operators.bigquery_dts.BigQueryDataTransferServiceStartTransferRunsOperator`. :class:`~airflow.providers.google.cloud.operators.bigquery_dts.BigQueryCreateDataTransferOperator` checks if automatic -scheduling option is present in passed configuration. If present then nothing is done, otherwise it's value is +scheduling option is present in passed configuration. If present then nothing is done, otherwise its value is set to ``True``. .. exampleinclude:: /../../tests/system/providers/google/cloud/bigquery/example_bigquery_dts.py diff --git a/docs/apache-airflow-providers-google/operators/cloud/kubernetes_engine.rst b/docs/apache-airflow-providers-google/operators/cloud/kubernetes_engine.rst index 373a196fd6672..8b51cdf06c8b1 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/kubernetes_engine.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/kubernetes_engine.rst @@ -135,7 +135,7 @@ Use of XCom ''''''''''' We can enable the usage of :ref:`XCom ` on the operator. This works by launching a sidecar container -with the pod specified. The sidecar is automatically mounted when the XCom usage is specified and it's mount point +with the pod specified. The sidecar is automatically mounted when the XCom usage is specified and its mount point is the path ``/airflow/xcom``. To provide values to the XCom, ensure your Pod writes it into a file called ``return.json`` in the sidecar. The contents of this can then be used downstream in your DAG. Here is an example of it being used: diff --git a/docs/apache-airflow-providers-microsoft-azure/operators/asb.rst b/docs/apache-airflow-providers-microsoft-azure/operators/asb.rst index 5ad6962418597..9adbbddc4c92e 100644 --- a/docs/apache-airflow-providers-microsoft-azure/operators/asb.rst +++ b/docs/apache-airflow-providers-microsoft-azure/operators/asb.rst @@ -70,7 +70,7 @@ Below is an example of using this operator to execute an Azure Service Bus Send Receive Message Azure Service Bus Queue ======================================== -To Receive Message or list of message or Batch message message in a Queue you can use +To Receive Message or list of message or Batch message in a Queue you can use :class:`~airflow.providers.microsoft.azure.operators.asb.AzureServiceBusReceiveMessageOperator`. Below is an example of using this operator to execute an Azure Service Bus Create Queue. diff --git a/docs/apache-airflow-providers-openlineage/guides/developer.rst b/docs/apache-airflow-providers-openlineage/guides/developer.rst index 4d5f7a5c67061..360dc2616b059 100644 --- a/docs/apache-airflow-providers-openlineage/guides/developer.rst +++ b/docs/apache-airflow-providers-openlineage/guides/developer.rst @@ -17,6 +17,9 @@ under the License. +.. _guides/developer:openlineage: + + Implementing OpenLineage in Operators ------------------------------------- @@ -61,7 +64,7 @@ This allows users to use your provider even if they do not have OpenLineage prov Second important point is to make sure your provider returns OpenLineage-compliant dataset names. It allows OpenLineage consumers to properly match information about datasets gathered from different sources. -The naming convention is described in the `OpenLineage docs `__. +The naming convention is described in the `OpenLineage naming docs `__. Third, OpenLineage implementation should not waste time of users that do not use it. This means not doing heavy processing or network calls in the ``execute`` method that aren't used by it. diff --git a/docs/apache-airflow-providers-openlineage/guides/user.rst b/docs/apache-airflow-providers-openlineage/guides/user.rst index 8aef3f17c683c..c2859f00a7cf2 100644 --- a/docs/apache-airflow-providers-openlineage/guides/user.rst +++ b/docs/apache-airflow-providers-openlineage/guides/user.rst @@ -20,19 +20,86 @@ Using OpenLineage integration ----------------------------- -Install -======= +Usage +===== -To use OpenLineage +No change to user DAG files is required to use OpenLineage. However, it needs to be configured. +Primary, and recommended method of configuring OpenLineage Airflow Provider is Airflow configuration. -Config -====== +At minimum, one thing that needs to be set up in every case is ``Transport`` - where do you wish for +your events to end up - for example `Marquez `_. The ``transport`` field in configuration is used for that purpose. -Primary method of configuring OpenLineage Airflow Provider is Airflow configuration. +.. code-block:: ini -One thing that needs to be set up in every case is ``Transport`` - where do you wish for -your events to end up. + [openlineage] + transport = '{"type": "http", "url": "http://example.com:5000"}' -Another option of configuration is using ``openlineage.yml`` file. -Detailed description of that configuration method is in OpenLineage docs -https://openlineage.io/docs/client/python#configuration + +If you want to look at OpenLineage events without sending them anywhere, you can set up ConsoleTransport - the events will end up in task logs. + +.. code-block:: ini + + [openlineage] + transport = '{"type": "console"}' + + +You can also configure OpenLineage transport using ``openlineage.yml`` file. +Detailed description of that configuration method is in `OpenLineage python docs `_. +To do that, you also need to set up path to the file in Airflow config, or point ``OPENLINEAGE_CONFIG`` variable to it: + +.. code-block:: ini + + [openlineage] + config_path = '/path/to/openlineage.yml' + +Lastly, you can set up http transport using ``OPENLINEAGE_URL`` environment variable, passing it the URL target of the OpenLineage consumer. + +It's also very useful to set up OpenLineage namespace for this particular instance. If not set, it's using ``default`` namespace. +That way, if you use multiple OpenLineage producers, events coming from them will be logically separated. + +.. code-block:: ini + + [openlineage] + transport = '{"type": "http", "url": "http://example.com:5000"}' + namespace = 'my-team-airflow-instance` + + +Additional Options +================== + +You can disable sending OpenLineage events without uninstalling OpenLineage provider by setting ``disabled`` to true or setting ``OPENLINEAGE_DISABLED`` +environment variable to True. + +.. code-block:: ini + + [openlineage] + transport = '{"type": "http", "url": "http://example.com:5000"}' + disabled = true + + +Several operators - for example Python, Bash - will by default include their source code in their OpenLineage events. To prevent that, set ``disable_source_code`` to true. + +.. code-block:: ini + + [openlineage] + transport = '{"type": "http", "url": "http://example.com:5000"}' + disable_source_code = true + +If you used OpenLineage previously, and use `Custom Extractors `_ feature, you can also use them in OpenLineage provider. +Register the extractors using ``extractors`` config option. + +.. code-block:: ini + + [openlineage] + transport = '{"type": "http", "url": "http://example.com:5000"}' + extractors = full.path.to.ExtractorClass;full.path.to.AnotherExtractorClass + + +Other +===== + +If you want to add OpenLineage coverage for particular operator, take a look at + +:ref:`guides/developer:openlineage` + +For more explanation visit `OpenLineage docs `_ diff --git a/docs/apache-airflow-providers-qubole/operators/qubole.rst b/docs/apache-airflow-providers-qubole/operators/qubole.rst index 162be4025ff83..31d6651a0cafd 100644 --- a/docs/apache-airflow-providers-qubole/operators/qubole.rst +++ b/docs/apache-airflow-providers-qubole/operators/qubole.rst @@ -71,7 +71,7 @@ To run jar file in your Hadoop cluster use Run Pig command """"""""""""""" -To run script script in *Pig Latin* in your Hadoop cluster use +To run script in *Pig Latin* in your Hadoop cluster use .. exampleinclude:: /../../tests/system/providers/qubole/example_qubole.py :language: python @@ -82,7 +82,7 @@ To run script script in *Pig Latin* in your Hadoop cluster use Run Shell command """"""""""""""""" -To run Shell-script script use +To run Shell-script use .. exampleinclude:: /../../tests/system/providers/qubole/example_qubole.py :language: python diff --git a/docs/apache-airflow-providers-yandex/changelog.rst b/docs/apache-airflow-providers-yandex/changelog.rst new file mode 100644 index 0000000000000..595a47e699439 --- /dev/null +++ b/docs/apache-airflow-providers-yandex/changelog.rst @@ -0,0 +1,19 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../airflow/providers/yandex/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-yandex/security.rst b/docs/apache-airflow-providers-yandex/security.rst new file mode 100644 index 0000000000000..66c6f79a4ecfc --- /dev/null +++ b/docs/apache-airflow-providers-yandex/security.rst @@ -0,0 +1,38 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Releasing security patches +-------------------------- + +Airflow providers are released independently from Airflow itself and the information about vulnerabilities +is published separately. You can upgrade providers independently from Airflow itself, following the +instructions found in :doc:`apache-airflow:installation/installing-from-pypi`. + +When we release Provider version, the development is always done from the ``main`` branch where we prepare +the next version. The provider uses strict `SemVer `_ versioning policy. Depending on +the scope of the change, Provider will get ''MAJOR'' version upgrade when there are +breaking changes, ``MINOR`` version upgrade when there are new features or ``PATCHLEVEL`` version upgrade +when there are only bug fixes (including security bugfixes) - and this is the only version that receives +security fixes by default, so you should upgrade to latest version of the provider if you want to receive +all released security fixes. + +The only exception to that rule is when we have a critical security fix and good reason to provide an +out-of-band release for the provider, in which case stakeholders in the provider might decide to cherry-pick +and prepare a branch for an older version of the provider following the +`mixed governance model `_ +and requires interested parties to cherry-pick and test the fixes. diff --git a/docs/apache-airflow-providers/howto/create-custom-providers.rst b/docs/apache-airflow-providers/howto/create-custom-providers.rst index 6fe27674f2267..7b2a2fcf4c1dd 100644 --- a/docs/apache-airflow-providers/howto/create-custom-providers.rst +++ b/docs/apache-airflow-providers/howto/create-custom-providers.rst @@ -86,7 +86,7 @@ Exposing customized functionality to the Airflow's core: * ``notifications`` - this field should contain the notification classes. See :doc:`apache-airflow:howto/notifications` for description of the notifications. -* ``executors`` - this field should contain the executor class class names. +* ``executors`` - this field should contain the executor class names. See :doc:`apache-airflow:core-concepts/executor/index` for description of the executors. * ``config`` - this field should contain dictionary that should conform to the diff --git a/docs/apache-airflow/administration-and-deployment/logging-monitoring/metrics.rst b/docs/apache-airflow/administration-and-deployment/logging-monitoring/metrics.rst index 224f4fdbeba3e..07f7dc651ad09 100644 --- a/docs/apache-airflow/administration-and-deployment/logging-monitoring/metrics.rst +++ b/docs/apache-airflow/administration-and-deployment/logging-monitoring/metrics.rst @@ -199,7 +199,7 @@ Gauges =================================================== ======================================================================== Name Description =================================================== ======================================================================== -``dagbag_size`` Number of DAGs found when the scheduler ran a scan based on it's +``dagbag_size`` Number of DAGs found when the scheduler ran a scan based on its configuration ``dag_processing.import_errors`` Number of errors from trying to parse DAG files ``dag_processing.total_parse_time`` Seconds taken to scan and import ``dag_processing.file_path_queue_size`` DAG files diff --git a/docs/apache-airflow/best-practices.rst b/docs/apache-airflow/best-practices.rst index 6cb2857d96755..d62a9d948fd76 100644 --- a/docs/apache-airflow/best-practices.rst +++ b/docs/apache-airflow/best-practices.rst @@ -331,7 +331,7 @@ Example of watcher pattern with trigger rules --------------------------------------------- The watcher pattern is how we call a DAG with a task that is "watching" the states of the other tasks. -It's primary purpose is to fail a DAG Run when any other task fail. +Its primary purpose is to fail a DAG Run when any other task fail. The need came from the Airflow system tests that are DAGs with different tasks (similarly like a test containing steps). Normally, when any task fails, all other tasks are not executed and the whole DAG Run gets failed status too. But diff --git a/docs/apache-airflow/deprecated-rest-api-ref.rst b/docs/apache-airflow/deprecated-rest-api-ref.rst index c0af5b90f6500..276175f97f0eb 100644 --- a/docs/apache-airflow/deprecated-rest-api-ref.rst +++ b/docs/apache-airflow/deprecated-rest-api-ref.rst @@ -21,7 +21,7 @@ Deprecated REST API .. warning:: This REST API is deprecated since version 2.0. Please consider using the :doc:`stable REST API `. - For more information on migration, see `UPDATING.md `_ + For more information on migration, see `UPDATING.md `_ Before Airflow 2.0 this REST API was known as the "experimental" API, but now that the :doc:`stable REST API ` is available, it has been renamed. diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index 03df112af07c8..4becdabaa08e1 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -214,6 +214,8 @@ These are extras that add dependencies needed for integration with external serv +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | vertica | ``pip install 'apache-airflow[vertica]'`` | Vertica hook support as an Airflow backend | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ +| yandex | ``pip install 'apache-airflow[yandex]'`` | Yandex.cloud hooks and operators | ++---------------------+-----------------------------------------------------+-----------------------------------------------------+ | zendesk | ``pip install 'apache-airflow[zendesk]'`` | Zendesk hooks | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ diff --git a/docs/apache-airflow/howto/docker-compose/index.rst b/docs/apache-airflow/howto/docker-compose/index.rst index 1ab56b2347887..696c83c765c21 100644 --- a/docs/apache-airflow/howto/docker-compose/index.rst +++ b/docs/apache-airflow/howto/docker-compose/index.rst @@ -364,7 +364,7 @@ runtime user id which is unknown at the time of building the image. | ``AIRFLOW_IMAGE_NAME`` | Airflow Image to use. | apache/airflow:|version| | +--------------------------------+-----------------------------------------------------+--------------------------+ | ``AIRFLOW_UID`` | UID of the user to run Airflow containers as. | ``50000`` | -| | Override if you want to use use non-default Airflow | | +| | Override if you want to use non-default Airflow | | | | UID (for example when you map folders from host, | | | | it should be set to result of ``id -u`` call. | | | | When it is changed, a user with the UID is | | diff --git a/docs/apache-airflow/howto/email-config.rst b/docs/apache-airflow/howto/email-config.rst index 014a2e2464fb2..c00d959d3bf2d 100644 --- a/docs/apache-airflow/howto/email-config.rst +++ b/docs/apache-airflow/howto/email-config.rst @@ -53,7 +53,7 @@ Equivalent environment variables look like: To configure SMTP settings, checkout the :ref:`SMTP ` section in the standard configuration. If you do not want to store the SMTP credentials in the config or in the environment variables, you can create a -connection called ``smtp_default`` of ``Email`` type, or choose a custom connection name and set the ``email_conn_id`` with it's name in +connection called ``smtp_default`` of ``Email`` type, or choose a custom connection name and set the ``email_conn_id`` with its name in the configuration & store SMTP username-password in it. Other SMTP settings like host, port etc always gets picked up from the configuration only. The connection can be of any type (for example 'HTTP connection'). diff --git a/docs/apache-airflow/howto/operator/datetime.rst b/docs/apache-airflow/howto/operator/datetime.rst index ac8644fe85972..53485f47cded8 100644 --- a/docs/apache-airflow/howto/operator/datetime.rst +++ b/docs/apache-airflow/howto/operator/datetime.rst @@ -35,7 +35,7 @@ Usage with current time The usages above might be useful in certain situations - for example when DAG is used to perform cleanups and maintenance and is not really supposed to be used for any DAGs that are supposed to be back-filled, -because the "current time" make back-filling non-idempotent, it's result depend on the time when the DAG +because the "current time" make back-filling non-idempotent, its result depend on the time when the DAG actually was run. It's also slightly non-deterministic potentially even if it is run on schedule. It can take some time between when the DAGRun was scheduled and executed and it might mean that even if the DAGRun was scheduled properly, the actual time used for branching decision will be different than the diff --git a/docs/apache-airflow/public-airflow-interface.rst b/docs/apache-airflow/public-airflow-interface.rst index add533f4ed2ad..e63ac8fdca39d 100644 --- a/docs/apache-airflow/public-airflow-interface.rst +++ b/docs/apache-airflow/public-airflow-interface.rst @@ -46,7 +46,7 @@ Airflow Interface and might change at any time. You can also use Airflow's Public Interface via the `Stable REST API `_ (based on the OpenAPI specification). For specific needs you can also use the -`Airflow Command Line Interface (CLI) `_ though it's behaviour might change +`Airflow Command Line Interface (CLI) `_ though its behaviour might change in details (such as output format and available flags) so if you want to rely on those in programmatic way, the Stable REST API is recommended. diff --git a/docs/apache-airflow/security/webserver.rst b/docs/apache-airflow/security/webserver.rst index 2862f7073ac3a..9e4c947ee7a88 100644 --- a/docs/apache-airflow/security/webserver.rst +++ b/docs/apache-airflow/security/webserver.rst @@ -176,7 +176,7 @@ Here is an example of what you might have in your webserver_config.py: AUTH_ROLES_SYNC_AT_LOGIN = True # Checks roles on every login AUTH_USER_REGISTRATION = True # allow users who are not already in the FAB DB to register # Make sure to replace this with the path to your security manager class - FAB_SECURITY_MANAGER_CLASS = "your_module.your_security_manager_class" + SECURITY_MANAGER_CLASS = "your_module.your_security_manager_class" AUTH_ROLES_MAPPING = { "Viewer": ["Viewer"], "Admin": ["Admin"], diff --git a/docs/apache-airflow/tutorial/fundamentals.rst b/docs/apache-airflow/tutorial/fundamentals.rst index d7d3acf2e8ebc..2c710530b2d6b 100644 --- a/docs/apache-airflow/tutorial/fundamentals.rst +++ b/docs/apache-airflow/tutorial/fundamentals.rst @@ -400,4 +400,4 @@ Here are a few things you might want to do next: .. seealso:: - Continue to the next step of the tutorial: :doc:`/tutorial/taskflow` - - Skip to the the :doc:`/core-concepts/index` section for detailed explanation of Airflow concepts such as DAGs, Tasks, Operators, and more + - Skip to the :doc:`/core-concepts/index` section for detailed explanation of Airflow concepts such as DAGs, Tasks, Operators, and more diff --git a/docs/docker-stack/entrypoint.rst b/docs/docker-stack/entrypoint.rst index 38b9ea22732f9..77648a6ba1e9b 100644 --- a/docs/docker-stack/entrypoint.rst +++ b/docs/docker-stack/entrypoint.rst @@ -57,7 +57,7 @@ The user can be any UID. In case UID is different than the default ``airflow`` (UID=50000), the user will be automatically created when entering the container. In order to accommodate a number of external libraries and projects, Airflow will automatically create -such an arbitrary user in (`/etc/passwd`) and make it's home directory point to ``/home/airflow``. +such an arbitrary user in (`/etc/passwd`) and make its home directory point to ``/home/airflow``. Many of 3rd-party libraries and packages require home directory of the user to be present, because they need to write some cache information there, so such a dynamic creation of a user is necessary. @@ -98,7 +98,7 @@ The entrypoint is waiting for a connection to the database independent of the da the stability of the environment. Waiting for connection involves executing ``airflow db check`` command, which means that a ``select 1 as is_alive;`` statement -is executed. Then it loops until the the command will be successful. +is executed. Then it loops until the command will be successful. It tries :envvar:`CONNECTION_CHECK_MAX_COUNT` times and sleeps :envvar:`CONNECTION_CHECK_SLEEP_TIME` between checks To disable check, set ``CONNECTION_CHECK_MAX_COUNT=0``. @@ -330,7 +330,7 @@ The entrypoint can also create webserver user automatically when you enter it. y production, it is only useful if you would like to run a quick test with the production image. You need to pass at least password to create such user via ``_AIRFLOW_WWW_USER_PASSWORD`` or :envvar:`_AIRFLOW_WWW_USER_PASSWORD_CMD` similarly like for other ``*_CMD`` variables, the content of -the ``*_CMD`` will be evaluated as shell command and it's output will be set as password. +the ``*_CMD`` will be evaluated as shell command and its output will be set as password. User creation will fail if none of the ``PASSWORD`` variables are set - there is no default for password for security reasons. diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 7a48585982c62..ec01067b12bfe 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -789,6 +789,7 @@ isfile ish isn isort +istio iterable iterables iteratively diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 15ee5d67e9758..0c5b09bd600bb 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -26,12 +26,13 @@ "apache-airflow>=2.4.0", "asgiref", "asgiref", - "boto3>=1.24.0", + "boto3>=1.28.0", + "botocore>=1.31.0", "jsonpath_ng>=1.5.3", - "mypy-boto3-appflow>=1.28.16", - "mypy-boto3-rds>=1.24.0", - "mypy-boto3-redshift-data>=1.24.0", - "mypy-boto3-s3>=1.24.0", + "mypy-boto3-appflow>=1.28.0", + "mypy-boto3-rds>=1.28.0", + "mypy-boto3-redshift-data>=1.28.0", + "mypy-boto3-s3>=1.28.0", "redshift_connector>=2.0.888", "sqlalchemy_redshift>=0.8.6", "watchtower~=2.0.1" @@ -559,7 +560,7 @@ "azure-datalake-store>=0.0.45", "azure-identity>=1.3.1", "azure-keyvault-secrets>=4.1.0", - "azure-kusto-data>=0.0.43,<0.1", + "azure-kusto-data>=4.1.0", "azure-mgmt-containerinstance>=1.5.0,<2.0", "azure-mgmt-datafactory>=1.0.0,<2.0", "azure-mgmt-datalake-store>=0.5.0", @@ -909,6 +910,14 @@ ], "excluded-python-versions": [] }, + "yandex": { + "deps": [ + "apache-airflow>=2.4.0", + "yandexcloud>=0.228.0" + ], + "cross-providers-deps": [], + "excluded-python-versions": [] + }, "zendesk": { "deps": [ "apache-airflow>=2.4.0", diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 5c81b1f82538e..bed222b5cca4c 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -2,7 +2,7 @@ # Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. main:42493af4045e28dcf52040810fbb7036 -build-docs:bfc91db55c595516fe84d56c96a5a8e0 +build-docs:173a48b2e23c17fa5adf394fc5c8a71e ci:find-backtracking-candidates:17fe56b867a745e5032a08dfcd3f73ee ci:fix-ownership:3e5a73533cc96045e72cb258783cfc96 ci:free-space:49af17b032039c05c41a7a8283f365cc @@ -36,25 +36,25 @@ prod-image:build:10cc859b7d898581bf9b2a24c19c1032 prod-image:pull:76f1f27e6119928412abecf153fce4bb prod-image:verify:bd2b78738a7c388dbad6076c41a9f906 prod-image:d9b47217a12f73f214bcd2938bbab84a -release-management:add-back-references:919b3bd706acf794e0e5e129fc587f6d +release-management:add-back-references:7e3a4e6d3f7932773cfb256059f7c2c8 release-management:create-minor-branch:a3834afc4aa5d1e98002c9e9e7a9931d release-management:generate-constraints:b8fcaf8f0acd35ed5dbd48659bdb6485 -release-management:generate-issue-content-providers:dc7149a2c6fc43d323e7c83865c0ca0b +release-management:generate-issue-content-providers:b810c1afe316edf547572b232759c004 release-management:generate-providers-metadata:d4e8e5cfaa024e3963af02d7a873048d release-management:install-provider-packages:a89493be1ae961c13469b5a25a605069 release-management:prepare-airflow-package:85d01c57e5b5ee0fb9e5f9d9706ed3b5 -release-management:prepare-provider-documentation:71dd952212206ca5c3b24bf8deaf13d6 -release-management:prepare-provider-packages:ed727969a3fa27e41c75c08381c9ebf4 -release-management:publish-docs:6d8ea0362b7a4394d8367e2105516cf8 +release-management:prepare-provider-documentation:01393745645e0307dcc15baaf24d3435 +release-management:prepare-provider-packages:93769f8db9fda9bb28fd0cd1a201cae8 +release-management:publish-docs:6540266086872d11fd6f6fba1c044756 release-management:release-prod-images:cfbfe8b19fee91fd90718f98ef2fd078 release-management:start-rc-process:b27bd524dd3c89f50a747b60a7e892c1 release-management:start-release:419f48f6a4ff4457cb9de7ff496aebbe release-management:update-constraints:02ec4b119150e3fdbac52026e94820ef release-management:verify-provider-packages:96dce5644aad6b37080acf77b3d8de3a -release-management:89b9da7b140208d6d15823d7ccf3c017 -sbom:generate-provider-requirements:9abe53200ea5f40e0bf7c27f6087f27f +release-management:fec2e6ac2786d18d0d2c2461a0ac0f33 +sbom:generate-provider-requirements:42cb9f199c7be30b5bf7102ea175d736 sbom:update-sbom-information:0ce56884e5f842e3e80d6619df1ccc64 -sbom:935d041028e847d3faf763a95b51063e +sbom:97ea9bb927149c1675345d3102a29ac7 setup:autocomplete:fffcd49e102e09ccd69b3841a9e3ea8e setup:check-all-params-in-groups:639dfb061b8da02dcaa4a57b567ff42a setup:config:fd32471ee31894decf91984615771add diff --git a/images/breeze/output_build-docs.svg b/images/breeze/output_build-docs.svg index 564a540f7b84e..79a5e1c9cdeb7 100644 --- a/images/breeze/output_build-docs.svg +++ b/images/breeze/output_build-docs.svg @@ -293,7 +293,7 @@ │apache-airflow-providers-ssh | apache-airflow-providers-tableau |                          â”‚ │apache-airflow-providers-tabular | apache-airflow-providers-telegram |                     â”‚ │apache-airflow-providers-trino | apache-airflow-providers-vertica |                        â”‚ -│apache-airflow-providers-zendesk)                                                          â”‚ +│apache-airflow-providers-yandex | apache-airflow-providers-zendesk)                        â”‚ │--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ │--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)[default: autodetect]│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management_add-back-references.svg b/images/breeze/output_release-management_add-back-references.svg index 75d6674bb0389..436c03db7742c 100644 --- a/images/breeze/output_release-management_add-back-references.svg +++ b/images/breeze/output_release-management_add-back-references.svg @@ -160,7 +160,7 @@                                                      papermill | plexus | postgres | presto | qubole | redis |                                                      salesforce | samba | segment | sendgrid | sftp | singularity |                                                      slack | smtp | snowflake | sqlite | ssh | tableau | tabular | -                                                     telegram | trino | vertica | zendesk]... +                                                     telegram | trino | vertica | yandex | zendesk]... Command to add back references for documentation to make it backward compatible. diff --git a/images/breeze/output_release-management_generate-issue-content-providers.svg b/images/breeze/output_release-management_generate-issue-content-providers.svg index 2e6d13bf24e22..0f30f62f7b105 100644 --- a/images/breeze/output_release-management_generate-issue-content-providers.svg +++ b/images/breeze/output_release-management_generate-issue-content-providers.svg @@ -189,7 +189,7 @@                                                                   redis | salesforce | samba | segment | sendgrid |                                                                   sftp | singularity | slack | smtp | snowflake |                                                                   sqlite | ssh | tableau | tabular | telegram | trino -                                                                  | vertica | zendesk]... +                                                                  | vertica | yandex | zendesk]... Generates content for issue to test the release. diff --git a/images/breeze/output_release-management_prepare-provider-documentation.svg b/images/breeze/output_release-management_prepare-provider-documentation.svg index f7d501809c110..95a393bb848ba 100644 --- a/images/breeze/output_release-management_prepare-provider-documentation.svg +++ b/images/breeze/output_release-management_prepare-provider-documentation.svg @@ -197,7 +197,7 @@                                                                 presto | qubole | redis | salesforce | samba | segment                                                                 | sendgrid | sftp | singularity | slack | smtp |                                                                 snowflake | sqlite | ssh | tableau | tabular | -                                                                telegram | trino | vertica | zendesk]... +                                                                telegram | trino | vertica | yandex | zendesk]... Prepare CHANGELOG, README and COMMITS information for providers. diff --git a/images/breeze/output_release-management_prepare-provider-packages.svg b/images/breeze/output_release-management_prepare-provider-packages.svg index 332e7d096a93d..8f938ecbb79d0 100644 --- a/images/breeze/output_release-management_prepare-provider-packages.svg +++ b/images/breeze/output_release-management_prepare-provider-packages.svg @@ -174,7 +174,7 @@                                                            presto | qubole | redis | salesforce | samba | segment |                                                            sendgrid | sftp | singularity | slack | smtp | snowflake |                                                            sqlite | ssh | tableau | tabular | telegram | trino | -                                                           vertica | zendesk]... +                                                           vertica | yandex | zendesk]... Prepare sdist/whl packages of Airflow Providers. diff --git a/images/breeze/output_release-management_publish-docs.svg b/images/breeze/output_release-management_publish-docs.svg index a75e335fe2cfe..4023fa18c6111 100644 --- a/images/breeze/output_release-management_publish-docs.svg +++ b/images/breeze/output_release-management_publish-docs.svg @@ -276,7 +276,7 @@ │apache-airflow-providers-ssh | apache-airflow-providers-tableau |                  â”‚ │apache-airflow-providers-tabular | apache-airflow-providers-telegram |             â”‚ │apache-airflow-providers-trino | apache-airflow-providers-vertica |                â”‚ -│apache-airflow-providers-zendesk)                                                  â”‚ +│apache-airflow-providers-yandex | apache-airflow-providers-zendesk)                â”‚ │*--airflow-site-directory-aLocal directory path of cloned airflow-site repo.(TEXT)[required]│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â•® diff --git a/images/breeze/output_sbom_generate-provider-requirements.svg b/images/breeze/output_sbom_generate-provider-requirements.svg index 66b8c6d01a097..dc65b329a12dd 100644 --- a/images/breeze/output_sbom_generate-provider-requirements.svg +++ b/images/breeze/output_sbom_generate-provider-requirements.svg @@ -162,7 +162,7 @@ │mysql | neo4j | odbc | openfaas | openlineage | opsgenie | oracle | pagerduty | papermill |│ │plexus | postgres | presto | qubole | redis | salesforce | samba | segment | sendgrid |    â”‚ │sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular | telegram│ -│| trino | vertica | zendesk)                                                               â”‚ +│| trino | vertica | yandex | zendesk)                                                      â”‚ │[required]                                                                                 â”‚ │--provider-versionProvider version to generate the requirements for(TEXT)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/images/ci_stop_arm_instance.sh b/scripts/ci/images/ci_stop_arm_instance.sh index 94e14623e011d..57a42c80a3259 100755 --- a/scripts/ci/images/ci_stop_arm_instance.sh +++ b/scripts/ci/images/ci_stop_arm_instance.sh @@ -24,8 +24,7 @@ function stop_arm_instance() { INSTANCE_ID=$(jq < "${INSTANCE_INFO}" ".Instances[0].InstanceId" -r) docker buildx rm --force airflow_cache || true aws ec2 terminate-instances --instance-ids "${INSTANCE_ID}" - cat ${AUTOSSH_LOGFILE} - + cat ${AUTOSSH_LOGFILE} || true } stop_arm_instance diff --git a/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py b/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py index 995ff18ba0407..400b958065ff7 100755 --- a/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py +++ b/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py @@ -51,6 +51,8 @@ *get_extra_docker_flags(MOUNT_SELECTED), "-e", "SKIP_ENVIRONMENT_INITIALIZATION=true", + "-e", + "PYTHONWARNINGS=default", "--pull", "never", airflow_image, diff --git a/scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py b/scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py index 6bdcc58040383..f0406d43749d6 100755 --- a/scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py +++ b/scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py @@ -74,7 +74,7 @@ def check_file(the_file: Path) -> int: if total_err_num: console.print( """ -[red]There are are some checkout instructions in github workflows that have no "persist_credentials" +[red]There are some checkout instructions in github workflows that have no "persist_credentials" set to False.[/] For security reasons - make sure all of the checkout actions have persist_credentials set, similar to: diff --git a/scripts/ci/pre_commit/pre_commit_update_common_sql_api_stubs.py b/scripts/ci/pre_commit/pre_commit_update_common_sql_api_stubs.py index 1a02ffad6a668..b8fead2861d5e 100755 --- a/scripts/ci/pre_commit/pre_commit_update_common_sql_api_stubs.py +++ b/scripts/ci/pre_commit/pre_commit_update_common_sql_api_stubs.py @@ -299,7 +299,7 @@ def compare_stub_files(generated_stub_path: Path, force_override: bool) -> tuple # This is automatically generated stub for the `common.sql` provider # # This file is generated automatically by the `update-common-sql-api stubs` pre-commit -# and the .pyi file represents part of the the "public" API that the +# and the .pyi file represents part of the "public" API that the # `common.sql` provider exposes to other providers. # # Any, potentially breaking change in the stubs will require deliberate manual action from the contributor diff --git a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py index e2fc0ca713362..b63a60b14b224 100755 --- a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py +++ b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py @@ -209,12 +209,28 @@ def check_if_different_provider_used(file_path: Path) -> None: console.print("[red]Errors found during verification. Exiting!") console.print() sys.exit(1) - DEPENDENCIES_JSON_FILE_PATH.write_text(json.dumps(unique_sorted_dependencies, indent=2) + "\n") - console.print( - f"[yellow]If you see changes to the {DEPENDENCIES_JSON_FILE_PATH} file - " - f"do not modify the file manually. Let pre-commit do the job!" - ) - console.print() - console.print("[green]Verification complete! Success!\n") - console.print(f"Written {DEPENDENCIES_JSON_FILE_PATH}") - console.print() + old_dependencies = DEPENDENCIES_JSON_FILE_PATH.read_text() + new_dependencies = json.dumps(unique_sorted_dependencies, indent=2) + "\n" + if new_dependencies != old_dependencies: + DEPENDENCIES_JSON_FILE_PATH.write_text(json.dumps(unique_sorted_dependencies, indent=2) + "\n") + if os.environ.get("CI"): + console.print() + console.print(f"[info]Written {DEPENDENCIES_JSON_FILE_PATH}") + console.print( + f"[yellow]You will need to run breeze locally and commit " + f"{DEPENDENCIES_JSON_FILE_PATH.relative_to(AIRFLOW_SOURCES_ROOT)}!\n" + ) + console.print() + else: + console.print() + console.print( + f"[yellow]Regenerated new dependencies. Please commit " + f"{DEPENDENCIES_JSON_FILE_PATH.relative_to(AIRFLOW_SOURCES_ROOT)}!\n" + ) + console.print(f"[info]Written {DEPENDENCIES_JSON_FILE_PATH}") + console.print() + else: + console.print( + "[green]No need to regenerate dependencies!\n[/]" + f"The {DEPENDENCIES_JSON_FILE_PATH.relative_to(AIRFLOW_SOURCES_ROOT)} is up to date!\n" + ) diff --git a/scripts/ci/pre_commit/pre_commit_version_heads_map.py b/scripts/ci/pre_commit/pre_commit_version_heads_map.py index 5ae542b1c11bd..b3461e78ccb4e 100755 --- a/scripts/ci/pre_commit/pre_commit_version_heads_map.py +++ b/scripts/ci/pre_commit/pre_commit_version_heads_map.py @@ -18,42 +18,69 @@ # under the License. from __future__ import annotations -import ast +import os import sys from pathlib import Path -from packaging.version import Version +import re2 +from packaging.version import parse as parse_version PROJECT_SOURCE_ROOT_DIR = Path(__file__).resolve().parent.parent.parent.parent DB_FILE = PROJECT_SOURCE_ROOT_DIR / "airflow" / "utils" / "db.py" +MIGRATION_PATH = PROJECT_SOURCE_ROOT_DIR / "airflow" / "migrations" / "versions" sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure common_precommit_utils is importable -from common_precommit_utils import read_airflow_version # noqa: E402 +def revision_heads_map(): + rh_map = {} + pattern = r'revision = "[a-fA-F0-9]+"' + airflow_version_pattern = r'airflow_version = "\d+\.\d+\.\d+"' + filenames = os.listdir(MIGRATION_PATH) -def read_revision_heads_map(): - revision_heads_map_ast_obj = ast.parse(open(DB_FILE).read()) + def sorting_key(filen): + prefix = filen.split("_")[0] + return int(prefix) if prefix.isdigit() else 0 - revision_heads_map_ast = [ - a - for a in revision_heads_map_ast_obj.body - if isinstance(a, ast.Assign) and a.targets[0].id == "REVISION_HEADS_MAP" - ][0] + sorted_filenames = sorted(filenames, key=sorting_key) - revision_heads_map = ast.literal_eval(revision_heads_map_ast.value) - - return revision_heads_map.keys() + for filename in sorted_filenames: + with open(os.path.join(MIGRATION_PATH, filename)) as file: + content = file.read() + revision_match = re2.search(pattern, content) + airflow_version_match = re2.search(airflow_version_pattern, content) + if revision_match and airflow_version_match: + revision = revision_match.group(0).split('"')[1] + version = airflow_version_match.group(0).split('"')[1] + if parse_version(version) >= parse_version("2.0.0"): + rh_map[version] = revision + return rh_map if __name__ == "__main__": - airflow_version = Version(read_airflow_version()) - if airflow_version.is_devrelease or "b" in (airflow_version.pre or ()): - exit(0) - versions = read_revision_heads_map() - if airflow_version.base_version not in versions: - print("Current airflow version is not in the REVISION_HEADS_MAP") - print("Current airflow version:", airflow_version) - print("Please add the version to the REVISION_HEADS_MAP at:", DB_FILE) - sys.exit(3) + with open(DB_FILE) as file: + content = file.read() + + pattern = r"_REVISION_HEADS_MAP = {[^}]+\}" + match = re2.search(pattern, content) + if not match: + print( + f"_REVISION_HEADS_MAP not found in {DB_FILE}. If this has been removed intentionally, " + "please update scripts/ci/pre_commit/pre_commit_version_heads_map.py" + ) + sys.exit(1) + + existing_revision_heads_map = match.group(0) + rh_map = revision_heads_map() + updated_revision_heads_map = "_REVISION_HEADS_MAP = {\n" + for k, v in rh_map.items(): + updated_revision_heads_map += f' "{k}": "{v}",\n' + updated_revision_heads_map += "}" + if existing_revision_heads_map != updated_revision_heads_map: + new_content = content.replace(existing_revision_heads_map, updated_revision_heads_map) + + with open(DB_FILE, "w") as file: + file.write(new_content) + print("_REVISION_HEADS_MAP updated in db.py. Please commit the changes.") + sys.exit(1) diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index f08e2eac21ace..67eb96defc2f3 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -400,10 +400,11 @@ else fi if [[ ${ENABLE_TEST_COVERAGE:="false"} == "true" ]]; then + _suffix="$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8)" EXTRA_PYTEST_ARGS+=( "--cov=airflow" "--cov-config=pyproject.toml" - "--cov-report=xml:/files/coverage-${TEST_TYPE/\[*\]/}-${BACKEND}.xml" + "--cov-report=xml:/files/coverage-${TEST_TYPE/\[*\]/}-${BACKEND}-${_suffix}.xml" ) fi diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index 16663f47389c0..288248b4eca49 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -297,6 +297,12 @@ function install_all_providers_from_pypi_with_eager_upgrade() { local res for provider_package in ${ALL_PROVIDERS_PACKAGES} do + # Until we release "yandex" provider with protobuf support we need to remove it from the list of providers + # to install, because it is impossible to find common requirements for already released yandex provider + # and current airflow + if [[ ${provider_package} == "apache-airflow-providers-yandex" ]]; then + continue + fi echo -n "Checking if ${provider_package} is available in PyPI: " res=$(curl --head -s -o /dev/null -w "%{http_code}" "https://pypi.org/project/${provider_package}/") if [[ ${res} == "200" ]]; then @@ -306,7 +312,11 @@ function install_all_providers_from_pypi_with_eager_upgrade() { echo "${COLOR_YELLOW}Skipped${COLOR_RESET}" fi done + + echo "Installing provider packages: ${packages_to_install[*]}" + + # we add eager requirements to make sure to take into account limitations that will allow us to # install all providers. We install only those packages that are available in PyPI - we might # Have some new providers in the works and they might not yet be simply available in PyPI diff --git a/scripts/in_container/run_provider_yaml_files_check.py b/scripts/in_container/run_provider_yaml_files_check.py index 3b5b3211352cc..721117cafc34a 100755 --- a/scripts/in_container/run_provider_yaml_files_check.py +++ b/scripts/in_container/run_provider_yaml_files_check.py @@ -26,6 +26,7 @@ import platform import sys import textwrap +import warnings from collections import Counter from enum import Enum from typing import Any, Iterable @@ -37,17 +38,22 @@ from tabulate import tabulate from airflow.cli.commands.info_command import Architecture +from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers_manager import ProvidersManager # Those are deprecated modules that contain removed Hooks/Sensors/Operators that we left in the code # so that users can get a very specific error message when they try to use them. -EXCLUDED_MODULES = [ +DEPRECATED_MODULES = [ "airflow.providers.apache.hdfs.sensors.hdfs", "airflow.providers.apache.hdfs.hooks.hdfs", "airflow.providers.cncf.kubernetes.triggers.kubernetes_pod", + "airflow.providers.cncf.kubernetes.operators.kubernetes_pod", ] +KNOWN_DEPRECATED_CLASSES = [ + "airflow.providers.google.cloud.links.dataproc.DataprocLink", +] try: from yaml import CSafeLoader as SafeLoader @@ -70,6 +76,13 @@ errors: list[str] = [] console = Console(width=400, color_system="standard") +# you need to enable warnings for all deprecations - needed by importlib library to show deprecations +if os.environ.get("PYTHONWARNINGS") != "default": + console.print( + "[red]Error: PYTHONWARNINGS not set[/]\n" + "You must set `PYTHONWARNINGS=default` environment variable to run this script" + ) + sys.exit(1) suspended_providers: set[str] = set() suspended_logos: set[str] = set() @@ -135,7 +148,14 @@ def check_integration_duplicates(yaml_files: dict[str, dict]): sys.exit(3) -def assert_sets_equal(set1, set2, allow_extra_in_set2=False): +def assert_sets_equal( + set1: set[str], + set_name_1: str, + set2: set[str], + set_name_2: str, + allow_extra_in_set2=False, + extra_message: str = "", +): try: difference1 = set1.difference(set2) except TypeError as e: @@ -152,6 +172,8 @@ def assert_sets_equal(set1, set2, allow_extra_in_set2=False): if difference1 or (difference2 and not allow_extra_in_set2): lines = [] + lines.append(f" Left set:{set_name_1}") + lines.append(f" Right set:{set_name_2}") if difference1: lines.append(" -- Items in the left set but not the right:") for item in sorted(difference1): @@ -162,6 +184,8 @@ def assert_sets_equal(set1, set2, allow_extra_in_set2=False): lines.append(f" {item!r}") standard_msg = "\n".join(lines) + if extra_message: + standard_msg += f"\n{extra_message}" raise AssertionError(standard_msg) @@ -173,12 +197,37 @@ class ObjectType(Enum): def check_if_object_exist(object_name: str, resource_type: str, yaml_file_path: str, object_type: ObjectType): try: if object_type == ObjectType.CLASS: - module_name, object_name = object_name.rsplit(".", maxsplit=1) - the_class = getattr(importlib.import_module(module_name), object_name) + module_name, class_name = object_name.rsplit(".", maxsplit=1) + with warnings.catch_warnings(record=True) as w: + the_class = getattr(importlib.import_module(module_name), class_name) + for warn in w: + if warn.category == AirflowProviderDeprecationWarning: + if object_name in KNOWN_DEPRECATED_CLASSES: + console.print( + f"[yellow]The {object_name} class is deprecated and we know about it. " + f"It should be removed in the future." + ) + continue + errors.append( + f"The `{class_name}` class in {resource_type} list in {yaml_file_path} " + f"is deprecated with this message: '{warn.message}'.\n" + f"[yellow]How to fix it[/]: Please remove it from provider.yaml and replace with " + f"the new class." + ) if the_class and inspect.isclass(the_class): return elif object_type == ObjectType.MODULE: - module = importlib.import_module(object_name) + with warnings.catch_warnings(record=True) as w: + module = importlib.import_module(object_name) + for warn in w: + if warn.category == AirflowProviderDeprecationWarning: + errors.append( + f"The `{object_name}` module in {resource_type} list in {yaml_file_path} " + f"is deprecated with this message: '{warn.message}'.\n" + f"[yellow]How to fix it[/]: Please remove it from provider.yaml and replace it " + f"with the new module. If you see warnings in classes - fix the classes so that " + f"they are not raising Deprecation Warnings when module is imported." + ) if inspect.ismodule(module): return else: @@ -230,23 +279,32 @@ def parse_module_data(provider_data, resource_type, yaml_file_path): return expected_modules, provider_package, resource_data -def check_correctness_of_list_of_sensors_operators_hook_modules(yaml_files: dict[str, dict]): - print("Checking completeness of list of {sensors, hooks, operators, triggers}") - print(" -- {sensors, hooks, operators, triggers} - Expected modules (left) : Current modules (right)") +def check_correctness_of_list_of_sensors_operators_hook_trigger_modules(yaml_files: dict[str, dict]): + print(" -- Checking completeness of list of {sensors, hooks, operators, triggers}") for (yaml_file_path, provider_data), resource_type in itertools.product( yaml_files.items(), ["sensors", "operators", "hooks", "triggers"] ): expected_modules, provider_package, resource_data = parse_module_data( provider_data, resource_type, yaml_file_path ) - expected_modules = {module for module in expected_modules if module not in EXCLUDED_MODULES} + expected_modules = {module for module in expected_modules if module not in DEPRECATED_MODULES} current_modules = {str(i) for r in resource_data for i in r.get("python-modules", [])} check_if_objects_exist_and_belong_to_package( current_modules, provider_package, yaml_file_path, resource_type, ObjectType.MODULE ) try: - assert_sets_equal(set(expected_modules), set(current_modules)) + package_name = os.fspath(ROOT_DIR.joinpath(yaml_file_path).parent.relative_to(ROOT_DIR)).replace( + "/", "." + ) + assert_sets_equal( + set(expected_modules), + f"Found list of {resource_type} modules in provider package: {package_name}", + set(current_modules), + f"Currently configured list of {resource_type} modules in {yaml_file_path}", + extra_message="[yellow]If there are deprecated modules in the list, please add them to " + f"DEPRECATED_MODULES in {pathlib.Path(__file__).relative_to(ROOT_DIR)}[/]", + ) except AssertionError as ex: nested_error = textwrap.indent(str(ex), " ") errors.append( @@ -275,19 +333,27 @@ def check_completeness_of_list_of_transfers(yaml_files: dict[str, dict]): print("Checking completeness of list of transfers") resource_type = "transfers" - print(" -- Expected transfers modules(Left): Current transfers Modules(Right)") + print(" -- Checking transfers modules") for yaml_file_path, provider_data in yaml_files.items(): expected_modules, provider_package, resource_data = parse_module_data( provider_data, resource_type, yaml_file_path ) - expected_modules = {module for module in expected_modules if module not in EXCLUDED_MODULES} + expected_modules = {module for module in expected_modules if module not in DEPRECATED_MODULES} current_modules = {r.get("python-module") for r in resource_data} check_if_objects_exist_and_belong_to_package( current_modules, provider_package, yaml_file_path, resource_type, ObjectType.MODULE ) try: - assert_sets_equal(set(expected_modules), set(current_modules)) + package_name = os.fspath(ROOT_DIR.joinpath(yaml_file_path).parent.relative_to(ROOT_DIR)).replace( + "/", "." + ) + assert_sets_equal( + set(expected_modules), + f"Found list of transfer modules in provider package: {package_name}", + set(current_modules), + f"Currently configured list of transfer modules in {yaml_file_path}", + ) except AssertionError as ex: nested_error = textwrap.indent(str(ex), " ") errors.append( @@ -296,13 +362,14 @@ def check_completeness_of_list_of_transfers(yaml_files: dict[str, dict]): ) -def check_hook_connection_classes(yaml_files: dict[str, dict]): +def check_hook_class_names(yaml_files: dict[str, dict]): print("Checking connection classes belong to package, exist and are classes") - resource_type = "hook-class-names" + resource_type = "connection-types" for yaml_file_path, provider_data in yaml_files.items(): provider_package = pathlib.Path(yaml_file_path).parent.as_posix().replace("/", ".") - hook_class_names = provider_data.get(resource_type) - if hook_class_names: + connection_types = provider_data.get(resource_type) + if connection_types: + hook_class_names = {connection_type["hook-class-name"] for connection_type in connection_types} check_if_objects_exist_and_belong_to_package( hook_class_names, provider_package, yaml_file_path, resource_type, ObjectType.CLASS ) @@ -336,6 +403,18 @@ def check_extra_link_classes(yaml_files: dict[str, dict]): ) +def check_notification_classes(yaml_files: dict[str, dict]): + print("Checking notifications belong to package, exist and are classes") + resource_type = "notifications" + for yaml_file_path, provider_data in yaml_files.items(): + provider_package = pathlib.Path(yaml_file_path).parent.as_posix().replace("/", ".") + notifications = provider_data.get(resource_type) + if notifications: + check_if_objects_exist_and_belong_to_package( + notifications, provider_package, yaml_file_path, resource_type, ObjectType.CLASS + ) + + def check_duplicates_in_list_of_transfers(yaml_files: dict[str, dict]): print("Checking for duplicates in list of transfers") errors = [] @@ -434,11 +513,20 @@ def check_doc_files(yaml_files: dict[str, dict]): } try: - print(" -- Checking document urls: expected (left), current (right)") - assert_sets_equal(set(expected_doc_urls), set(current_doc_urls)) - - print(" -- Checking logo urls: expected (left), current (right)") - assert_sets_equal(set(expected_logo_urls), set(current_logo_urls)) + print(" -- Checking document urls") + assert_sets_equal( + set(expected_doc_urls), + "Document urls found in airflow/docs", + set(current_doc_urls), + "Document urls configured in provider.yaml files", + ) + print(" -- Checking logo urls") + assert_sets_equal( + set(expected_logo_urls), + "Logo urls found in airflow/docs/integration-logos", + set(current_logo_urls), + "Logo urls configured in provider.yaml files", + ) except AssertionError as ex: print(ex) sys.exit(1) @@ -464,12 +552,15 @@ def check_providers_are_mentioned_in_issue_template(yaml_files: dict[str, dict]) issue_template = yaml.safe_load(issue_file) all_mentioned_providers = [match.value for match in jsonpath_expr.find(issue_template)] try: - print( - f" -- Checking providers: present in code (left), " - f"mentioned in {PROVIDER_ISSUE_TEMPLATE_PATH} (right)" - ) + print(f" -- Checking providers are mentioned in {PROVIDER_ISSUE_TEMPLATE_PATH}") # in case of suspended providers, we still want to have them in the issue template - assert_sets_equal(set(short_provider_names), set(all_mentioned_providers), allow_extra_in_set2=True) + assert_sets_equal( + set(short_provider_names), + "Provider names found in provider.yaml files", + set(all_mentioned_providers), + f"Provider names mentioned in {PROVIDER_ISSUE_TEMPLATE_PATH}", + allow_extra_in_set2=True, + ) except AssertionError as ex: print(ex) sys.exit(1) @@ -508,10 +599,11 @@ def check_providers_have_all_documentation_files(yaml_files: dict[str, dict]): check_completeness_of_list_of_transfers(all_parsed_yaml_files) check_duplicates_in_list_of_transfers(all_parsed_yaml_files) - check_hook_connection_classes(all_parsed_yaml_files) + check_hook_class_names(all_parsed_yaml_files) check_plugin_classes(all_parsed_yaml_files) check_extra_link_classes(all_parsed_yaml_files) - check_correctness_of_list_of_sensors_operators_hook_modules(all_parsed_yaml_files) + check_correctness_of_list_of_sensors_operators_hook_trigger_modules(all_parsed_yaml_files) + check_notification_classes(all_parsed_yaml_files) check_unique_provider_name(all_parsed_yaml_files) check_providers_have_all_documentation_files(all_parsed_yaml_files) diff --git a/setup.py b/setup.py index 9ded977624d14..2758cf0fda26b 100644 --- a/setup.py +++ b/setup.py @@ -868,7 +868,7 @@ def replace_extra_dependencies_with_provider_packages(extra: str, providers: lis So transitively 'salesforce' extra has all the dependencies it needs and in case the provider changes its dependencies, they will transitively change as well. - In the constraint mechanism we save both - provider versions and it's dependencies + In the constraint mechanism we save both - provider versions and its dependencies version, which means that installation using constraints is repeatable. For K8s and Celery which are both "Core executors" and "Providers" we have to diff --git a/tests/auth/managers/fab/cli_commands/__init__.py b/tests/auth/managers/fab/cli_commands/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/auth/managers/fab/cli_commands/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/cli/commands/test_role_command.py b/tests/auth/managers/fab/cli_commands/test_role_command.py similarity index 96% rename from tests/cli/commands/test_role_command.py rename to tests/auth/managers/fab/cli_commands/test_role_command.py index 544d8e9560627..5259ca8482fdb 100644 --- a/tests/cli/commands/test_role_command.py +++ b/tests/auth/managers/fab/cli_commands/test_role_command.py @@ -23,10 +23,11 @@ import pytest +from airflow.auth.managers.fab.cli_commands import role_command +from airflow.auth.managers.fab.cli_commands.utils import get_application_builder from airflow.auth.managers.fab.models import Role -from airflow.cli.commands import role_command +from airflow.cli import cli_parser from airflow.security import permissions -from airflow.utils.cli_app_builder import get_application_builder TEST_USER1_EMAIL = "test-user1@example.com" TEST_USER2_EMAIL = "test-user2@example.com" @@ -34,9 +35,8 @@ class TestCliRoles: @pytest.fixture(autouse=True) - def _set_attrs(self, dagbag, parser): - self.dagbag = dagbag - self.parser = parser + def _set_attrs(self): + self.parser = cli_parser.get_parser() with get_application_builder() as appbuilder: self.appbuilder = appbuilder self.clear_roles_and_roles() diff --git a/tests/cli/commands/test_sync_perm_command.py b/tests/auth/managers/fab/cli_commands/test_sync_perm_command.py similarity index 89% rename from tests/cli/commands/test_sync_perm_command.py rename to tests/auth/managers/fab/cli_commands/test_sync_perm_command.py index d55d4e372498a..d59fb34a8ced6 100644 --- a/tests/cli/commands/test_sync_perm_command.py +++ b/tests/auth/managers/fab/cli_commands/test_sync_perm_command.py @@ -19,8 +19,8 @@ from unittest import mock +from airflow.auth.managers.fab.cli_commands import sync_perm_command from airflow.cli import cli_parser -from airflow.cli.commands import sync_perm_command class TestCliSyncPerm: @@ -28,7 +28,7 @@ class TestCliSyncPerm: def setup_class(cls): cls.parser = cli_parser.get_parser() - @mock.patch("airflow.utils.cli_app_builder.get_application_builder") + @mock.patch("airflow.auth.managers.fab.cli_commands.utils.get_application_builder") def test_cli_sync_perm(self, mock_get_application_builder): mock_appbuilder = mock.MagicMock() mock_get_application_builder.return_value.__enter__.return_value = mock_appbuilder @@ -40,7 +40,7 @@ def test_cli_sync_perm(self, mock_get_application_builder): mock_appbuilder.sm.sync_roles.assert_called_once_with() mock_appbuilder.sm.create_dag_specific_permissions.assert_not_called() - @mock.patch("airflow.utils.cli_app_builder.get_application_builder") + @mock.patch("airflow.auth.managers.fab.cli_commands.utils.get_application_builder") def test_cli_sync_perm_include_dags(self, mock_get_application_builder): mock_appbuilder = mock.MagicMock() mock_get_application_builder.return_value.__enter__.return_value = mock_appbuilder diff --git a/tests/cli/commands/test_user_command.py b/tests/auth/managers/fab/cli_commands/test_user_command.py similarity index 98% rename from tests/cli/commands/test_user_command.py rename to tests/auth/managers/fab/cli_commands/test_user_command.py index 32f2e5db8840c..4daf0e6e36d05 100644 --- a/tests/cli/commands/test_user_command.py +++ b/tests/auth/managers/fab/cli_commands/test_user_command.py @@ -25,7 +25,8 @@ import pytest -from airflow.cli.commands import user_command +from airflow.auth.managers.fab.cli_commands import user_command +from airflow.cli import cli_parser from tests.test_utils.api_connexion_utils import delete_users TEST_USER1_EMAIL = "test-user1@example.com" @@ -44,10 +45,9 @@ def _does_user_belong_to_role(appbuilder, email, rolename): class TestCliUsers: @pytest.fixture(autouse=True) - def _set_attrs(self, app, dagbag, parser): + def _set_attrs(self, app): self.app = app - self.dagbag = dagbag - self.parser = parser + self.parser = cli_parser.get_parser() self.appbuilder = self.app.appbuilder delete_users(app) yield diff --git a/tests/auth/managers/fab/test_fab_auth_manager.py b/tests/auth/managers/fab/test_fab_auth_manager.py index cb4ef8ebc2113..66c279510e55e 100644 --- a/tests/auth/managers/fab/test_fab_auth_manager.py +++ b/tests/auth/managers/fab/test_fab_auth_manager.py @@ -85,14 +85,14 @@ def test_get_url_login_when_auth_view_not_defined(self, auth_manager): with pytest.raises(AirflowException, match="`auth_view` not defined in the security manager."): auth_manager.get_url_login() - @mock.patch("airflow.auth.managers.fab.fab_auth_manager.url_for") + @mock.patch.object(FabAuthManager, "url_for") def test_get_url_login(self, mock_url_for, auth_manager): auth_manager.security_manager.auth_view = Mock() auth_manager.security_manager.auth_view.endpoint = "test_endpoint" auth_manager.get_url_login() mock_url_for.assert_called_once_with("test_endpoint.login") - @mock.patch("airflow.auth.managers.fab.fab_auth_manager.url_for") + @mock.patch.object(FabAuthManager, "url_for") def test_get_url_login_with_next(self, mock_url_for, auth_manager): auth_manager.security_manager.auth_view = Mock() auth_manager.security_manager.auth_view.endpoint = "test_endpoint" @@ -103,7 +103,7 @@ def test_get_url_logout_when_auth_view_not_defined(self, auth_manager): with pytest.raises(AirflowException, match="`auth_view` not defined in the security manager."): auth_manager.get_url_logout() - @mock.patch("airflow.auth.managers.fab.fab_auth_manager.url_for") + @mock.patch.object(FabAuthManager, "url_for") def test_get_url_logout(self, mock_url_for, auth_manager): auth_manager.security_manager.auth_view = Mock() auth_manager.security_manager.auth_view.endpoint = "test_endpoint" @@ -113,7 +113,7 @@ def test_get_url_logout(self, mock_url_for, auth_manager): def test_get_url_user_profile_when_auth_view_not_defined(self, auth_manager): assert auth_manager.get_url_user_profile() is None - @mock.patch("airflow.auth.managers.fab.fab_auth_manager.url_for") + @mock.patch.object(FabAuthManager, "url_for") def test_get_url_user_profile(self, mock_url_for, auth_manager): expected_url = "test_url" mock_url_for.return_value = expected_url diff --git a/tests/dags_corrupted/test_impersonation_custom.py b/tests/dags_corrupted/test_impersonation_custom.py index 7fc9980d6cc63..d52342c65868f 100644 --- a/tests/dags_corrupted/test_impersonation_custom.py +++ b/tests/dags_corrupted/test_impersonation_custom.py @@ -40,7 +40,7 @@ def print_today(): date_time = FakeDatetime.utcnow() - print(f"Today is {date_time.strftime('%Y-%m-%d')}") + print(f"Today is {date_time:%Y-%m-%d}") def check_hive_conf(): diff --git a/tests/jobs/test_triggerer_job_logging.py b/tests/jobs/test_triggerer_job_logging.py index d11f4bb3686f7..52ff22c2f525c 100644 --- a/tests/jobs/test_triggerer_job_logging.py +++ b/tests/jobs/test_triggerer_job_logging.py @@ -171,7 +171,7 @@ def test_configure_trigger_log_handler_not_file_task_handler(cfg, cls, msg): """ No root handler configured. When non FileTaskHandler is configured, don't modify. - When when an incompatible subclass of FileTaskHandler is configured, don't modify. + When an incompatible subclass of FileTaskHandler is configured, don't modify. """ # reset handlers root_logger = logging.getLogger() diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index 87835c8adcc4f..342d08785197e 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -3822,7 +3822,7 @@ def test_clear_upstream_not_your_setup(self): the setup (and its teardown) will be cleared even though strictly speaking you don't "require" it since, depending on speed of execution, it might be torn down by t1 before / while w2 runs. It just gets cleared by virtue of it being upstream, and - that's what you requested. And it's teardown gets cleared too. But w1 doesn't. + that's what you requested. And its teardown gets cleared too. But w1 doesn't. """ with DAG(dag_id="test_dag", start_date=pendulum.now()) as dag: s1, w1, w2, t1 = self.make_tasks(dag, "s1, w1, w2, t1") diff --git a/tests/providers/amazon/aws/operators/test_appflow.py b/tests/providers/amazon/aws/operators/test_appflow.py index 2308810662f71..ce36faad5f132 100644 --- a/tests/providers/amazon/aws/operators/test_appflow.py +++ b/tests/providers/amazon/aws/operators/test_appflow.py @@ -114,9 +114,8 @@ def run_assertions_base(appflow_conn, tasks): def test_run(appflow_conn, ctx, waiter_mock): operator = AppflowRunOperator(**DUMP_COMMON_ARGS) operator.execute(ctx) # type: ignore - appflow_conn.describe_flow.assert_called_once_with(flowName=FLOW_NAME) - appflow_conn.describe_flow_execution_records.assert_called_once() appflow_conn.start_flow.assert_called_once_with(flowName=FLOW_NAME) + appflow_conn.describe_flow_execution_records.assert_called_once() def test_run_full(appflow_conn, ctx, waiter_mock): diff --git a/tests/providers/amazon/aws/utils/eks_test_utils.py b/tests/providers/amazon/aws/utils/eks_test_utils.py index ffab5d7fbfc38..d3f0c715ce09c 100644 --- a/tests/providers/amazon/aws/utils/eks_test_utils.py +++ b/tests/providers/amazon/aws/utils/eks_test_utils.py @@ -235,7 +235,7 @@ def convert_keys(original: dict) -> dict: def iso_date(input_datetime: datetime.datetime) -> str: - return input_datetime.strftime("%Y-%m-%dT%H:%M:%S") + "Z" + return f"{input_datetime:%Y-%m-%dT%H:%M:%S}Z" def generate_dict(prefix, count) -> dict: diff --git a/tests/providers/amazon/aws/waiters/test_custom_waiters.py b/tests/providers/amazon/aws/waiters/test_custom_waiters.py index 21c051f3b4901..09b6742f7ae32 100644 --- a/tests/providers/amazon/aws/waiters/test_custom_waiters.py +++ b/tests/providers/amazon/aws/waiters/test_custom_waiters.py @@ -27,6 +27,7 @@ from moto import mock_eks from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook +from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook from airflow.providers.amazon.aws.hooks.dynamodb import DynamoDBHook from airflow.providers.amazon.aws.hooks.ecs import EcsClusterStates, EcsHook, EcsTaskDefinitionStates from airflow.providers.amazon.aws.hooks.eks import EksHook @@ -295,3 +296,58 @@ def test_export_table_to_point_in_time_failed(self, mock_describe_export): ExportArn="LoremIpsumissimplydummytextoftheprintingandtypesettingindustry", WaiterConfig={"Delay": 0.01, "MaxAttempts": 3}, ) + + +class TestCustomBatchServiceWaiters: + """Test waiters from ``amazon/aws/waiters/batch.json``.""" + + JOB_ID = "test_job_id" + + @pytest.fixture(autouse=True) + def setup_test_cases(self, monkeypatch): + self.client = boto3.client("batch", region_name="eu-west-3") + monkeypatch.setattr(BatchClientHook, "conn", self.client) + + @pytest.fixture + def mock_describe_jobs(self): + """Mock ``BatchClientHook.Client.describe_jobs`` method.""" + with mock.patch.object(self.client, "describe_jobs") as m: + yield m + + def test_service_waiters(self): + hook_waiters = BatchClientHook(aws_conn_id=None).list_waiters() + assert "batch_job_complete" in hook_waiters + + @staticmethod + def describe_jobs(status: str): + """ + Helper function for generate minimal DescribeJobs response for a single job. + https://docs.aws.amazon.com/batch/latest/APIReference/API_DescribeJobs.html + """ + return { + "jobs": [ + { + "status": status, + }, + ], + } + + def test_job_succeeded(self, mock_describe_jobs): + """Test job succeeded""" + mock_describe_jobs.side_effect = [ + self.describe_jobs(BatchClientHook.RUNNING_STATE), + self.describe_jobs(BatchClientHook.SUCCESS_STATE), + ] + waiter = BatchClientHook(aws_conn_id=None).get_waiter("batch_job_complete") + waiter.wait(jobs=[self.JOB_ID], WaiterConfig={"Delay": 0.01, "MaxAttempts": 2}) + + def test_job_failed(self, mock_describe_jobs): + """Test job failed""" + mock_describe_jobs.side_effect = [ + self.describe_jobs(BatchClientHook.RUNNING_STATE), + self.describe_jobs(BatchClientHook.FAILURE_STATE), + ] + waiter = BatchClientHook(aws_conn_id=None).get_waiter("batch_job_complete") + + with pytest.raises(WaiterError, match="Waiter encountered a terminal failure state"): + waiter.wait(jobs=[self.JOB_ID], WaiterConfig={"Delay": 0.01, "MaxAttempts": 2}) diff --git a/tests/providers/apache/kylin/operators/test_kylin_cube.py b/tests/providers/apache/kylin/operators/test_kylin_cube.py index baa61ad80fc4a..58c1832a0cdd6 100644 --- a/tests/providers/apache/kylin/operators/test_kylin_cube.py +++ b/tests/providers/apache/kylin/operators/test_kylin_cube.py @@ -37,8 +37,8 @@ class TestKylinCubeOperator: "project": "learn_kylin", "cube": "kylin_sales_cube", "command": "build", - "start_time": datetime(2012, 1, 2, 0, 0).strftime("%s") + "000", - "end_time": datetime(2012, 1, 3, 0, 0).strftime("%s") + "000", + "start_time": str(int(datetime(2012, 1, 2, 0, 0).timestamp() * 1000)), + "end_time": str(int(datetime(2012, 1, 3, 0, 0).timestamp() * 1000)), } cube_command = [ "fullbuild", diff --git a/tests/providers/cncf/kubernetes/operators/test_pod.py b/tests/providers/cncf/kubernetes/operators/test_pod.py index 9eb3e2bcc108a..daf0ea90dd239 100644 --- a/tests/providers/cncf/kubernetes/operators/test_pod.py +++ b/tests/providers/cncf/kubernetes/operators/test_pod.py @@ -614,6 +614,88 @@ def test_termination_message_policy_default_value_correctly_set(self): pod = k.build_pod_request_obj(create_context(k)) assert pod.spec.containers[0].termination_message_policy == "File" + @pytest.mark.parametrize( + "task_kwargs, base_container_fail, expect_to_delete_pod", + [ + ({"on_finish_action": "delete_pod"}, True, True), + ({"on_finish_action": "delete_pod"}, False, True), + ({"on_finish_action": "keep_pod"}, False, False), + ({"on_finish_action": "keep_pod"}, True, False), + ({"on_finish_action": "delete_succeeded_pod"}, False, True), + ({"on_finish_action": "delete_succeeded_pod"}, True, False), + ], + ) + @patch(f"{KPO_MODULE}.KubernetesPodOperator.kill_istio_sidecar") + @patch(f"{KPO_MODULE}.KubernetesPodOperator.is_istio_enabled") + @patch(f"{POD_MANAGER_CLASS}.await_pod_completion") + @patch(f"{KPO_MODULE}.KubernetesPodOperator.find_pod") + def test_pod_with_istio_delete_after_await_container_error( + self, + find_pod_mock, + await_pod_completion_mock, + is_istio_enabled_mock, + kill_istio_sidecar_mock, + task_kwargs, + base_container_fail, + expect_to_delete_pod, + ): + """ + When KPO fails unexpectedly during await_container, we should still try to delete the pod, + and the pod we try to delete should be the one returned from find_pod earlier. + """ + sidecar = MagicMock() + sidecar.name = "istio-proxy" + sidecar.namespace = "default" + sidecar.image = "istio/proxyv2:1.18.2" + sidecar.args = [] + sidecar.state.running = True + + cont_status_1 = MagicMock() + cont_status_1.name = "base" + cont_status_1.state.running = False + cont_status_1.state.terminated.exit_code = 0 + if base_container_fail: + cont_status_1.state.terminated.exit_code = 1 + cont_status_1.state.terminated.message = "my-failure" + + cont_status_2 = MagicMock() + cont_status_2.name = "istio-proxy" + cont_status_2.state.running = True + cont_status_2.state.terminated = False + + await_pod_completion_mock.return_value.spec.containers = [sidecar] + await_pod_completion_mock.return_value.status.phase = "Running" + await_pod_completion_mock.return_value.status.container_statuses = [cont_status_1, cont_status_2] + await_pod_completion_mock.return_value.metadata.name = "pod-with-istio-sidecar" + await_pod_completion_mock.return_value.metadata.namespace = "default" + + find_pod_mock.return_value.spec.containers = [sidecar] + find_pod_mock.return_value.status.phase = "Running" + find_pod_mock.return_value.status.container_statuses = [cont_status_1, cont_status_2] + find_pod_mock.return_value.metadata.name = "pod-with-istio-sidecar" + find_pod_mock.return_value.metadata.namespace = "default" + + k = KubernetesPodOperator(task_id="task", **task_kwargs) + + context = create_context(k) + context["ti"].xcom_push = MagicMock() + if base_container_fail: + self.await_pod_mock.side_effect = AirflowException("fake failure") + with pytest.raises(AirflowException, match="my-failure"): + k.execute(context=context) + else: + k.execute(context=context) + + assert is_istio_enabled_mock(find_pod_mock.return_value) + if task_kwargs["on_finish_action"] == "delete_pod": + kill_istio_sidecar_mock.assert_called_with(await_pod_completion_mock.return_value) + elif expect_to_delete_pod and base_container_fail: + kill_istio_sidecar_mock.assert_called_with(find_pod_mock.return_value) + elif expect_to_delete_pod and not base_container_fail: + kill_istio_sidecar_mock.assert_called_with(await_pod_completion_mock.return_value) + else: + kill_istio_sidecar_mock.assert_not_called() + @pytest.mark.parametrize( "task_kwargs, should_be_deleted", [ diff --git a/tests/providers/cncf/kubernetes/utils/test_pod_manager.py b/tests/providers/cncf/kubernetes/utils/test_pod_manager.py index c915a03a4aa08..c6c42105b367b 100644 --- a/tests/providers/cncf/kubernetes/utils/test_pod_manager.py +++ b/tests/providers/cncf/kubernetes/utils/test_pod_manager.py @@ -19,6 +19,7 @@ import logging from datetime import datetime from json.decoder import JSONDecodeError +from types import SimpleNamespace from unittest import mock from unittest.mock import MagicMock @@ -36,6 +37,7 @@ PodManager, PodPhase, container_is_running, + container_is_succeeded, container_is_terminated, ) from airflow.utils.timezone import utc @@ -693,3 +695,60 @@ def test_read_pod( # second read with time_machine.travel(mock_read_pod_at_1): assert consumer.read_pod() == expected_read_pods[1] + + +def params_for_test_container_is_succeeded(): + """The `container_is_succeeded` method is designed to handle an assortment of bad objects + returned from `read_pod`. E.g. a None object, an object `e` such that `e.status` is None, + an object `e` such that `e.status.container_statuses` is None, and so on. This function + emits params used in `test_container_is_succeeded` to verify this behavior. + We create mock classes not derived from MagicMock because with an instance `e` of MagicMock, + tests like `e.hello is not None` are always True. + """ + + class RemotePodMock: + pass + + class ContainerStatusMock: + def __init__(self, name): + self.name = name + + def remote_pod(succeeded=None, not_succeeded=None): + e = RemotePodMock() + e.status = RemotePodMock() + e.status.container_statuses = [] + for r in not_succeeded or []: + e.status.container_statuses.append(container(r, False)) + for r in succeeded or []: + e.status.container_statuses.append(container(r, True)) + return e + + def container(name, succeeded): + c = ContainerStatusMock(name) + c.state = RemotePodMock() + c.state.terminated = SimpleNamespace(**{"exit_code": 0}) if succeeded else None + return c + + pod_mock_list = [] + pod_mock_list.append(pytest.param(None, False, id="None remote_pod")) + p = RemotePodMock() + p.status = None + pod_mock_list.append(pytest.param(p, False, id="None remote_pod.status")) + p = RemotePodMock() + p.status = RemotePodMock() + p.status.container_statuses = [] + pod_mock_list.append(pytest.param(p, False, id="empty remote_pod.status.container_statuses")) + pod_mock_list.append(pytest.param(remote_pod(), False, id="filter empty")) + pod_mock_list.append(pytest.param(remote_pod(None, ["base"]), False, id="filter 0 succeeded")) + pod_mock_list.append(pytest.param(remote_pod(["hello"], ["base"]), False, id="filter 1 not succeeded")) + pod_mock_list.append(pytest.param(remote_pod(["base"], ["hello"]), True, id="filter 1 succeeded")) + return pod_mock_list + + +@pytest.mark.parametrize("remote_pod, result", params_for_test_container_is_succeeded()) +def test_container_is_succeeded(remote_pod, result): + """The `container_is_succeeded` function is designed to handle an assortment of bad objects + returned from `read_pod`. E.g. a None object, an object `e` such that `e.status` is None, + an object `e` such that `e.status.container_statuses` is None, and so on. This test + verifies the expected behavior.""" + assert container_is_succeeded(remote_pod, "base") is result diff --git a/tests/providers/common/sql/hooks/test_sql.py b/tests/providers/common/sql/hooks/test_sql.py index 72684e5e00b8e..d866ddce0ec4c 100644 --- a/tests/providers/common/sql/hooks/test_sql.py +++ b/tests/providers/common/sql/hooks/test_sql.py @@ -156,7 +156,7 @@ def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: ([[1, 2], [11, 12]], [[3, 4], [13, 14]]), [[("id2",), ("value2",)]], [[3, 4], [13, 14]], - id="The return_last set set on multiple queries in list", + id="The return_last set on multiple queries in list", ), pytest.param( False, diff --git a/tests/providers/databricks/hooks/test_databricks_sql.py b/tests/providers/databricks/hooks/test_databricks_sql.py index ecc0385278557..ae2c07773cf5b 100644 --- a/tests/providers/databricks/hooks/test_databricks_sql.py +++ b/tests/providers/databricks/hooks/test_databricks_sql.py @@ -156,7 +156,7 @@ def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: ([[1, 2], [11, 12]], [[3, 4], [13, 14]]), [[("id2",), ("value2",)]], [[3, 4], [13, 14]], - id="The return_last set set on multiple queries in list", + id="The return_last set on multiple queries in list", ), pytest.param( False, diff --git a/tests/providers/exasol/hooks/test_sql.py b/tests/providers/exasol/hooks/test_sql.py index 465edc61314b6..6716e91708a40 100644 --- a/tests/providers/exasol/hooks/test_sql.py +++ b/tests/providers/exasol/hooks/test_sql.py @@ -218,7 +218,7 @@ def get_columns(fields: list[str]) -> dict[str, dict[str, Any]]: ] ], [[3, 4], [13, 14]], - id="The return_last set set on multiple queries in list", + id="The return_last set on multiple queries in list", ), pytest.param( False, diff --git a/tests/providers/http/sensors/test_http.py b/tests/providers/http/sensors/test_http.py index da070d91c2a93..b2fe5921a0a84 100644 --- a/tests/providers/http/sensors/test_http.py +++ b/tests/providers/http/sensors/test_http.py @@ -244,9 +244,7 @@ def test_sensor(self): endpoint="/search", request_params={"client": "ubuntu", "q": "airflow", "date": "{{ds}}"}, headers={}, - response_check=lambda response: ( - "apache/airflow/" + DEFAULT_DATE.strftime("%Y-%m-%d") in response.text - ), + response_check=lambda response: f"apache/airflow/{DEFAULT_DATE:%Y-%m-%d}" in response.text, poke_interval=5, timeout=15, dag=self.dag, diff --git a/tests/providers/microsoft/azure/hooks/test_adx.py b/tests/providers/microsoft/azure/hooks/test_adx.py index dd4787eb063e3..ad79d3e0c6484 100644 --- a/tests/providers/microsoft/azure/hooks/test_adx.py +++ b/tests/providers/microsoft/azure/hooks/test_adx.py @@ -23,7 +23,7 @@ from unittest.mock import patch import pytest -from azure.kusto.data.request import ClientRequestProperties, KustoClient, KustoConnectionStringBuilder +from azure.kusto.data import ClientRequestProperties, KustoClient, KustoConnectionStringBuilder from pytest import param from airflow.exceptions import AirflowException @@ -225,21 +225,23 @@ def test_get_ui_field_behaviour_placeholders(self): param("a://usr:pw@host?tenant=my-tenant&auth_method=AAD_APP", id="no-prefix"), ], ) - @patch("airflow.providers.microsoft.azure.hooks.adx.KustoConnectionStringBuilder") - def test_backcompat_prefix_works(self, mock_client, uri): - mock_with = mock_client.with_aad_application_key_authentication + def test_backcompat_prefix_works(self, uri): with patch.dict(os.environ, AIRFLOW_CONN_MY_CONN=uri): - AzureDataExplorerHook(azure_data_explorer_conn_id="my_conn") # get_conn is called in init - mock_with.assert_called_once_with("host", "usr", "pw", "my-tenant") + hook = AzureDataExplorerHook(azure_data_explorer_conn_id="my_conn") # get_conn is called in init + assert hook.connection._kcsb.data_source == "host" + assert hook.connection._kcsb.application_client_id == "usr" + assert hook.connection._kcsb.application_key == "pw" + assert hook.connection._kcsb.authority_id == "my-tenant" - @patch("airflow.providers.microsoft.azure.hooks.adx.KustoConnectionStringBuilder") - def test_backcompat_prefix_both_causes_warning(self, mock_client): - mock_with = mock_client.with_aad_application_key_authentication + def test_backcompat_prefix_both_causes_warning(self): with patch.dict( in_dict=os.environ, AIRFLOW_CONN_MY_CONN="a://usr:pw@host?tenant=my-tenant&auth_method=AAD_APP" "&extra__azure_data_explorer__auth_method=AAD_APP", ): with pytest.warns(Warning, match="Using value for `auth_method`"): - AzureDataExplorerHook(azure_data_explorer_conn_id="my_conn") # get_conn is called in init - mock_with.assert_called_once_with("host", "usr", "pw", "my-tenant") + hook = AzureDataExplorerHook(azure_data_explorer_conn_id="my_conn") + assert hook.connection._kcsb.data_source == "host" + assert hook.connection._kcsb.application_client_id == "usr" + assert hook.connection._kcsb.application_key == "pw" + assert hook.connection._kcsb.authority_id == "my-tenant" diff --git a/tests/providers/snowflake/hooks/test_sql.py b/tests/providers/snowflake/hooks/test_sql.py index 629cbadfdce85..b2143a1e1f0a9 100644 --- a/tests/providers/snowflake/hooks/test_sql.py +++ b/tests/providers/snowflake/hooks/test_sql.py @@ -151,7 +151,7 @@ def get_cursor_descriptions(fields: list[str]) -> list[tuple[str]]: [[("id2",), ("value2",)]], [[3, 4], [13, 14]], False, - id="The return_last set set on multiple queries in list", + id="The return_last set on multiple queries in list", ), pytest.param( False, diff --git a/tests/providers/yandex/operators/test_yandexcloud_dataproc.py b/tests/providers/yandex/operators/test_yandexcloud_dataproc.py index 8ffe2050a037c..879645daf60cb 100644 --- a/tests/providers/yandex/operators/test_yandexcloud_dataproc.py +++ b/tests/providers/yandex/operators/test_yandexcloud_dataproc.py @@ -130,6 +130,7 @@ def test_create_cluster(self, create_cluster_mock, *_): enable_ui_proxy=False, host_group_ids=None, security_group_ids=None, + labels=None, initialization_actions=None, ) context["task_instance"].xcom_push.assert_has_calls( diff --git a/tests/system/providers/amazon/CONTRIBUTING.md b/tests/system/providers/amazon/CONTRIBUTING.md index a06163ad58b35..b9aaaa66915c2 100644 --- a/tests/system/providers/amazon/CONTRIBUTING.md +++ b/tests/system/providers/amazon/CONTRIBUTING.md @@ -151,7 +151,7 @@ descriptive. For example, `example_redshift_cluster.py` and `example_redshift_s ## Environment ID -`ENV_ID` should be set via the the `SystemTestContextBuilder` and not manually. This +`ENV_ID` should be set via the `SystemTestContextBuilder` and not manually. This value should be used as part of any value which is test-specific such as the name of an S3 bucket being created. For example, `BUCKET_NAME = f'{ENV_ID}-test-bucket'`. diff --git a/tests/system/providers/amazon/aws/example_appflow.py b/tests/system/providers/amazon/aws/example_appflow.py index 4469c0290b539..8a6458aa56dba 100644 --- a/tests/system/providers/amazon/aws/example_appflow.py +++ b/tests/system/providers/amazon/aws/example_appflow.py @@ -27,7 +27,6 @@ AppflowRunBeforeOperator, AppflowRunDailyOperator, AppflowRunFullOperator, - AppflowRunOperator, ) from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder @@ -48,14 +47,6 @@ source_name = "salesforce" flow_name = f"{env_id}-salesforce-campaign" - # [START howto_operator_appflow_run] - campaign_dump = AppflowRunOperator( - task_id="campaign_dump", - source=source_name, - flow_name=flow_name, - ) - # [END howto_operator_appflow_run] - # [START howto_operator_appflow_run_full] campaign_dump_full = AppflowRunFullOperator( task_id="campaign_dump_full", @@ -111,7 +102,6 @@ # TEST SETUP test_context, # TEST BODY - campaign_dump, campaign_dump_full, campaign_dump_daily, campaign_dump_before, diff --git a/tests/system/providers/amazon/aws/example_appflow_run.py b/tests/system/providers/amazon/aws/example_appflow_run.py new file mode 100644 index 0000000000000..dacc549815b78 --- /dev/null +++ b/tests/system/providers/amazon/aws/example_appflow_run.py @@ -0,0 +1,204 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json +from datetime import datetime + +import boto3 + +from airflow import DAG +from airflow.decorators import task +from airflow.models.baseoperator import chain +from airflow.providers.amazon.aws.operators.appflow import ( + AppflowRunOperator, +) +from airflow.providers.amazon.aws.operators.s3 import ( + S3CreateBucketOperator, + S3CreateObjectOperator, + S3DeleteBucketOperator, +) +from airflow.utils.trigger_rule import TriggerRule +from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder + +sys_test_context_task = SystemTestContextBuilder().build() + +DAG_ID = "example_appflow_run" + + +@task +def create_s3_to_s3_flow(flow_name: str, bucket_name: str, source_folder: str): + """creates a flow that takes a CSV and converts it to a json containing the same data""" + client = boto3.client("appflow") + client.create_flow( + flowName=flow_name, + triggerConfig={"triggerType": "OnDemand"}, + sourceFlowConfig={ + "connectorType": "S3", + "sourceConnectorProperties": { + "S3": { + "bucketName": bucket_name, + "bucketPrefix": source_folder, + "s3InputFormatConfig": {"s3InputFileType": "CSV"}, + }, + }, + }, + destinationFlowConfigList=[ + { + "connectorType": "S3", + "destinationConnectorProperties": { + "S3": { + "bucketName": bucket_name, + "s3OutputFormatConfig": { + "fileType": "JSON", + "aggregationConfig": { + "aggregationType": "None", + }, + }, + } + }, + }, + ], + tasks=[ + { + "sourceFields": ["col1", "col2"], + "connectorOperator": {"S3": "PROJECTION"}, + "taskType": "Filter", + }, + { + "sourceFields": ["col1"], + "connectorOperator": {"S3": "NO_OP"}, + "destinationField": "col1", + "taskType": "Map", + "taskProperties": {"DESTINATION_DATA_TYPE": "string", "SOURCE_DATA_TYPE": "string"}, + }, + { + "sourceFields": ["col2"], + "connectorOperator": {"S3": "NO_OP"}, + "destinationField": "col2", + "taskType": "Map", + "taskProperties": {"DESTINATION_DATA_TYPE": "string", "SOURCE_DATA_TYPE": "string"}, + }, + ], + ) + + +@task +def setup_bucket_permissions(bucket_name): + s3 = boto3.client("s3") + s3.put_bucket_policy( + Bucket=bucket_name, + Policy=json.dumps( + { + "Version": "2008-10-17", + "Statement": [ + { + "Sid": "AllowAppFlowSourceActions", + "Effect": "Allow", + "Principal": {"Service": "appflow.amazonaws.com"}, + "Action": ["s3:ListBucket", "s3:GetObject"], + "Resource": [f"arn:aws:s3:::{bucket_name}", f"arn:aws:s3:::{bucket_name}/*"], + }, + { + "Sid": "AllowAppFlowDestinationActions", + "Effect": "Allow", + "Principal": {"Service": "appflow.amazonaws.com"}, + "Action": [ + "s3:PutObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts", + "s3:ListBucketMultipartUploads", + "s3:GetBucketAcl", + "s3:PutObjectAcl", + ], + "Resource": [f"arn:aws:s3:::{bucket_name}", f"arn:aws:s3:::{bucket_name}/*"], + }, + ], + } + ), + ) + + +@task(trigger_rule=TriggerRule.ALL_DONE) +def delete_flow(flow_name: str): + client = boto3.client("appflow") + client.delete_flow(flowName=flow_name, forceDelete=True) + + +with DAG( + dag_id=DAG_ID, + schedule="@once", + start_date=datetime(2022, 1, 1), + catchup=False, + tags=["example"], +) as dag: + test_context = sys_test_context_task() + env_id = test_context["ENV_ID"] + + flow_name = f"{env_id}-flow" + bucket_name = f"{env_id}-for-appflow" + source_folder = "source" + + create_bucket = S3CreateBucketOperator(task_id="create_bucket", bucket_name=bucket_name) + + upload_csv = S3CreateObjectOperator( + task_id="upload_csv", + s3_bucket=bucket_name, + s3_key="source_folder/data.csv", + data="""col1,col2\n"data1","data2"\n""", + replace=True, + ) + + # [START howto_operator_appflow_run] + run_flow = AppflowRunOperator( + task_id="run_flow", + flow_name=flow_name, + ) + # [END howto_operator_appflow_run] + run_flow.poll_interval = 1 + + delete_bucket = S3DeleteBucketOperator( + task_id="delete_bucket", + trigger_rule=TriggerRule.ALL_DONE, + bucket_name=bucket_name, + force_delete=True, + ) + + chain( + # TEST SETUP + test_context, + create_bucket, + setup_bucket_permissions(bucket_name), + upload_csv, + create_s3_to_s3_flow(flow_name, bucket_name, source_folder), + # TEST BODY + run_flow, + # TEARDOWN + delete_flow(flow_name), + delete_bucket, + ) + + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/tests/system/providers/google/cloud/datapipelines/resources/kinglear.txt b/tests/system/providers/google/cloud/datapipelines/resources/kinglear.txt index 91aef48fab7b9..a863ef33d227e 100644 --- a/tests/system/providers/google/cloud/datapipelines/resources/kinglear.txt +++ b/tests/system/providers/google/cloud/datapipelines/resources/kinglear.txt @@ -4044,7 +4044,7 @@ sound An unknown opposite. Thou art not vanquish'd, But cozen'd and beguil'd. Alb. Shut your mouth, dame, - Or with this paper shall I stop it. [Shows her her letter to + Or with this paper shall I stop it. [Shows her letter to Edmund.]- [To Edmund]. Hold, sir. [To Goneril] Thou worse than any name, read thine own evil. No tearing, lady! I perceive you know it. diff --git a/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py b/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py index 4ff49ec9aab74..e3d2dc711ee4e 100644 --- a/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py +++ b/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py @@ -56,7 +56,7 @@ # Sheet Finally, you need a Google Sheet you have access to, for testing you can -create a public sheet and get it's ID. +create a public sheet and get its ID. # Tear Down You can delete the db with diff --git a/tests/test_utils/mock_executor.py b/tests/test_utils/mock_executor.py index 96d42152c24ea..91e49e07febb7 100644 --- a/tests/test_utils/mock_executor.py +++ b/tests/test_utils/mock_executor.py @@ -95,7 +95,7 @@ def mock_task_fail(self, dag_id, task_id, run_id: str, try_number=1): FAILED. If the task identified by the tuple ``(dag_id, task_id, date, - try_number)`` is run by this executor it's state will be FAILED. + try_number)`` is run by this executor its state will be FAILED. """ assert isinstance(run_id, str) self.mock_task_results[TaskInstanceKey(dag_id, task_id, run_id, try_number)] = State.FAILED diff --git a/tests/test_utils/system_tests_class.py b/tests/test_utils/system_tests_class.py index 785cc1c815a54..7b07857ebe771 100644 --- a/tests/test_utils/system_tests_class.py +++ b/tests/test_utils/system_tests_class.py @@ -131,7 +131,7 @@ def _print_all_log_files(): def run_dag(self, dag_id: str, dag_folder: str = DEFAULT_DAG_FOLDER) -> None: """ - Runs example dag by it's ID. + Runs example dag by its ID. :param dag_id: id of a DAG to be run :param dag_folder: directory where to look for the specific DAG. Relative to AIRFLOW_HOME. diff --git a/tests/utils/test_db_cleanup.py b/tests/utils/test_db_cleanup.py index 80419cb9437a3..b6819342e7199 100644 --- a/tests/utils/test_db_cleanup.py +++ b/tests/utils/test_db_cleanup.py @@ -41,6 +41,7 @@ _cleanup_table, _confirm_drop_archives, _dump_table_to_file, + _get_archived_table_names, config_dict, drop_archived_tables, export_archived_records, @@ -231,7 +232,7 @@ def test__cleanup_table(self, table_name, date_add_kwargs, expected_to_delete, e Verify that _cleanup_table actually deletes the rows it should. TaskInstance represents the "normal" case. DagRun is the odd case where we want - to keep the last non-externally-triggered DagRun record even if if it should be + to keep the last non-externally-triggered DagRun record even if it should be deleted according to the provided timestamp. We also verify that the "on delete cascade" behavior is as expected. Some tables @@ -266,6 +267,34 @@ def test__cleanup_table(self, table_name, date_add_kwargs, expected_to_delete, e else: raise Exception("unexpected") + @pytest.mark.parametrize( + "skip_archive, expected_archives", + [param(True, 0, id="skip_archive"), param(False, 1, id="do_archive")], + ) + def test__skip_archive(self, skip_archive, expected_archives): + """ + Verify that running cleanup_table with drops the archives when requested. + """ + base_date = pendulum.DateTime(2022, 1, 1, tzinfo=pendulum.timezone("UTC")) + num_tis = 10 + create_tis( + base_date=base_date, + num_tis=num_tis, + ) + with create_session() as session: + clean_before_date = base_date.add(days=5) + _cleanup_table( + **config_dict["dag_run"].__dict__, + clean_before_timestamp=clean_before_date, + dry_run=False, + session=session, + table_names=["dag_run"], + skip_archive=skip_archive, + ) + model = config_dict["dag_run"].orm_model + assert len(session.query(model).all()) == 5 + assert len(_get_archived_table_names(["dag_run"], session)) == expected_archives + def test_no_models_missing(self): """ 1. Verify that for all tables in `airflow.models`, we either have them enabled in db cleanup, diff --git a/tests/utils/test_sqlalchemy.py b/tests/utils/test_sqlalchemy.py index 2bf4ad1be5bbd..84eb3c8093d52 100644 --- a/tests/utils/test_sqlalchemy.py +++ b/tests/utils/test_sqlalchemy.py @@ -272,7 +272,7 @@ def test_bind_processor(self, input, expected): mock_dialect.dbapi = None process = config_type.bind_processor(mock_dialect) assert pickle.loads(process(input)) == expected - assert pickle.loads(process(input)) == expected, "should should not mutate variable" + assert pickle.loads(process(input)) == expected, "should not mutate variable" @pytest.mark.parametrize( "input",