Rnhttr patch 1

.github/workflows/build-images.yml

@@ -99,7 +99,7 @@ jobs:
             }' --jq '.data.node.labels.nodes[]' | jq --slurp -c '[.[].name]' >> ${GITHUB_OUTPUT}
         if: github.event_name == 'pull_request_target'
       # Retrieve it to be able to determine which files has changed in the incoming commit of the PR
-      # we checkout the target commit and it's parent to be able to compare them
+      # we checkout the target commit and its parent to be able to compare them
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
       - uses: actions/checkout@v3

.github/workflows/ci.yml

@@ -886,7 +886,7 @@ jobs:
       DB_RESET: "false"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
       JOB_ID: "helm-tests"
-      COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
+      COVERAGE: "false"  # We do not need to run coverage on Helm tests
     if: >
       needs.build-info.outputs.needs-helm-tests == 'true' &&
       (github.repository == 'apache/airflow' || github.event_name != 'schedule') &&
@@ -1089,6 +1089,7 @@ jobs:
       MYSQL_VERSION: "${{matrix.mysql-version}}"
       BACKEND_VERSION: "${{matrix.mysql-version}}"
       JOB_ID: "mysql-${{matrix.mysql-version}}-${{matrix.python-version}}"
+      COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
     if: needs.build-info.outputs.run-tests == 'true'
     steps:
       - name: Cleanup repo

.pre-commit-config.yaml

@@ -412,7 +412,7 @@ repos:
         language: python
         entry: ./scripts/ci/pre_commit/pre_commit_version_heads_map.py
         pass_filenames: false
-        additional_dependencies: ['packaging']
+        additional_dependencies: ['packaging','google-re2']
       - id: update-version
         name: Update version to the latest version in the documentation
         entry: ./scripts/ci/pre_commit/pre_commit_update_versions.py

CONTRIBUTING.rst

@@ -679,7 +679,7 @@ ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm
 mysql, neo4j, odbc, openfaas, openlineage, opsgenie, oracle, otel, pagerduty, pandas, papermill,
 password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba,
 segment, sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, spark, sqlite, ssh, statsd,
-tableau, tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, zendesk
+tableau, tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk
   .. END EXTRAS HERE
 
 Provider packages

CONTRIBUTORS_QUICK_START.rst

@@ -104,7 +104,7 @@ Colima
 ------
 If you use Colima as your container runtimes engine, please follow the next steps:
 
-1. `Install buildx manually <https://github.com/docker/buildx#manual-download>`_ and follow it's instructions
+1. `Install buildx manually <https://github.com/docker/buildx#manual-download>`_ and follow its instructions
 
 2. Link the Colima socket to the default socket path. Note that this may break other Docker servers.
 
@@ -252,7 +252,7 @@ Typical development tasks
 #########################
 
 For many of the development tasks you will need ``Breeze`` to be configured. ``Breeze`` is a development
-environment which uses docker and docker-compose and it's main purpose is to provide a consistent
+environment which uses docker and docker-compose and its main purpose is to provide a consistent
 and repeatable environment for all the contributors and CI. When using ``Breeze`` you avoid the "works for me"
 syndrome - because not only others can reproduce easily what you do, but also the CI of Airflow uses
 the same environment to run all tests - so you should be able to easily reproduce the same failures you

Dockerfile

@@ -1336,7 +1336,7 @@ RUN if [[ -f /docker-context-files/requirements.txt ]]; then \
 
 ##############################################################################################
 # This is the actual Airflow image - much smaller than the build one. We copy
-# installed Airflow and all it's dependencies from the build image to make it smaller.
+# installed Airflow and all its dependencies from the build image to make it smaller.
 ##############################################################################################
 FROM ${PYTHON_BASE_IMAGE} as main
 

Dockerfile.ci

@@ -1020,10 +1020,11 @@ else
 fi
 
 if [[ ${ENABLE_TEST_COVERAGE:="false"} == "true" ]]; then
+    _suffix="$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8)"
     EXTRA_PYTEST_ARGS+=(
         "--cov=airflow"
         "--cov-config=pyproject.toml"
-        "--cov-report=xml:/files/coverage-${TEST_TYPE/\[*\]/}-${BACKEND}.xml"
+        "--cov-report=xml:/files/coverage-${TEST_TYPE/\[*\]/}-${BACKEND}-${_suffix}.xml"
     )
 fi
 
@@ -1395,7 +1396,7 @@ RUN echo "Airflow version: ${AIRFLOW_VERSION}"
 # force them on the main Airflow package. Currently we need no extra limits as PIP 23.1+ has much better
 # dependency resolution and we do not need to limit the versions of the dependencies
 # aiobotocore is limited temporarily until it stops backtracking pip
-ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="aiobotocore<2.6.0"
+ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=""
 ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
 ARG VERSION_SUFFIX_FOR_PYPI=""
 

INSTALL

@@ -106,7 +106,7 @@ ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm
 mysql, neo4j, odbc, openfaas, openlineage, opsgenie, oracle, otel, pagerduty, pandas, papermill,
 password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba,
 segment, sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, spark, sqlite, ssh, statsd,
-tableau, tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, zendesk
+tableau, tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk
 # END EXTRAS HERE
 
 # For installing Airflow in development environments - see CONTRIBUTING.rst

RELEASE_NOTES.rst

@@ -5439,7 +5439,7 @@ It has been removed.
 ``airflow.settings.CONTEXT_MANAGER_DAG``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-CONTEXT_MANAGER_DAG was removed from settings. It's role has been taken by ``DagContext`` in
+CONTEXT_MANAGER_DAG was removed from settings. Its role has been taken by ``DagContext`` in
 'airflow.models.dag'. One of the reasons was that settings should be rather static than store
 dynamic context from the DAG, but the main one is that moving the context out of settings allowed to
 untangle cyclic imports between DAG, BaseOperator, SerializedDAG, SerializedBaseOperator which was

TESTING.rst

@@ -451,9 +451,9 @@ Running Tests with provider packages
 
 Airflow 2.0 introduced the concept of splitting the monolithic Airflow package into separate
 providers packages. The main "apache-airflow" package contains the bare Airflow implementation,
-and additionally we have 70+ providers that we can install additionally to get integrations with
-external services. Those providers live in the same monorepo as Airflow, but we build separate
-packages for them and the main "apache-airflow" package does not contain the providers.
+and there are more than 70 additional providers that can be installed to integrate with
+external services. Such providers live in the same monorepo as Airflow but are built as 
+separate packages, distinct from the main "apache-airflow" package.
 
 Most of the development in Breeze happens by iterating on sources and when you run
 your tests during development, you usually do not want to build packages and install them separately.

airflow/auth/managers/base_auth_manager.py

@@ -20,11 +20,12 @@
 from abc import abstractmethod
 from typing import TYPE_CHECKING
 
-from airflow.auth.managers.models.base_user import BaseUser
 from airflow.exceptions import AirflowException
 from airflow.utils.log.logging_mixin import LoggingMixin
 
 if TYPE_CHECKING:
+    from airflow.auth.managers.models.base_user import BaseUser
+    from airflow.cli.cli_config import CLICommand
     from airflow.www.security import AirflowSecurityManager
 
 
@@ -38,6 +39,14 @@ class BaseAuthManager(LoggingMixin):
     def __init__(self):
         self._security_manager: AirflowSecurityManager | None = None
 
+    @staticmethod
+    def get_cli_commands() -> list[CLICommand]:
+        """Vends CLI commands to be included in Airflow CLI.
+
+        Override this method to expose commands via Airflow CLI to manage this auth manager.
+        """
+        return []
+
     @abstractmethod
     def get_user_name(self) -> str:
         """Return the username associated to the user in session."""

airflow/auth/managers/fab/cli_commands/__init__.py

@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

airflow/auth/managers/fab/cli_commands/definition.py

@@ -0,0 +1,220 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import textwrap
+
+from airflow.cli.cli_config import (
+    ARG_OUTPUT,
+    ARG_VERBOSE,
+    ActionCommand,
+    Arg,
+    lazy_load_command,
+)
+
+############
+# # ARGS # #
+############
+
+# users
+ARG_USERNAME = Arg(("-u", "--username"), help="Username of the user", required=True, type=str)
+ARG_USERNAME_OPTIONAL = Arg(("-u", "--username"), help="Username of the user", type=str)
+ARG_FIRSTNAME = Arg(("-f", "--firstname"), help="First name of the user", required=True, type=str)
+ARG_LASTNAME = Arg(("-l", "--lastname"), help="Last name of the user", required=True, type=str)
+ARG_ROLE = Arg(
+    ("-r", "--role"),
+    help="Role of the user. Existing roles include Admin, User, Op, Viewer, and Public",
+    required=True,
+    type=str,
+)
+ARG_EMAIL = Arg(("-e", "--email"), help="Email of the user", required=True, type=str)
+ARG_EMAIL_OPTIONAL = Arg(("-e", "--email"), help="Email of the user", type=str)
+ARG_PASSWORD = Arg(
+    ("-p", "--password"),
+    help="Password of the user, required to create a user without --use-random-password",
+    type=str,
+)
+ARG_USE_RANDOM_PASSWORD = Arg(
+    ("--use-random-password",),
+    help="Do not prompt for password. Use random string instead."
+    " Required to create a user without --password ",
+    default=False,
+    action="store_true",
+)
+ARG_USER_IMPORT = Arg(
+    ("import",),
+    metavar="FILEPATH",
+    help="Import users from JSON file. Example format::\n"
+    + textwrap.indent(
+        textwrap.dedent(
+            """
+            [
+                {
+                    "email": "foo@bar.org",
+                    "firstname": "Jon",
+                    "lastname": "Doe",
+                    "roles": ["Public"],
+                    "username": "jondoe"
+                }
+            ]"""
+        ),
+        " " * 4,
+    ),
+)
+ARG_USER_EXPORT = Arg(("export",), metavar="FILEPATH", help="Export all users to JSON file")
+
+# roles
+ARG_CREATE_ROLE = Arg(("-c", "--create"), help="Create a new role", action="store_true")
+ARG_LIST_ROLES = Arg(("-l", "--list"), help="List roles", action="store_true")
+ARG_ROLES = Arg(("role",), help="The name of a role", nargs="*")
+ARG_PERMISSIONS = Arg(("-p", "--permission"), help="Show role permissions", action="store_true")
+ARG_ROLE_RESOURCE = Arg(("-r", "--resource"), help="The name of permissions", nargs="*", required=True)
+ARG_ROLE_ACTION = Arg(("-a", "--action"), help="The action of permissions", nargs="*")
+ARG_ROLE_ACTION_REQUIRED = Arg(("-a", "--action"), help="The action of permissions", nargs="*", required=True)
+
+ARG_ROLE_IMPORT = Arg(("file",), help="Import roles from JSON file", nargs=None)
+ARG_ROLE_EXPORT = Arg(("file",), help="Export all roles to JSON file", nargs=None)
+ARG_ROLE_EXPORT_FMT = Arg(
+    ("-p", "--pretty"),
+    help="Format output JSON file by sorting role names and indenting by 4 spaces",
+    action="store_true",
+)
+
+# sync-perm
+ARG_INCLUDE_DAGS = Arg(
+    ("--include-dags",), help="If passed, DAG specific permissions will also be synced.", action="store_true"
+)
+
+################
+# # COMMANDS # #
+################
+
+USERS_COMMANDS = (
+    ActionCommand(
+        name="list",
+        help="List users",
+        func=lazy_load_command("airflow.auth.managers.fab.cli_commands.user_command.users_list"),
+        args=(ARG_OUTPUT, ARG_VERBOSE),
+    ),
+    ActionCommand(
+        name="create",
+        help="Create a user",
+        func=lazy_load_command("airflow.auth.managers.fab.cli_commands.user_command.users_create"),
+        args=(
+            ARG_ROLE,
+            ARG_USERNAME,
+            ARG_EMAIL,
+            ARG_FIRSTNAME,
+            ARG_LASTNAME,
+            ARG_PASSWORD,
+            ARG_USE_RANDOM_PASSWORD,
+            ARG_VERBOSE,
+        ),
+        epilog=(
+            "examples:\n"
+            'To create an user with "Admin" role and username equals to "admin", run:\n'
+            "\n"
+            "    $ airflow users create \\\n"
+            "          --username admin \\\n"
+            "          --firstname FIRST_NAME \\\n"
+            "          --lastname LAST_NAME \\\n"
+            "          --role Admin \\\n"
+            "          --email admin@example.org"
+        ),
+    ),
+    ActionCommand(
+        name="delete",
+        help="Delete a user",
+        func=lazy_load_command("airflow.auth.managers.fab.cli_commands.user_command.users_delete"),
+        args=(ARG_USERNAME_OPTIONAL, ARG_EMAIL_OPTIONAL, ARG_VERBOSE),
+    ),
+    ActionCommand(
+        name="add-role",
+        help="Add role to a user",
+        func=lazy_load_command("airflow.auth.managers.fab.cli_commands.user_command.add_role"),
+        args=(ARG_USERNAME_OPTIONAL, ARG_EMAIL_OPTIONAL, ARG_ROLE, ARG_VERBOSE),
+    ),
+    ActionCommand(
+        name="remove-role",
+        help="Remove role from a user",
+        func=lazy_load_command("airflow.auth.managers.fab.cli_commands.user_command.remove_role"),
+        args=(ARG_USERNAME_OPTIONAL, ARG_EMAIL_OPTIONAL, ARG_ROLE, ARG_VERBOSE),
+    ),
+    ActionCommand(
+        name="import",
+        help="Import users",
+        func=lazy_load_command("airflow.auth.managers.fab.cli_commands.user_command.users_import"),
+        args=(ARG_USER_IMPORT, ARG_VERBOSE),
+    ),
+    ActionCommand(
+        name="export",
+        help="Export all users",
+        func=lazy_load_command("airflow.auth.managers.fab.cli_commands.user_command.users_export"),
+        args=(ARG_USER_EXPORT, ARG_VERBOSE),
+    ),
+)
+ROLES_COMMANDS = (
+    ActionCommand(
+        name="list",
+        help="List roles",
+        func=lazy_load_command("airflow.auth.managers.fab.cli_commands.role_command.roles_list"),
+        args=(ARG_PERMISSIONS, ARG_OUTPUT, ARG_VERBOSE),
+    ),
+    ActionCommand(
+        name="create",
+        help="Create role",
+        func=lazy_load_command("airflow.auth.managers.fab.cli_commands.role_command.roles_create"),
+        args=(ARG_ROLES, ARG_VERBOSE),
+    ),
+    ActionCommand(
+        name="delete",
+        help="Delete role",
+        func=lazy_load_command("airflow.auth.managers.fab.cli_commands.role_command.roles_delete"),
+        args=(ARG_ROLES, ARG_VERBOSE),
+    ),
+    ActionCommand(
+        name="add-perms",
+        help="Add roles permissions",
+        func=lazy_load_command("airflow.auth.managers.fab.cli_commands.role_command.roles_add_perms"),
+        args=(ARG_ROLES, ARG_ROLE_RESOURCE, ARG_ROLE_ACTION_REQUIRED, ARG_VERBOSE),
+    ),
+    ActionCommand(
+        name="del-perms",
+        help="Delete roles permissions",
+        func=lazy_load_command("airflow.auth.managers.fab.cli_commands.role_command.roles_del_perms"),
+        args=(ARG_ROLES, ARG_ROLE_RESOURCE, ARG_ROLE_ACTION, ARG_VERBOSE),
+    ),
+    ActionCommand(
+        name="export",
+        help="Export roles (without permissions) from db to JSON file",
+        func=lazy_load_command("airflow.auth.managers.fab.cli_commands.role_command.roles_export"),
+        args=(ARG_ROLE_EXPORT, ARG_ROLE_EXPORT_FMT, ARG_VERBOSE),
+    ),
+    ActionCommand(
+        name="import",
+        help="Import roles (without permissions) from JSON file to db",
+        func=lazy_load_command("airflow.auth.managers.fab.cli_commands.role_command.roles_import"),
+        args=(ARG_ROLE_IMPORT, ARG_VERBOSE),
+    ),
+)
+
+SYNC_PERM_COMMAND = ActionCommand(
+    name="sync-perm",
+    help="Update permissions for existing roles and optionally DAGs",
+    func=lazy_load_command("airflow.auth.managers.fab.cli_commands.sync_perm_command.sync_perm"),
+    args=(ARG_INCLUDE_DAGS, ARG_VERBOSE),
+)