Merge branch 'main' into fix/core_import

.github/actions/build-ci-images/action.yml

@@ -34,12 +34,18 @@ runs:
     - name: "Build & Push AMD64 CI images ${{ env.IMAGE_TAG }} ${{ env.PYTHON_VERSIONS }}"
       shell: bash
       run: breeze ci-image build --push --tag-as-latest --run-in-parallel --upgrade-on-failure
-    - name: "Show dependencies to be upgraded"
+    - name: "Generate source constraints"
       shell: bash
       run: >
         breeze release-management generate-constraints --run-in-parallel
         --airflow-constraints-mode constraints-source-providers
       if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false'
+    - name: "Generate PyPI constraints"
+      shell: bash
+      run: >
+        breeze release-management generate-constraints --run-in-parallel
+        --airflow-constraints-mode constraints
+      if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false' and ${{ inputs.build-provider-packages != 'true' }}
     - name: "Print dependency upgrade summary"
       shell: bash
       run: |
@@ -48,6 +54,12 @@ runs:
           cat "files/constraints-${PYTHON_VERSION}/*.md" >> $GITHUB_STEP_SUMMARY || true
         done
       if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false'
+    - name: "Upload constraint artifacts"
+      uses: actions/upload-artifact@v3
+      with:
+        name: constraints
+        path: ./files/constraints-*/constraints-*.txt
+        retention-days: 7
     - name: "Fix ownership"
       shell: bash
       run: breeze ci fix-ownership

.github/actions/build-prod-images/action.yml

@@ -56,13 +56,30 @@ runs:
     - name: "Move dist packages to docker-context files"
       shell: bash
       run: mv -v ./dist/*.whl ./docker-context-files
-    - name: "Build & Push PROD images ${{ env.IMAGE_TAG }}:${{ env.PYTHON_VERSIONS }}"
+    - name: "Download constraints from the CI build"
+      uses: actions/download-artifact@v3
+      with:
+        name: constraints
+        path: ./docker-context-files
+      if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false'
+    - name: "Build & Push PROD images with source providers ${{ env.IMAGE_TAG }}:${{ env.PYTHON_VERSIONS }}"
+      shell: bash
+      run: >
+        breeze prod-image build --tag-as-latest --run-in-parallel --push
+        --install-packages-from-context --airflow-constraints-mode constraints-source-providers
+        --use-constraints-for-context-packages
+      env:
+        COMMIT_SHA: ${{ github.sha }}
+      if: ${{ inputs.build-provider-packages == 'true' }}
+    - name: "Build & Push PROD images with PyPi providers ${{ env.IMAGE_TAG }}:${{ env.PYTHON_VERSIONS }}"
       shell: bash
       run: >
         breeze prod-image build --tag-as-latest --run-in-parallel --push
-        --install-packages-from-context --upgrade-on-failure
+        --install-packages-from-context --airflow-constraints-mode constraints
+        --use-constraints-for-context-packages
       env:
         COMMIT_SHA: ${{ github.sha }}
+      if: ${{ inputs.build-provider-packages != 'true' }}
     - name: "Fix ownership"
       shell: bash
       run: breeze ci fix-ownership

.github/workflows/ci.yml

@@ -643,8 +643,16 @@ jobs:
       - name: "Publish docs"
         run: >
           breeze release-management publish-docs
-          --override-versioned
+          --override-versioned --run-in-parallel
           ${{ needs.build-info.outputs.docs-filter-list-as-string }}
+      - name: "Generate back references for providers"
+        run: breeze release-management add-back-references all-providers
+      - name: "Generate back references for apache-airflow"
+        run: breeze release-management add-back-references apache-airflow
+      - name: "Generate back references for docker-stack"
+        run: breeze release-management add-back-references docker-stack
+      - name: "Generate back references for helm-chart"
+        run: breeze release-management add-back-references helm-chart
       - name: Configure AWS credentials
         uses: ./.github/actions/configure-aws-credentials
         if: >

.pre-commit-config.yaml

@@ -412,6 +412,7 @@ repos:
         language: python
         entry: ./scripts/ci/pre_commit/pre_commit_version_heads_map.py
         pass_filenames: false
+        files: ^airflow/migrations/versions|^airflow/__init__.py$
         additional_dependencies: ['packaging','google-re2']
       - id: update-version
         name: Update version to the latest version in the documentation

BREEZE.rst

@@ -2068,6 +2068,9 @@ while publishing the documentation.
 The flag ``--airflow-site-directory`` takes the path of the cloned ``airflow-site``. The command will
 not proceed if this is an invalid path.
 
+When you have multi-processor machine docs publishing can be vastly sped up by using ``--run-in-parallel`` option when
+publishing docs for multiple providers.
+
 These are all available flags of ``release-management publish-docs`` command:
 
 .. image:: ./images/breeze/output_release-management_publish-docs.svg

Dockerfile

@@ -593,17 +593,42 @@ function install_airflow_and_providers_from_docker_context_files(){
         return
     fi
 
-    echo
-    echo "${COLOR_BLUE}Force re-installing airflow and providers from local files with eager upgrade${COLOR_RESET}"
-    echo
-    # force reinstall all airflow + provider package local files with eager upgrade
-    set -x
-    pip install "${pip_flags[@]}" --root-user-action ignore --upgrade --upgrade-strategy eager \
-        ${ADDITIONAL_PIP_INSTALL_FLAGS} \
-        ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} \
-        ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=}
-    set +x
+    if [[ ${USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES=} == "true" ]]; then
+        local python_version
+        python_version=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
+        local local_constraints_file=/docker-context-files/constraints-"${python_version}"/${AIRFLOW_CONSTRAINTS_MODE}-"${python_version}".txt
 
+        if [[ -f "${local_constraints_file}" ]]; then
+            echo
+            echo "${COLOR_BLUE}Installing docker-context-files packages with constraints found in ${local_constraints_file}${COLOR_RESET}"
+            echo
+            # force reinstall all airflow + provider packages with constraints found in
+            set -x
+            pip install "${pip_flags[@]}" --root-user-action ignore --upgrade \
+                ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint "${local_constraints_file}" \
+                ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages}
+            set +x
+        else
+            echo
+            echo "${COLOR_BLUE}Installing docker-context-files packages with constraints from GitHub${COLOR_RESET}"
+            echo
+            set -x
+            pip install "${pip_flags[@]}" --root-user-action ignore \
+                ${ADDITIONAL_PIP_INSTALL_FLAGS} \
+                --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" \
+                ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages}
+            set +x
+        fi
+    else
+        echo
+        echo "${COLOR_BLUE}Installing docker-context-files packages without constraints${COLOR_RESET}"
+        echo
+        set -x
+        pip install "${pip_flags[@]}" --root-user-action ignore \
+            ${ADDITIONAL_PIP_INSTALL_FLAGS} \
+            ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages}
+        set +x
+    fi
     common::install_pip_version
     pip check
 }
@@ -1280,6 +1305,12 @@ COPY --from=scripts common.sh install_pip_version.sh \
 # is installed from docker-context files rather than from PyPI)
 ARG INSTALL_PACKAGES_FROM_CONTEXT="false"
 
+# Normally constraints are not used when context packages are build - because we might have packages
+# that are conflicting with Airflow constraints, however there are cases when we want to use constraints
+# for example in CI builds when we already have source-package constraints - either from github branch or
+# from eager-upgraded constraints by the CI builds
+ARG USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES="false"
+
 # In case of Production build image segment we want to pre-install main version of airflow
 # dependencies from GitHub so that we do not have to always reinstall it from the scratch.
 # The Airflow (and providers in case INSTALL_PROVIDERS_FROM_SOURCES is "false")
@@ -1304,6 +1335,7 @@ ARG VERSION_SUFFIX_FOR_PYPI=""
 
 ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \
     INSTALL_PACKAGES_FROM_CONTEXT=${INSTALL_PACKAGES_FROM_CONTEXT} \
+    USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES=${USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES} \
     VERSION_SUFFIX_FOR_PYPI=${VERSION_SUFFIX_FOR_PYPI}
 
 WORKDIR ${AIRFLOW_HOME}

airflow/cli/commands/task_command.py

@@ -25,12 +25,13 @@
 import sys
 import textwrap
 from contextlib import contextmanager, redirect_stderr, redirect_stdout, suppress
-from typing import TYPE_CHECKING, Generator, Union, cast
+from typing import TYPE_CHECKING, Generator, Protocol, Union, cast
 
 import pendulum
 from pendulum.parsing.exceptions import ParserError
 from sqlalchemy import select
 from sqlalchemy.orm.exc import NoResultFound
+from typing_extensions import Literal
 
 from airflow import settings
 from airflow.cli.simple_table import AirflowConsole
@@ -49,7 +50,6 @@
 from airflow.settings import IS_K8S_EXECUTOR_POD
 from airflow.ti_deps.dep_context import DepContext
 from airflow.ti_deps.dependencies_deps import SCHEDULER_QUEUED_DEPS
-from airflow.typing_compat import Literal, Protocol
 from airflow.utils import cli as cli_utils
 from airflow.utils.cli import (
     get_dag,

airflow/dag_processing/manager.py

@@ -868,7 +868,7 @@ def _log_file_processing_stats(self, known_file_paths):
             rows.append((file_path, processor_pid, runtime, num_dags, num_errors, last_runtime, last_run))
 
         # Sort by longest last runtime. (Can't sort None values in python3)
-        rows = sorted(rows, key=lambda x: x[3] or 0.0)
+        rows.sort(key=lambda x: x[3] or 0.0)
 
         formatted_rows = []
         for file_path, pid, runtime, num_dags, num_errors, last_runtime, last_run in rows:
@@ -1170,7 +1170,7 @@ def prepare_file_path_queue(self):
         if is_mtime_mode:
             file_paths = sorted(files_with_mtime, key=files_with_mtime.get, reverse=True)
         elif list_mode == "alphabetical":
-            file_paths = sorted(file_paths)
+            file_paths.sort()
         elif list_mode == "random_seeded_by_host":
             # Shuffle the list seeded by hostname so multiple schedulers can work on different
             # set of files. Since we set the seed, the sort order will remain same per host

airflow/models/dag.py

@@ -131,13 +131,13 @@
     from pendulum.tz.timezone import Timezone
     from sqlalchemy.orm.query import Query
     from sqlalchemy.orm.session import Session
+    from typing_extensions import Literal
 
     from airflow.datasets import Dataset
     from airflow.decorators import TaskDecoratorCollection
     from airflow.models.dagbag import DagBag
     from airflow.models.operator import Operator
     from airflow.models.slamiss import SlaMiss
-    from airflow.typing_compat import Literal
     from airflow.utils.task_group import TaskGroup
 
 log = logging.getLogger(__name__)

airflow/models/dagrun.py

@@ -72,10 +72,10 @@
     from datetime import datetime
 
     from sqlalchemy.orm import Query, Session
+    from typing_extensions import Literal
 
     from airflow.models.dag import DAG
     from airflow.models.operator import Operator
-    from airflow.typing_compat import Literal
     from airflow.utils.types import ArgNotSet
 
     CreatedTasks = TypeVar("CreatedTasks", Iterator["dict[str, Any]"], Iterator[TI])

airflow/models/mappedoperator.py

@@ -25,6 +25,7 @@
 from typing import TYPE_CHECKING, Any, ClassVar, Collection, Iterable, Iterator, Mapping, Sequence, Union
 
 import attr
+from typing_extensions import Literal
 
 from airflow import settings
 from airflow.compat.functools import cache
@@ -51,7 +52,6 @@
 from airflow.models.pool import Pool
 from airflow.serialization.enums import DagAttributeTypes
 from airflow.ti_deps.deps.mapped_task_expanded import MappedTaskIsExpanded
-from airflow.typing_compat import Literal
 from airflow.utils.context import context_update_for_unmapped
 from airflow.utils.helpers import is_container, prevent_duplicates
 from airflow.utils.types import NOTSET

airflow/models/taskinstance.py

@@ -135,6 +135,7 @@
     from sqlalchemy.orm.session import Session
     from sqlalchemy.sql.elements import BooleanClauseList
     from sqlalchemy.sql.expression import ColumnOperators
+    from typing_extensions import Literal
 
     from airflow.models.abstractoperator import TaskStateChangeCallback
     from airflow.models.baseoperator import BaseOperator
@@ -143,7 +144,7 @@
     from airflow.models.dataset import DatasetEvent
     from airflow.models.operator import Operator
     from airflow.timetables.base import DataInterval
-    from airflow.typing_compat import Literal, TypeGuard
+    from airflow.typing_compat import TypeGuard
     from airflow.utils.task_group import TaskGroup
 
     # This is a workaround because mypy doesn't work with hybrid_property

airflow/providers/amazon/aws/sensors/sqs.py

@@ -19,9 +19,10 @@
 from __future__ import annotations
 
 from functools import cached_property
-from typing import TYPE_CHECKING, Any, Collection, Literal, Sequence
+from typing import TYPE_CHECKING, Any, Collection, Sequence
 
 from deprecated import deprecated
+from typing_extensions import Literal
 
 from airflow.configuration import conf
 from airflow.exceptions import AirflowException

airflow/providers/amazon/aws/transfers/sql_to_s3.py

@@ -20,7 +20,9 @@
 import enum
 from collections import namedtuple
 from tempfile import NamedTemporaryFile
-from typing import TYPE_CHECKING, Iterable, Literal, Mapping, Sequence
+from typing import TYPE_CHECKING, Iterable, Mapping, Sequence
+
+from typing_extensions import Literal
 
 from airflow.exceptions import AirflowException
 from airflow.hooks.base import BaseHook

airflow/providers/amazon/aws/triggers/sqs.py

@@ -17,7 +17,9 @@
 from __future__ import annotations
 
 import asyncio
-from typing import Any, AsyncIterator, Collection, Literal
+from typing import Any, AsyncIterator, Collection
+
+from typing_extensions import Literal
 
 from airflow.exceptions import AirflowException
 from airflow.providers.amazon.aws.hooks.base_aws import BaseAwsConnection

airflow/providers/amazon/aws/utils/sqs.py

@@ -18,9 +18,10 @@
 
 import json
 import logging
-from typing import Any, Literal
+from typing import Any
 
 from jsonpath_ng import parse
+from typing_extensions import Literal
 
 log = logging.getLogger(__name__)
 

airflow/providers/apache/beam/hooks/beam.py

@@ -23,15 +23,14 @@
 import copy
 import functools
 import json
-import logging
 import os
 import select
 import shlex
 import shutil
 import subprocess
 import tempfile
 import textwrap
-from typing import Callable
+from typing import TYPE_CHECKING, Callable
 
 from packaging.version import Version
 
@@ -40,6 +39,9 @@
 from airflow.providers.google.go_module_utils import init_module, install_dependencies
 from airflow.utils.python_virtualenv import prepare_virtualenv
 
+if TYPE_CHECKING:
+    import logging
+
 
 class BeamRunnerType:
     """

airflow/providers/apache/drill/hooks/drill.py

@@ -17,13 +17,15 @@
 # under the License.
 from __future__ import annotations
 
-from typing import Any, Iterable
+from typing import TYPE_CHECKING, Any, Iterable
 
 from sqlalchemy import create_engine
-from sqlalchemy.engine import Connection
 
 from airflow.providers.common.sql.hooks.sql import DbApiHook
 
+if TYPE_CHECKING:
+    from sqlalchemy.engine import Connection
+
 
 class DrillHook(DbApiHook):
     """

airflow/providers/apache/flink/operators/flink_kubernetes.py

@@ -20,12 +20,12 @@
 from functools import cached_property
 from typing import TYPE_CHECKING, Sequence
 
-from kubernetes.client import CoreV1Api
-
 from airflow.models import BaseOperator
 from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook
 
 if TYPE_CHECKING:
+    from kubernetes.client import CoreV1Api
+
     from airflow.utils.context import Context