Sync v2-9-stable with v2-9-test to release 2.9.3

.github/actions/post_tests_success/action.yml

@@ -35,7 +35,7 @@ runs:
         mkdir ./files/coverage-reposts
         mv ./files/coverage*.xml ./files/coverage-reposts/ || true
     - name: "Upload all coverage reports to codecov"
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       if: env.ENABLE_COVERAGE == 'true' && env.TEST_TYPES != 'Helm'
       with:
         name: coverage-${{env.JOB_ID}}

.github/workflows/ci.yml

@@ -304,7 +304,7 @@ jobs:
       packages: read
     secrets: inherit
     if: >
-      needs.build-info.outputs.skip-providers-tests != 'true' &&
+      needs.build-info.outputs.skip-provider-tests != 'true' &&
       needs.build-info.outputs.latest-versions-only != 'true'
     with:
       runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}

.pre-commit-config.yaml

@@ -211,7 +211,7 @@ repos:
         pass_filenames: false
         require_serial: true
   - repo: https://github.com/asottile/blacken-docs
-    rev: 1.16.0
+    rev: 1.18.0
     hooks:
       - id: blacken-docs
         name: Run black on Python code blocks in documentation files
@@ -224,7 +224,7 @@ repos:
         alias: blacken-docs
         additional_dependencies: [black==23.10.0]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-merge-conflict
         name: Check that merge conflicts are not being committed
@@ -297,7 +297,7 @@ repos:
          - --line-length
          - '99999'
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.3.0
     hooks:
       - id: codespell
         name: Run codespell to check for common misspellings in files
@@ -437,21 +437,21 @@ repos:
         files: ^docs/apache-airflow/extra-packages-ref\.rst$|^hatch_build.py
         pass_filenames: false
         entry: ./scripts/ci/pre_commit/check_extra_packages_ref.py
-        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.24.2', 'tabulate']
+        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.25.0', 'tabulate']
       - id: check-hatch-build-order
         name: Check order of dependencies in hatch_build.py
         language: python
         files: ^hatch_build.py$
         pass_filenames: false
         entry: ./scripts/ci/pre_commit/check_order_hatch_build.py
-        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.24.2']
+        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.25.0']
       - id: update-extras
         name: Update extras in documentation
         entry: ./scripts/ci/pre_commit/insert_extras.py
         language: python
         files: ^contributing-docs/12_airflow_dependencies_and_extras.rst$|^INSTALL$|^airflow/providers/.*/provider\.yaml$|^Dockerfile.*
         pass_filenames: false
-        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.24.2']
+        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.25.0']
       - id: check-extras-order
         name: Check order of extras in Dockerfile
         entry: ./scripts/ci/pre_commit/check_order_dockerfile_extras.py

Dockerfile

@@ -49,8 +49,8 @@ ARG AIRFLOW_VERSION="2.9.0"
 
 ARG PYTHON_BASE_IMAGE="python:3.8-slim-bookworm"
 
-ARG AIRFLOW_PIP_VERSION=24.0
-ARG AIRFLOW_UV_VERSION=0.2.2
+ARG AIRFLOW_PIP_VERSION=24.1.2
+ARG AIRFLOW_UV_VERSION=0.2.22
 ARG AIRFLOW_USE_UV="false"
 ARG UV_HTTP_TIMEOUT="300"
 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"

Dockerfile.ci

@@ -1003,10 +1003,15 @@ function check_boto_upgrade() {
     echo
     # shellcheck disable=SC2086
     ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs || true
-    # We need to include oss2 as dependency as otherwise jmespath will be bumped and it will not pass
-    # the pip check test, Similarly gcloud-aio-auth limit is needed to be included as it bumps cryptography
+    # We need to include few dependencies to pass pip check with other dependencies:
+    #   * oss2 as dependency as otherwise jmespath will be bumped (sync with alibaba provider)
+    #   * gcloud-aio-auth limit is needed to be included as it bumps cryptography (sync with google provider)
+    #   * requests needs to be limited to be compatible with apache beam (sync with apache-beam provider)
+    set -x
     # shellcheck disable=SC2086
-    ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore "oss2>=2.14.0" "gcloud-aio-auth>=4.0.0,<5.0.0"
+    ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \
+       "oss2>=2.14.0" "gcloud-aio-auth>=4.0.0,<5.0.0" "requests!=2.32.*,<3.0.0,>=2.24.0"
+    set +x
     pip check
 }
 
@@ -1034,7 +1039,7 @@ function check_pydantic() {
         echo "${COLOR_YELLOW}Uninstalling dependencies which are not compatible with Pydantic 1${COLOR_RESET}"
         echo
         # shellcheck disable=SC2086
-        ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} pyiceberg waeviate-client
+        ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} pyiceberg weaviate-client
         echo
         echo "${COLOR_YELLOW}Downgrading Pydantic to < 2${COLOR_RESET}"
         echo
@@ -1223,8 +1228,8 @@ ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main"
 # It can also be overwritten manually by setting the AIRFLOW_CI_BUILD_EPOCH environment variable.
 ARG AIRFLOW_CI_BUILD_EPOCH="10"
 ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
-ARG AIRFLOW_PIP_VERSION=24.0
-ARG AIRFLOW_UV_VERSION=0.2.2
+ARG AIRFLOW_PIP_VERSION=24.1.2
+ARG AIRFLOW_UV_VERSION=0.2.22
 ARG AIRFLOW_USE_UV="true"
 # Setup PIP
 # By default PIP install run without cache to make image smaller
@@ -1247,8 +1252,8 @@ ARG AIRFLOW_VERSION=""
 # Additional PIP flags passed to all pip install commands except reinstalling pip itself
 ARG ADDITIONAL_PIP_INSTALL_FLAGS=""
 
-ARG AIRFLOW_PIP_VERSION=24.0
-ARG AIRFLOW_UV_VERSION=0.2.2
+ARG AIRFLOW_PIP_VERSION=24.1.2
+ARG AIRFLOW_UV_VERSION=0.2.22
 ARG AIRFLOW_USE_UV="true"
 
 ENV AIRFLOW_REPO=${AIRFLOW_REPO}\

ISSUE_TRIAGE_PROCESS.rst

@@ -260,6 +260,23 @@ as long as the ``needs-triage`` label remains the triage team will keep an eye o
 if it needs to be accepted or closed/converted to Github Discussion.
 ``needs-triage`` label may also be applied manually by committers if they think a further action from the triage team is required.
 
+**area Label**
+
+``area:providers`` must be present for any provider issue. The ``provider:x`` is specifying the exact provider.
+We have ``provider:x`` for any provider that we own.
+``area:helm-chart`` must be for any helm chart issue.
+``area:core`` must be for any core issue. Additional labels like ``area:scheduler``, ``area:UI`` is specifying the exact
+core area relevant.
+
+This method allow us to quickly filter issues by the 3 major components of our code base: core, providers and helm-chart.
+
+**affected version Label**
+
+The ``affected_version:x`` will never be present with ``kind:feature`` as feature requests are not related to specific Airflow version.
+For bugs, The ``affected_version:x`` is expected to be used with core issues thus normally it appears with ``area:core`` label.
+When issue is reproducible on multiple Airflow versions we apply only the latest version with the label.
+This policy is best effort, we should try to have as little exceptions as possible.
+
 **Good First Issue**
 
 Issues which are relatively straight forward to solve, will be tagged with

LICENSE

@@ -215,7 +215,7 @@ Third party Apache 2.0 licenses
 
 The following components are provided under the Apache 2.0 License.
 See project link for details. The text of each license is also included
-at licenses/LICENSE-[project].txt.
+at 3rd-party-licenses/LICENSE-[project].txt.
 
     (ALv2 License) hue v4.3.0 (https://github.com/cloudera/hue/)
     (ALv2 License) jqclock v2.3.0 (https://github.com/JohnRDOrazio/jQuery-Clock-Plugin)
@@ -227,7 +227,7 @@ MIT licenses
 ========================================================================
 
 The following components are provided under the MIT License. See project link for details.
-The text of each license is also included at licenses/LICENSE-[project].txt.
+The text of each license is also included at 3rd-party-licenses/LICENSE-[project].txt.
 
     (MIT License) jquery v3.5.1 (https://jquery.org/license/)
     (MIT License) dagre-d3 v0.6.4 (https://github.com/cpettitt/dagre-d3)
@@ -243,11 +243,11 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
 BSD 3-Clause licenses
 ========================================================================
 The following components are provided under the BSD 3-Clause license. See project links for details.
-The text of each license is also included at licenses/LICENSE-[project].txt.
+The text of each license is also included at 3rd-party-licenses/LICENSE-[project].txt.
 
     (BSD 3 License) d3 v5.16.0 (https://d3js.org)
     (BSD 3 License) d3-shape v2.1.0 (https://github.com/d3/d3-shape)
     (BSD 3 License) cgroupspy 0.2.1 (https://github.com/cloudsigma/cgroupspy)
 
 ========================================================================
-See licenses/LICENSES-ui.txt for packages used in `/airflow/www`
+See 3rd-party-licenses/LICENSES-ui.txt for packages used in `/airflow/www`

README.md

@@ -98,7 +98,7 @@ Airflow is not a streaming solution, but it is often used to process real-time d
 
 Apache Airflow is tested with:
 
-|             | Main version (dev)         | Stable version (2.9.2)     |
+|             | Main version (dev)         | Stable version (2.9.3)     |
 |-------------|----------------------------|----------------------------|
 | Python      | 3.8, 3.9, 3.10, 3.11, 3.12 | 3.8, 3.9, 3.10, 3.11, 3.12 |
 | Platform    | AMD64/ARM64(\*)            | AMD64/ARM64(\*)            |
@@ -180,15 +180,15 @@ them to the appropriate format and workflow that your tool requires.
 
 
 ```bash
-pip install 'apache-airflow==2.9.2' \
- --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.2/constraints-3.8.txt"
+pip install 'apache-airflow==2.9.3' \
+ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.8.txt"
 ```
 
 2. Installing with extras (i.e., postgres, google)
 
 ```bash
-pip install 'apache-airflow[postgres,google]==2.8.3' \
- --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.2/constraints-3.8.txt"
+pip install 'apache-airflow[postgres,google]==2.9.3' \
+ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.8.txt"
 ```
 
 For information on installing provider packages, check
@@ -293,7 +293,7 @@ Apache Airflow version life cycle:
 
 | Version   | Current Patch/Minor   | State     | First Release   | Limited Support   | EOL/Terminated   |
 |-----------|-----------------------|-----------|-----------------|-------------------|------------------|
-| 2         | 2.9.2                 | Supported | Dec 17, 2020    | TBD               | TBD              |
+| 2         | 2.9.3                 | Supported | Dec 17, 2020    | TBD               | TBD              |
 | 1.10      | 1.10.15               | EOL       | Aug 27, 2018    | Dec 17, 2020      | June 17, 2021    |
 | 1.9       | 1.9.0                 | EOL       | Jan 03, 2018    | Aug 27, 2018      | Aug 27, 2018     |
 | 1.8       | 1.8.2                 | EOL       | Mar 19, 2017    | Jan 03, 2018      | Jan 03, 2018     |

RELEASE_NOTES.rst

@@ -21,6 +21,78 @@
 
 .. towncrier release notes start
 
+Airflow 2.9.3 (2024-07-15)
+--------------------------
+
+Significant Changes
+^^^^^^^^^^^^^^^^^^^
+
+Time unit for ``scheduled_duration`` and ``queued_duration`` changed (#37936)
+"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+
+``scheduled_duration`` and ``queued_duration`` metrics are now emitted in milliseconds instead of seconds.
+
+By convention all statsd metrics should be emitted in milliseconds, this is later expected in e.g. ``prometheus`` statsd-exporter.
+
+
+Support for OpenTelemetry Metrics is no longer "Experimental" (#40286)
+""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+
+Experimental support for OpenTelemetry was added in 2.7.0 since then fixes and improvements were added and now we announce the feature as stable.
+
+
+
+Bug Fixes
+"""""""""
+- Fix calendar view scroll (#40458)
+- Validating provider description for urls in provider list view (#40475)
+- Fix compatibility with old MySQL 8.0 (#40314)
+- Fix dag (un)pausing won't work on environment where dag files are missing (#40345)
+- Extra being passed to SQLalchemy (#40391)
+- Handle unsupported operand int + str when value of tag is int (job_id) (#40407)
+- Fix TriggeredDagRunOperator triggered link (#40336)
+- Add ``[webserver]update_fab_perms`` to deprecated configs (#40317)
+- Swap dag run link from legacy graph to grid with graph tab (#40241)
+- Change ``httpx`` to ``requests`` in ``file_task_handler`` (#39799)
+- Fix import future annotations in venv jinja template (#40208)
+- Ensures DAG params order regardless of backend (#40156)
+- Use a join for TI notes in TI batch API endpoint (#40028)
+- Improve trigger UI for string array format validation (#39993)
+- Disable jinja2 rendering for doc_md (#40522)
+- Skip checking sub dags list if taskinstance state is skipped (#40578)
+- Recognize quotes when parsing urls in logs (#40508)
+
+Doc Only Changes
+""""""""""""""""
+- Add notes about passing secrets via environment variables (#40519)
+- Revamp some confusing log messages (#40334)
+- Add more precise description of masking sensitive field names (#40512)
+- Add slightly more detailed guidance about upgrading to the docs (#40227)
+- Metrics allow_list complete example (#40120)
+- Add warning to deprecated api docs that access control isn't applied (#40129)
+- Simpler command to check local scheduler is alive (#40074)
+- Add a note and an example clarifying the usage of DAG-level params (#40541)
+- Fix highlight of example code in dags.rst (#40114)
+- Add warning about the PostgresOperator being deprecated (#40662)
+- Updating airflow download links to CDN based links (#40618)
+- Fix import statement for DatasetOrTimetable example (#40601)
+- Further clarify triage process (#40536)
+- Fix param order in PythonOperator docstring (#40122)
+- Update serializers.rst to mention that bytes are not supported (#40597)
+
+Miscellaneous
+"""""""""""""
+- Upgrade build installers and dependencies (#40177)
+- Bump braces from 3.0.2 to 3.0.3 in /airflow/www (#40180)
+- Upgrade to another version of trove-classifier (new CUDA classifiers) (#40564)
+- Rename "try_number" increments that are unrelated to the airflow concept (#39317)
+- Update trove classifiers to the latest version as build dependency (#40542)
+- Upgrade to latest version of hatchling as build dependency (#40387)
+- Fix bug in ``SchedulerJobRunner._process_executor_events`` (#40563)
+- Remove logging for "blocked" events (#40446)
+
+
+
 Airflow 2.9.2 (2024-06-10)
 --------------------------
 

airflow/__init__.py

@@ -17,7 +17,7 @@
 # under the License.
 from __future__ import annotations
 
-__version__ = "2.9.2"
+__version__ = "2.9.3"
 
 import os
 import sys

airflow/_vendor/README.md

@@ -12,7 +12,7 @@ All Vendored libraries must follow these rules:
 1. Vendored libraries must be pure Python--no compiling (so that we do not have to release multi-platform airflow packages on PyPI).
 2. Source code for the library is included in this directory.
 3. License must be included in this repo and in the [LICENSE](../../LICENSE) file and in the
-   [licenses](../../licenses) folder.
+   [licenses](../../3rd-party-licenses) folder.
 4. Requirements of the library become requirements of airflow core.
 5. Version of the library should be included in the [vendor.md](vendor.md) file.
 6. No modifications to the library may be made in the initial commit.
@@ -25,13 +25,12 @@ All Vendored libraries must follow these rules:
 
 Way to vendor a library or update a version:
 
-1. Update ``vendor.txt`` with the library, version, and SHA256 (`pypi` provides hashes as of recently)
+1. Update ``vendor.md`` with the library, version, and SHA256 (`pypi` provides hashes as of recently)
 2. Remove all old files and directories of the old version.
 3. Replace them with new files (only replace relevant python packages:move LICENSE )
-   * move license files to [licenses](../../licenses) folder
+   * move license files to [licenses](../../3rd-party-licenses) folder
    * remove README and any other supporting files (they can be found in PyPI)
    * make sure to add requirements to airflow's ``pyproject.toml`` with appropriate comment stating
      why the requirements are added and when they should be removed
 4. If you replace previous version, re-apply historical fixes from the "package" folder by
    cherry-picking them.
-

airflow/api/common/mark_tasks.py

@@ -144,8 +144,12 @@ def set_state(
     confirmed_infos = list(_iter_existing_dag_run_infos(dag, dag_run_ids, session=session))
     confirmed_dates = [info.logical_date for info in confirmed_infos]
 
-    sub_dag_run_ids = list(
-        _iter_subdag_run_ids(dag, session, DagRunState(state), task_ids, commit, confirmed_infos),
+    sub_dag_run_ids = (
+        list(
+            _iter_subdag_run_ids(dag, session, DagRunState(state), task_ids, commit, confirmed_infos),
+        )
+        if not state == TaskInstanceState.SKIPPED
+        else []
     )
 
     # now look for the task instances that are affected

airflow/api_connexion/endpoints/task_instance_endpoint.py

@@ -430,7 +430,9 @@ def get_task_instances_batch(session: Session = NEW_SESSION) -> APIResponse:
         ),
         isouter=True,
     ).add_columns(SlaMiss)
-    ti_query = base_query.options(joinedload(TI.rendered_task_instance_fields))
+    ti_query = base_query.options(
+        joinedload(TI.rendered_task_instance_fields), joinedload(TI.task_instance_note)
+    )
     # using execute because we want the SlaMiss entity. Scalars don't return None for missing entities
     task_instances = session.execute(ti_query).all()
 

airflow/api_connexion/openapi/v1.yaml

@@ -231,7 +231,7 @@ info:
     This means that the server encountered an unexpected condition that prevented it from
     fulfilling the request.
 
-  version: "2.9.2"
+  version: "2.9.3"
   license:
     name: Apache 2.0
     url: http://www.apache.org/licenses/LICENSE-2.0.html