From ac5c3e0c37831d067c0194ad6be22a909da95f0d Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Fri, 9 Aug 2024 07:24:54 +0100 Subject: [PATCH 001/161] Update default branches for 2-10 --- dev/breeze/src/airflow_breeze/branch_defaults.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/branch_defaults.py b/dev/breeze/src/airflow_breeze/branch_defaults.py index e8639d348cd8f..59f5a37787a74 100644 --- a/dev/breeze/src/airflow_breeze/branch_defaults.py +++ b/dev/breeze/src/airflow_breeze/branch_defaults.py @@ -38,6 +38,6 @@ from __future__ import annotations -AIRFLOW_BRANCH = "main" -DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "constraints-main" +AIRFLOW_BRANCH = "v2-10-test" +DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "constraints-2-10" DEBIAN_VERSION = "bookworm" From ac40b27bd7d694277b00a370538268ce653dc9f9 Mon Sep 17 00:00:00 2001 From: Tamara Janina Fingerlin <90063506+TJaniF@users.noreply.github.com> Date: Fri, 9 Aug 2024 17:54:57 +0200 Subject: [PATCH 002/161] Typo fix dataset guide (#41353) --- docs/apache-airflow/authoring-and-scheduling/datasets.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/apache-airflow/authoring-and-scheduling/datasets.rst b/docs/apache-airflow/authoring-and-scheduling/datasets.rst index 07eb571153853..a69c09bc13b0f 100644 --- a/docs/apache-airflow/authoring-and-scheduling/datasets.rst +++ b/docs/apache-airflow/authoring-and-scheduling/datasets.rst @@ -338,7 +338,7 @@ In this example, the DAG ``waiting_for_dataset_1_and_2`` will be triggered when ... -``quededEvent`` API endpoints are introduced to manipulate such records. +``queuedEvent`` API endpoints are introduced to manipulate such records. * Get a queued Dataset event for a DAG: ``/datasets/queuedEvent/{uri}`` * Get queued Dataset events for a DAG: ``/dags/{dag_id}/datasets/queuedEvent`` @@ -347,7 +347,7 @@ In this example, the DAG ``waiting_for_dataset_1_and_2`` will be triggered when * Get queued Dataset events for a Dataset: ``/dags/{dag_id}/datasets/queuedEvent/{uri}`` * Delete queued Dataset events for a Dataset: ``DELETE /dags/{dag_id}/datasets/queuedEvent/{uri}`` - For how to use REST API and the parameters needed for these endpoints, please refer to :doc:`Airflow API ` + For how to use REST API and the parameters needed for these endpoints, please refer to :doc:`Airflow API `. Advanced dataset scheduling with conditional expressions -------------------------------------------------------- @@ -444,7 +444,7 @@ The following example creates a dataset event against the S3 URI ``f"s3://bucket @task(outlets=[DatasetAlias("my-task-outputs")]) def my_task_with_metadata(): - s3_dataset = Dataset("s3://bucket/my-task}") + s3_dataset = Dataset("s3://bucket/my-task") yield Metadata(s3_dataset, extra={"k": "v"}, alias="my-task-outputs") Only one dataset event is emitted for an added dataset, even if it is added to the alias multiple times, or added to multiple aliases. However, if different ``extra`` values are passed, it can emit multiple dataset events. In the following example, two dataset events will be emitted. @@ -470,7 +470,7 @@ Only one dataset event is emitted for an added dataset, even if it is added to t Scheduling based on dataset aliases ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Since dataset events added to an alias are just simple dataset events, a downstream depending on the actual dataset can read dataset events of it normally, without considering the associated aliases. A downstream can also depend on a dataset alias. The authoring syntax is referencing the ``DatasetAlias`` by name, and the associated dataset events are picked up for scheduling. Note that a DAG can be triggered by a task with ``outlets=DatasetAlias("xxx")`` if and only if the alias is resolved into ``Dataset("s3://bucket/my-task")``. The DAG runs whenever a task with outlet ``DatasetAlias("out")`` gets associated with at least one dataset at runtime, regardless of the dataset's identity. The downstream DAG is not triggered if no datasets are associated to the alias for a particular given task run. This also means we can do conditional dataset-triggering. +Since dataset events added to an alias are just simple dataset events, a downstream DAG depending on the actual dataset can read dataset events of it normally, without considering the associated aliases. A downstream DAG can also depend on a dataset alias. The authoring syntax is referencing the ``DatasetAlias`` by name, and the associated dataset events are picked up for scheduling. Note that a DAG can be triggered by a task with ``outlets=DatasetAlias("xxx")`` if and only if the alias is resolved into ``Dataset("s3://bucket/my-task")``. The DAG runs whenever a task with outlet ``DatasetAlias("out")`` gets associated with at least one dataset at runtime, regardless of the dataset's identity. The downstream DAG is not triggered if no datasets are associated to the alias for a particular given task run. This also means we can do conditional dataset-triggering. The dataset alias is resolved to the datasets during DAG parsing. Thus, if the "min_file_process_interval" configuration is set to a high value, there is a possibility that the dataset alias may not be resolved. To resolve this issue, you can trigger DAG parsing. From 1002266c1d868fec34a4d2c038c4f023b7d97fee Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Fri, 9 Aug 2024 07:30:40 +0100 Subject: [PATCH 003/161] regenerate command hashes --- .../doc/images/output_ci-image_build.svg | 92 +++++++------- .../doc/images/output_ci-image_build.txt | 2 +- .../doc/images/output_ci_selective-check.svg | 24 ++-- .../doc/images/output_ci_selective-check.txt | 2 +- .../doc/images/output_prod-image_build.svg | 118 +++++++++--------- .../doc/images/output_prod-image_build.txt | 2 +- ...e-management_install-provider-packages.svg | 52 ++++---- ...e-management_install-provider-packages.txt | 2 +- ...se-management_verify-provider-packages.svg | 44 +++---- ...se-management_verify-provider-packages.txt | 2 +- dev/breeze/doc/images/output_shell.txt | 2 +- .../doc/images/output_start-airflow.svg | 98 +++++++-------- .../doc/images/output_start-airflow.txt | 2 +- .../doc/images/output_testing_db-tests.txt | 2 +- .../images/output_testing_non-db-tests.txt | 2 +- .../doc/images/output_testing_tests.txt | 2 +- 16 files changed, 224 insertions(+), 224 deletions(-) diff --git a/dev/breeze/doc/images/output_ci-image_build.svg b/dev/breeze/doc/images/output_ci-image_build.svg index d2089f482e0e2..ee363b3c9ac7b 100644 --- a/dev/breeze/doc/images/output_ci-image_build.svg +++ b/dev/breeze/doc/images/output_ci-image_build.svg @@ -354,97 +354,97 @@ Build CI image. Include building multiple images for all python versions. ╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ +│--python-pPython major/minor version used in Airflow image for images.│ │(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)                          │ │[default: 3.8]                                              │ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--upgrade-on-failure/--no-upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when       │ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--upgrade-on-failure/--no-upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when       │ │regular build fails. It is set to False by default on CI and True │ │by default locally.                                               │ │[default: upgrade-on-failure]                                     │ -│--image-tagTag the image after building it.(TEXT)[default: latest]│ -│--tag-as-latestTags the image as latest and update checksum of all files after   │ -│pulling. Useful when you build or pull image with --image-tag.    │ -│--docker-cache-cCache option for image used during the build.│ +│--image-tagTag the image after building it.(TEXT)[default: latest]│ +│--tag-as-latestTags the image as latest and update checksum of all files after   │ +│pulling. Useful when you build or pull image with --image-tag.    │ +│--docker-cache-cCache option for image used during the build.│ │(registry | local | disabled)                │ │[default: registry]                          │ -│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).│ +│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).│ │(TEXT)                                                         │ │[default: dev0]                                                │ -│--build-progressBuild progress.(auto | plain | tty)[default: auto]│ -│--docker-hostOptional - docker host to use when running docker commands. When  │ -│set, the `--builder` option is ignored when building images.      │ +│--build-progressBuild progress.(auto | plain | tty)[default: auto]│ +│--docker-hostOptional - docker host to use when running docker commands. When  │ +│set, the `--builder` option is ignored when building images.      │ │(TEXT)                                                            │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ │(INTEGER RANGE)                                                            │ │[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ │[default: 3.8 3.9 3.10 3.11 3.12]                                             │ -│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Advanced build options (for power users) ───────────────────────────────────────────────────────────────────────────╮ -│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`       │ +│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`       │ │itself).                                                                          │ │(TEXT)                                                                            │ -│--commit-shaCommit SHA that is used to build the images.(TEXT)│ -│--debian-versionDebian version used in Airflow image as base for building images.│ +│--commit-shaCommit SHA that is used to build the images.(TEXT)│ +│--debian-versionDebian version used in Airflow image as base for building images.│ │(bookworm | bullseye)                                            │ │[default: bookworm]                                              │ -│--install-mysql-client-typeWhich client to choose when installing.(mariadb | mysql)│ -│--python-imageIf specified this is the base python image used to build the image. Should be     │ +│--install-mysql-client-typeWhich client to choose when installing.(mariadb | mysql)│ +│--python-imageIf specified this is the base python image used to build the image. Should be     │ │something like: python:VERSION-slim-bookworm.                                     │ │(TEXT)                                                                            │ -│--use-uv/--no-use-uvUse uv instead of pip as packaging tool to build the image.[default: use-uv]│ -│--uv-http-timeoutTimeout for requests that UV makes (only used in case of UV builds).│ +│--use-uv/--no-use-uvUse uv instead of pip as packaging tool to build the image.[default: use-uv]│ +│--uv-http-timeoutTimeout for requests that UV makes (only used in case of UV builds).│ │(INTEGER RANGE)                                                     │ │[default: 300; x>=1]                                                │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Selecting constraint location (for power users) ────────────────────────────────────────────────────────────────────╮ -│--airflow-constraints-locationLocation of airflow constraints to use (remote URL or local context file).(TEXT)│ -│--airflow-constraints-modeMode of constraints for Airflow for CI image building.                 │ +│--airflow-constraints-locationLocation of airflow constraints to use (remote URL or local context file).(TEXT)│ +│--airflow-constraints-modeMode of constraints for Airflow for CI image building.                 │ │(constraints-source-providers | constraints | constraints-no-providers)│ │[default: constraints-source-providers]                                │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Choosing dependencies and extras (for power users) ─────────────────────────────────────────────────────────────────╮ -│--additional-airflow-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-airflow-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Backtracking options ───────────────────────────────────────────────────────────────────────────────────────────────╮ -│--build-timeout-minutesOptional timeout for the build in minutes. Useful to detect `pip`        │ +│--build-timeout-minutesOptional timeout for the build in minutes. Useful to detect `pip`        │ │backtracking problems.                                                   │ │(INTEGER)                                                                │ -│--eager-upgrade-additional-requirementsOptional additional requirements to upgrade eagerly to avoid backtracking│ +│--eager-upgrade-additional-requirementsOptional additional requirements to upgrade eagerly to avoid backtracking│ │(see `breeze ci find-backtracking-candidates`).                          │ │(TEXT)                                                                   │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ │[default: autodetect]                                         │ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--pushPush image after building it.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--pushPush image after building it.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ │image).                                                                                   │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_ci-image_build.txt b/dev/breeze/doc/images/output_ci-image_build.txt index 956086df7da3b..18c6a43c9a314 100644 --- a/dev/breeze/doc/images/output_ci-image_build.txt +++ b/dev/breeze/doc/images/output_ci-image_build.txt @@ -1 +1 @@ -2f9404d4776b9f417f14fd615e942534 +27c8608dc0d0c7351e647316c24c9dbd diff --git a/dev/breeze/doc/images/output_ci_selective-check.svg b/dev/breeze/doc/images/output_ci_selective-check.svg index 2ec3ba6a042ea..286f44b71569f 100644 --- a/dev/breeze/doc/images/output_ci_selective-check.svg +++ b/dev/breeze/doc/images/output_ci_selective-check.svg @@ -138,25 +138,25 @@ Checks what kind of tests should be run for an incoming commit. ╭─ Selective check flags ──────────────────────────────────────────────────────────────────────────────────────────────╮ -│--commit-refCommit-ish reference to the commit that should be checked(TEXT)│ -│--pr-labelsPython array formatted PR labels assigned to the PR(TEXT)│ -│--default-branchBranch against which the PR should be run(TEXT)[default: main]│ -│--default-constraints-branchConstraints Branch against which the PR should be run(TEXT)│ -│[default: constraints-main]                          │ +│--commit-refCommit-ish reference to the commit that should be checked(TEXT)│ +│--pr-labelsPython array formatted PR labels assigned to the PR(TEXT)│ +│--default-branchBranch against which the PR should be run(TEXT)[default: v2-10-test]│ +│--default-constraints-branchConstraints Branch against which the PR should be run(TEXT)│ +│[default: constraints-2-10]                          │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Github parameters ──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-event-nameName of the GitHub event that triggered the check                                          │ +│--github-event-nameName of the GitHub event that triggered the check                                          │ │(pull_request | pull_request_review | pull_request_target | pull_request_workflow | push | │ │schedule | workflow_dispatch | workflow_run)                                               │ │[default: pull_request]                                                                    │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--github-actorActor that triggered the event (Github user)(TEXT)│ -│--github-contextGithub context (JSON formatted) passed by Github Actions(TEXT)│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--github-actorActor that triggered the event (Github user)(TEXT)│ +│--github-contextGithub context (JSON formatted) passed by Github Actions(TEXT)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_ci_selective-check.txt b/dev/breeze/doc/images/output_ci_selective-check.txt index 22984f7955810..2eb3403f048f6 100644 --- a/dev/breeze/doc/images/output_ci_selective-check.txt +++ b/dev/breeze/doc/images/output_ci_selective-check.txt @@ -1 +1 @@ -6657ed5d42affb7264b5efcc86f17a2a +f9ac9fc50d20e8ecd823cad37888dc28 diff --git a/dev/breeze/doc/images/output_prod-image_build.svg b/dev/breeze/doc/images/output_prod-image_build.svg index 0e5a028346d9d..d791a37f14516 100644 --- a/dev/breeze/doc/images/output_prod-image_build.svg +++ b/dev/breeze/doc/images/output_prod-image_build.svg @@ -378,105 +378,105 @@ Build Production image. Include building multiple images for all or selected Python versions sequentially. ╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--build-progressBuild progress.(auto | plain | tty)[default: auto]│ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│--build-progressBuild progress.(auto | plain | tty)[default: auto]│ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ │[default: registry]                          │ -│--docker-hostOptional - docker host to use when running docker commands. When set, the `--builder`│ +│--docker-hostOptional - docker host to use when running docker commands. When set, the `--builder`│ │option is ignored when building images.                                              │ │(TEXT)                                                                               │ -│--image-tagTag the image after building it.(TEXT)[default: latest]│ -│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ -│--python-pPython major/minor version used in Airflow image for images.│ +│--image-tagTag the image after building it.(TEXT)[default: latest]│ +│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ +│--python-pPython major/minor version used in Airflow image for images.│ │(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)                          │ │[default: 3.8]                                              │ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when │ -│you build or pull image with --image-tag.                                            │ -│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when │ +│you build or pull image with --image-tag.                                            │ +│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ │(INTEGER RANGE)                                                            │ │[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ │[default: 3.8 3.9 3.10 3.11 3.12]                                             │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Advanced build options (for power users) ───────────────────────────────────────────────────────────────────────────╮ -│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`       │ +│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`       │ │itself).                                                                          │ │(TEXT)                                                                            │ -│--commit-shaCommit SHA that is used to build the images.(TEXT)│ -│--debian-versionDebian version used in Airflow image as base for building images.│ +│--commit-shaCommit SHA that is used to build the images.(TEXT)│ +│--debian-versionDebian version used in Airflow image as base for building images.│ │(bookworm | bullseye)                                            │ │[default: bookworm]                                              │ -│--python-imageIf specified this is the base python image used to build the image. Should be     │ +│--python-imageIf specified this is the base python image used to build the image. Should be     │ │something like: python:VERSION-slim-bookworm.                                     │ │(TEXT)                                                                            │ -│--use-uv/--no-use-uvUse uv instead of pip as packaging tool to build the image.[default: no-use-uv]│ -│--uv-http-timeoutTimeout for requests that UV makes (only used in case of UV builds).│ +│--use-uv/--no-use-uvUse uv instead of pip as packaging tool to build the image.[default: no-use-uv]│ +│--uv-http-timeoutTimeout for requests that UV makes (only used in case of UV builds).│ │(INTEGER RANGE)                                                     │ │[default: 300; x>=1]                                                │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Selecting constraint location (for power users) ────────────────────────────────────────────────────────────────────╮ -│--airflow-constraints-locationLocation of airflow constraints to use (remote URL or local context file).(TEXT)│ -│--airflow-constraints-modeMode of constraints for Airflow for PROD image building.               │ +│--airflow-constraints-locationLocation of airflow constraints to use (remote URL or local context file).(TEXT)│ +│--airflow-constraints-modeMode of constraints for Airflow for PROD image building.               │ │(constraints | constraints-no-providers | constraints-source-providers)│ │[default: constraints]                                                 │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Choosing dependencies and extras (for power users) ─────────────────────────────────────────────────────────────────╮ -│--airflow-extrasExtras to install by default.                                                   │ +│--airflow-extrasExtras to install by default.                                                   │ │(TEXT)                                                                          │ │[default:                                                                       │ │aiobotocore,amazon,async,celery,cncf-kubernetes,common-io,docker,elasticsearch,…│ -│--additional-airflow-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +│--additional-airflow-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Advanced customization options (for specific customization needs) ──────────────────────────────────────────────────╮ -│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)[default: .]│ -│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ -│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ -│Implies --disable-airflow-repo-cache.                                     │ -│--install-mysql-client-typeWhich client to choose when installing.(mariadb | mysql)│ -│--cleanup-contextClean up docker context files before running build (cannot be used        │ -│together with --install-packages-from-context).                           │ -│--use-constraints-for-context-packagesUses constraints for context packages installation - either from          │ +│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)[default: .]│ +│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ +│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ +│Implies --disable-airflow-repo-cache.                                     │ +│--install-mysql-client-typeWhich client to choose when installing.(mariadb | mysql)│ +│--cleanup-contextClean up docker context files before running build (cannot be used        │ +│together with --install-packages-from-context).                           │ +│--use-constraints-for-context-packagesUses constraints for context packages installation - either from          │ │constraints store in docker-context-files or from github.                 │ -│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ -│--disable-mysql-client-installationDo not install MySQL client.│ -│--disable-mssql-client-installationDo not install MsSQl client.│ -│--disable-postgres-client-installationDo not install Postgres client.│ +│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ +│--disable-mysql-client-installationDo not install MySQL client.│ +│--disable-mssql-client-installationDo not install MsSQl client.│ +│--disable-postgres-client-installationDo not install Postgres client.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ │[default: autodetect]                                         │ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--pushPush image after building it.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--pushPush image after building it.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ │image).                                                                                   │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_prod-image_build.txt b/dev/breeze/doc/images/output_prod-image_build.txt index 0ee4b9ab91157..b2e98a4f4824b 100644 --- a/dev/breeze/doc/images/output_prod-image_build.txt +++ b/dev/breeze/doc/images/output_prod-image_build.txt @@ -1 +1 @@ -613daeb0e2e5b0b117a3bf8e8eb8434f +92417f0412b52f7311c74867fce035e7 diff --git a/dev/breeze/doc/images/output_release-management_install-provider-packages.svg b/dev/breeze/doc/images/output_release-management_install-provider-packages.svg index 350a2d571f1dd..29e53f890d68a 100644 --- a/dev/breeze/doc/images/output_release-management_install-provider-packages.svg +++ b/dev/breeze/doc/images/output_release-management_install-provider-packages.svg @@ -249,62 +249,62 @@ Installs provider packages that can be found in dist. ╭─ Provider installation flags ────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ +│--python-pPython major/minor version used in Airflow image for images.│ │(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)                          │ │[default: 3.8]                                              │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =       │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =       │ │selected).                                                                                 │ │(selected | all | skip | remove | tests | providers-and-tests)                             │ │[default: selected]                                                                        │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Installing packages after entering shell ───────────────────────────────────────────────────────────────────────────╮ -│--airflow-constraints-locationLocation of airflow constraints to use (remote URL or local context file).│ +│--airflow-constraints-locationLocation of airflow constraints to use (remote URL or local context file).│ │(TEXT)                                                                    │ -│--airflow-constraints-modeMode of constraints for Airflow for CI image building.                 │ +│--airflow-constraints-modeMode of constraints for Airflow for CI image building.                 │ │(constraints-source-providers | constraints | constraints-no-providers)│ │[default: constraints-source-providers]                                │ -│--airflow-constraints-referenceConstraint reference to use for airflow installation (used in calculated       │ +│--airflow-constraints-referenceConstraint reference to use for airflow installation (used in calculated       │ │constraints URL).                                                              │ │(TEXT)                                                                         │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--airflow-skip-constraintsDo not use constraints when installing airflow.│ -│--install-selected-providersComma-separated list of providers selected to be installed (implies            │ -│--use-packages-from-dist).                                                     │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--airflow-skip-constraintsDo not use constraints when installing airflow.│ +│--install-selected-providersComma-separated list of providers selected to be installed (implies            │ +│--use-packages-from-dist).                                                     │ │(TEXT)                                                                         │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ │[default: wheel]                                      │ -│--providers-constraints-locationLocation of providers constraints to use (remote URL or local context file).│ +│--providers-constraints-locationLocation of providers constraints to use (remote URL or local context file).│ │(TEXT)                                                                      │ -│--providers-constraints-modeMode of constraints for Providers for CI image building.               │ +│--providers-constraints-modeMode of constraints for Providers for CI image building.               │ │(constraints-source-providers | constraints | constraints-no-providers)│ │[default: constraints-source-providers]                                │ -│--providers-constraints-referenceConstraint reference to use for providers installation (used in calculated     │ +│--providers-constraints-referenceConstraint reference to use for providers installation (used in calculated     │ │constraints URL). Can be 'default' in which case the default                   │ │constraints-reference is used.                                                 │ │(TEXT)                                                                         │ -│--providers-skip-constraintsDo not use constraints when installing providers.│ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be version (to │ +│--providers-skip-constraintsDo not use constraints when installing providers.│ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be version (to │ │install from PyPI), `none`, `wheel`, or `sdist` to install from `dist` folder, │ │or VCS URL to install from (https://pip.pypa.io/en/stable/topics/vcs-support/).│ -│Implies --mount-sources `remove`.                                              │ +│Implies --mount-sources `remove`.                                              │ │(none | wheel | sdist | <airflow_version>)                                     │ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ │folder when entering breeze.                                                   │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ │(INTEGER RANGE)                                                            │ │[default: 4; 1<=x<=8]                                                      │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -│--debug-resourcesWhether to show resource information while running in parallel.│ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +│--debug-resourcesWhether to show resource information while running in parallel.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_install-provider-packages.txt b/dev/breeze/doc/images/output_release-management_install-provider-packages.txt index 883ad7669971e..951ca7e9f026b 100644 --- a/dev/breeze/doc/images/output_release-management_install-provider-packages.txt +++ b/dev/breeze/doc/images/output_release-management_install-provider-packages.txt @@ -1 +1 @@ -7d8e9b3e73eda99894455d573ea940d7 +3db7c2188d442bd1d839873c66e18c32 diff --git a/dev/breeze/doc/images/output_release-management_verify-provider-packages.svg b/dev/breeze/doc/images/output_release-management_verify-provider-packages.svg index bb5aead938d11..f7118050dd39a 100644 --- a/dev/breeze/doc/images/output_release-management_verify-provider-packages.svg +++ b/dev/breeze/doc/images/output_release-management_verify-provider-packages.svg @@ -258,65 +258,65 @@ Verifies if all provider code is following expectations for providers. ╭─ Provider verification flags ────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ +│--python-pPython major/minor version used in Airflow image for images.│ │(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)                          │ │[default: 3.8]                                              │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =       │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =       │ │selected).                                                                                 │ │(selected | all | skip | remove | tests | providers-and-tests)                             │ │[default: selected]                                                                        │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Installing packages after entering shell ───────────────────────────────────────────────────────────────────────────╮ -│--airflow-constraints-locationLocation of airflow constraints to use (remote URL or   │ +│--airflow-constraints-locationLocation of airflow constraints to use (remote URL or   │ │local context file).                                    │ │(TEXT)                                                  │ -│--airflow-constraints-modeMode of constraints for Airflow for CI image building.  │ +│--airflow-constraints-modeMode of constraints for Airflow for CI image building.  │ │(constraints-source-providers | constraints |           │ │constraints-no-providers)                               │ │[default: constraints-source-providers]                 │ -│--airflow-constraints-referenceConstraint reference to use for airflow installation    │ +│--airflow-constraints-referenceConstraint reference to use for airflow installation    │ │(used in calculated constraints URL).                   │ │(TEXT)                                                  │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is │ │used                                                    │ │(TEXT)                                                  │ -│--airflow-skip-constraintsDo not use constraints when installing airflow.│ -│--install-airflow-with-constraints/--no-install-airflow…Install airflow in a separate step, with constraints    │ +│--airflow-skip-constraintsDo not use constraints when installing airflow.│ +│--install-airflow-with-constraints/--no-install-airflow…Install airflow in a separate step, with constraints    │ │determined from package or airflow version.             │ │[default: no-install-airflow-with-constraints]          │ -│--install-selected-providersComma-separated list of providers selected to be        │ -│installed (implies --use-packages-from-dist).           │ +│--install-selected-providersComma-separated list of providers selected to be        │ +│installed (implies --use-packages-from-dist).           │ │(TEXT)                                                  │ -│--package-formatFormat of packages that should be installed from dist.│ +│--package-formatFormat of packages that should be installed from dist.│ │(wheel | sdist)                                       │ │[default: wheel]                                      │ -│--providers-constraints-locationLocation of providers constraints to use (remote URL or │ +│--providers-constraints-locationLocation of providers constraints to use (remote URL or │ │local context file).                                    │ │(TEXT)                                                  │ -│--providers-constraints-modeMode of constraints for Providers for CI image building.│ +│--providers-constraints-modeMode of constraints for Providers for CI image building.│ │(constraints-source-providers | constraints |           │ │constraints-no-providers)                               │ │[default: constraints-source-providers]                 │ -│--providers-constraints-referenceConstraint reference to use for providers installation  │ +│--providers-constraints-referenceConstraint reference to use for providers installation  │ │(used in calculated constraints URL). Can be 'default'  │ │in which case the default constraints-reference is used.│ │(TEXT)                                                  │ -│--providers-skip-constraintsDo not use constraints when installing providers.│ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It  │ +│--providers-skip-constraintsDo not use constraints when installing providers.│ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It  │ │can also be version (to install from PyPI), `none`,     │ │`wheel`, or `sdist` to install from `dist` folder, or   │ │VCS URL to install from                                 │ │(https://pip.pypa.io/en/stable/topics/vcs-support/).    │ -│Implies --mount-sources `remove`.                       │ +│Implies --mount-sources `remove`.                       │ │(none | wheel | sdist | <airflow_version>)              │ -│--use-packages-from-distInstall all found packages (--package-format determines │ +│--use-packages-from-distInstall all found packages (--package-format determines │ │type) from 'dist' folder when entering breeze.          │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_verify-provider-packages.txt b/dev/breeze/doc/images/output_release-management_verify-provider-packages.txt index 42ab698416ead..96f5b1b758b6d 100644 --- a/dev/breeze/doc/images/output_release-management_verify-provider-packages.txt +++ b/dev/breeze/doc/images/output_release-management_verify-provider-packages.txt @@ -1 +1 @@ -f5c686ce50cae1ea4647b0f2202f731b +88910186cb3e385f972b7c02ba23282a diff --git a/dev/breeze/doc/images/output_shell.txt b/dev/breeze/doc/images/output_shell.txt index 876ed4ae8c6c5..321014a63d5ba 100644 --- a/dev/breeze/doc/images/output_shell.txt +++ b/dev/breeze/doc/images/output_shell.txt @@ -1 +1 @@ -56116218e3de632fa766f9c7368315a8 +e147d82143c5f3d4d22a7da9a3c181fa diff --git a/dev/breeze/doc/images/output_start-airflow.svg b/dev/breeze/doc/images/output_start-airflow.svg index 40079bf3aaba2..b959c9f9b0d0f 100644 --- a/dev/breeze/doc/images/output_start-airflow.svg +++ b/dev/breeze/doc/images/output_start-airflow.svg @@ -397,110 +397,110 @@ directory changed. ╭─ Execution mode ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ +│--python-pPython major/minor version used in Airflow image for images.│ │(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)                          │ │[default: 3.8]                                              │ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ -│--integrationIntegration(s) to enable when running (can be more than one).                       │ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ +│--integrationIntegration(s) to enable when running (can be more than one).                       │ │(all | all-testable | cassandra | celery | drill | kafka | kerberos | mongo | mssql │ │| openlineage | otel | pinot | qdrant | redis | statsd | trino | ydb)               │ -│--standalone-dag-processorRun standalone dag processor for start-airflow.│ -│--database-isolationRun airflow in database isolation mode.│ -│--load-example-dags-eEnable configuration to load example DAGs when starting Airflow.│ -│--load-default-connections-cEnable configuration to load default connections when starting Airflow.│ +│--standalone-dag-processorRun standalone dag processor for start-airflow.│ +│--database-isolationRun airflow in database isolation mode.│ +│--load-example-dags-eEnable configuration to load example DAGs when starting Airflow.│ +│--load-default-connections-cEnable configuration to load default connections when starting Airflow.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Docker Compose selection and cleanup ───────────────────────────────────────────────────────────────────────────────╮ -│--project-nameName of the docker-compose project to bring down. The `docker-compose` is for legacy  │ -│breeze project name and you can use `breeze down --project-name docker-compose` to    │ +│--project-nameName of the docker-compose project to bring down. The `docker-compose` is for legacy  │ +│breeze project name and you can use `breeze down --project-name docker-compose` to    │ │stop all containers belonging to it.                                                  │ │(breeze | pre-commit | docker-compose)                                                │ │[default: breeze]                                                                     │ -│--restart,--remove-orphansRestart all containers before entering shell (also removes orphan containers).│ -│--docker-hostOptional - docker host to use when running docker commands. When set, the `--builder` │ +│--restart,--remove-orphansRestart all containers before entering shell (also removes orphan containers).│ +│--docker-hostOptional - docker host to use when running docker commands. When set, the `--builder` │ │option is ignored when building images.                                               │ │(TEXT)                                                                                │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Database ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--backend-bDatabase backend to use. If 'none' is chosen, Breeze will start with an invalid database    │ +│--backend-bDatabase backend to use. If 'none' is chosen, Breeze will start with an invalid database    │ │configuration, meaning there will be no database available, and any attempts to connect to  │ │the Airflow database will fail.                                                             │ │(>sqlite< | mysql | postgres | none)                                                        │ │[default: sqlite]                                                                           │ -│--postgres-version-PVersion of Postgres used.(>12< | 13 | 14 | 15 | 16)[default: 12]│ -│--mysql-version-MVersion of MySQL used.(>8.0< | 8.4)[default: 8.0]│ -│--db-reset-dReset DB when entering the container.│ +│--postgres-version-PVersion of Postgres used.(>12< | 13 | 14 | 15 | 16)[default: 12]│ +│--mysql-version-MVersion of MySQL used.(>8.0< | 8.4)[default: 8.0]│ +│--db-reset-dReset DB when entering the container.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Choosing executor ──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--executorSpecify the executor to use with start-airflow command.│ +│--executorSpecify the executor to use with start-airflow command.│ │(LocalExecutor|CeleryExecutor|SequentialExecutor)      │ │[default: LocalExecutor]                               │ -│--celery-brokerSpecify the celery message broker(rabbitmq|redis)[default: redis]│ -│--celery-flowerStart celery flower│ +│--celery-brokerSpecify the celery message broker(rabbitmq|redis)[default: redis]│ +│--celery-flowerStart celery flower│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Asset compilation options ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--skip-assets-compilationSkips compilation of assets when starting airflow even if the content of www changed   │ -│(mutually exclusive with --dev-mode).                                                  │ -│--dev-modeStarts webserver in dev mode (assets are always recompiled in this case when starting) │ -│(mutually exclusive with --skip-assets-compilation).                                   │ +│--skip-assets-compilationSkips compilation of assets when starting airflow even if the content of www changed   │ +│(mutually exclusive with --dev-mode).                                                  │ +│--dev-modeStarts webserver in dev mode (assets are always recompiled in this case when starting) │ +│(mutually exclusive with --skip-assets-compilation).                                   │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Build CI image (before entering shell) ─────────────────────────────────────────────────────────────────────────────╮ -│--force-buildForce image build no matter if it is determined as needed.│ -│--image-tagTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ +│--force-buildForce image build no matter if it is determined as needed.│ +│--image-tagTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ │[default: latest]                                                              │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ │[default: autodetect]                                         │ -│--use-uv/--no-use-uvUse uv instead of pip as packaging tool to build the image.[default: use-uv]│ -│--uv-http-timeoutTimeout for requests that UV makes (only used in case of UV builds).(INTEGER RANGE)│ +│--use-uv/--no-use-uvUse uv instead of pip as packaging tool to build the image.[default: use-uv]│ +│--uv-http-timeoutTimeout for requests that UV makes (only used in case of UV builds).(INTEGER RANGE)│ │[default: 300; x>=1]                                                │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Mounting the sources and volumes ───────────────────────────────────────────────────────────────────────────────────╮ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default = selected).│ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default = selected).│ │(selected | all | skip | remove | tests | providers-and-tests)                                 │ │[default: selected]                                                                            │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Installing packages after entering shell ───────────────────────────────────────────────────────────────────────────╮ -│--airflow-constraints-locationLocation of airflow constraints to use (remote URL or local context file).│ +│--airflow-constraints-locationLocation of airflow constraints to use (remote URL or local context file).│ │(TEXT)                                                                    │ -│--airflow-constraints-modeMode of constraints for Airflow for CI image building.                 │ +│--airflow-constraints-modeMode of constraints for Airflow for CI image building.                 │ │(constraints-source-providers | constraints | constraints-no-providers)│ │[default: constraints-source-providers]                                │ -│--airflow-constraints-referenceConstraint reference to use for airflow installation (used in calculated       │ +│--airflow-constraints-referenceConstraint reference to use for airflow installation (used in calculated       │ │constraints URL).                                                              │ │(TEXT)                                                                         │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--airflow-skip-constraintsDo not use constraints when installing airflow.│ -│--install-selected-providersComma-separated list of providers selected to be installed (implies            │ -│--use-packages-from-dist).                                                     │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--airflow-skip-constraintsDo not use constraints when installing airflow.│ +│--install-selected-providersComma-separated list of providers selected to be installed (implies            │ +│--use-packages-from-dist).                                                     │ │(TEXT)                                                                         │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ │[default: wheel]                                      │ -│--providers-constraints-locationLocation of providers constraints to use (remote URL or local context file).│ +│--providers-constraints-locationLocation of providers constraints to use (remote URL or local context file).│ │(TEXT)                                                                      │ -│--providers-constraints-modeMode of constraints for Providers for CI image building.               │ +│--providers-constraints-modeMode of constraints for Providers for CI image building.               │ │(constraints-source-providers | constraints | constraints-no-providers)│ │[default: constraints-source-providers]                                │ -│--providers-constraints-referenceConstraint reference to use for providers installation (used in calculated     │ +│--providers-constraints-referenceConstraint reference to use for providers installation (used in calculated     │ │constraints URL). Can be 'default' in which case the default                   │ │constraints-reference is used.                                                 │ │(TEXT)                                                                         │ -│--providers-skip-constraintsDo not use constraints when installing providers.│ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be version (to │ +│--providers-skip-constraintsDo not use constraints when installing providers.│ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be version (to │ │install from PyPI), `none`, `wheel`, or `sdist` to install from `dist` folder, │ │or VCS URL to install from (https://pip.pypa.io/en/stable/topics/vcs-support/).│ -│Implies --mount-sources `remove`.                                              │ +│Implies --mount-sources `remove`.                                              │ │(none | wheel | sdist | <airflow_version>)                                     │ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ │folder when entering breeze.                                                   │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Other options ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--forward-credentials-fForward local credentials to container when running.│ +│--forward-credentials-fForward local credentials to container when running.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_start-airflow.txt b/dev/breeze/doc/images/output_start-airflow.txt index 3dfbef58dedf1..f48a6cb5268d4 100644 --- a/dev/breeze/doc/images/output_start-airflow.txt +++ b/dev/breeze/doc/images/output_start-airflow.txt @@ -1 +1 @@ -929ca9ec2fc8c229c302b161f610059b +057a38d65515c5db6201648467bc8178 diff --git a/dev/breeze/doc/images/output_testing_db-tests.txt b/dev/breeze/doc/images/output_testing_db-tests.txt index 753d881613a61..f541294aa6fdc 100644 --- a/dev/breeze/doc/images/output_testing_db-tests.txt +++ b/dev/breeze/doc/images/output_testing_db-tests.txt @@ -1 +1 @@ -d07def255ae142d31f4d4e07d8c6225a +ec6eb51a5f7de9ef6fbf04804497c12d diff --git a/dev/breeze/doc/images/output_testing_non-db-tests.txt b/dev/breeze/doc/images/output_testing_non-db-tests.txt index 26e669f152318..2d12e48e4ad8a 100644 --- a/dev/breeze/doc/images/output_testing_non-db-tests.txt +++ b/dev/breeze/doc/images/output_testing_non-db-tests.txt @@ -1 +1 @@ -08818852a363a43a9d4bbd319e13fd7a +f1d8ce62638d3f7e6a74184e70fa0e56 diff --git a/dev/breeze/doc/images/output_testing_tests.txt b/dev/breeze/doc/images/output_testing_tests.txt index d9c1a9943a6f2..fe753b6946331 100644 --- a/dev/breeze/doc/images/output_testing_tests.txt +++ b/dev/breeze/doc/images/output_testing_tests.txt @@ -1 +1 @@ -8c0f8ccb27bfdd44990a8997f9ec0116 +c69367b314fab4ad8a962e9e9f239f6a From f2a15d44258c653e321b53a33efb2e2e54280189 Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Fri, 9 Aug 2024 05:55:25 -0400 Subject: [PATCH 004/161] Fix Gantt Task Tries (#41342) (cherry picked from commit cb80bda5eedb3a52fa786df81d5e7eb14caf31ff) --- .../www/static/js/dag/details/gantt/Row.tsx | 35 ++++++++---- .../www/static/js/dag/details/gantt/index.tsx | 53 +++++++++++-------- airflow/www/static/js/dag/details/index.tsx | 2 + 3 files changed, 57 insertions(+), 33 deletions(-) diff --git a/airflow/www/static/js/dag/details/gantt/Row.tsx b/airflow/www/static/js/dag/details/gantt/Row.tsx index 2a51f9c6db5a7..9abd626962e70 100644 --- a/airflow/www/static/js/dag/details/gantt/Row.tsx +++ b/airflow/www/static/js/dag/details/gantt/Row.tsx @@ -69,6 +69,17 @@ const Row = ({ const isSelected = taskId === instance?.taskId; const isOpen = openGroupIds.includes(task.id || ""); + // Adjust gantt start/end if the instance dates are out of bounds + useEffect(() => { + if (setGanttDuration) { + setGanttDuration( + instance?.queuedDttm, + instance?.startDate, + instance?.endDate + ); + } + }, [instance, setGanttDuration]); + // Adjust gantt start/end if the ti history dates are out of bounds useEffect(() => { tiHistory?.taskInstances?.forEach( @@ -91,6 +102,7 @@ const Row = ({ > {!!instance && ( )} - {tiHistory?.taskInstances?.map((ti) => ( - - ))} + {tiHistory?.taskInstances?.map( + (ti) => + ti.tryNumber !== instance?.tryNumber && ( + + ) + )} {isOpen && !!task.children && diff --git a/airflow/www/static/js/dag/details/gantt/index.tsx b/airflow/www/static/js/dag/details/gantt/index.tsx index cb17d369ccc65..45c10d2b525e5 100644 --- a/airflow/www/static/js/dag/details/gantt/index.tsx +++ b/airflow/www/static/js/dag/details/gantt/index.tsx @@ -20,7 +20,6 @@ import React, { useCallback, useEffect, useRef, useState } from "react"; import { Alert, AlertIcon, Box, Divider, Text } from "@chakra-ui/react"; -import useSelection from "src/dag/useSelection"; import { useGridData } from "src/api"; import Time from "src/components/Time"; import { getDuration } from "src/datetime_utils"; @@ -28,19 +27,27 @@ import { getDuration } from "src/datetime_utils"; import Row from "./Row"; interface Props { + runId: string | null; + taskId: string | null; openGroupIds: string[]; gridScrollRef: React.RefObject; ganttScrollRef: React.RefObject; } -const Gantt = ({ openGroupIds, gridScrollRef, ganttScrollRef }: Props) => { +const Gantt = ({ + runId, + taskId, + openGroupIds, + gridScrollRef, + ganttScrollRef, +}: Props) => { const ganttRef = useRef(null); const [top, setTop] = useState(0); const [width, setWidth] = useState(500); const [height, setHeight] = useState("100%"); - const { - selected: { runId, taskId }, - } = useSelection(); + const [startDate, setStartDate] = useState(); + const [endDate, setEndDate] = useState(); + const { data: { dagRuns, groups }, } = useGridData(); @@ -106,15 +113,6 @@ const Gantt = ({ openGroupIds, gridScrollRef, ganttScrollRef }: Props) => { const dagRun = dagRuns.find((dr) => dr.runId === runId); - const [startDate, setStartDate] = useState( - dagRun?.queuedAt || dagRun?.startDate - ); - - const [endDate, setEndDate] = useState( - // @ts-ignore - dagRun?.endDate ?? moment().add(1, "s").toString() - ); - // Check if any task instance dates are outside the bounds of the dag run dates and update our min start and max end const setGanttDuration = useCallback( ( @@ -136,21 +134,30 @@ const Gantt = ({ openGroupIds, gridScrollRef, ganttScrollRef }: Props) => { if (end && (!endDate || Date.parse(end) > Date.parse(endDate))) { setEndDate(end); + } else if (!end) { + // @ts-ignore + setEndDate(moment().add(1, "s").toString()); } }, [startDate, endDate, setStartDate, setEndDate] ); + // Reset state when the dagrun changes useEffect(() => { - groups.children?.forEach((task) => { - const taskInstance = task.instances.find((ti) => ti.runId === runId); - setGanttDuration( - taskInstance?.queuedDttm, - taskInstance?.startDate, - taskInstance?.endDate - ); - }); - }, [groups.children, runId, setGanttDuration]); + if (startDate !== dagRun?.queuedAt && startDate !== dagRun?.startDate) { + setStartDate(dagRun?.queuedAt || dagRun?.startDate); + } + if (!endDate || endDate !== dagRun?.endDate) { + // @ts-ignore + setEndDate(dagRun?.endDate ?? moment().add(1, "s").toString()); + } + }, [ + dagRun?.queuedAt, + dagRun?.startDate, + dagRun?.endDate, + startDate, + endDate, + ]); const numBars = Math.round(width / 100); const runDuration = getDuration(startDate, endDate); diff --git a/airflow/www/static/js/dag/details/index.tsx b/airflow/www/static/js/dag/details/index.tsx index da2461d6bb21c..0046d339972a6 100644 --- a/airflow/www/static/js/dag/details/index.tsx +++ b/airflow/www/static/js/dag/details/index.tsx @@ -449,6 +449,8 @@ const Details = ({ openGroupIds={openGroupIds} gridScrollRef={gridScrollRef} ganttScrollRef={ganttScrollRef} + taskId={taskId} + runId={runId} /> From 0ee7aff654876bb23a5ba0c73d81859cf3b01ce9 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Fri, 9 Aug 2024 17:59:26 +0100 Subject: [PATCH 005/161] Revert "Send context using in venv operator (#41039)" (#41362) This reverts commit da553935d248f22695124c40777d3ea29e04d57f. (cherry picked from commit 0a06cc683a877d63305060766ae78f9ffe1fba0e) --- airflow/decorators/__init__.pyi | 6 - .../example_python_context_decorator.py | 92 ------------- .../example_python_context_operator.py | 91 ------------- airflow/operators/python.py | 36 ------ airflow/utils/python_virtualenv_script.jinja2 | 23 ---- docs/apache-airflow/howto/operator/python.rst | 92 ------------- tests/operators/test_python.py | 122 +----------------- 7 files changed, 1 insertion(+), 461 deletions(-) delete mode 100644 airflow/example_dags/example_python_context_decorator.py delete mode 100644 airflow/example_dags/example_python_context_operator.py diff --git a/airflow/decorators/__init__.pyi b/airflow/decorators/__init__.pyi index 089e453d02b43..faf77e8240d6c 100644 --- a/airflow/decorators/__init__.pyi +++ b/airflow/decorators/__init__.pyi @@ -125,7 +125,6 @@ class TaskDecoratorCollection: env_vars: dict[str, str] | None = None, inherit_env: bool = True, use_dill: bool = False, - use_airflow_context: bool = False, **kwargs, ) -> TaskDecorator: """Create a decorator to convert the decorated callable to a virtual environment task. @@ -177,7 +176,6 @@ class TaskDecoratorCollection: :param use_dill: Deprecated, use ``serializer`` instead. Whether to use dill to serialize the args and result (pickle is default). This allows more complex types but requires you to include dill in your requirements. - :param use_airflow_context: Whether to provide ``get_current_context()`` to the python_callable. """ @overload def virtualenv(self, python_callable: Callable[FParams, FReturn]) -> Task[FParams, FReturn]: ... @@ -194,7 +192,6 @@ class TaskDecoratorCollection: env_vars: dict[str, str] | None = None, inherit_env: bool = True, use_dill: bool = False, - use_airflow_context: bool = False, **kwargs, ) -> TaskDecorator: """Create a decorator to convert the decorated callable to a virtual environment task. @@ -228,7 +225,6 @@ class TaskDecoratorCollection: :param use_dill: Deprecated, use ``serializer`` instead. Whether to use dill to serialize the args and result (pickle is default). This allows more complex types but requires you to include dill in your requirements. - :param use_airflow_context: Whether to provide ``get_current_context()`` to the python_callable. """ @overload def branch( # type: ignore[misc] @@ -262,7 +258,6 @@ class TaskDecoratorCollection: venv_cache_path: None | str = None, show_return_value_in_logs: bool = True, use_dill: bool = False, - use_airflow_context: bool = False, **kwargs, ) -> TaskDecorator: """Create a decorator to wrap the decorated callable into a BranchPythonVirtualenvOperator. @@ -304,7 +299,6 @@ class TaskDecoratorCollection: :param use_dill: Deprecated, use ``serializer`` instead. Whether to use dill to serialize the args and result (pickle is default). This allows more complex types but requires you to include dill in your requirements. - :param use_airflow_context: Whether to provide ``get_current_context()`` to the python_callable. """ @overload def branch_virtualenv(self, python_callable: Callable[FParams, FReturn]) -> Task[FParams, FReturn]: ... diff --git a/airflow/example_dags/example_python_context_decorator.py b/airflow/example_dags/example_python_context_decorator.py deleted file mode 100644 index 497ee08e17cea..0000000000000 --- a/airflow/example_dags/example_python_context_decorator.py +++ /dev/null @@ -1,92 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Example DAG demonstrating the usage of the PythonOperator with `get_current_context()` to get the current context. - -Also, demonstrates the usage of the TaskFlow API. -""" - -from __future__ import annotations - -import sys - -import pendulum - -from airflow.decorators import dag, task - -SOME_EXTERNAL_PYTHON = sys.executable - - -@dag( - schedule=None, - start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), - catchup=False, - tags=["example"], -) -def example_python_context_decorator(): - # [START get_current_context] - @task(task_id="print_the_context") - def print_context() -> str: - """Print the Airflow context.""" - from pprint import pprint - - from airflow.operators.python import get_current_context - - context = get_current_context() - pprint(context) - return "Whatever you return gets printed in the logs" - - print_the_context = print_context() - # [END get_current_context] - - # [START get_current_context_venv] - @task.virtualenv(task_id="print_the_context_venv", use_airflow_context=True) - def print_context_venv() -> str: - """Print the Airflow context in venv.""" - from pprint import pprint - - from airflow.operators.python import get_current_context - - context = get_current_context() - pprint(context) - return "Whatever you return gets printed in the logs" - - print_the_context_venv = print_context_venv() - # [END get_current_context_venv] - - # [START get_current_context_external] - @task.external_python( - task_id="print_the_context_external", python=SOME_EXTERNAL_PYTHON, use_airflow_context=True - ) - def print_context_external() -> str: - """Print the Airflow context in external python.""" - from pprint import pprint - - from airflow.operators.python import get_current_context - - context = get_current_context() - pprint(context) - return "Whatever you return gets printed in the logs" - - print_the_context_external = print_context_external() - # [END get_current_context_external] - - _ = print_the_context >> [print_the_context_venv, print_the_context_external] - - -example_python_context_decorator() diff --git a/airflow/example_dags/example_python_context_operator.py b/airflow/example_dags/example_python_context_operator.py deleted file mode 100644 index f1b76c527cfd6..0000000000000 --- a/airflow/example_dags/example_python_context_operator.py +++ /dev/null @@ -1,91 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Example DAG demonstrating the usage of the PythonOperator with `get_current_context()` to get the current context. - -Also, demonstrates the usage of the classic Python operators. -""" - -from __future__ import annotations - -import sys - -import pendulum - -from airflow import DAG -from airflow.operators.python import ExternalPythonOperator, PythonOperator, PythonVirtualenvOperator - -SOME_EXTERNAL_PYTHON = sys.executable - -with DAG( - dag_id="example_python_context_operator", - schedule=None, - start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), - catchup=False, - tags=["example"], -) as dag: - # [START get_current_context] - def print_context() -> str: - """Print the Airflow context.""" - from pprint import pprint - - from airflow.operators.python import get_current_context - - context = get_current_context() - pprint(context) - return "Whatever you return gets printed in the logs" - - print_the_context = PythonOperator(task_id="print_the_context", python_callable=print_context) - # [END get_current_context] - - # [START get_current_context_venv] - def print_context_venv() -> str: - """Print the Airflow context in venv.""" - from pprint import pprint - - from airflow.operators.python import get_current_context - - context = get_current_context() - pprint(context) - return "Whatever you return gets printed in the logs" - - print_the_context_venv = PythonVirtualenvOperator( - task_id="print_the_context_venv", python_callable=print_context_venv, use_airflow_context=True - ) - # [END get_current_context_venv] - - # [START get_current_context_external] - def print_context_external() -> str: - """Print the Airflow context in external python.""" - from pprint import pprint - - from airflow.operators.python import get_current_context - - context = get_current_context() - pprint(context) - return "Whatever you return gets printed in the logs" - - print_the_context_external = ExternalPythonOperator( - task_id="print_the_context_external", - python_callable=print_context_external, - python=SOME_EXTERNAL_PYTHON, - use_airflow_context=True, - ) - # [END get_current_context_external] - - _ = print_the_context >> [print_the_context_venv, print_the_context_external] diff --git a/airflow/operators/python.py b/airflow/operators/python.py index ce6ccd3a40f7a..fdfe575fb927f 100644 --- a/airflow/operators/python.py +++ b/airflow/operators/python.py @@ -56,14 +56,12 @@ from airflow.utils.operator_helpers import ExecutionCallableRunner, KeywordParameters from airflow.utils.process_utils import execute_in_subprocess from airflow.utils.python_virtualenv import prepare_virtualenv, write_python_script -from airflow.utils.session import create_session log = logging.getLogger(__name__) if TYPE_CHECKING: from pendulum.datetime import DateTime - from airflow.serialization.enums import Encoding from airflow.utils.context import Context @@ -444,7 +442,6 @@ def __init__( env_vars: dict[str, str] | None = None, inherit_env: bool = True, use_dill: bool = False, - use_airflow_context: bool = False, **kwargs, ): if ( @@ -497,7 +494,6 @@ def __init__( ) self.env_vars = env_vars self.inherit_env = inherit_env - self.use_airflow_context = use_airflow_context @abstractmethod def _iter_serializable_context_keys(self): @@ -544,7 +540,6 @@ def _execute_python_callable_in_subprocess(self, python_path: Path): string_args_path = tmp_dir / "string_args.txt" script_path = tmp_dir / "script.py" termination_log_path = tmp_dir / "termination.log" - airflow_context_path = tmp_dir / "airflow_context.json" self._write_args(input_path) self._write_string_args(string_args_path) @@ -556,7 +551,6 @@ def _execute_python_callable_in_subprocess(self, python_path: Path): "pickling_library": self.serializer, "python_callable": self.python_callable.__name__, "python_callable_source": self.get_python_source(), - "use_airflow_context": self.use_airflow_context, } if inspect.getfile(self.python_callable) == self.dag.fileloc: @@ -567,23 +561,6 @@ def _execute_python_callable_in_subprocess(self, python_path: Path): filename=os.fspath(script_path), render_template_as_native_obj=self.dag.render_template_as_native_obj, ) - if self.use_airflow_context: - from airflow.serialization.serialized_objects import BaseSerialization - - context = get_current_context() - # TODO: `TaskInstance`` will also soon be serialized as expected. - # see more: - # https://github.com/apache/airflow/issues/40974 - # https://github.com/apache/airflow/pull/41067 - with create_session() as session: - # FIXME: DetachedInstanceError - dag_run, task_instance = context["dag_run"], context["task_instance"] - session.add_all([dag_run, task_instance]) - serializable_context: dict[Encoding, Any] = BaseSerialization.serialize( - context, use_pydantic_models=True - ) - with airflow_context_path.open("w+") as file: - json.dump(serializable_context, file) env_vars = dict(os.environ) if self.inherit_env else {} if self.env_vars: @@ -598,7 +575,6 @@ def _execute_python_callable_in_subprocess(self, python_path: Path): os.fspath(output_path), os.fspath(string_args_path), os.fspath(termination_log_path), - os.fspath(airflow_context_path), ], env=env_vars, ) @@ -690,7 +666,6 @@ class PythonVirtualenvOperator(_BasePythonVirtualenvOperator): :param use_dill: Deprecated, use ``serializer`` instead. Whether to use dill to serialize the args and result (pickle is default). This allows more complex types but requires you to include dill in your requirements. - :param use_airflow_context: Whether to provide ``get_current_context()`` to the python_callable. """ template_fields: Sequence[str] = tuple( @@ -719,7 +694,6 @@ def __init__( env_vars: dict[str, str] | None = None, inherit_env: bool = True, use_dill: bool = False, - use_airflow_context: bool = False, **kwargs, ): if ( @@ -741,9 +715,6 @@ def __init__( ) if not is_venv_installed(): raise AirflowException("PythonVirtualenvOperator requires virtualenv, please install it.") - if use_airflow_context and (not expect_airflow and not system_site_packages): - error_msg = "use_airflow_context is set to True, but expect_airflow and system_site_packages are set to False." - raise AirflowException(error_msg) if not requirements: self.requirements: list[str] = [] elif isinstance(requirements, str): @@ -773,7 +744,6 @@ def __init__( env_vars=env_vars, inherit_env=inherit_env, use_dill=use_dill, - use_airflow_context=use_airflow_context, **kwargs, ) @@ -992,7 +962,6 @@ class ExternalPythonOperator(_BasePythonVirtualenvOperator): :param use_dill: Deprecated, use ``serializer`` instead. Whether to use dill to serialize the args and result (pickle is default). This allows more complex types but requires you to include dill in your requirements. - :param use_airflow_context: Whether to provide ``get_current_context()`` to the python_callable. """ template_fields: Sequence[str] = tuple({"python"}.union(PythonOperator.template_fields)) @@ -1014,14 +983,10 @@ def __init__( env_vars: dict[str, str] | None = None, inherit_env: bool = True, use_dill: bool = False, - use_airflow_context: bool = False, **kwargs, ): if not python: raise ValueError("Python Path must be defined in ExternalPythonOperator") - if use_airflow_context and not expect_airflow: - error_msg = "use_airflow_context is set to True, but expect_airflow is set to False." - raise AirflowException(error_msg) self.python = python self.expect_pendulum = expect_pendulum super().__init__( @@ -1037,7 +1002,6 @@ def __init__( env_vars=env_vars, inherit_env=inherit_env, use_dill=use_dill, - use_airflow_context=use_airflow_context, **kwargs, ) diff --git a/airflow/utils/python_virtualenv_script.jinja2 b/airflow/utils/python_virtualenv_script.jinja2 index 22d68acd755b2..2ff417985e887 100644 --- a/airflow/utils/python_virtualenv_script.jinja2 +++ b/airflow/utils/python_virtualenv_script.jinja2 @@ -64,29 +64,6 @@ with open(sys.argv[3], "r") as file: virtualenv_string_args = list(map(lambda x: x.strip(), list(file))) {% endif %} -{% if use_airflow_context | default(false) -%} -if len(sys.argv) > 5: - import json - from types import ModuleType - - from airflow.operators import python as airflow_python - from airflow.serialization.serialized_objects import BaseSerialization - - - class _MockPython(ModuleType): - @staticmethod - def get_current_context(): - with open(sys.argv[5]) as file: - context = json.load(file) - return BaseSerialization.deserialize(context, use_pydantic_models=True) - - def __getattr__(self, name: str): - return getattr(airflow_python, name) - - - MockPython = _MockPython("MockPython") - sys.modules["airflow.operators.python"] = MockPython -{% endif %} try: res = {{ python_callable }}(*arg_dict["args"], **arg_dict["kwargs"]) diff --git a/docs/apache-airflow/howto/operator/python.rst b/docs/apache-airflow/howto/operator/python.rst index 5b5a60b6bcfe5..b8619cd38bce8 100644 --- a/docs/apache-airflow/howto/operator/python.rst +++ b/docs/apache-airflow/howto/operator/python.rst @@ -102,37 +102,6 @@ is evaluated as a :ref:`Jinja template `. :start-after: [START howto_operator_python_render_sql] :end-before: [END howto_operator_python_render_sql] -Context -^^^^^^^ - -The ``Context`` is a dictionary object that contains information -about the environment of the ``DagRun``. -For example, selecting ``task_instance`` will get the currently running ``TaskInstance`` object. - -It can be used implicitly, such as with ``**kwargs``, -but can also be used explicitly with ``get_current_context()``. -In this case, the type hint can be used for static analysis. - -.. tab-set:: - - .. tab-item:: @task - :sync: taskflow - - .. exampleinclude:: /../../airflow/example_dags/example_python_context_decorator.py - :language: python - :dedent: 4 - :start-after: [START get_current_context] - :end-before: [END get_current_context] - - .. tab-item:: PythonOperator - :sync: operator - - .. exampleinclude:: /../../airflow/example_dags/example_python_context_operator.py - :language: python - :dedent: 4 - :start-after: [START get_current_context] - :end-before: [END get_current_context] - .. _howto/operator:PythonVirtualenvOperator: PythonVirtualenvOperator @@ -234,42 +203,6 @@ In case you have problems during runtime with broken cached virtual environments Note that any modification of a cached virtual environment (like temp files in binary path, post-installing further requirements) might pollute a cached virtual environment and the operator is not maintaining or cleaning the cache path. -Context -^^^^^^^ - -With some limitations, you can also use ``Context`` in virtual environments. - -.. important:: - Using ``Context`` in a virtual environment is a bit of a challenge - because it involves library dependencies and serialization issues. - - You can bypass this to some extent by using :ref:`Jinja template variables ` and explicitly passing it as a parameter. - - You can also use ``get_current_context()`` in the same way as before, but with some limitations. - - * set ``use_airflow_context`` to ``True`` to call ``get_current_context()`` in the virtual environment. - - * set ``system_site_packages`` to ``True`` or set ``expect_airflow`` to ``True`` - -.. tab-set:: - - .. tab-item:: @task.virtualenv - :sync: taskflow - - .. exampleinclude:: /../../airflow/example_dags/example_python_context_decorator.py - :language: python - :dedent: 4 - :start-after: [START get_current_context_venv] - :end-before: [END get_current_context_venv] - - .. tab-item:: PythonVirtualenvOperator - :sync: operator - - .. exampleinclude:: /../../airflow/example_dags/example_python_context_operator.py - :language: python - :dedent: 4 - :start-after: [START get_current_context_venv] - :end-before: [END get_current_context_venv] .. _howto/operator:ExternalPythonOperator: @@ -334,31 +267,6 @@ If you want the context related to datetime objects like ``data_interval_start`` If you want to pass variables into the classic :class:`~airflow.operators.python.ExternalPythonOperator` use ``op_args`` and ``op_kwargs``. -Context -^^^^^^^ - -You can use ``Context`` under the same conditions as ``PythonVirtualenvOperator``. - -.. tab-set:: - - .. tab-item:: @task.external_python - :sync: taskflow - - .. exampleinclude:: /../../airflow/example_dags/example_python_context_decorator.py - :language: python - :dedent: 4 - :start-after: [START get_current_context_external] - :end-before: [END get_current_context_external] - - .. tab-item:: ExternalPythonOperator - :sync: operator - - .. exampleinclude:: /../../airflow/example_dags/example_python_context_operator.py - :language: python - :dedent: 4 - :start-after: [START get_current_context_external] - :end-before: [END get_current_context_external] - .. _howto/operator:PythonBranchOperator: PythonBranchOperator diff --git a/tests/operators/test_python.py b/tests/operators/test_python.py index 9148ae18b70bd..993d70cad3340 100644 --- a/tests/operators/test_python.py +++ b/tests/operators/test_python.py @@ -39,15 +39,10 @@ from slugify import slugify from airflow.decorators import task_group -from airflow.exceptions import ( - AirflowException, - DeserializingResultError, - RemovedInAirflow3Warning, -) +from airflow.exceptions import AirflowException, DeserializingResultError, RemovedInAirflow3Warning from airflow.models.baseoperator import BaseOperator from airflow.models.dag import DAG from airflow.models.taskinstance import TaskInstance, clear_task_instances, set_current_context -from airflow.operators.branch import BranchMixIn from airflow.operators.empty import EmptyOperator from airflow.operators.python import ( BranchExternalPythonOperator, @@ -1010,75 +1005,6 @@ def f(): task = self.run_as_task(f, env_vars={"MY_ENV_VAR": "EFGHI"}, inherit_env=True) assert task.execute_callable() == "EFGHI" - def test_branch_current_context(self): - if not issubclass(self.opcls, BranchMixIn): - pytest.skip("This test is only applicable to BranchMixIn") - - def test_current_context(self): - def f(): - from airflow.operators.python import get_current_context - from airflow.utils.context import Context - - context = get_current_context() - if not isinstance(context, Context): # type: ignore[misc] - error_msg = f"Expected Context, got {type(context)}" - raise TypeError(error_msg) - - return [] - - ti = self.run_as_task(f, return_ti=True, multiple_outputs=False, use_airflow_context=True) - assert ti.state == TaskInstanceState.SUCCESS - - def test_current_context_not_found_error(self): - def f(): - from airflow.operators.python import get_current_context - - get_current_context() - return [] - - with pytest.raises( - AirflowException, - match="Current context was requested but no context was found! " - "Are you running within an airflow task?", - ): - self.run_as_task(f, return_ti=True, multiple_outputs=False, use_airflow_context=False) - - def test_current_context_airflow_not_found_error(self): - airflow_flag: dict[str, bool] = {"expect_airflow": False} - error_msg = "use_airflow_context is set to True, but expect_airflow is set to False." - - if not issubclass(self.opcls, ExternalPythonOperator): - airflow_flag["system_site_packages"] = False - error_msg = "use_airflow_context is set to True, but expect_airflow and system_site_packages are set to False." - - def f(): - from airflow.operators.python import get_current_context - - get_current_context() - return [] - - with pytest.raises(AirflowException, match=error_msg): - self.run_as_task( - f, return_ti=True, multiple_outputs=False, use_airflow_context=True, **airflow_flag - ) - - def test_use_airflow_context_touch_other_variables(self): - def f(): - from airflow.operators.python import get_current_context - from airflow.utils.context import Context - - context = get_current_context() - if not isinstance(context, Context): # type: ignore[misc] - error_msg = f"Expected Context, got {type(context)}" - raise TypeError(error_msg) - - from airflow.operators.python import PythonOperator # noqa: F401 - - return [] - - ti = self.run_as_task(f, return_ti=True, multiple_outputs=False, use_airflow_context=True) - assert ti.state == TaskInstanceState.SUCCESS - venv_cache_path = tempfile.mkdtemp(prefix="venv_cache_path") @@ -1500,29 +1426,6 @@ def f( self.run_as_task(f, serializer=serializer, system_site_packages=False, requirements=None) - def test_current_context_system_site_packages(self, session): - def f(): - from airflow.operators.python import get_current_context - from airflow.utils.context import Context - - context = get_current_context() - if not isinstance(context, Context): # type: ignore[misc] - error_msg = f"Expected Context, got {type(context)}" - raise TypeError(error_msg) - - return [] - - ti = self.run_as_task( - f, - return_ti=True, - multiple_outputs=False, - use_airflow_context=True, - session=session, - expect_airflow=False, - system_site_packages=True, - ) - assert ti.state == TaskInstanceState.SUCCESS - # when venv tests are run in parallel to other test they create new processes and this might take # quite some time in shared docker environment and get some contention even between different containers @@ -1842,29 +1745,6 @@ def default_kwargs(*, python_version=DEFAULT_PYTHON_VERSION, **kwargs): kwargs["venv_cache_path"] = venv_cache_path return kwargs - def test_current_context_system_site_packages(self, session): - def f(): - from airflow.operators.python import get_current_context - from airflow.utils.context import Context - - context = get_current_context() - if not isinstance(context, Context): # type: ignore[misc] - error_msg = f"Expected Context, got {type(context)}" - raise TypeError(error_msg) - - return [] - - ti = self.run_as_task( - f, - return_ti=True, - multiple_outputs=False, - use_airflow_context=True, - session=session, - expect_airflow=False, - system_site_packages=True, - ) - assert ti.state == TaskInstanceState.SUCCESS - # when venv tests are run in parallel to other test they create new processes and this might take # quite some time in shared docker environment and get some contention even between different containers From 6b6224a7b8d76cbefd51768a2dd9ec205126e667 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Sun, 11 Aug 2024 14:23:33 +0100 Subject: [PATCH 006/161] Retain the function "resource_name_for_dag" for backwards compatibility (#41382) When using older FAB providers on the new airflow, this function is called in the old provider and is no longer available in the new airflow. This PR brings this back to fix issue in main and v2-10-test branch where all DAGs fail because of lack of this function (cherry picked from commit 0576f557f1fccf059cd9f70ec9fd9a1fb555cfec) --- airflow/security/permissions.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/airflow/security/permissions.py b/airflow/security/permissions.py index 058cde2927d27..45b56c342b44e 100644 --- a/airflow/security/permissions.py +++ b/airflow/security/permissions.py @@ -78,7 +78,6 @@ class ResourceDetails(TypedDict): # Keeping DAG_ACTIONS to keep the compatibility with outdated versions of FAB provider DAG_ACTIONS = {ACTION_CAN_READ, ACTION_CAN_EDIT, ACTION_CAN_DELETE} - RESOURCE_DETAILS_MAP = { RESOURCE_DAG: ResourceDetails( actions={ACTION_CAN_READ, ACTION_CAN_EDIT, ACTION_CAN_DELETE}, prefix=RESOURCE_DAG_PREFIX @@ -105,3 +104,20 @@ def resource_name(root_dag_id: str, resource: str) -> str: if root_dag_id.startswith(tuple(PREFIX_RESOURCES_MAP.keys())): return root_dag_id return f"{RESOURCE_DETAILS_MAP[resource]['prefix']}{root_dag_id}" + + +def resource_name_for_dag(root_dag_id: str) -> str: + """ + Return the resource name for a DAG id. + + Note that since a sub-DAG should follow the permission of its + parent DAG, you should pass ``DagModel.root_dag_id`` to this function, + for a subdag. A normal dag should pass the ``DagModel.dag_id``. + + Note: This function is kept for backwards compatibility. + """ + if root_dag_id == RESOURCE_DAG: + return root_dag_id + if root_dag_id.startswith(RESOURCE_DAG_PREFIX): + return root_dag_id + return f"{RESOURCE_DAG_PREFIX}{root_dag_id}" From 0d87d27d69a474282ef2ceea8180b02be08623b3 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Fri, 9 Aug 2024 07:45:18 +0100 Subject: [PATCH 007/161] Update version to 2.10.0 --- README.md | 24 +++++++++---------- airflow/__init__.py | 2 +- airflow/api_connexion/openapi/v1.yaml | 2 +- .../installation/supported-versions.rst | 2 +- docs/docker-stack/README.md | 10 ++++---- .../add-airflow-configuration/Dockerfile | 2 +- .../extending/add-apt-packages/Dockerfile | 2 +- .../add-build-essential-extend/Dockerfile | 2 +- .../extending/add-providers/Dockerfile | 2 +- .../add-pypi-packages-constraints/Dockerfile | 2 +- .../extending/add-pypi-packages-uv/Dockerfile | 2 +- .../extending/add-pypi-packages/Dockerfile | 2 +- .../add-requirement-packages/Dockerfile | 2 +- .../extending/custom-providers/Dockerfile | 2 +- .../extending/embedding-dags/Dockerfile | 2 +- .../extending/writable-directory/Dockerfile | 2 +- docs/docker-stack/entrypoint.rst | 14 +++++------ generated/PYPI_README.md | 22 ++++++++--------- scripts/ci/pre_commit/supported_versions.py | 2 +- 19 files changed, 50 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index f72e5ff9f7c55..78ecf1c39fe44 100644 --- a/README.md +++ b/README.md @@ -97,14 +97,14 @@ Airflow is not a streaming solution, but it is often used to process real-time d Apache Airflow is tested with: -| | Main version (dev) | Stable version (2.9.3) | -|------------|----------------------------|----------------------------| -| Python | 3.8, 3.9, 3.10, 3.11, 3.12 | 3.8, 3.9, 3.10, 3.11, 3.12 | -| Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | -| Kubernetes | 1.27, 1.28, 1.29, 1.30 | 1.26, 1.27, 1.28, 1.29 | -| PostgreSQL | 12, 13, 14, 15, 16 | 12, 13, 14, 15, 16 | -| MySQL | 8.0, 8.4, Innovation | 8.0, Innovation | -| SQLite | 3.15.0+ | 3.15.0+ | +| | Main version (dev) | Stable version (2.10.0) | +|-------------|------------------------------|----------------------------------| +| Python | 3.8, 3.9, 3.10, 3.11, 3.12 | 3.8, 3.9, 3.10, 3.11, 3.12 | +| Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | +| Kubernetes | 1.26, 1.27, 1.28, 1.29, 1.30 | 1.26, 1.27, 1.28, 1.29, 1.30 | +| PostgreSQL | 12, 13, 14, 15, 16 | 12, 13, 14, 15, 16 | +| MySQL | 8.0, 8.4, Innovation | 8.0, Innovation | +| SQLite | 3.15.0+ | 3.15.0+ | \* Experimental @@ -179,15 +179,15 @@ them to the appropriate format and workflow that your tool requires. ```bash -pip install 'apache-airflow==2.9.3' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.8.txt" +pip install 'apache-airflow==2.10.0' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.0/constraints-3.8.txt" ``` 2. Installing with extras (i.e., postgres, google) ```bash pip install 'apache-airflow[postgres,google]==2.8.3' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.8.txt" + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.0/constraints-3.8.txt" ``` For information on installing provider packages, check @@ -292,7 +292,7 @@ Apache Airflow version life cycle: | Version | Current Patch/Minor | State | First Release | Limited Support | EOL/Terminated | |-----------|-----------------------|-----------|-----------------|-------------------|------------------| -| 2 | 2.9.3 | Supported | Dec 17, 2020 | TBD | TBD | +| 2 | 2.10.0 | Supported | Dec 17, 2020 | TBD | TBD | | 1.10 | 1.10.15 | EOL | Aug 27, 2018 | Dec 17, 2020 | June 17, 2021 | | 1.9 | 1.9.0 | EOL | Jan 03, 2018 | Aug 27, 2018 | Aug 27, 2018 | | 1.8 | 1.8.2 | EOL | Mar 19, 2017 | Jan 03, 2018 | Jan 03, 2018 | diff --git a/airflow/__init__.py b/airflow/__init__.py index b2f58b5bc6dcb..f51fab3e39261 100644 --- a/airflow/__init__.py +++ b/airflow/__init__.py @@ -17,7 +17,7 @@ # under the License. from __future__ import annotations -__version__ = "2.10.0.dev0" +__version__ = "2.10.0" import os import sys diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index 0394da4f466cf..de99cccaa9225 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -231,7 +231,7 @@ info: This means that the server encountered an unexpected condition that prevented it from fulfilling the request. - version: "2.9.0.dev0" + version: "2.10.0" license: name: Apache 2.0 url: http://www.apache.org/licenses/LICENSE-2.0.html diff --git a/docs/apache-airflow/installation/supported-versions.rst b/docs/apache-airflow/installation/supported-versions.rst index 5d9f9df700bc5..0fbeb683526f6 100644 --- a/docs/apache-airflow/installation/supported-versions.rst +++ b/docs/apache-airflow/installation/supported-versions.rst @@ -29,7 +29,7 @@ Apache Airflow® version life cycle: ========= ===================== ========= =============== ================= ================ Version Current Patch/Minor State First Release Limited Support EOL/Terminated ========= ===================== ========= =============== ================= ================ -2 2.9.3 Supported Dec 17, 2020 TBD TBD +2 2.10.0 Supported Dec 17, 2020 TBD TBD 1.10 1.10.15 EOL Aug 27, 2018 Dec 17, 2020 June 17, 2021 1.9 1.9.0 EOL Jan 03, 2018 Aug 27, 2018 Aug 27, 2018 1.8 1.8.2 EOL Mar 19, 2017 Jan 03, 2018 Jan 03, 2018 diff --git a/docs/docker-stack/README.md b/docs/docker-stack/README.md index dcc407ccd3d84..44aebdca6366f 100644 --- a/docs/docker-stack/README.md +++ b/docs/docker-stack/README.md @@ -31,12 +31,12 @@ Every time a new version of Airflow is released, the images are prepared in the [apache/airflow DockerHub](https://hub.docker.com/r/apache/airflow) for all the supported Python versions. -You can find the following images there (Assuming Airflow version `2.10.0.dev0`): +You can find the following images there (Assuming Airflow version `2.10.0`): * `apache/airflow:latest` - the latest released Airflow image with default Python version (3.8 currently) * `apache/airflow:latest-pythonX.Y` - the latest released Airflow image with specific Python version -* `apache/airflow:2.10.0.dev0` - the versioned Airflow image with default Python version (3.8 currently) -* `apache/airflow:2.10.0.dev0-pythonX.Y` - the versioned Airflow image with specific Python version +* `apache/airflow:2.10.0` - the versioned Airflow image with default Python version (3.8 currently) +* `apache/airflow:2.10.0-pythonX.Y` - the versioned Airflow image with specific Python version Those are "reference" regular images. They contain the most common set of extras, dependencies and providers that are often used by the users and they are good to "try-things-out" when you want to just take Airflow for a spin, @@ -47,8 +47,8 @@ via [Building the image](https://airflow.apache.org/docs/docker-stack/build.html * `apache/airflow:slim-latest` - the latest released Airflow image with default Python version (3.8 currently) * `apache/airflow:slim-latest-pythonX.Y` - the latest released Airflow image with specific Python version -* `apache/airflow:slim-2.10.0.dev0` - the versioned Airflow image with default Python version (3.8 currently) -* `apache/airflow:slim-2.10.0.dev0-pythonX.Y` - the versioned Airflow image with specific Python version +* `apache/airflow:slim-2.10.0` - the versioned Airflow image with default Python version (3.8 currently) +* `apache/airflow:slim-2.10.0-pythonX.Y` - the versioned Airflow image with specific Python version The Apache Airflow image provided as convenience package is optimized for size, and it provides just a bare minimal set of the extras and dependencies installed and in most cases diff --git a/docs/docker-stack/docker-examples/extending/add-airflow-configuration/Dockerfile b/docs/docker-stack/docker-examples/extending/add-airflow-configuration/Dockerfile index 418ad5b64197e..c4b089d604332 100644 --- a/docs/docker-stack/docker-examples/extending/add-airflow-configuration/Dockerfile +++ b/docs/docker-stack/docker-examples/extending/add-airflow-configuration/Dockerfile @@ -15,7 +15,7 @@ # This is an example Dockerfile. It is not intended for PRODUCTION use # [START Dockerfile] -FROM apache/airflow:2.10.0.dev0 +FROM apache/airflow:2.10.0 ENV AIRFLOW__CORE__LOAD_EXAMPLES=True ENV AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=my_conn_string # [END Dockerfile] diff --git a/docs/docker-stack/docker-examples/extending/add-apt-packages/Dockerfile b/docs/docker-stack/docker-examples/extending/add-apt-packages/Dockerfile index dca6654ad3cb3..fda380ebd31d3 100644 --- a/docs/docker-stack/docker-examples/extending/add-apt-packages/Dockerfile +++ b/docs/docker-stack/docker-examples/extending/add-apt-packages/Dockerfile @@ -15,7 +15,7 @@ # This is an example Dockerfile. It is not intended for PRODUCTION use # [START Dockerfile] -FROM apache/airflow:2.10.0.dev0 +FROM apache/airflow:2.10.0 USER root RUN apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/docs/docker-stack/docker-examples/extending/add-build-essential-extend/Dockerfile b/docs/docker-stack/docker-examples/extending/add-build-essential-extend/Dockerfile index 4ad77eec9287e..e18e71899bf98 100644 --- a/docs/docker-stack/docker-examples/extending/add-build-essential-extend/Dockerfile +++ b/docs/docker-stack/docker-examples/extending/add-build-essential-extend/Dockerfile @@ -15,7 +15,7 @@ # This is an example Dockerfile. It is not intended for PRODUCTION use # [START Dockerfile] -FROM apache/airflow:2.10.0.dev0 +FROM apache/airflow:2.10.0 USER root RUN apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/docs/docker-stack/docker-examples/extending/add-providers/Dockerfile b/docs/docker-stack/docker-examples/extending/add-providers/Dockerfile index 7e0c718aba61f..3aa9dda0867b6 100644 --- a/docs/docker-stack/docker-examples/extending/add-providers/Dockerfile +++ b/docs/docker-stack/docker-examples/extending/add-providers/Dockerfile @@ -15,7 +15,7 @@ # This is an example Dockerfile. It is not intended for PRODUCTION use # [START Dockerfile] -FROM apache/airflow:2.10.0.dev0 +FROM apache/airflow:2.10.0 USER root RUN apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/docs/docker-stack/docker-examples/extending/add-pypi-packages-constraints/Dockerfile b/docs/docker-stack/docker-examples/extending/add-pypi-packages-constraints/Dockerfile index c046c0d514a05..edcef75577d41 100644 --- a/docs/docker-stack/docker-examples/extending/add-pypi-packages-constraints/Dockerfile +++ b/docs/docker-stack/docker-examples/extending/add-pypi-packages-constraints/Dockerfile @@ -15,6 +15,6 @@ # This is an example Dockerfile. It is not intended for PRODUCTION use # [START Dockerfile] -FROM apache/airflow:2.10.0.dev0 +FROM apache/airflow:2.10.0 RUN pip install --no-cache-dir "apache-airflow==${AIRFLOW_VERSION}" lxml --constraint "${HOME}/constraints.txt" # [END Dockerfile] diff --git a/docs/docker-stack/docker-examples/extending/add-pypi-packages-uv/Dockerfile b/docs/docker-stack/docker-examples/extending/add-pypi-packages-uv/Dockerfile index a0ce42eb17183..65d5d58b1e4c8 100644 --- a/docs/docker-stack/docker-examples/extending/add-pypi-packages-uv/Dockerfile +++ b/docs/docker-stack/docker-examples/extending/add-pypi-packages-uv/Dockerfile @@ -15,7 +15,7 @@ # This is an example Dockerfile. It is not intended for PRODUCTION use # [START Dockerfile] -FROM apache/airflow:2.10.0.dev0 +FROM apache/airflow:2.10.0 # The `uv` tools is Rust packaging tool that is much faster than `pip` and other installer # Support for uv as installation tool is experimental diff --git a/docs/docker-stack/docker-examples/extending/add-pypi-packages/Dockerfile b/docs/docker-stack/docker-examples/extending/add-pypi-packages/Dockerfile index b83ff5a59c8ec..2b97d78766c0f 100644 --- a/docs/docker-stack/docker-examples/extending/add-pypi-packages/Dockerfile +++ b/docs/docker-stack/docker-examples/extending/add-pypi-packages/Dockerfile @@ -15,6 +15,6 @@ # This is an example Dockerfile. It is not intended for PRODUCTION use # [START Dockerfile] -FROM apache/airflow:2.10.0.dev0 +FROM apache/airflow:2.10.0 RUN pip install --no-cache-dir "apache-airflow==${AIRFLOW_VERSION}" lxml # [END Dockerfile] diff --git a/docs/docker-stack/docker-examples/extending/add-requirement-packages/Dockerfile b/docs/docker-stack/docker-examples/extending/add-requirement-packages/Dockerfile index 9d7f42e959195..0bb72d83fab36 100644 --- a/docs/docker-stack/docker-examples/extending/add-requirement-packages/Dockerfile +++ b/docs/docker-stack/docker-examples/extending/add-requirement-packages/Dockerfile @@ -15,7 +15,7 @@ # This is an example Dockerfile. It is not intended for PRODUCTION use # [START Dockerfile] -FROM apache/airflow:2.10.0.dev0 +FROM apache/airflow:2.10.0 COPY requirements.txt / RUN pip install --no-cache-dir "apache-airflow==${AIRFLOW_VERSION}" -r /requirements.txt # [END Dockerfile] diff --git a/docs/docker-stack/docker-examples/extending/custom-providers/Dockerfile b/docs/docker-stack/docker-examples/extending/custom-providers/Dockerfile index 3b5c0d114bf43..cc45c302e3fd9 100644 --- a/docs/docker-stack/docker-examples/extending/custom-providers/Dockerfile +++ b/docs/docker-stack/docker-examples/extending/custom-providers/Dockerfile @@ -15,6 +15,6 @@ # This is an example Dockerfile. It is not intended for PRODUCTION use # [START Dockerfile] -FROM apache/airflow:2.10.0.dev0 +FROM apache/airflow:2.10.0 RUN pip install "apache-airflow==${AIRFLOW_VERSION}" --no-cache-dir apache-airflow-providers-docker==2.5.1 # [END Dockerfile] diff --git a/docs/docker-stack/docker-examples/extending/embedding-dags/Dockerfile b/docs/docker-stack/docker-examples/extending/embedding-dags/Dockerfile index 53065be3ab874..e2b8ca1c92a2d 100644 --- a/docs/docker-stack/docker-examples/extending/embedding-dags/Dockerfile +++ b/docs/docker-stack/docker-examples/extending/embedding-dags/Dockerfile @@ -15,7 +15,7 @@ # This is an example Dockerfile. It is not intended for PRODUCTION use # [START Dockerfile] -FROM apache/airflow:2.10.0.dev0 +FROM apache/airflow:2.10.0 COPY --chown=airflow:root test_dag.py /opt/airflow/dags diff --git a/docs/docker-stack/docker-examples/extending/writable-directory/Dockerfile b/docs/docker-stack/docker-examples/extending/writable-directory/Dockerfile index ae1288c80f6b1..af579ee69e27d 100644 --- a/docs/docker-stack/docker-examples/extending/writable-directory/Dockerfile +++ b/docs/docker-stack/docker-examples/extending/writable-directory/Dockerfile @@ -15,7 +15,7 @@ # This is an example Dockerfile. It is not intended for PRODUCTION use # [START Dockerfile] -FROM apache/airflow:2.10.0.dev0 +FROM apache/airflow:2.10.0 RUN umask 0002; \ mkdir -p ~/writeable-directory # [END Dockerfile] diff --git a/docs/docker-stack/entrypoint.rst b/docs/docker-stack/entrypoint.rst index a9dac6e27c817..cbd467b8b62b2 100644 --- a/docs/docker-stack/entrypoint.rst +++ b/docs/docker-stack/entrypoint.rst @@ -132,7 +132,7 @@ if you specify extra arguments. For example: .. code-block:: bash - docker run -it apache/airflow:2.10.0.dev0-python3.8 bash -c "ls -la" + docker run -it apache/airflow:2.10.0-python3.8 bash -c "ls -la" total 16 drwxr-xr-x 4 airflow root 4096 Jun 5 18:12 . drwxr-xr-x 1 root root 4096 Jun 5 18:12 .. @@ -144,7 +144,7 @@ you pass extra parameters. For example: .. code-block:: bash - > docker run -it apache/airflow:2.10.0.dev0-python3.8 python -c "print('test')" + > docker run -it apache/airflow:2.10.0-python3.8 python -c "print('test')" test If first argument equals to "airflow" - the rest of the arguments is treated as an airflow command @@ -152,13 +152,13 @@ to execute. Example: .. code-block:: bash - docker run -it apache/airflow:2.10.0.dev0-python3.8 airflow webserver + docker run -it apache/airflow:2.10.0-python3.8 airflow webserver If there are any other arguments - they are simply passed to the "airflow" command .. code-block:: bash - > docker run -it apache/airflow:2.10.0.dev0-python3.8 help + > docker run -it apache/airflow:2.10.0-python3.8 help usage: airflow [-h] GROUP_OR_COMMAND ... positional arguments: @@ -363,7 +363,7 @@ database and creating an ``admin/admin`` Admin user with the following command: --env "_AIRFLOW_DB_MIGRATE=true" \ --env "_AIRFLOW_WWW_USER_CREATE=true" \ --env "_AIRFLOW_WWW_USER_PASSWORD=admin" \ - apache/airflow:2.10.0.dev0-python3.8 webserver + apache/airflow:2.10.0-python3.8 webserver .. code-block:: bash @@ -372,7 +372,7 @@ database and creating an ``admin/admin`` Admin user with the following command: --env "_AIRFLOW_DB_MIGRATE=true" \ --env "_AIRFLOW_WWW_USER_CREATE=true" \ --env "_AIRFLOW_WWW_USER_PASSWORD_CMD=echo admin" \ - apache/airflow:2.10.0.dev0-python3.8 webserver + apache/airflow:2.10.0-python3.8 webserver The commands above perform initialization of the SQLite database, create admin user with admin password and Admin role. They also forward local port ``8080`` to the webserver port and finally start the webserver. @@ -412,6 +412,6 @@ Example: --env "_AIRFLOW_DB_MIGRATE=true" \ --env "_AIRFLOW_WWW_USER_CREATE=true" \ --env "_AIRFLOW_WWW_USER_PASSWORD_CMD=echo admin" \ - apache/airflow:2.10.0.dev0-python3.8 webserver + apache/airflow:2.10.0-python3.8 webserver This method is only available starting from Docker image of Airflow 2.1.1 and above. diff --git a/generated/PYPI_README.md b/generated/PYPI_README.md index 6492fcaace018..2bfb5554b21aa 100644 --- a/generated/PYPI_README.md +++ b/generated/PYPI_README.md @@ -54,14 +54,14 @@ Use Airflow to author workflows as directed acyclic graphs (DAGs) of tasks. The Apache Airflow is tested with: -| | Main version (dev) | Stable version (2.9.3) | -|------------|----------------------------|----------------------------| -| Python | 3.8, 3.9, 3.10, 3.11, 3.12 | 3.8, 3.9, 3.10, 3.11, 3.12 | -| Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | -| Kubernetes | 1.27, 1.28, 1.29, 1.30 | 1.26, 1.27, 1.28, 1.29 | -| PostgreSQL | 12, 13, 14, 15, 16 | 12, 13, 14, 15, 16 | -| MySQL | 8.0, 8.4, Innovation | 8.0, Innovation | -| SQLite | 3.15.0+ | 3.15.0+ | +| | Main version (dev) | Stable version (2.10.0) | +|-------------|------------------------------|----------------------------------| +| Python | 3.8, 3.9, 3.10, 3.11, 3.12 | 3.8, 3.9, 3.10, 3.11, 3.12 | +| Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | +| Kubernetes | 1.26, 1.27, 1.28, 1.29, 1.30 | 1.26, 1.27, 1.28, 1.29, 1.30 | +| PostgreSQL | 12, 13, 14, 15, 16 | 12, 13, 14, 15, 16 | +| MySQL | 8.0, 8.4, Innovation | 8.0, Innovation | +| SQLite | 3.15.0+ | 3.15.0+ | \* Experimental @@ -132,15 +132,15 @@ them to the appropriate format and workflow that your tool requires. ```bash -pip install 'apache-airflow==2.9.3' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.8.txt" +pip install 'apache-airflow==2.10.0' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.0/constraints-3.8.txt" ``` 2. Installing with extras (i.e., postgres, google) ```bash pip install 'apache-airflow[postgres,google]==2.8.3' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.8.txt" + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.0/constraints-3.8.txt" ``` For information on installing provider packages, check diff --git a/scripts/ci/pre_commit/supported_versions.py b/scripts/ci/pre_commit/supported_versions.py index 5bd530a1ed3bc..0fb68e5afc4b6 100755 --- a/scripts/ci/pre_commit/supported_versions.py +++ b/scripts/ci/pre_commit/supported_versions.py @@ -27,7 +27,7 @@ HEADERS = ("Version", "Current Patch/Minor", "State", "First Release", "Limited Support", "EOL/Terminated") SUPPORTED_VERSIONS = ( - ("2", "2.9.3", "Supported", "Dec 17, 2020", "TBD", "TBD"), + ("2", "2.10.0", "Supported", "Dec 17, 2020", "TBD", "TBD"), ("1.10", "1.10.15", "EOL", "Aug 27, 2018", "Dec 17, 2020", "June 17, 2021"), ("1.9", "1.9.0", "EOL", "Jan 03, 2018", "Aug 27, 2018", "Aug 27, 2018"), ("1.8", "1.8.2", "EOL", "Mar 19, 2017", "Jan 03, 2018", "Jan 03, 2018"), From 4fce874b95f1b0a16a8fb1033d6a798411227c08 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Fri, 9 Aug 2024 07:45:32 +0100 Subject: [PATCH 008/161] Update RELEASE_NOTES.rst --- RELEASE_NOTES.rst | 291 +++++++++++++++++++++++++++- airflow/reproducible_build.yaml | 4 +- newsfragments/37948.feature.rst | 1 - newsfragments/38891.significant.rst | 10 - newsfragments/39336.significant.rst | 7 - newsfragments/39823.bugfix.rst | 1 - newsfragments/40145.significant.rst | 5 - newsfragments/40379.improvement.rst | 1 - newsfragments/40701.feature.rst | 1 - newsfragments/40703.feature.rst | 1 - newsfragments/40874.significant.rst | 1 - newsfragments/41039.feature.rst | 1 - newsfragments/41116.feature.rst | 1 - 13 files changed, 292 insertions(+), 33 deletions(-) delete mode 100644 newsfragments/37948.feature.rst delete mode 100644 newsfragments/38891.significant.rst delete mode 100644 newsfragments/39336.significant.rst delete mode 100644 newsfragments/39823.bugfix.rst delete mode 100644 newsfragments/40145.significant.rst delete mode 100644 newsfragments/40379.improvement.rst delete mode 100644 newsfragments/40701.feature.rst delete mode 100644 newsfragments/40703.feature.rst delete mode 100644 newsfragments/40874.significant.rst delete mode 100644 newsfragments/41039.feature.rst delete mode 100644 newsfragments/41116.feature.rst diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index ddd4e986c56f6..c7be2d55efbdb 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -21,6 +21,296 @@ .. towncrier release notes start +Airflow 2.10.0 (2024-08-15) +--------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +Datasets no longer trigger inactive DAGs (#38891) +""""""""""""""""""""""""""""""""""""""""""""""""" + +Previously, when a DAG is paused or removed, incoming dataset events would still +trigger it, and the DAG would run when it is unpaused or added back in a DAG +file. This has been changed; a DAG's dataset schedule can now only be satisfied +by events that occur when the DAG is active. While this is a breaking change, +the previous behavior is considered a bug. + +The behavior of time-based scheduling is unchanged, including the timetable part +of ``DatasetOrTimeSchedule``. + +``try_number`` is no longer incremented during task execution (#39336) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Previously, the try number (``try_number``) was incremented at the beginning of task execution on the worker. This was problematic for many reasons. +For one it meant that the try number was incremented when it was not supposed to, namely when resuming from reschedule or deferral. And it also resulted in +the try number being "wrong" when the task had not yet started. The workarounds for these two issues caused a lot of confusion. + +Now, instead, the try number for a task run is determined at the time the task is scheduled, and does not change in flight, and it is never decremented. +So after the task runs, the observed try number remains the same as it was when the task was running; only when there is a "new try" will the try number be incremented again. + +One consequence of this change is, if users were "manually" running tasks (e.g. by calling ``ti.run()`` directly, or command line ``airflow tasks run``), +try number will no longer be incremented. Airflow assumes that tasks are always run after being scheduled by the scheduler, so we do not regard this as a breaking change. + +``/logout`` endpoint in FAB Auth Manager is now CSRF protected (#40145) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The ``/logout`` endpoint's method in FAB Auth Manager has been changed from ``GET`` to ``POST`` in all existing +AuthViews (``AuthDBView``, ``AuthLDAPView``, ``AuthOAuthView``, ``AuthOIDView``, ``AuthRemoteUserView``), and +now includes CSRF protection to enhance security and prevent unauthorized logouts. + +OpenTelemetry Traces for Apache Airflow (#37948). +""""""""""""""""""""""""""""""""""""""""""""""""" +This new feature adds capability for Apache Airflow to emit 1) airflow system traces of scheduler, +triggerer, executor, processor 2) DAG run traces for deployed DAG runs in OpenTelemetry format. Previously, only metrics were supported which emitted metrics in OpenTelemetry. +This new feature will add richer data for users to use OpenTelemetry standard to emit and send their trace data to OTLP compatible endpoints. + +Decorator for Task Flow ``(@skip_if, @run_if)`` to make it simple to apply whether or not to skip a Task. (#41116) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +This feature adds a decorator to make it simple to skip a Task. + +Using Multiple Executors Concurrently (#40701) +"""""""""""""""""""""""""""""""""""""""""""""" +Previously known as hybrid executors, this new feature allows Airflow to use multiple executors concurrently. DAGs, or even individual tasks, can be configured +to use a specific executor that suits its needs best. A single DAG can contain tasks all using different executors. Please see the Airflow documentation for +more details. Note: This feature is still experimental. See `documentation on Executor `_ for a more detailed description. + +Scarf based telemetry: Does Airflow collect any telemetry data? (#39510) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +Airflow integrates Scarf to collect basic usage data during operation. Deployments can opt-out of data collection by setting the ``[usage_data_collection]enabled`` option to False, or the SCARF_ANALYTICS=false environment variable. +See `FAQ on this `_ for more information. + + +New Features +"""""""""""" +- AIP-61 Hybrid Execution (`AIP-61 `_) +- AIP-62 Getting Lineage from Hook Instrumentation (`AIP-62 `_) +- AIP-64 TaskInstance Try History (`AIP-64 `_) +- AIP-44 Internal API (`AIP-44 `_) +- Enable ending the task directly from the triggerer without going into the worker. (#40084) +- Extend dataset dependencies (#40868) +- Feature/add token authentication to internal api (#40899) +- Add DatasetAlias to support dynamic Dataset Event Emission and Dataset Creation (#40478) +- Add example DAGs for inlet_events (#39893) +- Implement ``accessors`` to read dataset events defined as inlet (#39367) +- Decorator for Task Flow, to make it simple to apply whether or not to skip a Task. (#41116) +- Add start execution from triggerer support to dynamic task mapping (#39912) +- Add try_number to log table (#40739) +- Added ds_format_locale method in macros which allows localizing datetime formatting using Babel (#40746) +- Add DatasetAlias to support dynamic Dataset Event Emission and Dataset Creation (#40478, #40723, #40809, #41264, #40830, #40693, #41302) +- Use sentinel to mark dag as removed on re-serialization (#39825) +- Add parameter for the last number of queries to the DB in DAG file processing stats (#40323) +- Add prototype version dark mode for Airflow UI (#39355) +- Add ability to mark some tasks as successful in ``dag test`` (#40010) +- Allow use of callable for template_fields (#37028) +- Filter running/failed and active/paused dags on the home page(#39701) +- Add metrics about task CPU and memory usage (#39650) +- UI changes for DAG Re-parsing feature (#39636) +- Add Scarf based telemetry (#39510, #41318) +- Add dag re-parsing request endpoint (#39138) +- Redirect to new DAGRun after trigger from Grid view (#39569) +- Display ``endDate`` in task instance tooltip. (#39547) +- Implement ``accessors`` to read dataset events defined as inlet (#39367, #39893) +- Add color to log lines in UI for error and warnings based on keywords (#39006) +- Add Rendered k8s pod spec tab to ti details view (#39141) +- Make audit log before/after filterable (#39120) +- Consolidate grid collapse actions to a single full screen toggle (#39070) +- Implement Metadata to emit runtime extra (#38650) +- Add executor field to the DB and parameter to the operators (#38474) +- Implement context accessor for DatasetEvent extra (#38481) +- Add dataset event info to dag graph (#41012) +- Add button to toggle datasets on/off in dag graph (#41200) +- Add ``run_if`` & ``skip_if`` decorators (#41116) +- Add dag_stats rest api endpoint (#41017) +- Add listeners for Dag import errors (#39739) +- Allowing DateTimeSensorAsync, FileSensor and TimeSensorAsync to start execution from trigger during dynamic task mapping (#41182) + + +Improvements +"""""""""""" +- Allow set Dag Run resource into Dag Level permission: extends Dag's access_control feature to allow Dag Run resource permissions. (#40703) +- Improve security and error handling for the internal API (#40999) +- Datasets UI Improvements (#40871) +- Change DAG Audit log tab to Event Log (#40967) +- Make standalone dag file processor works in DB isolation mode (#40916) +- Show only the source on the consumer DAG page and only triggered DAG run in the producer DAG page (#41300) +- Update metrics names to allow multiple executors to report metrics (#40778) +- Format DAG run count (#39684) +- Update styles for ``renderedjson`` component (#40964) +- Improve ATTRIBUTE_REMOVED sentinel to use class and more context (#40920) +- Make XCom display as react json (#40640) +- Replace usages of task context logger with the log table (#40867) +- Rollback for all retry exceptions (#40882) (#40883) +- Support rendering ObjectStoragePath value (#40638) +- Add try_number and map_index as params for log event endpoint (#40845) +- Rotate fernet key in batches to limit memory usage (#40786) +- Add gauge metric for 'last_num_of_db_queries' parameter (#40833) +- Set parallelism log messages to warning level for better visibility (#39298) +- Add error handling for encoding the dag runs (#40222) +- Use params instead of dag_run.conf in example DAG (#40759) +- Load Example Plugins with Example DAGs (#39999) +- Stop deferring TimeDeltaSensorAsync task when the target_dttm is in the past (#40719) +- Send important executor logs to task logs (#40468) +- Open external links in new tabs (#40635) +- Attempt to add ReactJSON view to rendered templates (#40639) +- Speeding up regex match time for custom warnings (#40513) +- Refactor DAG.dataset_triggers into the timetable class (#39321) +- add next_kwargs to StartTriggerArgs (#40376) +- Improve UI error handling (#40350) +- Remove double warning in CLI when config value is deprecated (#40319) +- Implement XComArg concat() (#40172) +- Added ``get_extra_dejson`` method with nested parameter which allows you to specify if you want the nested json as string to be also deserialized (#39811) +- Add executor field to the task instance API (#40034) +- Support checking for db path absoluteness on Windows (#40069) +- Introduce StartTriggerArgs and prevent start trigger initialization in scheduler (#39585) +- Add task documentation to details tab in grid view (#39899) +- Allow executors to be specified with only the class name of the Executor (#40131) +- Remove obsolete conditional logic related to try_number (#40104) +- Allow Task Group Ids to be passed as branches in BranchMixIn (#38883) +- Javascript connection form will apply CodeMirror to all textarea's dynamically (#39812) +- Determine needs_expansion at time of serialization (#39604) +- Add indexes on dag_id column in referencing tables to speed up deletion of dag records (#39638) +- Add task failed dependencies to details page (#38449) +- Remove webserver try_number adjustment (#39623) +- Implement slicing in lazy sequence (#39483) +- Unify lazy db sequence implementations (#39426) +- Add ``__getattr__`` to task decorator stub (#39425) +- Allow passing labels to FAB Views registered via Plugins (#39444) +- Simpler error message when trying to offline migrate with sqlite (#39441) +- Add soft_fail to TriggerDagRunOperator (#39173) +- Rename "dataset event" in context to use "outlet" (#39397) +- Resolve ``RemovedIn20Warning`` in ``airflow task`` command (#39244) +- Determine fail_stop on client side when db isolated (#39258) +- Refactor cloudpickle support in Python operators/decorators (#39270) +- Update trigger kwargs migration to specify existing_nullable (#39361) +- Allowing tasks to start execution directly from triggerer without going to worker (#38674) +- Better ``db migrate`` error messages (#39268) +- Add stacklevel into the ``suppress_and_warn`` warning (#39263) +- Support searching by dag_display_name (#39008) +- Allow sort by on all fields in MappedInstances.tsx (#38090) +- Expose count of scheduled tasks in metrics (#38899) +- Use ``declarative_base`` from ``sqlalchemy.orm`` instead of ``sqlalchemy.ext.declarative`` (#39134) +- Add example DAG to demonstrate emitting approaches (#38821) +- Give ``on_task_instance_failed`` access to the error that caused the failure (#38155) +- Simplify dataset serialization (#38694) +- Add heartbeat recovery message to jobs (#34457) +- Remove select_column option in TaskInstance.get_task_instance (#38571) +- Don't create session in get_dag if not reading dags from database (#38553) +- Add a migration script for encrypted trigger kwargs (#38358) +- Implement render_templates on TaskInstancePydantic (#38559) +- Handle optional session in _refresh_from_db (#38572) +- Make type annotation less confusing in task_command.py (#38561) +- Use fetch_dagrun directly to avoid session creation (#38557) +- Added ``output_processor`` parameter to ``BashProcessor`` (#40843) +- Improve serialization for Database Isolation Mode (#41239) +- Only orphan non-orphaned Datasets (#40806) +- Adjust gantt width based on task history dates (#41192) +- Enable scrolling on legend with high number of elements. (#41187) + +Bug Fixes +""""""""" +- Bugfix for get_parsing_context() when ran with LocalExecutor (#40738) +- Validating provider documentation urls before displaying in views (#40933) +- Move import to make PythonOperator working on Windows (#40424) +- Fix dataset_with_extra_from_classic_operator example DAG (#40747) +- Call listener on_task_instance_failed() after ti state is changed (#41053) +- Add ``never_fail`` in BaseSensor (#40915) +- Fix tasks API endpoint when DAG doesn't have ``start_date`` (#40878) +- Fix and adjust URL generation for UI grid and older runs (#40764) +- Rotate fernet key optimization (#40758) +- Fix class instance vs. class type in validate_database_executor_compatibility() call (#40626) +- Clean up dark mode (#40466) +- Validate expected types for args for DAG, BaseOperator and TaskGroup (#40269) +- Exponential Backoff Not Functioning in BaseSensorOperator Reschedule Mode (#39823) +- local task job: add timeout, to not kill on_task_instance_success listener prematurely (#39890) +- Move Post Execution Log Grouping behind Exception Print (#40146) +- Fix triggerer race condition in HA setting (#38666) +- Pass triggered or existing DAG Run logical date to DagStateTrigger (#39960) +- Passing ``external_task_group_id`` to ``WorkflowTrigger`` (#39617) +- ECS Executor: Set tasks to RUNNING state once active (#39212) +- Only heartbeat if necessary in backfill loop (#39399) +- Fix trigger kwarg encryption migration (#39246) +- Fix decryption of trigger kwargs when downgrading. (#38743) +- Fix wrong link in TriggeredDagRuns (#41166) +- Pass MapIndex to LogLink component for external log systems (#41125) +- Add NonCachingRotatingFileHandler for worker task (#41064) +- Add argument include_xcom in method resolve an optional value (#41062) +- Sanitizing file names in example_bash_decorator DAG (#40949) +- Show dataset aliases in dependency graphs (#41128) +- Render Dataset Conditions in DAG Graph view (#41137) +- Add task duration plot across dagruns (#40755) +- Add start execution from trigger support for existing core sensors (#41021) +- add example dag for dataset_alias (#41037) +- Add dataset alias unique constraint and remove wrong dataset alias removing logic (#41097) +- Set "has_outlet_datasets" to true if "dataset alias" exists (#41091) +- Make HookLineageCollector group datasets by (#41034) +- Enhance start_trigger_args serialization (#40993) +- Refactor ``BaseSensorOperator`` introduce ``skip_policy`` parameter (#40924) +- Fix viewing logs from triggerer when task is deferred (#41272) +- Refactor how triggered dag run url is replaced (#41259) +- Added support for additional sql alchemy session args (#41048) +- Allow empty list in TriggerDagRun failed_state (#41249) +- Clean up the exception handler when run_as_user is the airflow user (#41241) +- Collapse docs when click and folded (#41214) +- Update updated_at when saving to db as session.merge does not trigger on-update (#40782) +- Fix query count statistics when parsing DAF file (#41149) +- Method Resolution Order in operators without ``__init__`` (#41086) +- Ensure try_number incremented for empty operator (#40426) + +Miscellaneous +""""""""""""" +- Remove the Experimental flag from ``OTel`` Traces (#40874) +- Bump packaging version to 23.0 in order to fix issue with older otel (#40865) +- Simplify _auth_manager_is_authorized_map function (#40803) +- Use correct unknown executor exception in scheduler job (#40700) +- Add D1 ``pydocstyle`` rules to pyproject.toml (#40569) +- Enable enforcing ``pydocstyle`` rule D213 in ruff. (#40448, #40464) +- Update ``Dag.test()`` to run with an executor if desired (#40205) +- Update jest and babel minor versions (#40203) +- Refactor BashOperator and Bash decorator for consistency and simplicity (#39871) +- Add ``AirflowInternalRuntimeError`` for raise ``non catchable`` errors (#38778) +- ruff version bump 0.4.5 (#39849) +- Bump ``pytest`` to 8.0+ (#39450) +- Remove stale comment about TI index (#39470) +- Configure ``back_populates`` between ``DagScheduleDatasetReference.dag`` and ``DagModel.schedule_dataset_references`` (#39392) +- Remove deprecation warnings in endpoints.py (#39389) +- Fix SQLA deprecations in Airflow core (#39211) +- Use class-bound attribute directly in SA (#39198, #39195) +- Fix stacklevel for TaskContextLogger (#39142) +- Capture warnings during collect DAGs (#39109) +- Resolve ``B028`` (no-explicit-stacklevel) in core (#39123) +- Rename model ``ImportError`` to ``ParseImportError`` for avoid shadowing with builtin exception (#39116) +- Add option to support cloudpickle in PythonVenv/External Operator (#38531) +- Suppress ``SubDagOperator`` examples warnings (#39057) +- Add log for running callback (#38892) +- Use ``model_dump`` instead of ``dict`` for serialize Pydantic V2 model (#38933) +- Widen cheat sheet column to avoid wrapping commands (#38888) +- Update hatchling to latest version (1.22.5) (#38780) +- bump uv to 0.1.29 (#38758) +- Add missing serializations found during provider tests fixing (#41252) +- Bump ``ws`` from 7.5.5 to 7.5.10 in /airflow/www (#40288) +- Improve typing for allowed/failed_states in TriggerDagRunOperator (#39855) + +Doc Only Changes +"""""""""""""""" +- Add ``filesystems`` and ``dataset-uris`` to "how to create your own provider" page (#40801) +- Fix (TM) to (R) in Airflow repository (#40783) +- Set ``otel_on`` to True in example airflow.cfg (#40712) +- Add warning for _AIRFLOW_PATCH_GEVENT (#40677) +- Update multi-team diagram proposal after Airflow 3 discussions (#40671) +- Add stronger warning that MSSQL is not supported and no longer functional (#40565) +- Fix misleading mac menu structure in howto (#40440) +- Update k8s supported version in docs (#39878) +- Add compatibility note for Listeners (#39544) +- Update edge label image in documentation example with the new graph view (#38802) +- Update UI doc screenshots (#38680) +- Add section "Manipulating queued dataset events through REST API" (#41022) +- Add information about lack of security guarantees for docker compose (#41072) +- Add links to example dags in use params section (#41031) +- Change ``task_id`` from ``send_email`` to ``send_email_notification`` in ``taskflow.rst`` (#41060) +- Remove unnecessary nginx redirect rule from reverse proxy documentation (#38953) + + Airflow 2.9.3 (2024-07-15) -------------------------- @@ -3573,7 +3863,6 @@ Bug Fixes - Remove custom signal handling in Triggerer (#23274) - Override pool for TaskInstance when pool is passed from cli. (#23258) - Show warning if '/' is used in a DAG run ID (#23106) -- Use kubernetes queue in kubernetes hybrid executors (#23048) - Add tags inside try block. (#21784) Doc only changes diff --git a/airflow/reproducible_build.yaml b/airflow/reproducible_build.yaml index b6f92ca9a4706..b97922aef5920 100644 --- a/airflow/reproducible_build.yaml +++ b/airflow/reproducible_build.yaml @@ -1,2 +1,2 @@ -release-notes-hash: fdd42ae58b946146d51d09ea6e5c28cd -source-date-epoch: 1721131067 +release-notes-hash: 1ea4fc9c7b9e956aa5b6746ade1858c5 +source-date-epoch: 1723407822 diff --git a/newsfragments/37948.feature.rst b/newsfragments/37948.feature.rst deleted file mode 100644 index 440788c264523..0000000000000 --- a/newsfragments/37948.feature.rst +++ /dev/null @@ -1 +0,0 @@ -OpenTelemetry Traces for Apache Airflow. This new feature adds capability for Apache Airflow to emit 1) airflow system traces of scheduler, triggerer, executor, processor 2) DAG run traces for deployed DAG runs in OpenTelemetry format. Previously, only metrics were supported which emitted metrics in OpenTelemetry. This new feature will add richer data for users to use OpenTelemetry standard to emitt and send their trace data to OTLP compatible endpoints. diff --git a/newsfragments/38891.significant.rst b/newsfragments/38891.significant.rst deleted file mode 100644 index 82caa4cdfc60d..0000000000000 --- a/newsfragments/38891.significant.rst +++ /dev/null @@ -1,10 +0,0 @@ -Datasets no longer trigger inactive DAGs - -Previously, when a DAG is paused or removed, incoming dataset events would still -trigger it, and the DAG would run when it is unpaused or added back in a DAG -file. This has been changed; a DAG's dataset schedule can now only be satisfied -by events that occur when the DAG is active. While this is a breaking change, -the previous behavior is considered a bug. - -The behavior of time-based scheduling is unchanged, including the timetable part -of ``DatasetOrTimeSchedule``. diff --git a/newsfragments/39336.significant.rst b/newsfragments/39336.significant.rst deleted file mode 100644 index 750a1807881e4..0000000000000 --- a/newsfragments/39336.significant.rst +++ /dev/null @@ -1,7 +0,0 @@ -``try_number`` is no longer incremented during task execution - -Previously, the try number (``try_number``) was incremented at the beginning of task execution on the worker. This was problematic for many reasons. For one it meant that the try number was incremented when it was not supposed to, namely when resuming from reschedule or deferral. And it also resulted in the try number being "wrong" when the task had not yet started. The workarounds for these two issues caused a lot of confusion. - -Now, instead, the try number for a task run is determined at the time the task is scheduled, and does not change in flight, and it is never decremented. So after the task runs, the observed try number remains the same as it was when the task was running; only when there is a "new try" will the try number be incremented again. - -One consequence of this change is, if users were "manually" running tasks (e.g. by calling ``ti.run()`` directly, or command line ``airflow tasks run``), try number will no longer be incremented. Airflow assumes that tasks are always run after being scheduled by the scheduler, so we do not regard this as a breaking change. diff --git a/newsfragments/39823.bugfix.rst b/newsfragments/39823.bugfix.rst deleted file mode 100644 index 7a774258a4732..0000000000000 --- a/newsfragments/39823.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed ``BaseSensorOperator`` with exponential backoff and reschedule mode by estimating try number based on ``run_duration``; previously, sensors had a fixed reschedule interval. diff --git a/newsfragments/40145.significant.rst b/newsfragments/40145.significant.rst deleted file mode 100644 index beedfc7746d43..0000000000000 --- a/newsfragments/40145.significant.rst +++ /dev/null @@ -1,5 +0,0 @@ -``/logout`` endpoint in FAB Auth Manager is now CSRF protected - -The ``/logout`` endpoint's method in FAB Auth Manager has been changed from ``GET`` to ``POST`` in all existing -AuthViews (``AuthDBView``, ``AuthLDAPView``, ``AuthOAuthView``, ``AuthOIDView``, ``AuthRemoteUserView``), and -now includes CSRF protection to enhance security and prevent unauthorized logouts. diff --git a/newsfragments/40379.improvement.rst b/newsfragments/40379.improvement.rst deleted file mode 100644 index ecccde2065a1d..0000000000000 --- a/newsfragments/40379.improvement.rst +++ /dev/null @@ -1 +0,0 @@ -``chunk_size`` parameter is added to ``LocalFilesystemToGCSOperator``, enabling file uploads in multiple chunks of a specified size. diff --git a/newsfragments/40701.feature.rst b/newsfragments/40701.feature.rst deleted file mode 100644 index 48f928962862a..0000000000000 --- a/newsfragments/40701.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Using Multiple Executors Concurrently: Previously known as hybrid executors, this new feature allows Airflow to use multiple executors concurrently. DAGs, or even individual tasks, can be configured to use a specific executor that suits its needs best. A single DAG can contain tasks all using different executors. Please see the Airflow documentation for more details. Note: This feature is still experimental. diff --git a/newsfragments/40703.feature.rst b/newsfragments/40703.feature.rst deleted file mode 100644 index 4fd2fddf7e66a..0000000000000 --- a/newsfragments/40703.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Allow set Dag Run resource into Dag Level permission: extends Dag's access_control feature to allow Dag Run resource permissions. diff --git a/newsfragments/40874.significant.rst b/newsfragments/40874.significant.rst deleted file mode 100644 index 0677d131d557e..0000000000000 --- a/newsfragments/40874.significant.rst +++ /dev/null @@ -1 +0,0 @@ -Support for OpenTelemetry Traces is no longer "Experimental" diff --git a/newsfragments/41039.feature.rst b/newsfragments/41039.feature.rst deleted file mode 100644 index c696d25f874a8..0000000000000 --- a/newsfragments/41039.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Enable ``get_current_context()`` to work in virtual environments. The following ``Operators`` are affected: ``PythonVirtualenvOperator``, ``BranchPythonVirtualenvOperator``, ``ExternalPythonOperator``, ``BranchExternalPythonOperator`` diff --git a/newsfragments/41116.feature.rst b/newsfragments/41116.feature.rst deleted file mode 100644 index f5fa13d5295f4..0000000000000 --- a/newsfragments/41116.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Decorator for Task Flow, to make it simple to apply whether or not to skip a Task. From a7d48cb850ac300f83303e55e5065c9d322bb9c1 Mon Sep 17 00:00:00 2001 From: Jens Scheffler <95105677+jscheffl@users.noreply.github.com> Date: Fri, 9 Aug 2024 19:48:31 +0200 Subject: [PATCH 009/161] Fix tests/models/test_taskinstance.py for Database Isolation Tests (#41344) (cherry picked from commit f811ac304e3d49b015ffb1e326863fe1cce14523) --- airflow/models/taskinstance.py | 24 +++- .../serialization/pydantic/taskinstance.py | 15 ++ tests/conftest.py | 35 ++++- tests/models/test_taskinstance.py | 129 ++++++++++++++++-- 4 files changed, 184 insertions(+), 19 deletions(-) diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index f53fb5e27481f..baa4e3eed0c20 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -1404,6 +1404,25 @@ def _get_previous_execution_date( return pendulum.instance(prev_ti.execution_date) if prev_ti and prev_ti.execution_date else None +def _get_previous_start_date( + *, + task_instance: TaskInstance | TaskInstancePydantic, + state: DagRunState | None, + session: Session, +) -> pendulum.DateTime | None: + """ + Return the start date from property previous_ti_success. + + :param task_instance: the task instance + :param state: If passed, it only take into account instances of a specific state. + :param session: SQLAlchemy ORM Session + """ + log.debug("previous_start_date was called") + prev_ti = task_instance.get_previous_ti(state=state, session=session) + # prev_ti may not exist and prev_ti.start_date may be None. + return pendulum.instance(prev_ti.start_date) if prev_ti and prev_ti.start_date else None + + def _email_alert( *, task_instance: TaskInstance | TaskInstancePydantic, exception, task: BaseOperator ) -> None: @@ -2533,10 +2552,7 @@ def get_previous_start_date( :param state: If passed, it only take into account instances of a specific state. :param session: SQLAlchemy ORM Session """ - self.log.debug("previous_start_date was called") - prev_ti = self.get_previous_ti(state=state, session=session) - # prev_ti may not exist and prev_ti.start_date may be None. - return pendulum.instance(prev_ti.start_date) if prev_ti and prev_ti.start_date else None + return _get_previous_start_date(task_instance=self, state=state, session=session) @property def previous_start_date_success(self) -> pendulum.DateTime | None: diff --git a/airflow/serialization/pydantic/taskinstance.py b/airflow/serialization/pydantic/taskinstance.py index 2af5dcbecaf11..e89d21cd98da6 100644 --- a/airflow/serialization/pydantic/taskinstance.py +++ b/airflow/serialization/pydantic/taskinstance.py @@ -381,6 +381,21 @@ def get_previous_execution_date( return _get_previous_execution_date(task_instance=self, state=state, session=session) + def get_previous_start_date( + self, + state: DagRunState | None = None, + session: Session | None = None, + ) -> pendulum.DateTime | None: + """ + Return the execution date from property previous_ti_success. + + :param state: If passed, it only take into account instances of a specific state. + :param session: SQLAlchemy ORM Session + """ + from airflow.models.taskinstance import _get_previous_start_date + + return _get_previous_start_date(task_instance=self, state=state, session=session) + def email_alert(self, exception, task: BaseOperator) -> None: """ Send alert email with exception information. diff --git a/tests/conftest.py b/tests/conftest.py index 560146140918f..56acc986eb27a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1082,6 +1082,7 @@ def create_task_instance(dag_maker, create_dummy_dag): Uses ``create_dummy_dag`` to create the dag structure. """ + from airflow.operators.empty import EmptyOperator def maker( execution_date=None, @@ -1091,6 +1092,19 @@ def maker( run_type=None, data_interval=None, external_executor_id=None, + dag_id="dag", + task_id="op1", + task_display_name=None, + max_active_tis_per_dag=16, + max_active_tis_per_dagrun=None, + pool="default_pool", + executor_config=None, + trigger_rule="all_done", + on_success_callback=None, + on_execute_callback=None, + on_failure_callback=None, + on_retry_callback=None, + email=None, map_index=-1, **kwargs, ) -> TaskInstance: @@ -1098,7 +1112,26 @@ def maker( from airflow.utils import timezone execution_date = timezone.utcnow() - _, task = create_dummy_dag(with_dagrun_type=None, **kwargs) + with dag_maker(dag_id, **kwargs): + op_kwargs = {} + from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS + + if AIRFLOW_V_2_9_PLUS: + op_kwargs["task_display_name"] = task_display_name + task = EmptyOperator( + task_id=task_id, + max_active_tis_per_dag=max_active_tis_per_dag, + max_active_tis_per_dagrun=max_active_tis_per_dagrun, + executor_config=executor_config or {}, + on_success_callback=on_success_callback, + on_execute_callback=on_execute_callback, + on_failure_callback=on_failure_callback, + on_retry_callback=on_retry_callback, + email=email, + pool=pool, + trigger_rule=trigger_rule, + **op_kwargs, + ) dagrun_kwargs = {"execution_date": execution_date, "state": dagrun_state} if run_id is not None: diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index f490dd61dc689..c2993e9ce8d95 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -206,6 +206,7 @@ def test_set_task_dates(self, dag_maker): assert op3.start_date == DEFAULT_DATE + datetime.timedelta(days=1) assert op3.end_date == DEFAULT_DATE + datetime.timedelta(days=9) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_current_state(self, create_task_instance, session): ti = create_task_instance(session=session) assert ti.current_state(session=session) is None @@ -243,6 +244,7 @@ def test_set_dag(self, dag_maker): assert op.dag is dag assert op in dag.tasks + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_infer_dag(self, create_dummy_dag): op1 = EmptyOperator(task_id="test_op_1") op2 = EmptyOperator(task_id="test_op_2") @@ -287,6 +289,7 @@ def test_init_on_load(self, create_task_instance): assert ti.log.name == "airflow.task" assert not ti.test_mode + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode, mock not on server side @patch.object(DAG, "get_concurrency_reached") def test_requeue_over_dag_concurrency(self, mock_concurrency_reached, create_task_instance, dag_maker): mock_concurrency_reached.return_value = True @@ -309,6 +312,7 @@ def test_requeue_over_max_active_tis_per_dag(self, create_task_instance): max_active_runs=1, max_active_tasks=2, dagrun_state=State.QUEUED, + serialized=True, ) ti.run() @@ -322,6 +326,7 @@ def test_requeue_over_max_active_tis_per_dagrun(self, create_task_instance): max_active_runs=1, max_active_tasks=2, dagrun_state=State.QUEUED, + serialized=True, ) ti.run() @@ -334,6 +339,7 @@ def test_requeue_over_pool_concurrency(self, create_task_instance, test_pool): max_active_tis_per_dag=0, max_active_runs=1, max_active_tasks=2, + serialized=True, ) with create_session() as session: test_pool.slots = 0 @@ -341,6 +347,7 @@ def test_requeue_over_pool_concurrency(self, create_task_instance, test_pool): ti.run() assert ti.state == State.NONE + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.usefixtures("test_pool") def test_not_requeue_non_requeueable_task_instance(self, dag_maker): # Use BaseSensorOperator because sensor got @@ -376,6 +383,7 @@ def test_not_requeue_non_requeueable_task_instance(self, dag_maker): for dep_patch, _ in patch_dict.values(): dep_patch.stop() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_mark_non_runnable_task_as_success(self, create_task_instance): """ test that running task with mark_success param update task state @@ -399,6 +407,7 @@ def test_run_pooling_task(self, create_task_instance): dag_id="test_run_pooling_task", task_id="test_run_pooling_task_op", pool="test_pool", + serialized=True, ) ti.run() @@ -420,6 +429,7 @@ def test_pool_slots_property(self): pool_slots=0, ) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @provide_session def test_ti_updates_with_task(self, create_task_instance, session=None): """ @@ -462,6 +472,7 @@ def test_run_pooling_task_with_mark_success(self, create_task_instance): ti = create_task_instance( dag_id="test_run_pooling_task_with_mark_success", task_id="test_run_pooling_task_with_mark_success_op", + serialized=True, ) ti.run(mark_success=True) @@ -476,7 +487,7 @@ def test_run_pooling_task_with_skip(self, dag_maker): def raise_skip_exception(): raise AirflowSkipException - with dag_maker(dag_id="test_run_pooling_task_with_skip"): + with dag_maker(dag_id="test_run_pooling_task_with_skip", serialized=True): task = PythonOperator( task_id="test_run_pooling_task_with_skip", python_callable=raise_skip_exception, @@ -488,6 +499,7 @@ def raise_skip_exception(): ti.run() assert State.SKIPPED == ti.state + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_task_sigterm_calls_on_failure_callback(self, dag_maker, caplog): """ Test that ensures that tasks call on_failure_callback when they receive sigterm @@ -518,7 +530,7 @@ def test_task_sigterm_works_with_retries(self, dag_maker): def task_function(ti): os.kill(ti.pid, signal.SIGTERM) - with dag_maker("test_mark_failure_2"): + with dag_maker("test_mark_failure_2", serialized=True): task = PythonOperator( task_id="test_on_failure", python_callable=task_function, @@ -534,6 +546,7 @@ def task_function(ti): ti.refresh_from_db() assert ti.state == State.UP_FOR_RETRY + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode as DB access in code @pytest.mark.parametrize("state", [State.SUCCESS, State.FAILED, State.SKIPPED]) def test_task_sigterm_doesnt_change_state_of_finished_tasks(self, state, dag_maker): session = settings.Session() @@ -558,6 +571,7 @@ def task_function(ti): ti.refresh_from_db() ti.state == state + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "state, exception, retries", [ @@ -605,6 +619,7 @@ def _raise_if_exception(): assert ti.next_kwargs is None assert ti.state == state + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_retry_delay(self, dag_maker, time_machine): """ Test that retry delays are respected @@ -651,6 +666,7 @@ def run_with_error(ti): assert ti.state == State.FAILED assert ti.try_number == 3 + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_retry_handling(self, dag_maker, session): """ Test that task retries are handled properly @@ -778,6 +794,7 @@ def test_next_retry_datetime_short_or_zero_intervals(self, dag_maker, seconds): date = ti.next_retry_datetime() assert date == ti.end_date + datetime.timedelta(seconds=1) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_reschedule_handling(self, dag_maker, task_reschedules_for_ti): """ Test that task reschedules are handled properly @@ -886,6 +903,7 @@ def run_ti_and_assert( done, fail = True, False run_ti_and_assert(date4, date3, date4, 60, State.SUCCESS, 2, 1) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_mapped_reschedule_handling(self, dag_maker, task_reschedules_for_ti): """ Test that mapped task reschedules are handled properly @@ -989,6 +1007,7 @@ def run_ti_and_assert( done, fail = True, False run_ti_and_assert(date4, date3, date4, 60, State.SUCCESS, 2, 1) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.usefixtures("test_pool") def test_mapped_task_reschedule_handling_clear_reschedules(self, dag_maker, task_reschedules_for_ti): """ @@ -1053,6 +1072,7 @@ def run_ti_and_assert( # Check that reschedules for ti have also been cleared. assert not task_reschedules_for_ti(ti) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.usefixtures("test_pool") def test_reschedule_handling_clear_reschedules(self, dag_maker, task_reschedules_for_ti): """ @@ -1118,7 +1138,7 @@ def run_ti_and_assert( assert not task_reschedules_for_ti(ti) def test_depends_on_past(self, dag_maker): - with dag_maker(dag_id="test_depends_on_past"): + with dag_maker(dag_id="test_depends_on_past", serialized=True): task = EmptyOperator( task_id="test_dop_task", depends_on_past=True, @@ -1554,6 +1574,7 @@ def test_respects_prev_dagrun_dep(self, create_task_instance): ): assert ti.are_dependencies_met() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "downstream_ti_state, expected_are_dependents_done", [ @@ -1579,6 +1600,7 @@ def test_are_dependents_done( session.flush() assert ti.are_dependents_done(session) == expected_are_dependents_done + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_xcom_pull(self, dag_maker): """Test xcom_pull, using different filtering methods.""" with dag_maker(dag_id="test_xcom") as dag: @@ -1610,6 +1632,7 @@ def test_xcom_pull(self, dag_maker): result = ti1.xcom_pull(task_ids=["test_xcom_1", "test_xcom_2"], key="foo") assert result == ["bar", "baz"] + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_xcom_pull_mapped(self, dag_maker, session): with dag_maker(dag_id="test_xcom", session=session): # Use the private _expand() method to avoid the empty kwargs check. @@ -1651,6 +1674,7 @@ def test_xcom_pull_after_success(self, create_task_instance): schedule="@monthly", task_id="test_xcom", pool="test_xcom", + serialized=True, ) ti.run(mark_success=True) @@ -1666,6 +1690,7 @@ def test_xcom_pull_after_success(self, create_task_instance): ti.run(ignore_all_deps=True) assert ti.xcom_pull(task_ids="test_xcom", key=key) is None + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_xcom_pull_after_deferral(self, create_task_instance, session): """ tests xcom will not clear before a task runs its next method after deferral. @@ -1691,6 +1716,7 @@ def test_xcom_pull_after_deferral(self, create_task_instance, session): ti.run(ignore_all_deps=True) assert ti.xcom_pull(task_ids="test_xcom", key=key) == value + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_xcom_pull_different_execution_date(self, create_task_instance): """ tests xcom fetch behavior with different execution dates, using @@ -1729,7 +1755,7 @@ def test_xcom_push_flag(self, dag_maker): value = "hello" task_id = "test_no_xcom_push" - with dag_maker(dag_id="test_xcom"): + with dag_maker(dag_id="test_xcom", serialized=True): # nothing saved to XCom task = PythonOperator( task_id=task_id, @@ -1748,7 +1774,7 @@ def test_xcom_without_multiple_outputs(self, dag_maker): value = {"key1": "value1", "key2": "value2"} task_id = "test_xcom_push_without_multiple_outputs" - with dag_maker(dag_id="test_xcom"): + with dag_maker(dag_id="test_xcom", serialized=True): task = PythonOperator( task_id=task_id, python_callable=lambda: value, @@ -1766,7 +1792,7 @@ def test_xcom_with_multiple_outputs(self, dag_maker): value = {"key1": "value1", "key2": "value2"} task_id = "test_xcom_push_with_multiple_outputs" - with dag_maker(dag_id="test_xcom"): + with dag_maker(dag_id="test_xcom", serialized=True): task = PythonOperator( task_id=task_id, python_callable=lambda: value, @@ -1787,7 +1813,7 @@ def test_xcom_with_multiple_outputs_and_no_mapping_result(self, dag_maker): value = "value" task_id = "test_xcom_push_with_multiple_outputs" - with dag_maker(dag_id="test_xcom"): + with dag_maker(dag_id="test_xcom", serialized=True): task = PythonOperator( task_id=task_id, python_callable=lambda: value, @@ -1816,7 +1842,7 @@ def post_execute(self, context, result=None): if result == "error": raise TestError("expected error.") - with dag_maker(dag_id="test_post_execute_dag"): + with dag_maker(dag_id="test_post_execute_dag", serialized=True): task = TestOperator( task_id="test_operator", python_callable=lambda: "error", @@ -1826,6 +1852,7 @@ def post_execute(self, context, result=None): with pytest.raises(TestError): ti.run() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_check_and_change_state_before_execution(self, create_task_instance): expected_external_executor_id = "banana" ti = create_task_instance( @@ -1844,6 +1871,7 @@ def test_check_and_change_state_before_execution(self, create_task_instance): assert ti_from_deserialized_task.state == State.RUNNING assert ti_from_deserialized_task.try_number == 0 + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_check_and_change_state_before_execution_provided_id_overrides(self, create_task_instance): expected_external_executor_id = "banana" ti = create_task_instance( @@ -1865,6 +1893,7 @@ def test_check_and_change_state_before_execution_provided_id_overrides(self, cre assert ti_from_deserialized_task.state == State.RUNNING assert ti_from_deserialized_task.try_number == 0 + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_check_and_change_state_before_execution_with_exec_id(self, create_task_instance): expected_external_executor_id = "minions" ti = create_task_instance(dag_id="test_check_and_change_state_before_execution") @@ -1883,6 +1912,7 @@ def test_check_and_change_state_before_execution_with_exec_id(self, create_task_ assert ti_from_deserialized_task.state == State.RUNNING assert ti_from_deserialized_task.try_number == 0 + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_check_and_change_state_before_execution_dep_not_met(self, create_task_instance): ti = create_task_instance(dag_id="test_check_and_change_state_before_execution") task2 = EmptyOperator(task_id="task2", dag=ti.task.dag, start_date=DEFAULT_DATE) @@ -1893,6 +1923,7 @@ def test_check_and_change_state_before_execution_dep_not_met(self, create_task_i ti2 = TI(task=serialized_dag.get_task(task2.task_id), run_id=ti.run_id) assert not ti2.check_and_change_state_before_execution() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_check_and_change_state_before_execution_dep_not_met_already_running(self, create_task_instance): """return False if the task instance state is running""" ti = create_task_instance(dag_id="test_check_and_change_state_before_execution") @@ -1908,6 +1939,7 @@ def test_check_and_change_state_before_execution_dep_not_met_already_running(sel assert ti_from_deserialized_task.state == State.RUNNING assert ti_from_deserialized_task.external_executor_id is None + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_check_and_change_state_before_execution_dep_not_met_not_runnable_state( self, create_task_instance ): @@ -1938,6 +1970,7 @@ def test_try_number(self, create_task_instance): ti.state = State.SUCCESS assert ti.try_number == 2 # unaffected by state + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_get_num_running_task_instances(self, create_task_instance): session = settings.Session() @@ -1973,6 +2006,7 @@ def test_get_num_running_task_instances(self, create_task_instance): assert 1 == ti2.get_num_running_task_instances(session=session) assert 1 == ti3.get_num_running_task_instances(session=session) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_get_num_running_task_instances_per_dagrun(self, create_task_instance, dag_maker): session = settings.Session() @@ -2070,6 +2104,7 @@ def test_overwrite_params_with_dag_run_conf_none(self, create_task_instance): params = process_params(ti.task.dag, ti.task, dag_run, suppress_exception=False) assert params["override"] is False + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("use_native_obj", [True, False]) @patch("airflow.models.taskinstance.send_email") def test_email_alert(self, mock_send_email, dag_maker, use_native_obj): @@ -2087,6 +2122,7 @@ def test_email_alert(self, mock_send_email, dag_maker, use_native_obj): assert "test_email_alert" in body assert "Try 0" in body + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @conf_vars( { ("email", "subject_template"): "/subject/path", @@ -2114,6 +2150,7 @@ def test_email_alert_with_config(self, mock_send_email, dag_maker): assert "template: test_email_alert_with_config" == title assert "template: test_email_alert_with_config" == body + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @patch("airflow.models.taskinstance.send_email") def test_email_alert_with_filenotfound_config(self, mock_send_email, dag_maker): with dag_maker(dag_id="test_failure_email"): @@ -2144,6 +2181,7 @@ def test_email_alert_with_filenotfound_config(self, mock_send_email, dag_maker): assert title_default == title_error assert body_default == body_error + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("task_id", ["test_email_alert", "test_email_alert__1"]) @patch("airflow.models.taskinstance.send_email") def test_failure_mapped_taskflow(self, mock_send_email, dag_maker, session, task_id): @@ -2192,6 +2230,7 @@ def test_set_duration_empty_dates(self): ti.set_duration() assert ti.duration is None + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_success_callback_no_race_condition(self, create_task_instance): callback_wrapper = CallbackWrapper() ti = create_task_instance( @@ -2212,6 +2251,7 @@ def test_success_callback_no_race_condition(self, create_task_instance): ti.refresh_from_db() assert ti.state == State.SUCCESS + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_outlet_datasets(self, create_task_instance): """ Verify that when we have an outlet dataset on a task, and the task @@ -2270,6 +2310,7 @@ def test_outlet_datasets(self, create_task_instance): ) assert all([event.timestamp < ddrq_timestamp for (ddrq_timestamp,) in ddrq_timestamps]) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_outlet_datasets_failed(self, create_task_instance): """ Verify that when we have an outlet dataset on a task, and the task @@ -2301,6 +2342,7 @@ def test_outlet_datasets_failed(self, create_task_instance): # check that no dataset events were generated assert session.query(DatasetEvent).count() == 0 + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_mapped_current_state(self, dag_maker): with dag_maker(dag_id="test_mapped_current_state") as _: from airflow.decorators import task @@ -2324,6 +2366,7 @@ def raise_an_exception(placeholder: int): task_instance.run() assert task_instance.current_state() == TaskInstanceState.SUCCESS + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_outlet_datasets_skipped(self): """ Verify that when we have an outlet dataset on a task, and the task @@ -2354,6 +2397,7 @@ def test_outlet_datasets_skipped(self): # check that no dataset events were generated assert session.query(DatasetEvent).count() == 0 + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_outlet_dataset_extra(self, dag_maker, session): from airflow.datasets import Dataset @@ -2395,6 +2439,7 @@ def _write2_post_execute(context, _): assert events["write2"].dataset.uri == "test_outlet_dataset_extra_2" assert events["write2"].extra == {"x": 1} + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_outlet_dataset_extra_ignore_different(self, dag_maker, session): from airflow.datasets import Dataset @@ -2416,6 +2461,7 @@ def write(*, outlet_events): assert event.source_task_id == "write" assert event.extra == {"one": 1} + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_outlet_dataset_extra_yield(self, dag_maker, session): from airflow.datasets import Dataset from airflow.datasets.metadata import Metadata @@ -2465,6 +2511,7 @@ def _write2_post_execute(context, result): assert events["write2"].dataset.uri == "test_outlet_dataset_extra_2" assert events["write2"].extra == {"x": 1} + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_outlet_dataset_alias(self, dag_maker, session): from airflow.datasets import Dataset, DatasetAlias @@ -2513,6 +2560,7 @@ def producer(*, outlet_events): assert len(dsa_obj.datasets) == 1 assert dsa_obj.datasets[0].uri == ds_uri + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_outlet_multiple_dataset_alias(self, dag_maker, session): from airflow.datasets import Dataset, DatasetAlias @@ -2573,6 +2621,7 @@ def producer(*, outlet_events): assert len(dsa_obj.datasets) == 1 assert dsa_obj.datasets[0].uri == ds_uri + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_outlet_dataset_alias_through_metadata(self, dag_maker, session): from airflow.datasets import DatasetAlias from airflow.datasets.metadata import Metadata @@ -2617,6 +2666,7 @@ def producer(*, outlet_events): assert len(dsa_obj.datasets) == 1 assert dsa_obj.datasets[0].uri == ds_uri + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_outlet_dataset_alias_dataset_not_exists(self, dag_maker, session): from airflow.datasets import Dataset, DatasetAlias @@ -2656,6 +2706,7 @@ def producer(*, outlet_events): assert len(dsa_obj.datasets) == 1 assert dsa_obj.datasets[0].uri == ds_uri + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_inlet_dataset_extra(self, dag_maker, session): from airflow.datasets import Dataset @@ -2709,6 +2760,7 @@ def read(*, inlet_events): assert not dr.task_instance_scheduling_decisions(session=session).schedulable_tis assert read_task_evaluated + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_inlet_dataset_alias_extra(self, dag_maker, session): ds_uri = "test_inlet_dataset_extra_ds" dsa_name = "test_inlet_dataset_extra_dsa" @@ -2796,6 +2848,7 @@ def read(*, inlet_events): # Should be done. assert not dr.task_instance_scheduling_decisions(session=session).schedulable_tis + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "slicer, expected", [ @@ -2849,6 +2902,7 @@ def read(*, inlet_events): assert not dr.task_instance_scheduling_decisions(session=session).schedulable_tis assert result == expected + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "slicer, expected", [ @@ -2909,6 +2963,7 @@ def read(*, inlet_events): assert not dr.task_instance_scheduling_decisions(session=session).schedulable_tis assert result == expected + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_changing_of_dataset_when_ddrq_is_already_populated(self, dag_maker): """ Test that when a task that produces dataset has ran, that changing the consumer @@ -3029,6 +3084,7 @@ def test_previous_execution_date_success(self, schedule_interval, catchup, dag_m assert ti_list[3].get_previous_execution_date(state=State.SUCCESS) == ti_list[1].execution_date assert ti_list[3].get_previous_execution_date(state=State.SUCCESS) != ti_list[2].execution_date + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("schedule_interval, catchup", _prev_dates_param_list) def test_previous_start_date_success(self, schedule_interval, catchup, dag_maker) -> None: scenario = [State.FAILED, State.SUCCESS, State.FAILED, State.SUCCESS] @@ -3044,7 +3100,7 @@ def test_get_previous_start_date_none(self, dag_maker): """ Test that get_previous_start_date() can handle TaskInstance with no start_date. """ - with dag_maker("test_get_previous_start_date_none", schedule=None) as dag: + with dag_maker("test_get_previous_start_date_none", schedule=None, serialized=True): task = EmptyOperator(task_id="op") day_1 = DEFAULT_DATE @@ -3059,7 +3115,7 @@ def test_get_previous_start_date_none(self, dag_maker): run_type=DagRunType.MANUAL, ) - dagrun_2 = dag.create_dagrun( + dagrun_2 = dag_maker.create_dagrun( execution_date=day_2, state=State.RUNNING, run_type=DagRunType.MANUAL, @@ -3074,6 +3130,7 @@ def test_get_previous_start_date_none(self, dag_maker): assert ti_2.get_previous_start_date() == ti_1.start_date assert ti_1.start_date is None + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_context_triggering_dataset_events_none(self, session, create_task_instance): ti = create_task_instance() template_context = ti.get_template_context() @@ -3083,6 +3140,7 @@ def test_context_triggering_dataset_events_none(self, session, create_task_insta assert template_context["triggering_dataset_events"] == {} + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_context_triggering_dataset_events(self, create_dummy_dag, session): ds1 = DatasetModel(id=1, uri="one") ds2 = DatasetModel(id=2, uri="two") @@ -3135,6 +3193,7 @@ def test_pendulum_template_dates(self, create_task_instance): dag_id="test_pendulum_template_dates", task_id="test_pendulum_template_dates_task", schedule="0 12 * * *", + serialized=True, ) template_context = ti.get_template_context() @@ -3167,6 +3226,7 @@ def test_template_render_deprecated(self, create_task_instance, session): result = ti.task.render_template("Execution date: {{ execution_date }}", template_context) assert result.startswith("Execution date: ") + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "content, expected_output", [ @@ -3288,7 +3348,7 @@ def test_template_with_json_variable_missing(self, create_task_instance, session ], ) def test_deprecated_context(self, field, expected, create_task_instance): - ti = create_task_instance(execution_date=DEFAULT_DATE) + ti = create_task_instance(execution_date=DEFAULT_DATE, serialized=True) context = ti.get_template_context() with pytest.deprecated_call() as recorder: assert context[field] == expected @@ -3375,6 +3435,7 @@ def on_finish_callable(context): assert "Executing on_finish_callable callback" in caplog.text assert "Error when executing on_finish_callable callback" in caplog.text + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @provide_session def test_handle_failure(self, create_dummy_dag, session=None): start_date = timezone.datetime(2016, 6, 1) @@ -3472,6 +3533,7 @@ def test_handle_failure(self, create_dummy_dag, session=None): assert "task_instance" in context_arg_3 mock_on_retry_3.assert_not_called() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_handle_failure_updates_queued_task_updates_state(self, dag_maker): session = settings.Session() with dag_maker(): @@ -3485,6 +3547,7 @@ def test_handle_failure_updates_queued_task_updates_state(self, dag_maker): ti.handle_failure("test queued ti", test_mode=True) assert ti.state == State.UP_FOR_RETRY + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @patch.object(Stats, "incr") def test_handle_failure_no_task(self, Stats_incr, dag_maker): """ @@ -3519,6 +3582,7 @@ def test_handle_failure_no_task(self, Stats_incr, dag_maker): "operator_failures", tags={**expected_stats_tags, "operator": "EmptyOperator"} ) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_handle_failure_task_undefined(self, create_task_instance): """ When the loaded taskinstance does not use refresh_from_task, the task may be undefined. @@ -3529,6 +3593,7 @@ def test_handle_failure_task_undefined(self, create_task_instance): del ti.task ti.handle_failure("test ti.task undefined") + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @provide_session def test_handle_failure_fail_stop(self, create_dummy_dag, session=None): start_date = timezone.datetime(2016, 6, 1) @@ -3586,6 +3651,7 @@ def test_handle_failure_fail_stop(self, create_dummy_dag, session=None): for i in range(len(states)): assert tasks[i].state == exp_states[i] + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_does_not_retry_on_airflow_fail_exception(self, dag_maker): def fail(): raise AirflowFailException("hopeless") @@ -3602,6 +3668,7 @@ def fail(): ti.run() assert State.FAILED == ti.state + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_retries_on_other_exceptions(self, dag_maker): def fail(): raise AirflowException("maybe this will pass?") @@ -3618,6 +3685,7 @@ def fail(): ti.run() assert State.UP_FOR_RETRY == ti.state + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @patch.object(TaskInstance, "logger") def test_stacktrace_on_failure_starts_with_task_execute_method(self, mock_get_log, dag_maker): def fail(): @@ -3703,6 +3771,7 @@ def f(*args, **kwargs): ti.refresh_from_db() assert ti.state == expected_state + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_get_current_context_works_in_template(self, dag_maker): def user_defined_macro(): from airflow.operators.python import get_current_context @@ -3816,6 +3885,7 @@ def test_generate_command_specific_param(self): ) assert assert_command == generate_command + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @provide_session def test_get_rendered_template_fields(self, dag_maker, session=None): with dag_maker("test-dag", session=session) as dag: @@ -3839,6 +3909,7 @@ def test_get_rendered_template_fields(self, dag_maker, session=None): with create_session() as session: session.query(RenderedTaskInstanceFields).delete() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_set_state_up_for_retry(self, create_task_instance): ti = create_task_instance(state=State.RUNNING) @@ -3937,6 +4008,7 @@ def test_operator_field_with_serialization(self, create_task_instance): assert ser_ti.operator == "EmptyOperator" assert ser_ti.task.operator_name == "EmptyOperator" + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_clear_db_references(self, session, create_task_instance): tables = [TaskFail, RenderedTaskInstanceFields, XCom] ti = create_task_instance() @@ -3968,7 +4040,7 @@ def raise_skip_exception(): callback_function = mock.MagicMock() callback_function.__name__ = "callback_function" - with dag_maker(dag_id="test_skipped_task"): + with dag_maker(dag_id="test_skipped_task", serialized=True): task = PythonOperator( task_id="test_skipped_task", python_callable=raise_skip_exception, @@ -3983,7 +4055,7 @@ def raise_skip_exception(): assert callback_function.called def test_task_instance_history_is_created_when_ti_goes_for_retry(self, dag_maker, session): - with dag_maker(): + with dag_maker(serialized=True): task = BashOperator( task_id="test_history_tab", bash_command="ech", @@ -4070,6 +4142,7 @@ def teardown_method(self) -> None: self._clean() +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("mode", ["poke", "reschedule"]) @pytest.mark.parametrize("retries", [0, 1]) def test_sensor_timeout(mode, retries, dag_maker): @@ -4099,6 +4172,7 @@ def timeout(): assert ti.state == State.FAILED +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("mode", ["poke", "reschedule"]) @pytest.mark.parametrize("retries", [0, 1]) def test_mapped_sensor_timeout(mode, retries, dag_maker): @@ -4137,7 +4211,7 @@ def test_mapped_sensor_works(mode, retries, dag_maker): def timeout(ti): return 1 - with dag_maker(dag_id=f"test_sensor_timeout_{mode}_{retries}"): + with dag_maker(dag_id=f"test_sensor_timeout_{mode}_{retries}", serialized=True): PythonSensor.partial( task_id="test_raise_sensor_timeout", python_callable=timeout, @@ -4157,6 +4231,7 @@ def setup_class(self): with create_session() as session: session.query(TaskMap).delete() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("xcom_value", [[1, 2, 3], {"a": 1, "b": 2}, "abc"]) def test_not_recorded_if_leaf(self, dag_maker, xcom_value): """Return value should not be recorded if there are no downstreams.""" @@ -4173,6 +4248,7 @@ def push_something(): assert dag_maker.session.query(TaskMap).count() == 0 + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("xcom_value", [[1, 2, 3], {"a": 1, "b": 2}, "abc"]) def test_not_recorded_if_not_used(self, dag_maker, xcom_value): """Return value should not be recorded if no downstreams are mapped.""" @@ -4193,6 +4269,7 @@ def completely_different(): assert dag_maker.session.query(TaskMap).count() == 0 + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("xcom_1", [[1, 2, 3], {"a": 1, "b": 2}, "abc"]) @pytest.mark.parametrize("xcom_4", [[1, 2, 3], {"a": 1, "b": 2}]) def test_not_recorded_if_irrelevant(self, dag_maker, xcom_1, xcom_4): @@ -4241,6 +4318,7 @@ def tg(arg): tis["push_4"].run() assert dag_maker.session.query(TaskMap).count() == 2 + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "return_value, exception_type, error_message", [ @@ -4270,6 +4348,7 @@ def pull_something(value): assert ti.state == TaskInstanceState.FAILED assert str(ctx.value) == error_message + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "return_value, exception_type, error_message", [ @@ -4301,6 +4380,7 @@ def push(): assert ti.state == TaskInstanceState.FAILED assert str(ctx.value) == error_message + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "return_value, exception_type, error_message", [ @@ -4336,6 +4416,7 @@ def tg(arg): assert ti.state == TaskInstanceState.FAILED assert str(ctx.value) == error_message + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "return_value, exception_type, error_message", [ @@ -4371,6 +4452,7 @@ def tg(arg): assert ti.state == TaskInstanceState.FAILED assert str(ctx.value) == error_message + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "create_upstream", [ @@ -4406,6 +4488,7 @@ def pull(v): ti.run() assert str(ctx.value) == "expand_kwargs() expects a list[dict], not list[int]" + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "downstream, error_message", [ @@ -4461,6 +4544,7 @@ def pull(arg1, arg2): ti.run() ti.xcom_pull(task_ids=downstream, map_indexes=1, session=session) == ["b", "c"] + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_error_if_upstream_does_not_push(self, dag_maker): """Fail the upstream task if it fails to push the XCom used for task mapping.""" with dag_maker(dag_id="test_not_recorded_for_unused") as dag: @@ -4483,6 +4567,7 @@ def pull_something(value): assert ti.state == TaskInstanceState.FAILED assert str(ctx.value) == "did not push XCom for task mapping" + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @conf_vars({("core", "max_map_length"): "1"}) def test_error_if_unmappable_length(self, dag_maker): """If an unmappable return value is used to map, fail the task that pushed the XCom.""" @@ -4506,6 +4591,7 @@ def pull_something(value): assert ti.state == TaskInstanceState.FAILED assert str(ctx.value) == "unmappable return value length: 2 > 1" + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "xcom_value, expected_length, expected_keys", [ @@ -4539,6 +4625,7 @@ def pull_something(value): assert task_map.length == expected_length assert task_map.keys == expected_keys + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_no_error_on_changing_from_non_mapped_to_mapped(self, dag_maker, session): """If a task changes from non-mapped to mapped, don't fail on integrity error.""" with dag_maker(dag_id="test_no_error_on_changing_from_non_mapped_to_mapped") as dag: @@ -4574,6 +4661,7 @@ def add_two(x): class TestMappedTaskInstanceReceiveValue: + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "literal, expected_outputs", [ @@ -4607,6 +4695,7 @@ def show(value): ti.run() assert outputs == expected_outputs + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "upstream_return, expected_outputs", [ @@ -4643,6 +4732,7 @@ def show(value): ti.run() assert outputs == expected_outputs + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_map_product(self, dag_maker, session): outputs = [] @@ -4684,6 +4774,7 @@ def show(number, letter): (2, ("c", "z")), ] + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_map_product_same(self, dag_maker, session): """Test a mapped task can refer to the same source multiple times.""" outputs = [] @@ -4717,6 +4808,7 @@ def show(a, b): ti.run() assert outputs == [(1, 1), (1, 2), (2, 1), (2, 2)] + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_map_literal_cross_product(self, dag_maker, session): """Test a mapped task with literal cross product args expand properly.""" outputs = [] @@ -4752,6 +4844,7 @@ def show(a, b): ti.run() assert outputs == [(2, 5), (2, 10), (4, 5), (4, 10), (8, 5), (8, 10)] + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_map_in_group(self, tmp_path: pathlib.Path, dag_maker, session): out = tmp_path.joinpath("out") out.touch() @@ -4819,6 +4912,7 @@ def _get_lazy_xcom_access_expected_sql_lines() -> list[str]: raise RuntimeError(f"unknown backend {backend!r}") +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_lazy_xcom_access_does_not_pickle_session(dag_maker, session): with dag_maker(session=session): EmptyOperator(task_id="t") @@ -4848,6 +4942,7 @@ def test_lazy_xcom_access_does_not_pickle_session(dag_maker, session): assert list(processed) == [123] +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @mock.patch("airflow.models.taskinstance.XCom.deserialize_value", side_effect=XCom.deserialize_value) def test_ti_xcom_pull_on_mapped_operator_return_lazy_iterable(mock_deserialize_value, dag_maker, session): """Ensure we access XCom lazily when pulling from a mapped operator.""" @@ -4884,6 +4979,7 @@ def test_ti_xcom_pull_on_mapped_operator_return_lazy_iterable(mock_deserialize_v next(it) +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_ti_mapped_depends_on_mapped_xcom_arg(dag_maker, session): with dag_maker(session=session) as dag: @@ -4909,6 +5005,7 @@ def add_one(x): assert [x.value for x in query.order_by(None).order_by(XCom.map_index)] == [3, 4, 5] +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_mapped_upstream_return_none_should_skip(dag_maker, session): results = set() @@ -4965,6 +5062,7 @@ def get_extra_env(): assert "get_extra_env" in echo_task.upstream_task_ids +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_mapped_task_does_not_error_in_mini_scheduler_if_upstreams_are_not_done(dag_maker, caplog, session): """ This tests that when scheduling child tasks of a task and there's a mapped downstream task, @@ -5007,6 +5105,7 @@ def last_task(): assert "0 downstream tasks scheduled from follow-on schedule" in caplog.text +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_empty_operator_is_not_considered_in_mini_scheduler(dag_maker, caplog, session): """ This tests verify that operators with inherits_from_empty_operator are not considered by mini scheduler. @@ -5051,6 +5150,7 @@ def second_task(): assert "2 downstream tasks scheduled from follow-on schedule" in caplog.text +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_mapped_task_expands_in_mini_scheduler_if_upstreams_are_done(dag_maker, caplog, session): """Test that mini scheduler expands mapped task""" with dag_maker() as dag: @@ -5092,6 +5192,7 @@ def last_task(): assert "3 downstream tasks scheduled from follow-on schedule" in caplog.text +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_mini_scheduler_not_skip_mapped_downstream_until_all_upstreams_finish(dag_maker, session): with dag_maker(session=session): From dc0548e2a5a141d7f139488619e82cafef1102a3 Mon Sep 17 00:00:00 2001 From: Jens Scheffler <95105677+jscheffl@users.noreply.github.com> Date: Fri, 9 Aug 2024 20:54:19 +0200 Subject: [PATCH 010/161] Fix TriggerDagRunOperator Tests for Database Isolation Tests (#41298) * Attempt to fix TriggerDagRunOperator for Database Isolation Tests * Finalize making tests run for triggerdagrunoperator in db isolation mode * Adjust query count assert for adjustments to serialization * Review feedback (cherry picked from commit 6b810b89c3f63dd2d2cf107c568be40ba9da0ba2) --- airflow/api/common/trigger_dag.py | 8 + .../endpoints/rpc_api_endpoint.py | 2 + airflow/exceptions.py | 22 + airflow/models/dag.py | 4 + airflow/operators/trigger_dagrun.py | 14 + airflow/serialization/serialized_objects.py | 7 +- tests/models/test_dag.py | 2 +- tests/operators/test_trigger_dagrun.py | 675 ++++++++++-------- 8 files changed, 451 insertions(+), 283 deletions(-) diff --git a/airflow/api/common/trigger_dag.py b/airflow/api/common/trigger_dag.py index 86513f78333c2..f22755ec640ea 100644 --- a/airflow/api/common/trigger_dag.py +++ b/airflow/api/common/trigger_dag.py @@ -22,15 +22,19 @@ import json from typing import TYPE_CHECKING +from airflow.api_internal.internal_api_call import internal_api_call from airflow.exceptions import DagNotFound, DagRunAlreadyExists from airflow.models import DagBag, DagModel, DagRun from airflow.utils import timezone +from airflow.utils.session import NEW_SESSION, provide_session from airflow.utils.state import DagRunState from airflow.utils.types import DagRunType if TYPE_CHECKING: from datetime import datetime + from sqlalchemy.orm.session import Session + def _trigger_dag( dag_id: str, @@ -103,12 +107,15 @@ def _trigger_dag( return dag_runs +@internal_api_call +@provide_session def trigger_dag( dag_id: str, run_id: str | None = None, conf: dict | str | None = None, execution_date: datetime | None = None, replace_microseconds: bool = True, + session: Session = NEW_SESSION, ) -> DagRun | None: """ Triggers execution of DAG specified by dag_id. @@ -118,6 +125,7 @@ def trigger_dag( :param conf: configuration :param execution_date: date of execution :param replace_microseconds: whether microseconds should be zeroed + :param session: Unused. Only added in compatibility with database isolation mode :return: first dag run triggered - even if more than one Dag Runs were triggered or None """ dag_model = DagModel.get_current(dag_id) diff --git a/airflow/api_internal/endpoints/rpc_api_endpoint.py b/airflow/api_internal/endpoints/rpc_api_endpoint.py index be4699fa6c7dd..ad65157ef9415 100644 --- a/airflow/api_internal/endpoints/rpc_api_endpoint.py +++ b/airflow/api_internal/endpoints/rpc_api_endpoint.py @@ -53,6 +53,7 @@ @functools.lru_cache def initialize_method_map() -> dict[str, Callable]: + from airflow.api.common.trigger_dag import trigger_dag from airflow.cli.commands.task_command import _get_ti_db_access from airflow.dag_processing.manager import DagFileProcessorManager from airflow.dag_processing.processor import DagFileProcessor @@ -92,6 +93,7 @@ def initialize_method_map() -> dict[str, Callable]: _add_log, _xcom_pull, _record_task_map_for_downstreams, + trigger_dag, DagCode.remove_deleted_code, DagModel.deactivate_deleted_dags, DagModel.get_paused_dag_ids, diff --git a/airflow/exceptions.py b/airflow/exceptions.py index 40a62ad20854c..3831d909fc272 100644 --- a/airflow/exceptions.py +++ b/airflow/exceptions.py @@ -239,6 +239,28 @@ def __init__(self, dag_run: DagRun, execution_date: datetime.datetime, run_id: s f"A DAG Run already exists for DAG {dag_run.dag_id} at {execution_date} with run id {run_id}" ) self.dag_run = dag_run + self.execution_date = execution_date + self.run_id = run_id + + def serialize(self): + cls = self.__class__ + # Note the DagRun object will be detached here and fails serialization, we need to create a new one + from airflow.models import DagRun + + dag_run = DagRun( + state=self.dag_run.state, + dag_id=self.dag_run.dag_id, + run_id=self.dag_run.run_id, + external_trigger=self.dag_run.external_trigger, + run_type=self.dag_run.run_type, + execution_date=self.dag_run.execution_date, + ) + dag_run.id = self.dag_run.id + return ( + f"{cls.__module__}.{cls.__name__}", + (), + {"dag_run": dag_run, "execution_date": self.execution_date, "run_id": self.run_id}, + ) class DagFileExists(AirflowBadRequest): diff --git a/airflow/models/dag.py b/airflow/models/dag.py index c9380494a034d..1c9d351c1d292 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -115,6 +115,7 @@ TaskInstanceKey, clear_task_instances, ) +from airflow.models.tasklog import LogTemplate from airflow.secrets.local_filesystem import LocalFilesystemBackend from airflow.security import permissions from airflow.settings import json @@ -338,6 +339,9 @@ def _create_orm_dagrun( creating_job_id=creating_job_id, data_interval=data_interval, ) + # Load defaults into the following two fields to ensure result can be serialized detached + run.log_template_id = int(session.scalar(select(func.max(LogTemplate.__table__.c.id)))) + run.consumed_dataset_events = [] session.add(run) session.flush() run.dag = dag diff --git a/airflow/operators/trigger_dagrun.py b/airflow/operators/trigger_dagrun.py index 35d387738a0d3..2521297dcf936 100644 --- a/airflow/operators/trigger_dagrun.py +++ b/airflow/operators/trigger_dagrun.py @@ -27,6 +27,7 @@ from sqlalchemy.orm.exc import NoResultFound from airflow.api.common.trigger_dag import trigger_dag +from airflow.api_internal.internal_api_call import InternalApiConfig from airflow.configuration import conf from airflow.exceptions import ( AirflowException, @@ -83,6 +84,8 @@ class TriggerDagRunOperator(BaseOperator): """ Triggers a DAG run for a specified DAG ID. + Note that if database isolation mode is enabled, not all features are supported. + :param trigger_dag_id: The ``dag_id`` of the DAG to trigger (templated). :param trigger_run_id: The run ID to use for the triggered DAG run (templated). If not provided, a run ID will be automatically generated. @@ -174,6 +177,14 @@ def __init__( self.logical_date = logical_date def execute(self, context: Context): + if InternalApiConfig.get_use_internal_api(): + if self.reset_dag_run: + raise AirflowException("Parameter reset_dag_run=True is broken with Database Isolation Mode.") + if self.wait_for_completion: + raise AirflowException( + "Parameter wait_for_completion=True is broken with Database Isolation Mode." + ) + if isinstance(self.logical_date, datetime.datetime): parsed_logical_date = self.logical_date elif isinstance(self.logical_date, str): @@ -210,6 +221,7 @@ def execute(self, context: Context): if dag_model is None: raise DagNotFound(f"Dag id {self.trigger_dag_id} not found in DagModel") + # Note: here execution fails on database isolation mode. Needs structural changes for AIP-72 dag_bag = DagBag(dag_folder=dag_model.fileloc, read_dags_from_db=True) dag = dag_bag.get_dag(self.trigger_dag_id) dag.clear(start_date=dag_run.logical_date, end_date=dag_run.logical_date) @@ -250,6 +262,7 @@ def execute(self, context: Context): ) time.sleep(self.poke_interval) + # Note: here execution fails on database isolation mode. Needs structural changes for AIP-72 dag_run.refresh_from_db() state = dag_run.state if state in self.failed_states: @@ -263,6 +276,7 @@ def execute_complete(self, context: Context, session: Session, event: tuple[str, # This logical_date is parsed from the return trigger event provided_logical_date = event[1]["execution_dates"][0] try: + # Note: here execution fails on database isolation mode. Needs structural changes for AIP-72 dag_run = session.execute( select(DagRun).where( DagRun.dag_id == self.trigger_dag_id, DagRun.execution_date == provided_logical_date diff --git a/airflow/serialization/serialized_objects.py b/airflow/serialization/serialized_objects.py index 94631c993c122..d110271c3da08 100644 --- a/airflow/serialization/serialized_objects.py +++ b/airflow/serialization/serialized_objects.py @@ -1447,7 +1447,12 @@ def get_custom_dep() -> list[DagDependency]: @classmethod def _is_excluded(cls, var: Any, attrname: str, op: DAGNode): - if var is not None and op.has_dag() and attrname.endswith("_date"): + if ( + var is not None + and op.has_dag() + and op.dag.__class__ is not AttributeRemoved + and attrname.endswith("_date") + ): # If this date is the same as the matching field in the dag, then # don't store it again at the task level. dag_date = getattr(op.dag, attrname, None) diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index 376d5c5beb170..3d39a7290d909 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -3293,7 +3293,7 @@ def test_count_number_queries(self, tasks_count): dag = DAG("test_dagrun_query_count", start_date=DEFAULT_DATE) for i in range(tasks_count): EmptyOperator(task_id=f"dummy_task_{i}", owner="test", dag=dag) - with assert_queries_count(2): + with assert_queries_count(3): dag.create_dagrun( run_id="test_dagrun_query_count", state=State.RUNNING, diff --git a/tests/operators/test_trigger_dagrun.py b/tests/operators/test_trigger_dagrun.py index 341b34fe46fc6..349bba463800f 100644 --- a/tests/operators/test_trigger_dagrun.py +++ b/tests/operators/test_trigger_dagrun.py @@ -17,7 +17,6 @@ # under the License. from __future__ import annotations -import pathlib import tempfile from datetime import datetime from unittest import mock @@ -26,13 +25,14 @@ import pytest from airflow.exceptions import AirflowException, DagRunAlreadyExists, RemovedInAirflow3Warning, TaskDeferred -from airflow.models.dag import DAG, DagModel +from airflow.models.dag import DagModel from airflow.models.dagbag import DagBag from airflow.models.dagrun import DagRun from airflow.models.log import Log from airflow.models.serialized_dag import SerializedDagModel from airflow.models.taskinstance import TaskInstance from airflow.operators.trigger_dagrun import TriggerDagRunOperator +from airflow.settings import TracebackSessionForTests from airflow.triggers.external_task import DagStateTrigger from airflow.utils import timezone from airflow.utils.session import create_session @@ -67,15 +67,18 @@ def setup_method(self): self._tmpfile = f.name f.write(DAG_SCRIPT) f.flush() + self.f_name = f.name with create_session() as session: session.add(DagModel(dag_id=TRIGGERED_DAG_ID, fileloc=self._tmpfile)) session.commit() - self.dag = DAG(TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}) - dagbag = DagBag(f.name, read_dags_from_db=False, include_examples=False) - dagbag.bag_dag(self.dag, root_dag=self.dag) - dagbag.sync_to_db() + def re_sync_triggered_dag_to_db(self, dag, dag_maker): + TracebackSessionForTests.set_allow_db_access(dag_maker.session, True) + dagbag = DagBag(self.f_name, read_dags_from_db=False, include_examples=False) + dagbag.bag_dag(dag, root_dag=dag) + dagbag.sync_to_db(session=dag_maker.session) + TracebackSessionForTests.set_allow_db_access(dag_maker.session, False) def teardown_method(self): """Cleanup state after testing in DB.""" @@ -86,7 +89,7 @@ def teardown_method(self): synchronize_session=False ) - pathlib.Path(self._tmpfile).unlink() + # pathlib.Path(self._tmpfile).unlink() def assert_extra_link(self, triggered_dag_run, triggering_task, session): """ @@ -115,24 +118,32 @@ def assert_extra_link(self, triggered_dag_run, triggering_task, session): } assert expected_args in args - def test_trigger_dagrun(self): + def test_trigger_dagrun(self, dag_maker): """Test TriggerDagRunOperator.""" - task = TriggerDagRunOperator(task_id="test_task", trigger_dag_id=TRIGGERED_DAG_ID, dag=self.dag) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator(task_id="test_task", trigger_dag_id=TRIGGERED_DAG_ID) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) - with create_session() as session: - dagrun = session.query(DagRun).filter(DagRun.dag_id == TRIGGERED_DAG_ID).one() - assert dagrun.external_trigger - assert dagrun.run_id == DagRun.generate_run_id(DagRunType.MANUAL, dagrun.logical_date) - self.assert_extra_link(dagrun, task, session) + dagrun = dag_maker.session.query(DagRun).filter(DagRun.dag_id == TRIGGERED_DAG_ID).one() + assert dagrun.external_trigger + assert dagrun.run_id == DagRun.generate_run_id(DagRunType.MANUAL, dagrun.logical_date) + self.assert_extra_link(dagrun, task, dag_maker.session) - def test_trigger_dagrun_custom_run_id(self): - task = TriggerDagRunOperator( - task_id="test_task", - trigger_dag_id=TRIGGERED_DAG_ID, - trigger_run_id="custom_run_id", - dag=self.dag, - ) + def test_trigger_dagrun_custom_run_id(self, dag_maker): + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_task", + trigger_dag_id=TRIGGERED_DAG_ID, + trigger_run_id="custom_run_id", + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE) with create_session() as session: @@ -140,15 +151,19 @@ def test_trigger_dagrun_custom_run_id(self): assert len(dagruns) == 1 assert dagruns[0].run_id == "custom_run_id" - def test_trigger_dagrun_with_logical_date(self): + def test_trigger_dagrun_with_logical_date(self, dag_maker): """Test TriggerDagRunOperator with custom logical_date.""" custom_logical_date = timezone.datetime(2021, 1, 2, 3, 4, 5) - task = TriggerDagRunOperator( - task_id="test_trigger_dagrun_with_logical_date", - trigger_dag_id=TRIGGERED_DAG_ID, - logical_date=custom_logical_date, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_trigger_dagrun_with_logical_date", + trigger_dag_id=TRIGGERED_DAG_ID, + logical_date=custom_logical_date, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) with create_session() as session: @@ -158,78 +173,91 @@ def test_trigger_dagrun_with_logical_date(self): assert dagrun.run_id == DagRun.generate_run_id(DagRunType.MANUAL, custom_logical_date) self.assert_extra_link(dagrun, task, session) - def test_trigger_dagrun_twice(self): + @pytest.mark.skip_if_database_isolation_mode # Known to be broken in db isolation mode + def test_trigger_dagrun_twice(self, dag_maker): """Test TriggerDagRunOperator with custom logical_date.""" utc_now = timezone.utcnow() - task = TriggerDagRunOperator( - task_id="test_trigger_dagrun_with_logical_date", - trigger_dag_id=TRIGGERED_DAG_ID, - logical_date=utc_now, - dag=self.dag, - poke_interval=1, - reset_dag_run=True, - wait_for_completion=True, - ) run_id = f"manual__{utc_now.isoformat()}" - with create_session() as session: - dag_run = DagRun( - dag_id=TRIGGERED_DAG_ID, - execution_date=utc_now, - state=State.SUCCESS, - run_type="manual", - run_id=run_id, + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_trigger_dagrun_with_logical_date", + trigger_dag_id=TRIGGERED_DAG_ID, + trigger_run_id=run_id, + logical_date=utc_now, + poke_interval=1, + reset_dag_run=True, + wait_for_completion=True, ) - session.add(dag_run) - session.commit() - task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() + dag_run = DagRun( + dag_id=TRIGGERED_DAG_ID, + execution_date=utc_now, + state=State.SUCCESS, + run_type="manual", + run_id=run_id, + ) + dag_maker.session.add(dag_run) + dag_maker.session.commit() + task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) - dagruns = session.query(DagRun).filter(DagRun.dag_id == TRIGGERED_DAG_ID).all() - assert len(dagruns) == 1 - triggered_dag_run = dagruns[0] - assert triggered_dag_run.external_trigger - assert triggered_dag_run.logical_date == utc_now - self.assert_extra_link(triggered_dag_run, task, session) + dagruns = dag_maker.session.query(DagRun).filter(DagRun.dag_id == TRIGGERED_DAG_ID).all() + assert len(dagruns) == 1 + triggered_dag_run = dagruns[0] + assert triggered_dag_run.external_trigger + assert triggered_dag_run.logical_date == utc_now + self.assert_extra_link(triggered_dag_run, task, dag_maker.session) - def test_trigger_dagrun_with_scheduled_dag_run(self): + @pytest.mark.skip_if_database_isolation_mode # Known to be broken in db isolation mode + def test_trigger_dagrun_with_scheduled_dag_run(self, dag_maker): """Test TriggerDagRunOperator with custom logical_date and scheduled dag_run.""" utc_now = timezone.utcnow() - task = TriggerDagRunOperator( - task_id="test_trigger_dagrun_with_logical_date", - trigger_dag_id=TRIGGERED_DAG_ID, - logical_date=utc_now, - dag=self.dag, - poke_interval=1, - reset_dag_run=True, - wait_for_completion=True, - ) - run_id = f"scheduled__{utc_now.isoformat()}" - with create_session() as session: - dag_run = DagRun( - dag_id=TRIGGERED_DAG_ID, - execution_date=utc_now, - state=State.SUCCESS, - run_type="scheduled", - run_id=run_id, + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_trigger_dagrun_with_logical_date", + trigger_dag_id=TRIGGERED_DAG_ID, + logical_date=utc_now, + poke_interval=1, + reset_dag_run=True, + wait_for_completion=True, ) - session.add(dag_run) - session.commit() - task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() + run_id = f"scheduled__{utc_now.isoformat()}" + dag_run = DagRun( + dag_id=TRIGGERED_DAG_ID, + execution_date=utc_now, + state=State.SUCCESS, + run_type="scheduled", + run_id=run_id, + ) + dag_maker.session.add(dag_run) + dag_maker.session.commit() + task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) - dagruns = session.query(DagRun).filter(DagRun.dag_id == TRIGGERED_DAG_ID).all() - assert len(dagruns) == 1 - triggered_dag_run = dagruns[0] - assert triggered_dag_run.external_trigger - assert triggered_dag_run.logical_date == utc_now - self.assert_extra_link(triggered_dag_run, task, session) + dagruns = dag_maker.session.query(DagRun).filter(DagRun.dag_id == TRIGGERED_DAG_ID).all() + assert len(dagruns) == 1 + triggered_dag_run = dagruns[0] + assert triggered_dag_run.external_trigger + assert triggered_dag_run.logical_date == utc_now + self.assert_extra_link(triggered_dag_run, task, dag_maker.session) - def test_trigger_dagrun_with_templated_logical_date(self): + def test_trigger_dagrun_with_templated_logical_date(self, dag_maker): """Test TriggerDagRunOperator with templated logical_date.""" - task = TriggerDagRunOperator( - task_id="test_trigger_dagrun_with_str_logical_date", - trigger_dag_id=TRIGGERED_DAG_ID, - logical_date="{{ logical_date }}", - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_trigger_dagrun_with_str_logical_date", + trigger_dag_id=TRIGGERED_DAG_ID, + logical_date="{{ logical_date }}", + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) with create_session() as session: @@ -240,14 +268,18 @@ def test_trigger_dagrun_with_templated_logical_date(self): assert triggered_dag_run.logical_date == DEFAULT_DATE self.assert_extra_link(triggered_dag_run, task, session) - def test_trigger_dagrun_operator_conf(self): + def test_trigger_dagrun_operator_conf(self, dag_maker): """Test passing conf to the triggered DagRun.""" - task = TriggerDagRunOperator( - task_id="test_trigger_dagrun_with_str_logical_date", - trigger_dag_id=TRIGGERED_DAG_ID, - conf={"foo": "bar"}, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_trigger_dagrun_with_str_logical_date", + trigger_dag_id=TRIGGERED_DAG_ID, + conf={"foo": "bar"}, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) with create_session() as session: @@ -255,25 +287,33 @@ def test_trigger_dagrun_operator_conf(self): assert len(dagruns) == 1 assert dagruns[0].conf == {"foo": "bar"} - def test_trigger_dagrun_operator_templated_invalid_conf(self): + def test_trigger_dagrun_operator_templated_invalid_conf(self, dag_maker): """Test passing a conf that is not JSON Serializable raise error.""" - task = TriggerDagRunOperator( - task_id="test_trigger_dagrun_with_invalid_conf", - trigger_dag_id=TRIGGERED_DAG_ID, - conf={"foo": "{{ dag.dag_id }}", "datetime": timezone.utcnow()}, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_trigger_dagrun_with_invalid_conf", + trigger_dag_id=TRIGGERED_DAG_ID, + conf={"foo": "{{ dag.dag_id }}", "datetime": timezone.utcnow()}, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() with pytest.raises(AirflowException, match="^conf parameter should be JSON Serializable$"): task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE) - def test_trigger_dagrun_operator_templated_conf(self): + def test_trigger_dagrun_operator_templated_conf(self, dag_maker): """Test passing a templated conf to the triggered DagRun.""" - task = TriggerDagRunOperator( - task_id="test_trigger_dagrun_with_str_logical_date", - trigger_dag_id=TRIGGERED_DAG_ID, - conf={"foo": "{{ dag.dag_id }}"}, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_trigger_dagrun_with_str_logical_date", + trigger_dag_id=TRIGGERED_DAG_ID, + conf={"foo": "{{ dag.dag_id }}"}, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) with create_session() as session: @@ -281,17 +321,21 @@ def test_trigger_dagrun_operator_templated_conf(self): assert len(dagruns) == 1 assert dagruns[0].conf == {"foo": TEST_DAG_ID} - def test_trigger_dagrun_with_reset_dag_run_false(self): + def test_trigger_dagrun_with_reset_dag_run_false(self, dag_maker): """Test TriggerDagRunOperator without reset_dag_run.""" logical_date = DEFAULT_DATE - task = TriggerDagRunOperator( - task_id="test_task", - trigger_dag_id=TRIGGERED_DAG_ID, - trigger_run_id=None, - logical_date=None, - reset_dag_run=False, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_task", + trigger_dag_id=TRIGGERED_DAG_ID, + trigger_run_id=None, + logical_date=None, + reset_dag_run=False, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() task.run(start_date=logical_date, end_date=logical_date, ignore_ti_state=True) task.run(start_date=logical_date, end_date=logical_date, ignore_ti_state=True) @@ -307,39 +351,50 @@ def test_trigger_dagrun_with_reset_dag_run_false(self): ("dummy_run_id", DEFAULT_DATE), ], ) - def test_trigger_dagrun_with_reset_dag_run_false_fail(self, trigger_run_id, trigger_logical_date): + def test_trigger_dagrun_with_reset_dag_run_false_fail( + self, trigger_run_id, trigger_logical_date, dag_maker + ): """Test TriggerDagRunOperator without reset_dag_run but triggered dag fails.""" logical_date = DEFAULT_DATE - task = TriggerDagRunOperator( - task_id="test_task", - trigger_dag_id=TRIGGERED_DAG_ID, - trigger_run_id=trigger_run_id, - logical_date=trigger_logical_date, - reset_dag_run=False, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_task", + trigger_dag_id=TRIGGERED_DAG_ID, + trigger_run_id=trigger_run_id, + logical_date=trigger_logical_date, + reset_dag_run=False, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() task.run(start_date=logical_date, end_date=logical_date, ignore_ti_state=True) with pytest.raises(DagRunAlreadyExists): task.run(start_date=logical_date, end_date=logical_date, ignore_ti_state=True) - def test_trigger_dagrun_with_skip_when_already_exists(self): + def test_trigger_dagrun_with_skip_when_already_exists(self, dag_maker): """Test TriggerDagRunOperator with skip_when_already_exists.""" execution_date = DEFAULT_DATE - task = TriggerDagRunOperator( - task_id="test_task", - trigger_dag_id=TRIGGERED_DAG_ID, - trigger_run_id="dummy_run_id", - execution_date=None, - reset_dag_run=False, - skip_when_already_exists=True, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_task", + trigger_dag_id=TRIGGERED_DAG_ID, + trigger_run_id="dummy_run_id", + execution_date=None, + reset_dag_run=False, + skip_when_already_exists=True, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dr: DagRun = dag_maker.create_dagrun() task.run(start_date=execution_date, end_date=execution_date, ignore_ti_state=True) - assert task.get_task_instances()[0].state == TaskInstanceState.SUCCESS + assert dr.get_task_instance("test_task").state == TaskInstanceState.SUCCESS task.run(start_date=execution_date, end_date=execution_date, ignore_ti_state=True) - assert task.get_task_instances()[0].state == TaskInstanceState.SKIPPED + assert dr.get_task_instance("test_task").state == TaskInstanceState.SKIPPED + @pytest.mark.skip_if_database_isolation_mode # Known to be broken in db isolation mode @pytest.mark.parametrize( "trigger_run_id, trigger_logical_date, expected_dagruns_count", [ @@ -350,18 +405,22 @@ def test_trigger_dagrun_with_skip_when_already_exists(self): ], ) def test_trigger_dagrun_with_reset_dag_run_true( - self, trigger_run_id, trigger_logical_date, expected_dagruns_count + self, trigger_run_id, trigger_logical_date, expected_dagruns_count, dag_maker ): """Test TriggerDagRunOperator with reset_dag_run.""" logical_date = DEFAULT_DATE - task = TriggerDagRunOperator( - task_id="test_task", - trigger_dag_id=TRIGGERED_DAG_ID, - trigger_run_id=trigger_run_id, - logical_date=trigger_logical_date, - reset_dag_run=True, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_task", + trigger_dag_id=TRIGGERED_DAG_ID, + trigger_run_id=trigger_run_id, + logical_date=trigger_logical_date, + reset_dag_run=True, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() task.run(start_date=logical_date, end_date=logical_date, ignore_ti_state=True) task.run(start_date=logical_date, end_date=logical_date, ignore_ti_state=True) @@ -370,106 +429,132 @@ def test_trigger_dagrun_with_reset_dag_run_true( assert len(dag_runs) == expected_dagruns_count assert dag_runs[0].external_trigger - def test_trigger_dagrun_with_wait_for_completion_true(self): + @pytest.mark.skip_if_database_isolation_mode # Known to be broken in db isolation mode + def test_trigger_dagrun_with_wait_for_completion_true(self, dag_maker): """Test TriggerDagRunOperator with wait_for_completion.""" logical_date = DEFAULT_DATE - task = TriggerDagRunOperator( - task_id="test_task", - trigger_dag_id=TRIGGERED_DAG_ID, - logical_date=logical_date, - wait_for_completion=True, - poke_interval=10, - allowed_states=[State.QUEUED], - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_task", + trigger_dag_id=TRIGGERED_DAG_ID, + logical_date=logical_date, + wait_for_completion=True, + poke_interval=10, + allowed_states=[State.QUEUED], + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() task.run(start_date=logical_date, end_date=logical_date) with create_session() as session: dagruns = session.query(DagRun).filter(DagRun.dag_id == TRIGGERED_DAG_ID).all() assert len(dagruns) == 1 - def test_trigger_dagrun_with_wait_for_completion_true_fail(self): + @pytest.mark.skip_if_database_isolation_mode # Known to be broken in db isolation mode + def test_trigger_dagrun_with_wait_for_completion_true_fail(self, dag_maker): """Test TriggerDagRunOperator with wait_for_completion but triggered dag fails.""" logical_date = DEFAULT_DATE - task = TriggerDagRunOperator( - task_id="test_task", - trigger_dag_id=TRIGGERED_DAG_ID, - logical_date=logical_date, - wait_for_completion=True, - poke_interval=10, - failed_states=[State.QUEUED], - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_task", + trigger_dag_id=TRIGGERED_DAG_ID, + logical_date=logical_date, + wait_for_completion=True, + poke_interval=10, + failed_states=[State.QUEUED], + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() with pytest.raises(AirflowException): task.run(start_date=logical_date, end_date=logical_date) - def test_trigger_dagrun_triggering_itself(self): + def test_trigger_dagrun_triggering_itself(self, dag_maker): """Test TriggerDagRunOperator that triggers itself""" logical_date = DEFAULT_DATE - task = TriggerDagRunOperator( - task_id="test_task", - trigger_dag_id=self.dag.dag_id, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_task", + trigger_dag_id=TEST_DAG_ID, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() task.run(start_date=logical_date, end_date=logical_date) - with create_session() as session: - dagruns = ( - session.query(DagRun) - .filter(DagRun.dag_id == self.dag.dag_id) - .order_by(DagRun.execution_date) - .all() - ) - assert len(dagruns) == 2 - triggered_dag_run = dagruns[1] - assert triggered_dag_run.state == State.QUEUED - self.assert_extra_link(triggered_dag_run, task, session) + dagruns = ( + dag_maker.session.query(DagRun) + .filter(DagRun.dag_id == TEST_DAG_ID) + .order_by(DagRun.execution_date) + .all() + ) + assert len(dagruns) == 2 + triggered_dag_run = dagruns[1] + assert triggered_dag_run.state == State.QUEUED - def test_trigger_dagrun_triggering_itself_with_logical_date(self): + def test_trigger_dagrun_triggering_itself_with_logical_date(self, dag_maker): """Test TriggerDagRunOperator that triggers itself with logical date, fails with DagRunAlreadyExists""" logical_date = DEFAULT_DATE - task = TriggerDagRunOperator( - task_id="test_task", - trigger_dag_id=self.dag.dag_id, - logical_date=logical_date, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_task", + trigger_dag_id=TEST_DAG_ID, + logical_date=logical_date, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() with pytest.raises(DagRunAlreadyExists): task.run(start_date=logical_date, end_date=logical_date) - def test_trigger_dagrun_with_wait_for_completion_true_defer_false(self): + @pytest.mark.skip_if_database_isolation_mode # Known to be broken in db isolation mode + def test_trigger_dagrun_with_wait_for_completion_true_defer_false(self, dag_maker): """Test TriggerDagRunOperator with wait_for_completion.""" logical_date = DEFAULT_DATE - task = TriggerDagRunOperator( - task_id="test_task", - trigger_dag_id=TRIGGERED_DAG_ID, - logical_date=logical_date, - wait_for_completion=True, - poke_interval=10, - allowed_states=[State.QUEUED], - deferrable=False, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_task", + trigger_dag_id=TRIGGERED_DAG_ID, + logical_date=logical_date, + wait_for_completion=True, + poke_interval=10, + allowed_states=[State.QUEUED], + deferrable=False, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() task.run(start_date=logical_date, end_date=logical_date) with create_session() as session: dagruns = session.query(DagRun).filter(DagRun.dag_id == TRIGGERED_DAG_ID).all() assert len(dagruns) == 1 - def test_trigger_dagrun_with_wait_for_completion_true_defer_true(self): + @pytest.mark.skip_if_database_isolation_mode # Known to be broken in db isolation mode + def test_trigger_dagrun_with_wait_for_completion_true_defer_true(self, dag_maker): """Test TriggerDagRunOperator with wait_for_completion.""" logical_date = DEFAULT_DATE - task = TriggerDagRunOperator( - task_id="test_task", - trigger_dag_id=TRIGGERED_DAG_ID, - logical_date=logical_date, - wait_for_completion=True, - poke_interval=10, - allowed_states=[State.QUEUED], - deferrable=True, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_task", + trigger_dag_id=TRIGGERED_DAG_ID, + logical_date=logical_date, + wait_for_completion=True, + poke_interval=10, + allowed_states=[State.QUEUED], + deferrable=True, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() task.run(start_date=logical_date, end_date=logical_date) @@ -485,19 +570,24 @@ def test_trigger_dagrun_with_wait_for_completion_true_defer_true(self): task.execute_complete(context={}, event=trigger.serialize()) - def test_trigger_dagrun_with_wait_for_completion_true_defer_true_failure(self): + @pytest.mark.skip_if_database_isolation_mode # Known to be broken in db isolation mode + def test_trigger_dagrun_with_wait_for_completion_true_defer_true_failure(self, dag_maker): """Test TriggerDagRunOperator wait_for_completion dag run in non defined state.""" logical_date = DEFAULT_DATE - task = TriggerDagRunOperator( - task_id="test_task", - trigger_dag_id=TRIGGERED_DAG_ID, - logical_date=logical_date, - wait_for_completion=True, - poke_interval=10, - allowed_states=[State.SUCCESS], - deferrable=True, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_task", + trigger_dag_id=TRIGGERED_DAG_ID, + logical_date=logical_date, + wait_for_completion=True, + poke_interval=10, + allowed_states=[State.SUCCESS], + deferrable=True, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() task.run(start_date=logical_date, end_date=logical_date) @@ -517,20 +607,25 @@ def test_trigger_dagrun_with_wait_for_completion_true_defer_true_failure(self): event=trigger.serialize(), ) - def test_trigger_dagrun_with_wait_for_completion_true_defer_true_failure_2(self): + @pytest.mark.skip_if_database_isolation_mode # Known to be broken in db isolation mode + def test_trigger_dagrun_with_wait_for_completion_true_defer_true_failure_2(self, dag_maker): """Test TriggerDagRunOperator wait_for_completion dag run in failed state.""" logical_date = DEFAULT_DATE - task = TriggerDagRunOperator( - task_id="test_task", - trigger_dag_id=TRIGGERED_DAG_ID, - logical_date=logical_date, - wait_for_completion=True, - poke_interval=10, - allowed_states=[State.SUCCESS], - failed_states=[State.QUEUED], - deferrable=True, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_task", + trigger_dag_id=TRIGGERED_DAG_ID, + logical_date=logical_date, + wait_for_completion=True, + poke_interval=10, + allowed_states=[State.SUCCESS], + failed_states=[State.QUEUED], + deferrable=True, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() task.run(start_date=logical_date, end_date=logical_date) @@ -548,19 +643,23 @@ def test_trigger_dagrun_with_wait_for_completion_true_defer_true_failure_2(self) with pytest.raises(AirflowException, match="failed with failed state"): task.execute_complete(context={}, event=trigger.serialize()) - def test_trigger_dagrun_with_execution_date(self): + def test_trigger_dagrun_with_execution_date(self, dag_maker): """Test TriggerDagRunOperator with custom execution_date (deprecated parameter)""" custom_execution_date = timezone.datetime(2021, 1, 2, 3, 4, 5) - with pytest.warns( - RemovedInAirflow3Warning, - match="Parameter 'execution_date' is deprecated. Use 'logical_date' instead.", - ): - task = TriggerDagRunOperator( - task_id="test_trigger_dagrun_with_execution_date", - trigger_dag_id=TRIGGERED_DAG_ID, - execution_date=custom_execution_date, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + with pytest.warns( + RemovedInAirflow3Warning, + match="Parameter 'execution_date' is deprecated. Use 'logical_date' instead.", + ): + task = TriggerDagRunOperator( + task_id="test_trigger_dagrun_with_execution_date", + trigger_dag_id=TRIGGERED_DAG_ID, + execution_date=custom_execution_date, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) with create_session() as session: @@ -570,6 +669,7 @@ def test_trigger_dagrun_with_execution_date(self): assert dagrun.run_id == DagRun.generate_run_id(DagRunType.MANUAL, custom_execution_date) self.assert_extra_link(dagrun, task, session) + @pytest.mark.skip_if_database_isolation_mode # Known to be broken in db isolation mode @pytest.mark.parametrize( argnames=["trigger_logical_date"], argvalues=[ @@ -577,18 +677,22 @@ def test_trigger_dagrun_with_execution_date(self): pytest.param(None, id="logical_date=None"), ], ) - def test_dagstatetrigger_execution_dates(self, trigger_logical_date): + def test_dagstatetrigger_execution_dates(self, trigger_logical_date, dag_maker): """Ensure that the DagStateTrigger is called with the triggered DAG's logical date.""" - task = TriggerDagRunOperator( - task_id="test_task", - trigger_dag_id=TRIGGERED_DAG_ID, - logical_date=trigger_logical_date, - wait_for_completion=True, - poke_interval=5, - allowed_states=[DagRunState.QUEUED], - deferrable=True, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_task", + trigger_dag_id=TRIGGERED_DAG_ID, + logical_date=trigger_logical_date, + wait_for_completion=True, + poke_interval=5, + allowed_states=[DagRunState.QUEUED], + deferrable=True, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() mock_task_defer = mock.MagicMock(side_effect=task.defer) with mock.patch.object(TriggerDagRunOperator, "defer", mock_task_defer), pytest.raises(TaskDeferred): @@ -602,19 +706,24 @@ def test_dagstatetrigger_execution_dates(self, trigger_logical_date): pendulum.instance(dagruns[0].logical_date) ] - def test_dagstatetrigger_execution_dates_with_clear_and_reset(self): + @pytest.mark.skip_if_database_isolation_mode # Known to be broken in db isolation mode + def test_dagstatetrigger_execution_dates_with_clear_and_reset(self, dag_maker): """Check DagStateTrigger is called with the triggered DAG's logical date on subsequent defers.""" - task = TriggerDagRunOperator( - task_id="test_task", - trigger_dag_id=TRIGGERED_DAG_ID, - trigger_run_id="custom_run_id", - wait_for_completion=True, - poke_interval=5, - allowed_states=[DagRunState.QUEUED], - deferrable=True, - reset_dag_run=True, - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_task", + trigger_dag_id=TRIGGERED_DAG_ID, + trigger_run_id="custom_run_id", + wait_for_completion=True, + poke_interval=5, + allowed_states=[DagRunState.QUEUED], + deferrable=True, + reset_dag_run=True, + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() mock_task_defer = mock.MagicMock(side_effect=task.defer) with mock.patch.object(TriggerDagRunOperator, "defer", mock_task_defer), pytest.raises(TaskDeferred): @@ -647,16 +756,20 @@ def test_dagstatetrigger_execution_dates_with_clear_and_reset(self): pendulum.instance(triggered_logical_date) ] - def test_trigger_dagrun_with_no_failed_state(self): + def test_trigger_dagrun_with_no_failed_state(self, dag_maker): logical_date = DEFAULT_DATE - task = TriggerDagRunOperator( - task_id="test_task", - trigger_dag_id=TRIGGERED_DAG_ID, - logical_date=logical_date, - wait_for_completion=True, - poke_interval=10, - failed_states=[], - dag=self.dag, - ) + with dag_maker( + TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True + ) as dag: + task = TriggerDagRunOperator( + task_id="test_task", + trigger_dag_id=TRIGGERED_DAG_ID, + logical_date=logical_date, + wait_for_completion=True, + poke_interval=10, + failed_states=[], + ) + self.re_sync_triggered_dag_to_db(dag, dag_maker) + dag_maker.create_dagrun() assert task.failed_states == [] From 2740c1c16a4d48ee371801bce811f0397547dd8f Mon Sep 17 00:00:00 2001 From: Jens Scheffler <95105677+jscheffl@users.noreply.github.com> Date: Sat, 10 Aug 2024 16:21:13 +0200 Subject: [PATCH 011/161] Fix core tests from start to SkipMixin for Database Isolation Mode (#41369) * Skip core tests from start to SkipMixin for Database Isolation Mode * Skip core tests from start to SkipMixin for Database Isolation Mode, uups * Skip core tests from start to SkipMixin for Database Isolation Mode, uups (cherry picked from commit b87f9878ff801575b35a30220c0b2e285a2cf1b8) --- tests/jobs/test_base_job.py | 1 + tests/jobs/test_local_task_job.py | 25 ++++++++++++++++++++++--- tests/jobs/test_triggerer_job.py | 12 ++++++++++++ tests/models/test_baseoperator.py | 2 ++ tests/models/test_baseoperatormeta.py | 6 ++++++ tests/models/test_dagbag.py | 11 +++++++++++ tests/models/test_mappedoperator.py | 25 +++++++++++++++++++++++++ tests/models/test_param.py | 3 +++ tests/models/test_renderedtifields.py | 8 +++++++- tests/models/test_serialized_dag.py | 9 +++++++++ tests/models/test_skipmixin.py | 9 ++++++--- 11 files changed, 104 insertions(+), 7 deletions(-) diff --git a/tests/jobs/test_base_job.py b/tests/jobs/test_base_job.py index e956d12889ca1..e9c9fe94ce737 100644 --- a/tests/jobs/test_base_job.py +++ b/tests/jobs/test_base_job.py @@ -267,6 +267,7 @@ def test_essential_attr(self, mock_getuser, mock_hostname, mock_init_executors, assert test_job.executor == mock_sequential_executor assert test_job.executors == [mock_sequential_executor] + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_heartbeat(self, frozen_sleep, monkeypatch): monkeypatch.setattr("airflow.jobs.job.sleep", frozen_sleep) with create_session() as session: diff --git a/tests/jobs/test_local_task_job.py b/tests/jobs/test_local_task_job.py index aefb77997e517..a4e7c4f387752 100644 --- a/tests/jobs/test_local_task_job.py +++ b/tests/jobs/test_local_task_job.py @@ -109,7 +109,7 @@ def test_localtaskjob_essential_attr(self, dag_maker): of LocalTaskJob can be assigned with proper values without intervention """ - with dag_maker("test_localtaskjob_essential_attr"): + with dag_maker("test_localtaskjob_essential_attr", serialized=True): op1 = EmptyOperator(task_id="op1") dr = dag_maker.create_dagrun() @@ -127,6 +127,7 @@ def test_localtaskjob_essential_attr(self, dag_maker): check_result_2 = [getattr(job1, attr) is not None for attr in essential_attr] assert all(check_result_2) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_localtaskjob_heartbeat(self, dag_maker): session = settings.Session() with dag_maker("test_localtaskjob_heartbeat"): @@ -173,6 +174,7 @@ def test_localtaskjob_heartbeat(self, dag_maker): assert not job1.task_runner.run_as_user job_runner.heartbeat_callback() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @mock.patch("subprocess.check_call") @mock.patch("airflow.jobs.local_task_job_runner.psutil") def test_localtaskjob_heartbeat_with_run_as_user(self, psutil_mock, _, dag_maker): @@ -227,6 +229,7 @@ def test_localtaskjob_heartbeat_with_run_as_user(self, psutil_mock, _, dag_maker assert ti.pid != job1.task_runner.process.pid job_runner.heartbeat_callback() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @conf_vars({("core", "default_impersonation"): "testuser"}) @mock.patch("subprocess.check_call") @mock.patch("airflow.jobs.local_task_job_runner.psutil") @@ -282,6 +285,7 @@ def test_localtaskjob_heartbeat_with_default_impersonation(self, psutil_mock, _, assert ti.pid != job1.task_runner.process.pid job_runner.heartbeat_callback() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_heartbeat_failed_fast(self): """ Test that task heartbeat will sleep when it fails fast @@ -323,6 +327,7 @@ def test_heartbeat_failed_fast(self): delta = (time2 - time1).total_seconds() assert abs(delta - job.heartrate) < 0.8 + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @conf_vars({("core", "task_success_overtime"): "1"}) def test_mark_success_no_kill(self, caplog, get_test_dag, session): """ @@ -354,6 +359,7 @@ def test_mark_success_no_kill(self, caplog, get_test_dag, session): "State of this instance has been externally set to success. Terminating instance." in caplog.text ) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_localtaskjob_double_trigger(self): dag = self.dagbag.dags.get("test_localtaskjob_double_trigger") task = dag.get_task("test_localtaskjob_double_trigger_task") @@ -392,6 +398,7 @@ def test_localtaskjob_double_trigger(self): session.close() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @patch.object(StandardTaskRunner, "return_code") @mock.patch("airflow.jobs.scheduler_job_runner.Stats.incr", autospec=True) def test_local_task_return_code_metric(self, mock_stats_incr, mock_return_code, create_dummy_dag): @@ -424,6 +431,7 @@ def test_local_task_return_code_metric(self, mock_stats_incr, mock_return_code, ] ) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @patch.object(StandardTaskRunner, "return_code") def test_localtaskjob_maintain_heart_rate(self, mock_return_code, caplog, create_dummy_dag): dag, task = create_dummy_dag("test_localtaskjob_double_trigger") @@ -456,6 +464,7 @@ def test_localtaskjob_maintain_heart_rate(self, mock_return_code, caplog, create assert time_end - time_start < job1.heartrate assert "Task exited with return code 0" in caplog.text + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_mark_failure_on_failure_callback(self, caplog, get_test_dag): """ Test that ensures that mark_failure in the UI fails @@ -488,6 +497,7 @@ def test_mark_failure_on_failure_callback(self, caplog, get_test_dag): "State of this instance has been externally set to failed. Terminating instance." ) in caplog.text + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_dagrun_timeout_logged_in_task_logs(self, caplog, get_test_dag): """ Test that ensures that if a running task is externally skipped (due to a dagrun timeout) @@ -520,6 +530,7 @@ def test_dagrun_timeout_logged_in_task_logs(self, caplog, get_test_dag): assert ti.state == State.SKIPPED assert "DagRun timed out after " in caplog.text + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_failure_callback_called_by_airflow_run_raw_process(self, monkeypatch, tmp_path, get_test_dag): """ Ensure failure callback of a task is run by the airflow run --raw process @@ -555,6 +566,7 @@ def test_failure_callback_called_by_airflow_run_raw_process(self, monkeypatch, t assert m, "pid expected in output." assert os.getpid() != int(m.group(1)) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @conf_vars({("core", "task_success_overtime"): "5"}) def test_mark_success_on_success_callback(self, caplog, get_test_dag): """ @@ -586,6 +598,7 @@ def test_mark_success_on_success_callback(self, caplog, get_test_dag): "State of this instance has been externally set to success. Terminating instance." in caplog.text ) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_success_listeners_executed(self, caplog, get_test_dag): """ Test that ensures that when listeners are executed, the task is not killed before they finish @@ -623,6 +636,7 @@ def test_success_listeners_executed(self, caplog, get_test_dag): ) lm.clear() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @conf_vars({("core", "task_success_overtime"): "3"}) def test_success_slow_listeners_executed_kill(self, caplog, get_test_dag): """ @@ -659,6 +673,7 @@ def test_success_slow_listeners_executed_kill(self, caplog, get_test_dag): ) lm.clear() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @conf_vars({("core", "task_success_overtime"): "3"}) def test_success_slow_task_not_killed_by_overtime_but_regular_timeout(self, caplog, get_test_dag): """ @@ -698,6 +713,7 @@ def test_success_slow_task_not_killed_by_overtime_but_regular_timeout(self, capl ) lm.clear() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("signal_type", [signal.SIGTERM, signal.SIGKILL]) def test_process_os_signal_calls_on_failure_callback( self, monkeypatch, tmp_path, get_test_dag, signal_type @@ -792,6 +808,7 @@ def send_signal(ti, signal_sent, sig): lines = f.readlines() assert len(lines) == 0 + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "conf, init_state, first_run_state, second_run_state, task_ids_to_run, error_message", [ @@ -876,6 +893,7 @@ def test_fast_follow( if scheduler_job_runner.processor_agent: scheduler_job_runner.processor_agent.end() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @conf_vars({("scheduler", "schedule_after_task_execution"): "True"}) def test_mini_scheduler_works_with_wait_for_upstream(self, caplog, get_test_dag): dag = get_test_dag("test_dagrun_fast_follow") @@ -944,7 +962,7 @@ def task_function(ti): os.kill(psutil.Process(os.getpid()).ppid(), signal.SIGSEGV) - with dag_maker(dag_id="test_segmentation_fault"): + with dag_maker(dag_id="test_segmentation_fault", serialized=True): task = PythonOperator( task_id="test_sigsegv", python_callable=task_function, @@ -975,7 +993,7 @@ def test_number_of_queries_single_loop(mock_get_task_runner, dag_maker): mock_get_task_runner.return_value.return_code.side_effects = [[0], codes] unique_prefix = str(uuid.uuid4()) - with dag_maker(dag_id=f"{unique_prefix}_test_number_of_queries"): + with dag_maker(dag_id=f"{unique_prefix}_test_number_of_queries", serialized=True): task = EmptyOperator(task_id="test_state_succeeded1") dr = dag_maker.create_dagrun(run_id=unique_prefix, state=State.NONE) @@ -992,6 +1010,7 @@ def test_number_of_queries_single_loop(mock_get_task_runner, dag_maker): class TestSigtermOnRunner: """Test receive SIGTERM on Task Runner.""" + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "daemon", [pytest.param(True, id="daemon"), pytest.param(False, id="non-daemon")] ) diff --git a/tests/jobs/test_triggerer_job.py b/tests/jobs/test_triggerer_job.py index 10d4196ac97e3..28fc00694b400 100644 --- a/tests/jobs/test_triggerer_job.py +++ b/tests/jobs/test_triggerer_job.py @@ -113,6 +113,7 @@ def create_trigger_in_db(session, trigger, operator=None): return dag_model, run, trigger_orm, task_instance +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_trigger_logging_sensitive_info(session, caplog): """ Checks that when a trigger fires, it doesn't log any sensitive @@ -176,6 +177,7 @@ def test_is_alive(): assert not triggerer_job.is_alive(), "Completed jobs even with recent heartbeat should not be alive" +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_is_needed(session): """Checks the triggerer-is-needed logic""" # No triggers, no need @@ -219,6 +221,7 @@ def test_capacity_decode(): TriggererJobRunner(job=job, capacity=input_str) +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_trigger_lifecycle(session): """ Checks that the triggerer will correctly see a new Trigger in the database @@ -309,6 +312,7 @@ def test_update_trigger_with_triggerer_argument_change( assert "got an unexpected keyword argument 'not_exists_arg'" in caplog.text +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.asyncio async def test_trigger_create_race_condition_38599(session, tmp_path): """ @@ -389,6 +393,7 @@ async def test_trigger_create_race_condition_38599(session, tmp_path): assert path.read_text() == "hi\n" +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_trigger_create_race_condition_18392(session, tmp_path): """ This verifies the resolution of race condition documented in github issue #18392. @@ -499,6 +504,7 @@ def handle_events(self): assert len(instances) == 1 +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_trigger_from_dead_triggerer(session, create_task_instance): """ Checks that the triggerer will correctly claim a Trigger that is assigned to a @@ -526,6 +532,7 @@ def test_trigger_from_dead_triggerer(session, create_task_instance): assert [x for x, y in job_runner.trigger_runner.to_create] == [1] +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_trigger_from_expired_triggerer(session, create_task_instance): """ Checks that the triggerer will correctly claim a Trigger that is assigned to a @@ -560,6 +567,7 @@ def test_trigger_from_expired_triggerer(session, create_task_instance): assert [x for x, y in job_runner.trigger_runner.to_create] == [1] +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_trigger_runner_exception_stops_triggerer(session): """ Checks that if an exception occurs when creating triggers, that the triggerer @@ -603,6 +611,7 @@ async def create_triggers(self): thread.join() +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_trigger_firing(session): """ Checks that when a trigger fires, it correctly makes it into the @@ -633,6 +642,7 @@ def test_trigger_firing(session): job_runner.trigger_runner.join(30) +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_trigger_failing(session): """ Checks that when a trigger fails, it correctly makes it into the @@ -667,6 +677,7 @@ def test_trigger_failing(session): job_runner.trigger_runner.join(30) +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_trigger_cleanup(session): """ Checks that the triggerer will correctly clean up triggers that do not @@ -686,6 +697,7 @@ def test_trigger_cleanup(session): assert session.query(Trigger).count() == 0 +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_invalid_trigger(session, dag_maker): """ Checks that the triggerer will correctly fail task instances that depend on diff --git a/tests/models/test_baseoperator.py b/tests/models/test_baseoperator.py index b94a1b9f819d6..89b268af1f8b1 100644 --- a/tests/models/test_baseoperator.py +++ b/tests/models/test_baseoperator.py @@ -1037,6 +1037,7 @@ def get_states(dr): return dict(ti_dict) +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.db_test def test_teardown_and_fail_stop(dag_maker): """ @@ -1082,6 +1083,7 @@ def my_teardown(): assert states == expected +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.db_test def test_get_task_instances(session): import pendulum diff --git a/tests/models/test_baseoperatormeta.py b/tests/models/test_baseoperatormeta.py index 7c719189aadd9..6c6567b23899e 100644 --- a/tests/models/test_baseoperatormeta.py +++ b/tests/models/test_baseoperatormeta.py @@ -47,6 +47,7 @@ def setup_method(self): def teardown_method(self, method): ExecutorSafeguard.test_mode = conf.getboolean("core", "unit_test_mode") + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.db_test def test_executor_when_classic_operator_called_from_dag(self, dag_maker): with dag_maker() as dag: @@ -55,6 +56,7 @@ def test_executor_when_classic_operator_called_from_dag(self, dag_maker): dag_run = dag.test() assert dag_run.state == DagRunState.SUCCESS + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "state, exception, retries", [ @@ -101,6 +103,7 @@ def _raise_if_exception(): assert ti.next_kwargs is None assert ti.state == state + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.db_test def test_executor_when_classic_operator_called_from_decorated_task_with_allow_nested_operators_false( self, dag_maker @@ -117,6 +120,7 @@ def say_hello(**context): dag_run = dag.test() assert dag_run.state == DagRunState.FAILED + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.db_test @patch.object(HelloWorldOperator, "log") def test_executor_when_classic_operator_called_from_decorated_task_without_allow_nested_operators( @@ -139,6 +143,7 @@ def say_hello(**context): "HelloWorldOperator.execute cannot be called outside TaskInstance!" ) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.db_test def test_executor_when_classic_operator_called_from_python_operator_with_allow_nested_operators_false( self, @@ -159,6 +164,7 @@ def say_hello(**context): dag_run = dag.test() assert dag_run.state == DagRunState.FAILED + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.db_test @patch.object(HelloWorldOperator, "log") def test_executor_when_classic_operator_called_from_python_operator_without_allow_nested_operators( diff --git a/tests/models/test_dagbag.py b/tests/models/test_dagbag.py index 836ede04df4fe..936852dd082e4 100644 --- a/tests/models/test_dagbag.py +++ b/tests/models/test_dagbag.py @@ -94,6 +94,7 @@ def test_get_non_existing_dag(self, tmp_path): non_existing_dag_id = "non_existing_dag_id" assert dagbag.get_dag(non_existing_dag_id) is None + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_serialized_dag_not_existing_doesnt_raise(self, tmp_path): """ test that retrieving a non existing dag id returns None without crashing @@ -459,6 +460,7 @@ def process_file(self, filepath, only_if_updated=True, safe_mode=True): assert dag_id == dag.dag_id assert 2 == dagbag.process_file_calls + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_dag_removed_if_serialized_dag_is_removed(self, dag_maker, tmp_path): """ Test that if a DAG does not exist in serialized_dag table (as the DAG file was removed), @@ -789,6 +791,7 @@ def test_process_file_with_none(self, tmp_path): assert [] == dagbag.process_file(None) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_deactivate_unknown_dags(self): """ Test that dag_ids not passed into deactivate_unknown_dags @@ -812,6 +815,7 @@ def test_deactivate_unknown_dags(self): with create_session() as session: session.query(DagModel).filter(DagModel.dag_id == "test_deactivate_unknown_dags").delete() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_serialized_dags_are_written_to_db_on_sync(self): """ Test that when dagbag.sync_to_db is called the DAGs are Serialized and written to DB @@ -832,6 +836,7 @@ def test_serialized_dags_are_written_to_db_on_sync(self): new_serialized_dags_count = session.query(func.count(SerializedDagModel.dag_id)).scalar() assert new_serialized_dags_count == 1 + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @patch("airflow.models.serialized_dag.SerializedDagModel.write_dag") def test_serialized_dag_errors_are_import_errors(self, mock_serialize, caplog): """ @@ -899,6 +904,7 @@ def test_sync_to_db_is_retried(self, mock_bulk_write_to_db, mock_s10n_write_dag, ] ) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @patch("airflow.models.dagbag.settings.MIN_SERIALIZED_DAG_UPDATE_INTERVAL", 5) @patch("airflow.models.dagbag.DagBag._sync_perm_for_dag") def test_sync_to_db_syncs_dag_specific_perms_on_update(self, mock_sync_perm_for_dag): @@ -932,6 +938,7 @@ def _sync_to_db(): _sync_to_db() mock_sync_perm_for_dag.assert_called_once_with(dag, session=session) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @patch("airflow.www.security_appless.ApplessAirflowSecurityManager") def test_sync_perm_for_dag(self, mock_security_manager): """ @@ -968,6 +975,7 @@ def _sync_perms(): "test_example_bash_operator", {"Public": {"DAGs": {"can_read"}}} ) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @patch("airflow.www.security_appless.ApplessAirflowSecurityManager") def test_sync_perm_for_dag_with_dict_access_control(self, mock_security_manager): """ @@ -1004,6 +1012,7 @@ def _sync_perms(): "test_example_bash_operator", {"Public": {"DAGs": {"can_read"}, "DAG Runs": {"can_create"}}} ) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @patch("airflow.models.dagbag.settings.MIN_SERIALIZED_DAG_UPDATE_INTERVAL", 5) @patch("airflow.models.dagbag.settings.MIN_SERIALIZED_DAG_FETCH_INTERVAL", 5) def test_get_dag_with_dag_serialization(self): @@ -1043,6 +1052,7 @@ def test_get_dag_with_dag_serialization(self): assert set(updated_ser_dag_1.tags) == {"example", "example2", "new_tag"} assert updated_ser_dag_1_update_time > ser_dag_1_update_time + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @patch("airflow.models.dagbag.settings.MIN_SERIALIZED_DAG_UPDATE_INTERVAL", 5) @patch("airflow.models.dagbag.settings.MIN_SERIALIZED_DAG_FETCH_INTERVAL", 5) def test_get_dag_refresh_race_condition(self): @@ -1091,6 +1101,7 @@ def test_get_dag_refresh_race_condition(self): assert set(updated_ser_dag.tags) == {"example", "example2", "new_tag"} assert updated_ser_dag_update_time > ser_dag_update_time + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_collect_dags_from_db(self): """DAGs are collected from Database""" db.clear_db_dags() diff --git a/tests/models/test_mappedoperator.py b/tests/models/test_mappedoperator.py index 9f31652424aeb..2ee597879064c 100644 --- a/tests/models/test_mappedoperator.py +++ b/tests/models/test_mappedoperator.py @@ -70,6 +70,7 @@ def test_task_mapping_with_dag(): assert mapped.downstream_list == [finish] +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @patch("airflow.models.abstractoperator.AbstractOperator.render_template") def test_task_mapping_with_dag_and_list_of_pandas_dataframe(mock_render_template, caplog): class UnrenderableClass: @@ -159,6 +160,7 @@ def test_map_xcom_arg(): assert task1.downstream_list == [mapped] +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_map_xcom_arg_multiple_upstream_xcoms(dag_maker, session): """Test that the correct number of downstream tasks are generated when mapping with an XComArg""" @@ -218,6 +220,7 @@ def test_partial_on_class_invalid_ctor_args() -> None: MockOperator.partial(task_id="a", foo="bar", bar=2) +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( ["num_existing_tis", "expected"], ( @@ -285,6 +288,7 @@ def test_expand_mapped_task_instance(dag_maker, session, num_existing_tis, expec assert indices == expected +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_expand_mapped_task_failed_state_in_db(dag_maker, session): """ This test tries to recreate a faulty state in the database and checks if we can recover from it. @@ -336,6 +340,7 @@ def test_expand_mapped_task_failed_state_in_db(dag_maker, session): assert indices == [(0, "success"), (1, "success")] +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_expand_mapped_task_instance_skipped_on_zero(dag_maker, session): with dag_maker(session=session): task1 = BaseOperator(task_id="op1") @@ -401,6 +406,7 @@ def test_mapped_expand_against_params(dag_maker, dag_params, task_params, expect assert t.expand_input.value == {"params": [{"c": "x"}, {"d": 1}]} +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_mapped_render_template_fields_validating_operator(dag_maker, session, tmp_path): file_template_dir = tmp_path / "path" / "to" file_template_dir.mkdir(parents=True, exist_ok=True) @@ -466,6 +472,7 @@ def execute(self, context): assert mapped_ti.task.file_template == "loaded data", "Should be templated!" +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_mapped_expand_kwargs_render_template_fields_validating_operator(dag_maker, session, tmp_path): file_template_dir = tmp_path / "path" / "to" file_template_dir.mkdir(parents=True, exist_ok=True) @@ -515,6 +522,7 @@ def execute(self, context): assert mapped_ti.task.file_template == "loaded data", "Should be templated!" +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_mapped_render_nested_template_fields(dag_maker, session): with dag_maker(session=session): MockOperatorWithNestedFields.partial( @@ -539,6 +547,7 @@ def test_mapped_render_nested_template_fields(dag_maker, session): assert ti.task.arg2.field_2 == "value_2" +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( ["num_existing_tis", "expected"], ( @@ -658,6 +667,7 @@ def task1(map_name): return task1.expand(map_name=map_names) +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "template, expected_rendered_names", [ @@ -706,6 +716,7 @@ def test_expand_mapped_task_instance_with_named_index( assert indices == expected_rendered_names +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "map_index, expected", [ @@ -872,6 +883,7 @@ def inner(*args, **kwargs): else: return PythonOperator(**kwargs) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("type_", ["taskflow", "classic"]) def test_one_to_many_work_failed(self, type_, dag_maker): """ @@ -922,6 +934,7 @@ def my_work(val): } assert states == expected + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("type_", ["taskflow", "classic"]) def test_many_one_explicit_odd_setup_mapped_setups_fail(self, type_, dag_maker): """ @@ -1008,6 +1021,7 @@ def my_work(val): } assert states == expected + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("type_", ["taskflow", "classic"]) def test_many_one_explicit_odd_setup_all_setups_fail(self, type_, dag_maker): """ @@ -1105,6 +1119,7 @@ def my_work(val): } assert states == expected + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("type_", ["taskflow", "classic"]) def test_many_one_explicit_odd_setup_one_mapped_fails(self, type_, dag_maker): """ @@ -1217,6 +1232,7 @@ def my_teardown_callable(val): } assert states == expected + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("type_", ["taskflow", "classic"]) def test_one_to_many_as_teardown(self, type_, dag_maker): """ @@ -1272,6 +1288,7 @@ def my_work(val): } assert states == expected + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("type_", ["taskflow", "classic"]) def test_one_to_many_as_teardown_on_failure_fail_dagrun(self, type_, dag_maker): """ @@ -1336,6 +1353,7 @@ def my_teardown_callable(val): } assert states == expected + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("type_", ["taskflow", "classic"]) def test_mapped_task_group_simple(self, type_, dag_maker, session): """ @@ -1410,6 +1428,7 @@ def file_transforms(filename): assert states == expected + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("type_", ["taskflow", "classic"]) def test_mapped_task_group_work_fail_or_skip(self, type_, dag_maker): """ @@ -1481,6 +1500,7 @@ def file_transforms(filename): } assert states == expected + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("type_", ["taskflow", "classic"]) def test_teardown_many_one_explicit(self, type_, dag_maker): """-- passing @@ -1541,6 +1561,7 @@ def my_work(val): } assert states == expected + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_one_to_many_with_teardown_and_fail_stop(self, dag_maker): """ With fail_stop enabled, the teardown for an already-completed setup @@ -1577,6 +1598,7 @@ def my_teardown(val): } assert states == expected + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_one_to_many_with_teardown_and_fail_stop_more_tasks(self, dag_maker): """ when fail_stop enabled, teardowns should run according to their setups. @@ -1619,6 +1641,7 @@ def my_teardown(val): } assert states == expected + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_one_to_many_with_teardown_and_fail_stop_more_tasks_mapped_setup(self, dag_maker): """ when fail_stop enabled, teardowns should run according to their setups. @@ -1668,6 +1691,7 @@ def my_teardown(val): } assert states == expected + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_skip_one_mapped_task_from_task_group_with_generator(self, dag_maker): with dag_maker() as dag: @@ -1699,6 +1723,7 @@ def group(n: int) -> None: } assert states == expected + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_skip_one_mapped_task_from_task_group(self, dag_maker): with dag_maker() as dag: diff --git a/tests/models/test_param.py b/tests/models/test_param.py index 89421f0dc2620..18d4c190ad28a 100644 --- a/tests/models/test_param.py +++ b/tests/models/test_param.py @@ -323,6 +323,7 @@ def setup_class(self): def teardown_method(self): self.clean_db() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.db_test def test_dag_param_resolves(self, dag_maker): """Test dagparam resolves on operator execution""" @@ -345,6 +346,7 @@ def return_num(num): ti = dr.get_task_instances()[0] assert ti.xcom_pull() == self.VALUE + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.db_test def test_dag_param_overwrite(self, dag_maker): """Test dag param is overwritten from dagrun config""" @@ -370,6 +372,7 @@ def return_num(num): ti = dr.get_task_instances()[0] assert ti.xcom_pull() == new_value + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.db_test def test_dag_param_default(self, dag_maker): """Test dag param is retrieved from default config""" diff --git a/tests/models/test_renderedtifields.py b/tests/models/test_renderedtifields.py index 3e631a0ba9d49..b8c45193814aa 100644 --- a/tests/models/test_renderedtifields.py +++ b/tests/models/test_renderedtifields.py @@ -90,6 +90,7 @@ def setup_method(self): def teardown_method(self): self.clean_db() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "templated_field, expected_rendered_field", [ @@ -169,6 +170,7 @@ def test_get_templated_fields(self, templated_field, expected_rendered_field, da # Fetching them will return None assert RTIF.get_templated_fields(ti=ti2) is None + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.enable_redact def test_secrets_are_masked_when_large_string(self, dag_maker): """ @@ -186,6 +188,7 @@ def test_secrets_are_masked_when_large_string(self, dag_maker): rtif = RTIF(ti=ti) assert "***" in rtif.rendered_fields.get("bash_command") + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @mock.patch("airflow.models.BaseOperator.render_template") def test_pandas_dataframes_works_with_the_string_compare(self, render_mock, dag_maker): """Test that rendered dataframe gets passed through the serialized template fields.""" @@ -209,6 +212,7 @@ def consume_pd(data): rtif = RTIF(ti=ti2) rtif.write() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "rtif_num, num_to_keep, remaining_rtifs, expected_query_count", [ @@ -254,6 +258,7 @@ def test_delete_old_records( result = session.query(RTIF).filter(RTIF.dag_id == dag.dag_id, RTIF.task_id == task.task_id).all() assert remaining_rtifs == len(result) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "num_runs, num_to_keep, remaining_rtifs, expected_query_count", [ @@ -297,6 +302,7 @@ def test_delete_old_records_mapped( # Check that we have _all_ the data for each row assert len(result) == remaining_rtifs * 2 + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_write(self, dag_maker): """ Test records can be written and overwritten @@ -357,7 +363,7 @@ def test_write(self, dag_maker): @mock.patch.dict(os.environ, {"AIRFLOW_VAR_API_KEY": "secret"}) @mock.patch("airflow.utils.log.secrets_masker.redact", autospec=True) def test_redact(self, redact, dag_maker): - with dag_maker("test_ritf_redact"): + with dag_maker("test_ritf_redact", serialized=True): task = BashOperator( task_id="test", bash_command="echo {{ var.value.api_key }}", diff --git a/tests/models/test_serialized_dag.py b/tests/models/test_serialized_dag.py index 848d26119506e..531ffb031925e 100644 --- a/tests/models/test_serialized_dag.py +++ b/tests/models/test_serialized_dag.py @@ -74,6 +74,7 @@ def _write_example_dags(self): SDM.write_dag(dag) return example_dags + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_write_dag(self): """DAGs can be written into database""" example_dags = self._write_example_dags() @@ -87,6 +88,7 @@ def test_write_dag(self): # Verifies JSON schema. SerializedDAG.validate_schema(result.data) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_serialized_dag_is_updated_if_dag_is_changed(self): """Test Serialized DAG is updated if DAG is changed""" example_dags = make_example_dags(example_dags_module) @@ -118,6 +120,7 @@ def test_serialized_dag_is_updated_if_dag_is_changed(self): assert s_dag_2.data["dag"]["tags"] == ["example", "example2", "new_tag"] assert dag_updated is True + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_serialized_dag_is_updated_if_processor_subdir_changed(self): """Test Serialized DAG is updated if processor_subdir is changed""" example_dags = make_example_dags(example_dags_module) @@ -145,6 +148,7 @@ def test_serialized_dag_is_updated_if_processor_subdir_changed(self): assert s_dag.processor_subdir != s_dag_2.processor_subdir assert dag_updated is True + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_read_dags(self): """DAGs can be read from database.""" example_dags = self._write_example_dags() @@ -156,6 +160,7 @@ def test_read_dags(self): assert serialized_dag.dag_id == dag.dag_id assert set(serialized_dag.task_dict) == set(dag.task_dict) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_remove_dags_by_id(self): """DAGs can be removed from database.""" example_dags_list = list(self._write_example_dags().values()) @@ -167,6 +172,7 @@ def test_remove_dags_by_id(self): SDM.remove_dag(dag_removed_by_id.dag_id) assert not SDM.has_dag(dag_removed_by_id.dag_id) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_remove_dags_by_filepath(self): """DAGs can be removed from database.""" example_dags_list = list(self._write_example_dags().values()) @@ -181,6 +187,7 @@ def test_remove_dags_by_filepath(self): SDM.remove_deleted_dags(example_dag_files, processor_subdir="/tmp/test") assert not SDM.has_dag(dag_removed_by_file.dag_id) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_bulk_sync_to_db(self): dags = [ DAG("dag_1"), @@ -190,6 +197,7 @@ def test_bulk_sync_to_db(self): with assert_queries_count(10): SDM.bulk_sync_to_db(dags) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("dag_dependencies_fields", [{"dag_dependencies": None}, {}]) def test_get_dag_dependencies_default_to_empty(self, dag_dependencies_fields): """Test a pre-2.1.0 serialized DAG can deserialize DAG dependencies.""" @@ -206,6 +214,7 @@ def test_get_dag_dependencies_default_to_empty(self, dag_dependencies_fields): expected_dependencies = {dag_id: [] for dag_id in example_dags} assert SDM.get_dag_dependencies() == expected_dependencies + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_order_of_dag_params_is_stable(self): """ This asserts that we have logic in place which guarantees the order diff --git a/tests/models/test_skipmixin.py b/tests/models/test_skipmixin.py index 465d15130f4de..62d8c4d059baf 100644 --- a/tests/models/test_skipmixin.py +++ b/tests/models/test_skipmixin.py @@ -53,6 +53,7 @@ def setup_method(self): def teardown_method(self): self.clean_db() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @patch("airflow.utils.timezone.utcnow") def test_skip(self, mock_now, dag_maker): session = settings.Session() @@ -75,14 +76,13 @@ def test_skip(self, mock_now, dag_maker): TI.end_date == now, ).one() + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @patch("airflow.utils.timezone.utcnow") def test_skip_none_dagrun(self, mock_now, dag_maker): - session = settings.Session() now = datetime.datetime.now(tz=pendulum.timezone("UTC")) mock_now.return_value = now with dag_maker( "dag", - session=session, ): tasks = [EmptyOperator(task_id="task")] dag_maker.create_dagrun(execution_date=now) @@ -93,6 +93,7 @@ def test_skip_none_dagrun(self, mock_now, dag_maker): ): SkipMixin().skip(dag_run=None, execution_date=now, tasks=tasks) + session = dag_maker.session session.query(TI).filter( TI.dag_id == "dag", TI.task_id == "task", @@ -121,6 +122,7 @@ def test_skip_none_tasks(self): def test_skip_all_except(self, dag_maker, branch_task_ids, expected_states): with dag_maker( "dag_test_skip_all_except", + serialized=True, ): task1 = EmptyOperator(task_id="task1") task2 = EmptyOperator(task_id="task2") @@ -143,6 +145,7 @@ def get_state(ti): assert executed_states == expected_states + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_mapped_tasks_skip_all_except(self, dag_maker): with dag_maker("dag_test_skip_all_except") as dag: @@ -209,7 +212,7 @@ def test_raise_exception_on_not_accepted_iterable_branch_task_ids_type(self, dag ], ) def test_raise_exception_on_not_valid_branch_task_ids(self, dag_maker, branch_task_ids): - with dag_maker("dag_test_skip_all_except_wrong_type"): + with dag_maker("dag_test_skip_all_except_wrong_type", serialized=True): task1 = EmptyOperator(task_id="task1") task2 = EmptyOperator(task_id="task2") task3 = EmptyOperator(task_id="task3") From 9f3c1aedb7b4ff3924acf6a113a953da47283545 Mon Sep 17 00:00:00 2001 From: Bugra Ozturk Date: Sun, 11 Aug 2024 20:01:43 +0200 Subject: [PATCH 012/161] Fix Variable and KubernetesJobOperator Tests for Database Isolation Tests (#41370) Fixing remaining Variable tests for db isolation mode, also fixing secret backend haven't called from EnvironmentVariablesBackend, Metastore and custom ones. This caused side effect to move the Variable.get() method to internal API (cherry picked from commit c98d1a177be4b260d44df75f6e97f752bb381cce) --- airflow/models/variable.py | 13 ++++-- tests/models/test_variable.py | 40 ++++++++++--------- .../cncf/kubernetes/operators/test_job.py | 1 + 3 files changed, 32 insertions(+), 22 deletions(-) diff --git a/airflow/models/variable.py b/airflow/models/variable.py index 4a9530e5d9523..63b71303bc803 100644 --- a/airflow/models/variable.py +++ b/airflow/models/variable.py @@ -110,12 +110,13 @@ def setdefault(cls, key, default, description=None, deserialize_json=False): :param description: Default value to set Description of the Variable :param deserialize_json: Store this as a JSON encoded value in the DB and un-encode it when retrieving a value + :param session: Session :return: Mixed """ obj = Variable.get(key, default_var=None, deserialize_json=deserialize_json) if obj is None: if default is not None: - Variable.set(key, default, description=description, serialize_json=deserialize_json) + Variable.set(key=key, value=default, description=description, serialize_json=deserialize_json) return default else: raise ValueError("Default Value must be set") @@ -170,9 +171,10 @@ def set( :param value: Value to set for the Variable :param description: Description of the Variable :param serialize_json: Serialize the value to a JSON string + :param session: Session """ # check if the secret exists in the custom secrets' backend. - Variable.check_for_write_conflict(key) + Variable.check_for_write_conflict(key=key) if serialize_json: stored_value = json.dumps(value, indent=2) else: @@ -201,8 +203,9 @@ def update( :param key: Variable Key :param value: Value to set for the Variable :param serialize_json: Serialize the value to a JSON string + :param session: Session """ - Variable.check_for_write_conflict(key) + Variable.check_for_write_conflict(key=key) if Variable.get_variable_from_secrets(key=key) is None: raise KeyError(f"Variable {key} does not exist") @@ -210,7 +213,9 @@ def update( if obj is None: raise AttributeError(f"Variable {key} does not exist in the Database and cannot be updated.") - Variable.set(key, value, description=obj.description, serialize_json=serialize_json) + Variable.set( + key=key, value=value, description=obj.description, serialize_json=serialize_json, session=session + ) @staticmethod @provide_session diff --git a/tests/models/test_variable.py b/tests/models/test_variable.py index b3e327dab521b..3ec2691e5af95 100644 --- a/tests/models/test_variable.py +++ b/tests/models/test_variable.py @@ -47,7 +47,7 @@ def setup_test_cases(self): db.clear_db_variables() crypto._fernet = None - @conf_vars({("core", "fernet_key"): ""}) + @conf_vars({("core", "fernet_key"): "", ("core", "unit_test_mode"): "True"}) def test_variable_no_encryption(self, session): """ Test variables without encryption @@ -100,12 +100,13 @@ def test_variable_set_get_round_trip(self): Variable.set("tested_var_set_id", "Monday morning breakfast") assert "Monday morning breakfast" == Variable.get("tested_var_set_id") + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_variable_set_with_env_variable(self, caplog, session): caplog.set_level(logging.WARNING, logger=variable.log.name) Variable.set(key="key", value="db-value", session=session) with mock.patch.dict("os.environ", AIRFLOW_VAR_KEY="env-value"): # setting value while shadowed by an env variable will generate a warning - Variable.set("key", "new-db-value") + Variable.set(key="key", value="new-db-value", session=session) # value set above is not returned because the env variable value takes priority assert "env-value" == Variable.get("key") # invalidate the cache to re-evaluate value @@ -120,6 +121,7 @@ def test_variable_set_with_env_variable(self, caplog, session): "EnvironmentVariablesBackend" ) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @mock.patch("airflow.models.variable.ensure_secrets_loaded") def test_variable_set_with_extra_secret_backend(self, mock_ensure_secrets, caplog, session): caplog.set_level(logging.WARNING, logger=variable.log.name) @@ -137,11 +139,11 @@ def test_variable_set_with_extra_secret_backend(self, mock_ensure_secrets, caplo "will be updated, but to read it you have to delete the conflicting variable from " "MockSecretsBackend" ) - Variable.delete("key") + Variable.delete(key="key", session=session) def test_variable_set_get_round_trip_json(self): value = {"a": 17, "b": 47} - Variable.set("tested_var_set_id", value, serialize_json=True) + Variable.set(key="tested_var_set_id", value=value, serialize_json=True) assert value == Variable.get("tested_var_set_id", deserialize_json=True) def test_variable_update(self, session): @@ -184,9 +186,9 @@ def test_get_non_existing_var_should_raise_key_error(self): with pytest.raises(KeyError): Variable.get("thisIdDoesNotExist") - def test_update_non_existing_var_should_raise_key_error(self): + def test_update_non_existing_var_should_raise_key_error(self, session): with pytest.raises(KeyError): - Variable.update("thisIdDoesNotExist", "value") + Variable.update(key="thisIdDoesNotExist", value="value", session=session) def test_get_non_existing_var_with_none_default_should_return_none(self): assert Variable.get("thisIdDoesNotExist", default_var=None) is None @@ -197,42 +199,45 @@ def test_get_non_existing_var_should_not_deserialize_json_default(self): "thisIdDoesNotExist", default_var=default_value, deserialize_json=True ) - def test_variable_setdefault_round_trip(self): + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode + def test_variable_setdefault_round_trip(self, session): key = "tested_var_setdefault_1_id" value = "Monday morning breakfast in Paris" - Variable.setdefault(key, value) + Variable.setdefault(key=key, default=value) assert value == Variable.get(key) - def test_variable_setdefault_round_trip_json(self): + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode + def test_variable_setdefault_round_trip_json(self, session): key = "tested_var_setdefault_2_id" value = {"city": "Paris", "Happiness": True} - Variable.setdefault(key, value, deserialize_json=True) + Variable.setdefault(key=key, default=value, deserialize_json=True) assert value == Variable.get(key, deserialize_json=True) + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_variable_setdefault_existing_json(self, session): key = "tested_var_setdefault_2_id" value = {"city": "Paris", "Happiness": True} Variable.set(key=key, value=value, serialize_json=True, session=session) - val = Variable.setdefault(key, value, deserialize_json=True) + val = Variable.setdefault(key=key, default=value, deserialize_json=True) # Check the returned value, and the stored value are handled correctly. assert value == val assert value == Variable.get(key, deserialize_json=True) - def test_variable_delete(self): + def test_variable_delete(self, session): key = "tested_var_delete" value = "to be deleted" # No-op if the variable doesn't exist - Variable.delete(key) + Variable.delete(key=key, session=session) with pytest.raises(KeyError): Variable.get(key) # Set the variable - Variable.set(key, value) + Variable.set(key=key, value=value, session=session) assert value == Variable.get(key) # Delete the variable - Variable.delete(key) + Variable.delete(key=key, session=session) with pytest.raises(KeyError): Variable.get(key) @@ -276,7 +281,7 @@ def test_caching_caches(self, mock_ensure_secrets: mock.Mock): mock_backend.get_variable.assert_called_once() # second call was not made because of cache assert first == second - def test_cache_invalidation_on_set(self): + def test_cache_invalidation_on_set(self, session): with mock.patch.dict("os.environ", AIRFLOW_VAR_KEY="from_env"): a = Variable.get("key") # value is saved in cache with mock.patch.dict("os.environ", AIRFLOW_VAR_KEY="from_env_two"): @@ -284,7 +289,7 @@ def test_cache_invalidation_on_set(self): assert a == b # setting a new value invalidates the cache - Variable.set("key", "new_value") + Variable.set(key="key", value="new_value", session=session) c = Variable.get("key") # cache should not be used @@ -312,7 +317,6 @@ def test_masking_only_secret_values(variable_value, deserialize_json, expected_m ) session.add(var) session.flush() - # Make sure we re-load it, not just get the cached object back session.expunge(var) _secrets_masker().patterns = set() diff --git a/tests/providers/cncf/kubernetes/operators/test_job.py b/tests/providers/cncf/kubernetes/operators/test_job.py index ac888b706cc19..307a4ff8425b3 100644 --- a/tests/providers/cncf/kubernetes/operators/test_job.py +++ b/tests/providers/cncf/kubernetes/operators/test_job.py @@ -116,6 +116,7 @@ def test_templates(self, create_task_instance_of_operator, session): cmds="{{ dag.dag_id }}", image="{{ dag.dag_id }}", annotations={"dag-id": "{{ dag.dag_id }}"}, + session=session, ) session.add(ti) From ef147dd8c403cb93c5419092cab2d20012ad3a72 Mon Sep 17 00:00:00 2001 From: Jens Scheffler <95105677+jscheffl@users.noreply.github.com> Date: Sun, 11 Aug 2024 20:07:13 +0200 Subject: [PATCH 013/161] Fix pytests for Core except Variable for DB Isolation Mode (#41375) (cherry picked from commit 60cbea509707a4390ea9ea4239c737f0a44e8aea) --- tests/models/test_trigger.py | 5 +++++ tests/models/test_xcom_arg.py | 1 + tests/models/test_xcom_arg_map.py | 8 ++++++++ .../utils/test_task_handler_with_custom_formatter.py | 12 ++++++------ 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/models/test_trigger.py b/tests/models/test_trigger.py index 5a8ef28df0a64..4aa5b8b581b8f 100644 --- a/tests/models/test_trigger.py +++ b/tests/models/test_trigger.py @@ -99,6 +99,7 @@ def test_clean_unused(session, create_task_instance): assert session.query(Trigger).one().id == trigger1.id +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_submit_event(session, create_task_instance): """ Tests that events submitted to a trigger re-wake their dependent @@ -126,6 +127,7 @@ def test_submit_event(session, create_task_instance): assert updated_task_instance.next_kwargs == {"event": 42, "cheesecake": True} +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_submit_failure(session, create_task_instance): """ Tests that failures submitted to a trigger fail their dependent @@ -150,6 +152,7 @@ def test_submit_failure(session, create_task_instance): assert updated_task_instance.next_method == "__fail__" +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "event_cls, expected", [ @@ -300,6 +303,7 @@ def test_assign_unassigned(session, create_task_instance): ) +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_get_sorted_triggers_same_priority_weight(session, create_task_instance): """ Tests that triggers are sorted by the creation_date if they have the same priority. @@ -350,6 +354,7 @@ def test_get_sorted_triggers_same_priority_weight(session, create_task_instance) assert trigger_ids_query == [(1,), (2,)] +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_get_sorted_triggers_different_priority_weights(session, create_task_instance): """ Tests that triggers are sorted by the priority_weight. diff --git a/tests/models/test_xcom_arg.py b/tests/models/test_xcom_arg.py index 2652a1032b7d8..6108c5e81930f 100644 --- a/tests/models/test_xcom_arg.py +++ b/tests/models/test_xcom_arg.py @@ -184,6 +184,7 @@ def push_xcom_value(key, value, **context): dag.run() +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "fillvalue, expected_results", [ diff --git a/tests/models/test_xcom_arg_map.py b/tests/models/test_xcom_arg_map.py index b2e885e940833..26df335215bcc 100644 --- a/tests/models/test_xcom_arg_map.py +++ b/tests/models/test_xcom_arg_map.py @@ -29,6 +29,7 @@ pytestmark = pytest.mark.db_test +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_xcom_map(dag_maker, session): results = set() with dag_maker(session=session) as dag: @@ -64,6 +65,7 @@ def pull(value): assert results == {"aa", "bb", "cc"} +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_xcom_map_transform_to_none(dag_maker, session): results = set() @@ -98,6 +100,7 @@ def c_to_none(v): assert results == {"a", "b", None} +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_xcom_convert_to_kwargs_fails_task(dag_maker, session): results = set() @@ -145,6 +148,7 @@ def c_to_none(v): ] +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_xcom_map_error_fails_task(dag_maker, session): with dag_maker(session=session) as dag: @@ -241,6 +245,7 @@ def test_task_map_variant(): assert task_map.variant == TaskMapVariant.DICT +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_xcom_map_raise_to_skip(dag_maker, session): result = None @@ -285,6 +290,7 @@ def skip_c(v): assert result == ["a", "b"] +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_xcom_map_nest(dag_maker, session): results = set() @@ -318,6 +324,7 @@ def pull(value): assert results == {"aa", "bb", "cc"} +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_xcom_map_zip_nest(dag_maker, session): results = set() @@ -364,6 +371,7 @@ def convert_zipped(zipped): assert results == {"aa", "bbbb", "cccccc", "dddddddd"} +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_xcom_concat(dag_maker, session): from airflow.models.xcom_arg import _ConcatResult diff --git a/tests/utils/test_task_handler_with_custom_formatter.py b/tests/utils/test_task_handler_with_custom_formatter.py index c6c6565f54fca..9eb183c6be3fe 100644 --- a/tests/utils/test_task_handler_with_custom_formatter.py +++ b/tests/utils/test_task_handler_with_custom_formatter.py @@ -22,7 +22,6 @@ import pytest from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG -from airflow.models.dag import DAG from airflow.models.taskinstance import TaskInstance from airflow.operators.empty import EmptyOperator from airflow.utils.log.logging_mixin import set_context @@ -59,11 +58,11 @@ def custom_task_log_handler_config(): @pytest.fixture -def task_instance(): - dag = DAG(DAG_ID, start_date=DEFAULT_DATE) - task = EmptyOperator(task_id=TASK_ID, dag=dag) - dagrun = dag.create_dagrun( - DagRunState.RUNNING, +def task_instance(dag_maker): + with dag_maker(DAG_ID, start_date=DEFAULT_DATE, serialized=True) as dag: + task = EmptyOperator(task_id=TASK_ID) + dagrun = dag_maker.create_dagrun( + state=DagRunState.RUNNING, execution_date=DEFAULT_DATE, run_type=DagRunType.MANUAL, data_interval=dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE), @@ -103,6 +102,7 @@ def test_custom_formatter_default_format(task_instance): assert_prefix_once(task_instance, "") +@pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @conf_vars({("logging", "task_log_prefix_template"): "{{ ti.dag_id }}-{{ ti.task_id }}"}) def test_custom_formatter_custom_format_not_affected_by_config(task_instance): """Certifies that the prefix is only added once, even after repeated calls""" From 34e2c9c694b9fc924a0dab0665cd080ba6e97d76 Mon Sep 17 00:00:00 2001 From: Gopal Dirisala <39794726+dirrao@users.noreply.github.com> Date: Mon, 12 Aug 2024 00:26:55 +0530 Subject: [PATCH 014/161] bump uv version to 0.2.34 (#41334) (cherry picked from commit b4a92f802e0f6c1aac09953e4c490a149b5e7c12) --- Dockerfile | 2 +- Dockerfile.ci | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3cc7f0fc69924..8c4a43274fd33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,7 +50,7 @@ ARG AIRFLOW_VERSION="2.9.3" ARG PYTHON_BASE_IMAGE="python:3.8-slim-bookworm" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.2.33 +ARG AIRFLOW_UV_VERSION=0.2.34 ARG AIRFLOW_USE_UV="false" ARG UV_HTTP_TIMEOUT="300" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" diff --git a/Dockerfile.ci b/Dockerfile.ci index 1ef5d38389ffd..14ccb669f62aa 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1310,7 +1310,7 @@ ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main" ARG AIRFLOW_CI_BUILD_EPOCH="10" ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.2.33 +ARG AIRFLOW_UV_VERSION=0.2.34 ARG AIRFLOW_USE_UV="true" # Setup PIP # By default PIP install run without cache to make image smaller @@ -1334,7 +1334,7 @@ ARG AIRFLOW_VERSION="" ARG ADDITIONAL_PIP_INSTALL_FLAGS="" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.2.33 +ARG AIRFLOW_UV_VERSION=0.2.34 ARG AIRFLOW_USE_UV="true" ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ From 09567dcc61d89c4247610ec63d0414dfade5d368 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 11 Aug 2024 23:20:39 +0200 Subject: [PATCH 015/161] Skip docs publishing on non-main brnaches (#41385) (cherry picked from commit 54c165cee03808bcf6a83708b5a58f2b205a3d9d) --- .github/workflows/static-checks-mypy-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/static-checks-mypy-docs.yml b/.github/workflows/static-checks-mypy-docs.yml index e464e17c9986e..9a1e4ac4ac7f9 100644 --- a/.github/workflows/static-checks-mypy-docs.yml +++ b/.github/workflows/static-checks-mypy-docs.yml @@ -243,7 +243,7 @@ jobs: INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}" PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" VERBOSE: "true" - if: inputs.canary-run == 'true' + if: inputs.canary-run == 'true' && inputs.branch == 'main' steps: - name: "Cleanup repo" shell: bash From 6c6797c5d9ab173d2860cf628543b866fa5873a5 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 11 Aug 2024 23:36:18 +0200 Subject: [PATCH 016/161] Fix mypy checks for new azure libraries (#41386) (cherry picked from commit 68a6a0583b5100c1f313d7bff3dc664205d9a1ad) --- airflow/providers/microsoft/azure/hooks/wasb.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/airflow/providers/microsoft/azure/hooks/wasb.py b/airflow/providers/microsoft/azure/hooks/wasb.py index 237594139e3c0..36cafd9933bdc 100644 --- a/airflow/providers/microsoft/azure/hooks/wasb.py +++ b/airflow/providers/microsoft/azure/hooks/wasb.py @@ -213,7 +213,8 @@ def get_conn(self) -> BlobServiceClient: **extra, ) - def _get_container_client(self, container_name: str) -> ContainerClient: + # TODO: rework the interface as it might also return AsyncContainerClient + def _get_container_client(self, container_name: str) -> ContainerClient: # type: ignore[override] """ Instantiate a container client. @@ -222,7 +223,7 @@ def _get_container_client(self, container_name: str) -> ContainerClient: """ return self.blob_service_client.get_container_client(container_name) - def _get_blob_client(self, container_name: str, blob_name: str) -> BlobClient: + def _get_blob_client(self, container_name: str, blob_name: str) -> BlobClient | AsyncBlobClient: """ Instantiate a blob client. @@ -415,7 +416,8 @@ def upload( self.create_container(container_name) blob_client = self._get_blob_client(container_name, blob_name) - return blob_client.upload_blob(data, blob_type, length=length, **kwargs) + # TODO: rework the interface as it might also return Awaitable + return blob_client.upload_blob(data, blob_type, length=length, **kwargs) # type: ignore[return-value] def download( self, container_name, blob_name, offset: int | None = None, length: int | None = None, **kwargs @@ -430,7 +432,8 @@ def download( :param length: Number of bytes to read from the stream. """ blob_client = self._get_blob_client(container_name, blob_name) - return blob_client.download_blob(offset=offset, length=length, **kwargs) + # TODO: rework the interface as it might also return Awaitable + return blob_client.download_blob(offset=offset, length=length, **kwargs) # type: ignore[return-value] def create_container(self, container_name: str) -> None: """ @@ -656,7 +659,8 @@ async def check_for_blob_async(self, container_name: str, blob_name: str, **kwar return False return True - def _get_container_client(self, container_name: str) -> AsyncContainerClient: + # TODO: rework the interface as in parent Hook it returns ContainerClient + def _get_container_client(self, container_name: str) -> AsyncContainerClient: # type: ignore[override] """ Instantiate a container client. From 8ea4eb1ce016d41c63b322f1448c799bd2730e81 Mon Sep 17 00:00:00 2001 From: Bugra Ozturk Date: Mon, 12 Aug 2024 01:10:54 +0200 Subject: [PATCH 017/161] Fix tests/decorators/test_python.py for database isolation tests (#41387) * Pass serialized parameter for dag_maker * Serialisation of object is on __exit__ moving out the dag definition out of dag_maker context (cherry picked from commit 278f3c4b6eaf823a0c6eb538a6f61d21e0f955d8) --- tests/decorators/test_python.py | 80 +++++++++++++++++---------------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/tests/decorators/test_python.py b/tests/decorators/test_python.py index fb8c75b72b4c3..9d2b9a14c82b4 100644 --- a/tests/decorators/test_python.py +++ b/tests/decorators/test_python.py @@ -1000,50 +1000,52 @@ def down(a: str): def test_teardown_trigger_rule_selective_application(dag_maker, session): - with dag_maker(session=session) as dag: - - @dag.task - def my_work(): - return "abc" - - @setup - @dag.task - def my_setup(): - return "abc" - - @teardown - @dag.task - def my_teardown(): - return "abc" - - work_task = my_work() - setup_task = my_setup() - teardown_task = my_teardown() + with dag_maker(session=session, serialized=True) as created_dag: + dag = created_dag + + @dag.task + def my_work(): + return "abc" + + @setup + @dag.task + def my_setup(): + return "abc" + + @teardown + @dag.task + def my_teardown(): + return "abc" + + work_task = my_work() + setup_task = my_setup() + teardown_task = my_teardown() assert work_task.operator.trigger_rule == TriggerRule.ALL_SUCCESS assert setup_task.operator.trigger_rule == TriggerRule.ALL_SUCCESS assert teardown_task.operator.trigger_rule == TriggerRule.ALL_DONE_SETUP_SUCCESS def test_teardown_trigger_rule_override_behavior(dag_maker, session): - with dag_maker(session=session) as dag: - - @dag.task(trigger_rule=TriggerRule.ONE_SUCCESS) - def my_work(): - return "abc" - - @setup - @dag.task(trigger_rule=TriggerRule.ONE_SUCCESS) - def my_setup(): - return "abc" - - @teardown - @dag.task(trigger_rule=TriggerRule.ONE_SUCCESS) - def my_teardown(): - return "abc" - - work_task = my_work() - setup_task = my_setup() - with pytest.raises(Exception, match="Trigger rule not configurable for teardown tasks."): - my_teardown() + with dag_maker(session=session, serialized=True) as created_dag: + dag = created_dag + + @dag.task(trigger_rule=TriggerRule.ONE_SUCCESS) + def my_work(): + return "abc" + + @setup + @dag.task(trigger_rule=TriggerRule.ONE_SUCCESS) + def my_setup(): + return "abc" + + @teardown + @dag.task(trigger_rule=TriggerRule.ONE_SUCCESS) + def my_teardown(): + return "abc" + + work_task = my_work() + setup_task = my_setup() + with pytest.raises(Exception, match="Trigger rule not configurable for teardown tasks."): + my_teardown() assert work_task.operator.trigger_rule == TriggerRule.ONE_SUCCESS assert setup_task.operator.trigger_rule == TriggerRule.ONE_SUCCESS From e001b88f5875cfd7e295891a0bbdbc75a3dccbfb Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 12 Aug 2024 17:24:59 +0800 Subject: [PATCH 018/161] fix DagPriorityParsingRequest unique constraint error when dataset aliases are resolved into new datasets (#41398) * fix(datasets/manager): fix DagPriorityParsingRequest unique constraint error when dataset aliases are resolved into new datasets this happens when dynamic task mapping is used * refactor(dataset/manager): reword debug log Co-authored-by: Ephraim Anierobi * refactor(dataset/manager): remove unnecessary logging Co-authored-by: Ephraim Anierobi --------- Co-authored-by: Ephraim Anierobi (cherry picked from commit bf64cb686be28f5e60e0b06f5fae790481c2efcd) --- airflow/datasets/manager.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/airflow/datasets/manager.py b/airflow/datasets/manager.py index 29f95ef4c742a..058eef6ab8922 100644 --- a/airflow/datasets/manager.py +++ b/airflow/datasets/manager.py @@ -140,10 +140,8 @@ def register_dataset_change( dags_to_reparse = dags_to_queue_from_dataset_alias - dags_to_queue_from_dataset if dags_to_reparse: - session.add_all( - DagPriorityParsingRequest(fileloc=fileloc) - for fileloc in {dag.fileloc for dag in dags_to_reparse} - ) + file_locs = {dag.fileloc for dag in dags_to_reparse} + cls._send_dag_priority_parsing_request(file_locs, session) session.flush() cls.notify_dataset_changed(dataset=dataset) @@ -208,6 +206,35 @@ def _postgres_queue_dagruns(cls, dataset_id: int, dags_to_queue: set[DagModel], stmt = insert(DatasetDagRunQueue).values(dataset_id=dataset_id).on_conflict_do_nothing() session.execute(stmt, values) + @classmethod + def _send_dag_priority_parsing_request(cls, file_locs: Iterable[str], session: Session) -> None: + if session.bind.dialect.name == "postgresql": + return cls._postgres_send_dag_priority_parsing_request(file_locs, session) + return cls._slow_path_send_dag_priority_parsing_request(file_locs, session) + + @classmethod + def _slow_path_send_dag_priority_parsing_request(cls, file_locs: Iterable[str], session: Session) -> None: + def _send_dag_priority_parsing_request_if_needed(fileloc: str) -> str | None: + # Don't error whole transaction when a single DagPriorityParsingRequest item conflicts. + # https://docs.sqlalchemy.org/en/14/orm/session_transaction.html#using-savepoint + req = DagPriorityParsingRequest(fileloc=fileloc) + try: + with session.begin_nested(): + session.merge(req) + except exc.IntegrityError: + cls.logger().debug("Skipping request %s, already present", req, exc_info=True) + return None + return req.fileloc + + (_send_dag_priority_parsing_request_if_needed(fileloc) for fileloc in file_locs) + + @classmethod + def _postgres_send_dag_priority_parsing_request(cls, file_locs: Iterable[str], session: Session) -> None: + from sqlalchemy.dialects.postgresql import insert + + stmt = insert(DagPriorityParsingRequest).on_conflict_do_nothing() + session.execute(stmt, {"fileloc": fileloc for fileloc in file_locs}) + def resolve_dataset_manager() -> DatasetManager: """Retrieve the dataset manager.""" From 535f27b1373a14839767f5abe767472ad9377b86 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 20 Aug 2024 21:22:27 +0200 Subject: [PATCH 019/161] Make PROD image building works in non-main PRs (#41480) (#41484) (#41623) The PROD image building fails currently in non-main because it attempts to build source provider packages rather than use them from PyPi when PR is run against "v-test" branch. This PR fixes it: * PROD images in non-main-targetted build will pull providers from PyPI rather than build them * they use PyPI constraints to install the providers * they use UV - which should speed up building of the images (cherry picked from commit 4d5f1c42a7873329b1b6b8b9b39db2c3033b46df) (cherry picked from commit bf0d412531e099ecb918beb04287d30b72dc5682) --- .github/workflows/build-images.yml | 4 ++-- .github/workflows/ci.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index bd10e73aac65c..1256fd2f0da6e 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -241,14 +241,14 @@ jobs: pull-request-target: "true" is-committer-build: ${{ needs.build-info.outputs.is-committer-build }} push-image: "true" - use-uv: ${{ needs.build-info.outputs.default-branch == 'main' && 'true' || 'false' }} + use-uv: "true" image-tag: ${{ needs.build-info.outputs.image-tag }} platform: "linux/amd64" python-versions: ${{ needs.build-info.outputs.python-versions }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} branch: ${{ needs.build-info.outputs.default-branch }} constraints-branch: ${{ needs.build-info.outputs.constraints-branch }} - build-provider-packages: "true" + build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68aa51bf860f5..a1fa32e3ad7da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -541,7 +541,7 @@ jobs: default-python-version: ${{ needs.build-info.outputs.default-python-version }} branch: ${{ needs.build-info.outputs.default-branch }} push-image: "true" - use-uv: ${{ needs.build-info.outputs.default-branch == 'main' && 'true' || 'false' }} + use-uv: "true" build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} From 91f8265b6820eca747634dce39ae0d70aada2640 Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Thu, 22 Aug 2024 14:30:09 +0530 Subject: [PATCH 020/161] Sync v2-10-stable with v2-10-test to release python client v2.10.0 (#41610) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Enable pull requests to be run from v*test branches (#41474) (#41476) Since we switch from direct push of cherry-picking to open PRs against v*test branch, we should enable PRs to run for the target branch. (cherry picked from commit a9363e6a30d73a647ed7d45c92d46d1f6f98513f) * Prevent provider lowest-dependency tests to run in non-main branch (#41478) (#41481) When running tests in v2-10-test branch, lowest depenency tests are run for providers - because when calculating separate tests, the "skip_provider_tests" has not been used to filter them out. This PR fixes it. (cherry picked from commit 75da5074969ec874040ea094d5afe00b7f02be76) * Make PROD image building works in non-main PRs (#41480) (#41484) The PROD image building fails currently in non-main because it attempts to build source provider packages rather than use them from PyPi when PR is run against "v-test" branch. This PR fixes it: * PROD images in non-main-targetted build will pull providers from PyPI rather than build them * they use PyPI constraints to install the providers * they use UV - which should speed up building of the images (cherry picked from commit 4d5f1c42a7873329b1b6b8b9b39db2c3033b46df) * Add WebEncoder for trigger page rendering to avoid render failure (#41350) (#41485) Co-authored-by: M. Olcay Tercanlı * Incorrect try number subtraction producing invalid span id for OTEL airflow (issue #41501) (#41502) (#41535) * Fix for issue #39336 * removed unnecessary import (cherry picked from commit dd3c3a7a43102c967d76cdcfe1f2f8ebeef4e212) Co-authored-by: Howard Yoo <32691630+howardyoo@users.noreply.github.com> * Fix failing pydantic v1 tests (#41534) (#41541) We need to exclude some versions of Pydantic v1 because it conflicts with aws provider. (cherry picked from commit a033c5f15a033c751419506ea77ffdbacdd37705) * Fix Non-DB test calculation for main builds (#41499) (#41543) Pytest has a weird behaviour that it will not collect tests from parent folder when subfolder of it is specified after the parent folder. This caused some non-db tests from providers folder have been skipped during main build. The issue in Pytest 8.2 (used to work before) is tracked at https://github.com/pytest-dev/pytest/issues/12605 (cherry picked from commit d48982692c54d024d7c05e1efb7cd2adeb7d896c) * Add changelog for airflow python client 2.10.0 (#41583) (#41584) * Add changelog for airflow python client 2.10.0 * Update client version (cherry picked from commit 317a28ed435960e7184e357a2f128806c34612fa) * Make all test pass in Database Isolation mode (#41567) This adds dedicated "DatabaseIsolation" test to airflow v2-10-test branch.. The DatabaseIsolation test will run all "db-tests" with enabled DB isolation mode and running `internal-api` component - groups of tests marked with "skip-if-database-isolation" will be skipped. * Upgrade build and chart dependencies (#41570) (#41588) (cherry picked from commit c88192c466cb91842310f82a61eaa48b39439bef) Co-authored-by: Jarek Potiuk * Limit watchtower as depenendcy as 3.3.0 breaks moin. (#41612) (cherry picked from commit 1b602d50266184d118db52a674baeab29b1f5688) * Enable running Pull Requests against v2-10-stable branch (#41624) (cherry picked from commit e306e7f7bc1ef12aeab0fc09e018accda3684a2f) * Fix tests/models/test_variable.py for database isolation mode (#41414) * Fix tests/models/test_variable.py for database isolation mode * Review feedback (cherry picked from commit 736ebfe3fe2bd67406d5a50dacbfa1e43767d4ce) * Make latest botocore tests green (#41626) The latest botocore tests are conflicting with a few requirements and until apache-beam upcoming version is released we need to do some manual exclusions. Those exclusions should make latest botocore test green again. (cherry picked from commit a13ccbbdec8e59f30218f604fca8cbb999fcb757) * Simpler task retrieval for taskinstance test (#41389) The test has been updated for DB isolation but the retrieval of task was not intuitive and it could lead to flaky tests possibly (cherry picked from commit f25adf14ad486bac818fe3fdcd61eb3355e8ec9b) * Skip database isolation case for task mapping taskinstance tests (#41471) Related: #41067 (cherry picked from commit 7718bd7a6ed7fb476e4920315b6d11f1ac465f44) * Skipping tests for db isolation because similar tests were skipped (#41450) (cherry picked from commit e94b508b946471420488cc466d92301b54b4c5ae) --------- Co-authored-by: Jarek Potiuk Co-authored-by: Brent Bovenzi Co-authored-by: M. Olcay Tercanlı Co-authored-by: Howard Yoo <32691630+howardyoo@users.noreply.github.com> Co-authored-by: Jens Scheffler <95105677+jscheffl@users.noreply.github.com> Co-authored-by: Bugra Ozturk --- .github/workflows/ci.yml | 2 +- .github/workflows/run-unit-tests.yml | 6 ++ .github/workflows/special-tests.yml | 23 +++++++ Dockerfile | 2 +- Dockerfile.ci | 16 +++-- .../endpoints/rpc_api_endpoint.py | 15 +++-- airflow/api_internal/internal_api_call.py | 2 +- airflow/executors/base_executor.py | 2 +- airflow/executors/local_executor.py | 2 +- airflow/executors/sequential_executor.py | 2 +- airflow/jobs/scheduler_job_runner.py | 2 +- airflow/models/baseoperator.py | 3 +- airflow/models/variable.py | 66 ++++++++++++++++++- airflow/providers/amazon/provider.yaml | 2 +- airflow/serialization/enums.py | 1 + airflow/serialization/serialized_objects.py | 16 ++++- airflow/settings.py | 51 +++++++++----- airflow/traces/utils.py | 7 +- airflow/www/views.py | 3 +- chart/values.schema.json | 2 +- chart/values.yaml | 2 +- clients/python/CHANGELOG.md | 20 ++++++ clients/python/version.txt | 2 +- .../commands/testing_commands.py | 2 + .../templates/CHANGELOG_TEMPLATE.rst.jinja2 | 6 +- .../src/airflow_breeze/utils/run_tests.py | 18 +++-- .../airflow_breeze/utils/selective_checks.py | 4 ++ .../tests/test_pytest_args_for_test_types.py | 13 ++++ dev/breeze/tests/test_selective_checks.py | 5 +- generated/provider_dependencies.json | 2 +- pyproject.toml | 3 + scripts/docker/entrypoint_ci.sh | 12 ++-- tests/always/test_example_dags.py | 20 ++---- .../cli/commands/test_internal_api_command.py | 3 + tests/cli/commands/test_webserver_command.py | 3 + tests/decorators/test_bash.py | 3 + .../decorators/test_branch_external_python.py | 3 +- tests/decorators/test_branch_python.py | 3 +- tests/decorators/test_branch_virtualenv.py | 3 +- tests/decorators/test_condition.py | 3 +- tests/decorators/test_external_python.py | 2 +- tests/decorators/test_python.py | 11 +++- tests/decorators/test_python_virtualenv.py | 4 +- tests/decorators/test_sensor.py | 3 +- tests/decorators/test_short_circuit.py | 2 +- tests/models/test_taskinstance.py | 9 ++- tests/models/test_variable.py | 10 ++- tests/operators/test_python.py | 3 + tests/providers/opensearch/conftest.py | 11 +++- .../opensearch/hooks/test_opensearch.py | 3 +- .../opensearch/operators/test_opensearch.py | 3 + tests/sensors/test_external_task_sensor.py | 2 + tests/serialization/test_dag_serialization.py | 2 + tests/www/views/test_views_trigger_dag.py | 28 ++++++++ 54 files changed, 350 insertions(+), 98 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1fa32e3ad7da..866f8f253d401 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ on: # yamllint disable-line rule:truthy push: branches: ['v[0-9]+-[0-9]+-test'] pull_request: - branches: ['main'] + branches: ['main', 'v[0-9]+-[0-9]+-test', 'v[0-9]+-[0-9]+-stable'] workflow_dispatch: permissions: # All other permissions are set to none diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 7828e50ed7e95..2989a952d9ea2 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -104,6 +104,11 @@ on: # yamllint disable-line rule:truthy required: false default: "true" type: string + database-isolation: + description: "Whether to enable database isolattion or not (true/false)" + required: false + default: "false" + type: string force-lowest-dependencies: description: "Whether to force lowest dependencies for the tests or not (true/false)" required: false @@ -152,6 +157,7 @@ jobs: PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}" UPGRADE_BOTO: "${{ inputs.upgrade-boto }}" AIRFLOW_MONITOR_DELAY_TIME_IN_SECONDS: "${{inputs.monitor-delay-time-in-seconds}}" + DATABASE_ISOLATION: "${{ inputs.database-isolation }}" VERBOSE: "true" steps: - name: "Cleanup repo" diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index 000b5aa3d958b..e09b813acf916 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -193,6 +193,29 @@ jobs: run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} + tests-database-isolation: + name: "Database isolation test" + uses: ./.github/workflows/run-unit-tests.yml + permissions: + contents: read + packages: read + secrets: inherit + with: + runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + enable-aip-44: "true" + database-isolation: "true" + test-name: "DatabaseIsolation-Postgres" + test-scope: "DB" + backend: "postgres" + image-tag: ${{ inputs.image-tag }} + python-versions: "['${{ inputs.default-python-version }}']" + backend-versions: "['${{ inputs.default-postgres-version }}']" + excludes: "[]" + parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ inputs.run-coverage }} + debug-resources: ${{ inputs.debug-resources }} + tests-quarantined: name: "Quarantined test" uses: ./.github/workflows/run-unit-tests.yml diff --git a/Dockerfile b/Dockerfile index 8c4a43274fd33..5eb2b3355695f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,7 +50,7 @@ ARG AIRFLOW_VERSION="2.9.3" ARG PYTHON_BASE_IMAGE="python:3.8-slim-bookworm" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.2.34 +ARG AIRFLOW_UV_VERSION=0.2.37 ARG AIRFLOW_USE_UV="false" ARG UV_HTTP_TIMEOUT="300" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" diff --git a/Dockerfile.ci b/Dockerfile.ci index 14ccb669f62aa..f46890cd8145c 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1022,16 +1022,17 @@ function check_boto_upgrade() { echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" echo # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs yandexcloud || true + ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs yandexcloud opensearch-py || true # We need to include few dependencies to pass pip check with other dependencies: # * oss2 as dependency as otherwise jmespath will be bumped (sync with alibaba provider) - # * gcloud-aio-auth limit is needed to be included as it bumps cryptography (sync with google provider) + # * cryptography is kept for snowflake-connector-python limitation (sync with snowflake provider) # * requests needs to be limited to be compatible with apache beam (sync with apache-beam provider) # * yandexcloud requirements for requests does not match those of apache.beam and latest botocore # Both requests and yandexcloud exclusion above might be removed after # https://github.com/apache/beam/issues/32080 is addressed - # When you remove yandexcloud from the above list, also remove it from "test_example_dags.py" - # in "tests/always". + # This is already addressed and planned for 2.59.0 release. + # When you remove yandexcloud and opensearch from the above list, you can also remove the + # optional providers_dependencies exclusions from "test_example_dags.py" in "tests/always". set -x # shellcheck disable=SC2086 ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \ @@ -1068,8 +1069,9 @@ function check_pydantic() { echo echo "${COLOR_YELLOW}Downgrading Pydantic to < 2${COLOR_RESET}" echo + # Pydantic 1.10.17/1.10.15 conflicts with aws-sam-translator so we need to exclude it # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "pydantic<2.0.0" + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "pydantic<2.0.0,!=1.10.17,!=1.10.15" pip check else echo @@ -1310,7 +1312,7 @@ ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main" ARG AIRFLOW_CI_BUILD_EPOCH="10" ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.2.34 +ARG AIRFLOW_UV_VERSION=0.2.37 ARG AIRFLOW_USE_UV="true" # Setup PIP # By default PIP install run without cache to make image smaller @@ -1334,7 +1336,7 @@ ARG AIRFLOW_VERSION="" ARG ADDITIONAL_PIP_INSTALL_FLAGS="" ARG AIRFLOW_PIP_VERSION=24.2 -ARG AIRFLOW_UV_VERSION=0.2.34 +ARG AIRFLOW_UV_VERSION=0.2.37 ARG AIRFLOW_USE_UV="true" ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ diff --git a/airflow/api_internal/endpoints/rpc_api_endpoint.py b/airflow/api_internal/endpoints/rpc_api_endpoint.py index ad65157ef9415..c3d8b671fbb08 100644 --- a/airflow/api_internal/endpoints/rpc_api_endpoint.py +++ b/airflow/api_internal/endpoints/rpc_api_endpoint.py @@ -126,9 +126,9 @@ def initialize_method_map() -> dict[str, Callable]: # XCom.get_many, # Not supported because it returns query XCom.clear, XCom.set, - Variable.set, - Variable.update, - Variable.delete, + Variable._set, + Variable._update, + Variable._delete, DAG.fetch_callback, DAG.fetch_dagrun, DagRun.fetch_task_instances, @@ -228,19 +228,20 @@ def internal_airflow_api(body: dict[str, Any]) -> APIResponse: except Exception: return log_and_build_error_response(message="Error deserializing parameters.", status=400) - log.info("Calling method %s\nparams: %s", method_name, params) + log.debug("Calling method %s\nparams: %s", method_name, params) try: # Session must be created there as it may be needed by serializer for lazy-loaded fields. with create_session() as session: output = handler(**params, session=session) output_json = BaseSerialization.serialize(output, use_pydantic_models=True) response = json.dumps(output_json) if output_json is not None else None - log.info("Sending response: %s", response) + log.debug("Sending response: %s", response) return Response(response=response, headers={"Content-Type": "application/json"}) - except AirflowException as e: # In case of AirflowException transport the exception class back to caller + # In case of AirflowException or other selective known types, transport the exception class back to caller + except (KeyError, AttributeError, AirflowException) as e: exception_json = BaseSerialization.serialize(e, use_pydantic_models=True) response = json.dumps(exception_json) - log.info("Sending exception response: %s", response) + log.debug("Sending exception response: %s", response) return Response(response=response, headers={"Content-Type": "application/json"}) except Exception: return log_and_build_error_response(message=f"Error executing method '{method_name}'.", status=500) diff --git a/airflow/api_internal/internal_api_call.py b/airflow/api_internal/internal_api_call.py index fc0945b3c0fe0..8838377877bec 100644 --- a/airflow/api_internal/internal_api_call.py +++ b/airflow/api_internal/internal_api_call.py @@ -159,7 +159,7 @@ def wrapper(*args, **kwargs): if result is None or result == b"": return None result = BaseSerialization.deserialize(json.loads(result), use_pydantic_models=True) - if isinstance(result, AirflowException): + if isinstance(result, (KeyError, AttributeError, AirflowException)): raise result return result diff --git a/airflow/executors/base_executor.py b/airflow/executors/base_executor.py index dd0b8a66d2857..57568af199710 100644 --- a/airflow/executors/base_executor.py +++ b/airflow/executors/base_executor.py @@ -467,7 +467,7 @@ def success(self, key: TaskInstanceKey, info=None) -> None: span.set_attribute("dag_id", key.dag_id) span.set_attribute("run_id", key.run_id) span.set_attribute("task_id", key.task_id) - span.set_attribute("try_number", key.try_number - 1) + span.set_attribute("try_number", key.try_number) self.change_state(key, TaskInstanceState.SUCCESS, info) diff --git a/airflow/executors/local_executor.py b/airflow/executors/local_executor.py index afa51b1d86bb4..32bba4208273b 100644 --- a/airflow/executors/local_executor.py +++ b/airflow/executors/local_executor.py @@ -277,7 +277,7 @@ def execute_async( span.set_attribute("dag_id", key.dag_id) span.set_attribute("run_id", key.run_id) span.set_attribute("task_id", key.task_id) - span.set_attribute("try_number", key.try_number - 1) + span.set_attribute("try_number", key.try_number) span.set_attribute("commands_to_run", str(command)) local_worker = LocalWorker(self.executor.result_queue, key=key, command=command) diff --git a/airflow/executors/sequential_executor.py b/airflow/executors/sequential_executor.py index 1b145892ebc7e..5e9542d9158b1 100644 --- a/airflow/executors/sequential_executor.py +++ b/airflow/executors/sequential_executor.py @@ -76,7 +76,7 @@ def execute_async( span.set_attribute("dag_id", key.dag_id) span.set_attribute("run_id", key.run_id) span.set_attribute("task_id", key.task_id) - span.set_attribute("try_number", key.try_number - 1) + span.set_attribute("try_number", key.try_number) span.set_attribute("commands_to_run", str(self.commands_to_run)) def sync(self) -> None: diff --git a/airflow/jobs/scheduler_job_runner.py b/airflow/jobs/scheduler_job_runner.py index 163bf5b71449b..ba5f90c68b772 100644 --- a/airflow/jobs/scheduler_job_runner.py +++ b/airflow/jobs/scheduler_job_runner.py @@ -837,7 +837,7 @@ def _process_executor_events(self, executor: BaseExecutor, session: Session) -> span.set_attribute("hostname", ti.hostname) span.set_attribute("log_url", ti.log_url) span.set_attribute("operator", str(ti.operator)) - span.set_attribute("try_number", ti.try_number - 1) + span.set_attribute("try_number", ti.try_number) span.set_attribute("executor_state", state) span.set_attribute("job_id", ti.job_id) span.set_attribute("pool", ti.pool) diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py index 7ffa596ec67a1..f21db0c675fd3 100644 --- a/airflow/models/baseoperator.py +++ b/airflow/models/baseoperator.py @@ -1516,10 +1516,11 @@ def run( data_interval=info.data_interval, ) ti = TaskInstance(self, run_id=dr.run_id) + session.add(ti) ti.dag_run = dr session.add(dr) session.flush() - + session.commit() ti.run( mark_success=mark_success, ignore_depends_on_past=ignore_depends_on_past, diff --git a/airflow/models/variable.py b/airflow/models/variable.py index 63b71303bc803..563cac46e8c84 100644 --- a/airflow/models/variable.py +++ b/airflow/models/variable.py @@ -154,7 +154,6 @@ def get( @staticmethod @provide_session - @internal_api_call def set( key: str, value: Any, @@ -167,6 +166,35 @@ def set( This operation overwrites an existing variable. + :param key: Variable Key + :param value: Value to set for the Variable + :param description: Description of the Variable + :param serialize_json: Serialize the value to a JSON string + :param session: Session + """ + Variable._set( + key=key, value=value, description=description, serialize_json=serialize_json, session=session + ) + # invalidate key in cache for faster propagation + # we cannot save the value set because it's possible that it's shadowed by a custom backend + # (see call to check_for_write_conflict above) + SecretCache.invalidate_variable(key) + + @staticmethod + @provide_session + @internal_api_call + def _set( + key: str, + value: Any, + description: str | None = None, + serialize_json: bool = False, + session: Session = None, + ) -> None: + """ + Set a value for an Airflow Variable with a given Key. + + This operation overwrites an existing variable. + :param key: Variable Key :param value: Value to set for the Variable :param description: Description of the Variable @@ -190,7 +218,6 @@ def set( @staticmethod @provide_session - @internal_api_call def update( key: str, value: Any, @@ -200,6 +227,27 @@ def update( """ Update a given Airflow Variable with the Provided value. + :param key: Variable Key + :param value: Value to set for the Variable + :param serialize_json: Serialize the value to a JSON string + :param session: Session + """ + Variable._update(key=key, value=value, serialize_json=serialize_json, session=session) + # We need to invalidate the cache for internal API cases on the client side + SecretCache.invalidate_variable(key) + + @staticmethod + @provide_session + @internal_api_call + def _update( + key: str, + value: Any, + serialize_json: bool = False, + session: Session = None, + ) -> None: + """ + Update a given Airflow Variable with the Provided value. + :param key: Variable Key :param value: Value to set for the Variable :param serialize_json: Serialize the value to a JSON string @@ -219,11 +267,23 @@ def update( @staticmethod @provide_session - @internal_api_call def delete(key: str, session: Session = None) -> int: """ Delete an Airflow Variable for a given key. + :param key: Variable Keys + """ + rows = Variable._delete(key=key, session=session) + SecretCache.invalidate_variable(key) + return rows + + @staticmethod + @provide_session + @internal_api_call + def _delete(key: str, session: Session = None) -> int: + """ + Delete an Airflow Variable for a given key. + :param key: Variable Keys """ rows = session.execute(delete(Variable).where(Variable.key == key)).rowcount diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index c69542ef334dc..1c6a86f1805fa 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -101,7 +101,7 @@ dependencies: - botocore>=1.34.90 - inflection>=0.5.1 # Allow a wider range of watchtower versions for flexibility among users - - watchtower>=3.0.0,<4 + - watchtower>=3.0.0,!=3.3.0,<4 - jsonpath_ng>=1.5.3 - redshift_connector>=2.0.918 - sqlalchemy_redshift>=0.8.6 diff --git a/airflow/serialization/enums.py b/airflow/serialization/enums.py index a5bd5e3646e83..f216ce7316103 100644 --- a/airflow/serialization/enums.py +++ b/airflow/serialization/enums.py @@ -46,6 +46,7 @@ class DagAttributeTypes(str, Enum): RELATIVEDELTA = "relativedelta" BASE_TRIGGER = "base_trigger" AIRFLOW_EXC_SER = "airflow_exc_ser" + BASE_EXC_SER = "base_exc_ser" DICT = "dict" SET = "set" TUPLE = "tuple" diff --git a/airflow/serialization/serialized_objects.py b/airflow/serialization/serialized_objects.py index d110271c3da08..a3886aa49acef 100644 --- a/airflow/serialization/serialized_objects.py +++ b/airflow/serialization/serialized_objects.py @@ -692,6 +692,15 @@ def serialize( ), type_=DAT.AIRFLOW_EXC_SER, ) + elif isinstance(var, (KeyError, AttributeError)): + return cls._encode( + cls.serialize( + {"exc_cls_name": var.__class__.__name__, "args": [var.args], "kwargs": {}}, + use_pydantic_models=use_pydantic_models, + strict=strict, + ), + type_=DAT.BASE_EXC_SER, + ) elif isinstance(var, BaseTrigger): return cls._encode( cls.serialize(var.serialize(), use_pydantic_models=use_pydantic_models, strict=strict), @@ -834,13 +843,16 @@ def deserialize(cls, encoded_var: Any, use_pydantic_models=False) -> Any: return decode_timezone(var) elif type_ == DAT.RELATIVEDELTA: return decode_relativedelta(var) - elif type_ == DAT.AIRFLOW_EXC_SER: + elif type_ == DAT.AIRFLOW_EXC_SER or type_ == DAT.BASE_EXC_SER: deser = cls.deserialize(var, use_pydantic_models=use_pydantic_models) exc_cls_name = deser["exc_cls_name"] args = deser["args"] kwargs = deser["kwargs"] del deser - exc_cls = import_string(exc_cls_name) + if type_ == DAT.AIRFLOW_EXC_SER: + exc_cls = import_string(exc_cls_name) + else: + exc_cls = import_string(f"builtins.{exc_cls_name}") return exc_cls(*args, **kwargs) elif type_ == DAT.BASE_TRIGGER: tr_cls_name, kwargs = cls.deserialize(var, use_pydantic_models=use_pydantic_models) diff --git a/airflow/settings.py b/airflow/settings.py index 751bb3876037e..175a63f69d2f4 100644 --- a/airflow/settings.py +++ b/airflow/settings.py @@ -313,6 +313,8 @@ def remove(*args, **kwargs): AIRFLOW_SETTINGS_PATH = os.path.join(AIRFLOW_PATH, "airflow", "settings.py") AIRFLOW_UTILS_SESSION_PATH = os.path.join(AIRFLOW_PATH, "airflow", "utils", "session.py") AIRFLOW_MODELS_BASEOPERATOR_PATH = os.path.join(AIRFLOW_PATH, "airflow", "models", "baseoperator.py") +AIRFLOW_MODELS_DAG_PATH = os.path.join(AIRFLOW_PATH, "airflow", "models", "dag.py") +AIRFLOW_DB_UTILS_PATH = os.path.join(AIRFLOW_PATH, "airflow", "utils", "db.py") class TracebackSessionForTests: @@ -370,6 +372,9 @@ def is_called_from_test_code(self) -> tuple[bool, traceback.FrameSummary | None] :return: True if the object was created from test code, False otherwise. """ self.traceback = traceback.extract_stack() + if any(filename.endswith("_pytest/fixtures.py") for filename, _, _, _ in self.traceback): + # This is a fixture call + return True, None airflow_frames = [ tb for tb in self.traceback @@ -378,24 +383,30 @@ def is_called_from_test_code(self) -> tuple[bool, traceback.FrameSummary | None] and not tb.filename == AIRFLOW_UTILS_SESSION_PATH ] if any( - filename.endswith("conftest.py") or filename.endswith("tests/test_utils/db.py") - for filename, _, _, _ in airflow_frames + filename.endswith("conftest.py") + or filename.endswith("tests/test_utils/db.py") + or (filename.startswith(AIRFLOW_TESTS_PATH) and name in ("setup_method", "teardown_method")) + for filename, _, name, _ in airflow_frames ): # This is a fixture call or testing utilities return True, None - if ( - len(airflow_frames) >= 2 - and airflow_frames[-2].filename.startswith(AIRFLOW_TESTS_PATH) - and airflow_frames[-1].filename == AIRFLOW_MODELS_BASEOPERATOR_PATH - and airflow_frames[-1].name == "run" - ): - # This is baseoperator run method that is called directly from the test code and this is - # usual pattern where we create a session in the test code to create dag_runs for tests. - # If `run` code will be run inside a real "airflow" code the stack trace would be longer - # and it would not be directly called from the test code. Also if subsequently any of the - # run_task() method called later from the task code will attempt to execute any DB - # method, the stack trace will be longer and we will catch it as "illegal" call. - return True, None + if len(airflow_frames) >= 2 and airflow_frames[-2].filename.startswith(AIRFLOW_TESTS_PATH): + # Let's look at what we are calling directly from the test code + current_filename, current_method_name = airflow_frames[-1].filename, airflow_frames[-1].name + if (current_filename, current_method_name) in ( + (AIRFLOW_MODELS_BASEOPERATOR_PATH, "run"), + (AIRFLOW_MODELS_DAG_PATH, "create_dagrun"), + ): + # This is baseoperator run method that is called directly from the test code and this is + # usual pattern where we create a session in the test code to create dag_runs for tests. + # If `run` code will be run inside a real "airflow" code the stack trace would be longer + # and it would not be directly called from the test code. Also if subsequently any of the + # run_task() method called later from the task code will attempt to execute any DB + # method, the stack trace will be longer and we will catch it as "illegal" call. + return True, None + if current_filename == AIRFLOW_DB_UTILS_PATH: + # This is a util method called directly from the test code + return True, None for tb in airflow_frames[::-1]: if tb.filename.startswith(AIRFLOW_PATH): if tb.filename.startswith(AIRFLOW_TESTS_PATH): @@ -407,6 +418,16 @@ def is_called_from_test_code(self) -> tuple[bool, traceback.FrameSummary | None] # The traceback line will be always 3rd (two bottom ones are Airflow) return False, self.traceback[-2] + def get_bind( + self, + mapper=None, + clause=None, + bind=None, + _sa_skip_events=None, + _sa_skip_for_implicit_returning=False, + ): + pass + def _is_sqlite_db_path_relative(sqla_conn_str: str) -> bool: """Determine whether the database connection URI specifies a relative path.""" diff --git a/airflow/traces/utils.py b/airflow/traces/utils.py index afab2591d5146..9932c249f0772 100644 --- a/airflow/traces/utils.py +++ b/airflow/traces/utils.py @@ -22,7 +22,6 @@ from airflow.traces import NO_TRACE_ID from airflow.utils.hashlib_wrapper import md5 -from airflow.utils.state import TaskInstanceState if TYPE_CHECKING: from airflow.models import DagRun, TaskInstance @@ -75,12 +74,8 @@ def gen_dag_span_id(dag_run: DagRun, as_int: bool = False) -> str | int: def gen_span_id(ti: TaskInstance, as_int: bool = False) -> str | int: """Generate span id from the task instance.""" dag_run = ti.dag_run - if ti.state == TaskInstanceState.SUCCESS or ti.state == TaskInstanceState.FAILED: - try_number = ti.try_number - 1 - else: - try_number = ti.try_number return _gen_id( - [dag_run.dag_id, dag_run.run_id, ti.task_id, str(try_number)], + [dag_run.dag_id, dag_run.run_id, ti.task_id, str(ti.try_number)], as_int, SPAN_ID, ) diff --git a/airflow/www/views.py b/airflow/www/views.py index 236beed4511a3..a485f84ed4b1c 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -2163,7 +2163,7 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): .limit(num_recent_confs) ) recent_confs = { - run_id: json.dumps(run_conf) + run_id: json.dumps(run_conf, cls=utils_json.WebEncoder) for run_id, run_conf in ((run.run_id, run.conf) for run in recent_runs) if isinstance(run_conf, dict) and any(run_conf) } @@ -2198,6 +2198,7 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): }, indent=4, ensure_ascii=False, + cls=utils_json.WebEncoder, ) except TypeError: flash("Could not pre-populate conf field due to non-JSON-serializable data-types") diff --git a/chart/values.schema.json b/chart/values.schema.json index 5f6a3b55d89f5..4ba434c3417bf 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -671,7 +671,7 @@ "tag": { "description": "The StatsD image tag.", "type": "string", - "default": "v0.26.1" + "default": "v0.27.1" }, "pullPolicy": { "description": "The StatsD image pull policy.", diff --git a/chart/values.yaml b/chart/values.yaml index 76130e55bb5a0..a28c3c6d54033 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -105,7 +105,7 @@ images: pullPolicy: IfNotPresent statsd: repository: quay.io/prometheus/statsd-exporter - tag: v0.26.1 + tag: v0.27.1 pullPolicy: IfNotPresent redis: repository: redis diff --git a/clients/python/CHANGELOG.md b/clients/python/CHANGELOG.md index d56d692d3c4e2..6a254029a4760 100644 --- a/clients/python/CHANGELOG.md +++ b/clients/python/CHANGELOG.md @@ -17,6 +17,26 @@ under the License. --> +# v2.10.0 + +## Major changes: + + - Add dag_stats rest api endpoint ([#41017](https://github.com/apache/airflow/pull/41017)) + - AIP-64: Add task instance history list endpoint ([#40988](https://github.com/apache/airflow/pull/40988)) + - Change DAG Audit log tab to Event Log ([#40967](https://github.com/apache/airflow/pull/40967)) + - AIP-64: Add REST API endpoints for TI try level details ([#40441](https://github.com/apache/airflow/pull/40441)) + - Make XCom display as react json ([#40640](https://github.com/apache/airflow/pull/40640)) + - Replace usages of task context logger with the log table ([#40867](https://github.com/apache/airflow/pull/40867)) + - Fix tasks API endpoint when DAG doesn't have `start_date` ([#40878](https://github.com/apache/airflow/pull/40878)) + - Add try_number to log table ([#40739](https://github.com/apache/airflow/pull/40739)) + - Add executor field to the task instance API ([#40034](https://github.com/apache/airflow/pull/40034)) + - Add task documentation to details tab in grid view. ([#39899](https://github.com/apache/airflow/pull/39899)) + - Add max_consecutive_failed_dag_runs in API spec ([#39830](https://github.com/apache/airflow/pull/39830)) + - Add task failed dependencies to details page. ([#38449](https://github.com/apache/airflow/pull/38449)) + - Add dag re-parsing request endpoint ([#39138](https://github.com/apache/airflow/pull/39138)) + - Reorder OpenAPI Spec tags alphabetically ([#38717](https://github.com/apache/airflow/pull/38717)) + + # v2.9.1 ## Major changes: diff --git a/clients/python/version.txt b/clients/python/version.txt index dedcc7d4335da..10c2c0c3d6213 100644 --- a/clients/python/version.txt +++ b/clients/python/version.txt @@ -1 +1 @@ -2.9.1 +2.10.0 diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 51ca4bea5d636..cef51d975219e 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -206,6 +206,7 @@ def _run_test( helm_test_package=None, keep_env_variables=shell_params.keep_env_variables, no_db_cleanup=shell_params.no_db_cleanup, + database_isolation=shell_params.database_isolation, ) ) run_cmd.extend(list(extra_pytest_args)) @@ -968,6 +969,7 @@ def helm_tests( helm_test_package=helm_test_package, keep_env_variables=False, no_db_cleanup=False, + database_isolation=False, ) cmd = ["docker", "compose", "run", "--service-ports", "--rm", "airflow", *pytest_args, *extra_pytest_args] result = run_command(cmd, check=False, env=env, output_outside_the_group=True) diff --git a/dev/breeze/src/airflow_breeze/templates/CHANGELOG_TEMPLATE.rst.jinja2 b/dev/breeze/src/airflow_breeze/templates/CHANGELOG_TEMPLATE.rst.jinja2 index b8a966448c07b..e51939c57571c 100644 --- a/dev/breeze/src/airflow_breeze/templates/CHANGELOG_TEMPLATE.rst.jinja2 +++ b/dev/breeze/src/airflow_breeze/templates/CHANGELOG_TEMPLATE.rst.jinja2 @@ -40,7 +40,7 @@ Features {%- endif %} -{%- if classified_changes.fixes %} +{%- if classified_changes and classified_changes.fixes %} Bug Fixes ~~~~~~~~~ @@ -50,7 +50,7 @@ Bug Fixes {%- endif %} -{%- if classified_changes.misc %} +{%- if classified_changes and classified_changes.misc %} Misc ~~~~ @@ -62,7 +62,7 @@ Misc .. Below changes are excluded from the changelog. Move them to appropriate section above if needed. Do not delete the lines(!): -{%- if classified_changes.other %} +{%- if classified_changes and classified_changes.other %} {%- for other in classified_changes.other %} * ``{{ other.message_without_backticks | safe }}`` {%- endfor %} diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py b/dev/breeze/src/airflow_breeze/utils/run_tests.py index fe099efbc024a..73cbb430817cc 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_tests.py +++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py @@ -311,6 +311,7 @@ def generate_args_for_pytest( helm_test_package: str | None, keep_env_variables: bool, no_db_cleanup: bool, + database_isolation: bool, ): result_log_file, warnings_file, coverage_file = test_paths(test_type, backend, helm_test_package) if skip_db_tests: @@ -327,12 +328,13 @@ def generate_args_for_pytest( helm_test_package=helm_test_package, python_version=python_version, ) + max_fail = 50 args.extend( [ "--verbosity=0", "--strict-markers", "--durations=100", - "--maxfail=50", + f"--maxfail={max_fail}", "--color=yes", f"--junitxml={result_log_file}", # timeouts in seconds for individual tests @@ -374,7 +376,7 @@ def generate_args_for_pytest( args.extend(get_excluded_provider_args(python_version)) if use_xdist: args.extend(["-n", str(parallelism) if parallelism else "auto"]) - # We have to disabke coverage for Python 3.12 because of the issue with coverage that takes too long, despite + # We have to disable coverage for Python 3.12 because of the issue with coverage that takes too long, despite # Using experimental support for Python 3.12 PEP 669. The coverage.py is not yet fully compatible with the # full scope of PEP-669. That will be fully done when https://github.com/nedbat/coveragepy/issues/1746 is # resolve for now we are disabling coverage for Python 3.12, and it causes slower execution and occasional @@ -417,5 +419,13 @@ def convert_parallel_types_to_folders( python_version=python_version, ) ) - # leave only folders, strip --pytest-args - return [arg for arg in args if arg.startswith("test")] + # leave only folders, strip --pytest-args that exclude some folders with `-' prefix + folders = [arg for arg in args if arg.startswith("test")] + # remove specific provider sub-folders if "tests/providers" is already in the list + # This workarounds pytest issues where it will only run tests from specific subfolders + # if both parent and child folders are in the list + # The issue in Pytest (changed behaviour in Pytest 8.2 is tracked here + # https://github.com/pytest-dev/pytest/issues/12605 + if "tests/providers" in folders: + folders = [folder for folder in folders if not folder.startswith("tests/providers/")] + return folders diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 62f5a20abc4e0..224e76c251921 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -861,6 +861,10 @@ def separate_test_types_list_as_string(self) -> str | None: if "Providers" in current_test_types: current_test_types.remove("Providers") current_test_types.update({f"Providers[{provider}]" for provider in get_available_packages()}) + if self.skip_provider_tests: + current_test_types = { + test_type for test_type in current_test_types if not test_type.startswith("Providers") + } return " ".join(sorted(current_test_types)) @cached_property diff --git a/dev/breeze/tests/test_pytest_args_for_test_types.py b/dev/breeze/tests/test_pytest_args_for_test_types.py index a64dccbd06f2a..fbb3785949e85 100644 --- a/dev/breeze/tests/test_pytest_args_for_test_types.py +++ b/dev/breeze/tests/test_pytest_args_for_test_types.py @@ -329,6 +329,19 @@ def test_pytest_args_for_helm_test_types(helm_test_package: str, pytest_args: li ], True, ), + ( + "Core Providers[-amazon,google] Providers[amazon] Providers[google]", + [ + "tests/core", + "tests/executors", + "tests/jobs", + "tests/models", + "tests/ti_deps", + "tests/utils", + "tests/providers", + ], + False, + ), ], ) def test_folders_for_parallel_test_types( diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 7c0ca940949cd..6bee6bbc7e308 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1075,7 +1075,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ ), ( pytest.param( - ("INTHEWILD.md",), + ("INTHEWILD.md", "tests/providers/asana.py"), ("full tests needed",), "v2-7-stable", { @@ -1097,6 +1097,9 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "parallel-test-types-list-as-string": "API Always BranchExternalPython " "BranchPythonVenv CLI Core ExternalPython Operators Other PlainAsserts " "PythonVenv Serialization WWW", + "separate-test-types-list-as-string": "API Always BranchExternalPython " + "BranchPythonVenv CLI Core ExternalPython Operators Other PlainAsserts " + "PythonVenv Serialization WWW", "needs-mypy": "true", "mypy-folders": "['airflow', 'docs', 'dev']", }, diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 58fb34b2adf02..1ac11366c643d 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -41,7 +41,7 @@ "jsonpath_ng>=1.5.3", "redshift_connector>=2.0.918", "sqlalchemy_redshift>=0.8.6", - "watchtower>=3.0.0,<4" + "watchtower>=3.0.0,!=3.3.0,<4" ], "devel-deps": [ "aiobotocore>=2.13.0", diff --git a/pyproject.toml b/pyproject.toml index 621a9e48b8e7d..194c4b268c88c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -405,6 +405,9 @@ combine-as-imports = true "tests/providers/google/cloud/operators/vertex_ai/test_generative_model.py" = ["E402"] "tests/providers/google/cloud/triggers/test_vertex_ai.py" = ["E402"] "tests/providers/openai/hooks/test_openai.py" = ["E402"] +"tests/providers/opensearch/conftest.py" = ["E402"] +"tests/providers/opensearch/hooks/test_opensearch.py" = ["E402"] +"tests/providers/opensearch/operators/test_opensearch.py" = ["E402"] "tests/providers/openai/operators/test_openai.py" = ["E402"] "tests/providers/qdrant/hooks/test_qdrant.py" = ["E402"] "tests/providers/qdrant/operators/test_qdrant.py" = ["E402"] diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 1ff3ef0cd2cce..6e9b36507ba68 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -242,16 +242,17 @@ function check_boto_upgrade() { echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" echo # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs yandexcloud || true + ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs yandexcloud opensearch-py || true # We need to include few dependencies to pass pip check with other dependencies: # * oss2 as dependency as otherwise jmespath will be bumped (sync with alibaba provider) - # * gcloud-aio-auth limit is needed to be included as it bumps cryptography (sync with google provider) + # * cryptography is kept for snowflake-connector-python limitation (sync with snowflake provider) # * requests needs to be limited to be compatible with apache beam (sync with apache-beam provider) # * yandexcloud requirements for requests does not match those of apache.beam and latest botocore # Both requests and yandexcloud exclusion above might be removed after # https://github.com/apache/beam/issues/32080 is addressed - # When you remove yandexcloud from the above list, also remove it from "test_example_dags.py" - # in "tests/always". + # This is already addressed and planned for 2.59.0 release. + # When you remove yandexcloud and opensearch from the above list, you can also remove the + # optional providers_dependencies exclusions from "test_example_dags.py" in "tests/always". set -x # shellcheck disable=SC2086 ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \ @@ -289,8 +290,9 @@ function check_pydantic() { echo echo "${COLOR_YELLOW}Downgrading Pydantic to < 2${COLOR_RESET}" echo + # Pydantic 1.10.17/1.10.15 conflicts with aws-sam-translator so we need to exclude it # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "pydantic<2.0.0" + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "pydantic<2.0.0,!=1.10.17,!=1.10.15" pip check else echo diff --git a/tests/always/test_example_dags.py b/tests/always/test_example_dags.py index 2b5f37631a427..b8e3edb99b82d 100644 --- a/tests/always/test_example_dags.py +++ b/tests/always/test_example_dags.py @@ -17,6 +17,7 @@ from __future__ import annotations import os +import re import sys from glob import glob from importlib import metadata as importlib_metadata @@ -39,8 +40,11 @@ # Some examples or system tests may depend on additional packages # that are not included in certain CI checks. # The format of the dictionary is as follows: - # key: the prefix of the file to be excluded, + # key: the regexp matching the file to be excluded, # value: a dictionary containing package distributions with an optional version specifier, e.g., >=2.3.4 + ".*example_bedrock_retrieve_and_generate.py": {"opensearch-py": None}, + ".*example_opensearch.py": {"opensearch-py": None}, + r".*example_yandexcloud.*\.py": {"yandexcloud": None}, } IGNORE_AIRFLOW_PROVIDER_DEPRECATION_WARNING: tuple[str, ...] = ( # Certain examples or system tests may trigger AirflowProviderDeprecationWarnings. @@ -124,13 +128,6 @@ def example_not_excluded_dags(xfail_db_exception: bool = False): for prefix in PROVIDERS_PREFIXES for provider in suspended_providers_folders ] - temporary_excluded_upgrade_boto_providers_folders = [ - AIRFLOW_SOURCES_ROOT.joinpath(prefix, provider).as_posix() - for prefix in PROVIDERS_PREFIXES - # TODO - remove me when https://github.com/apache/beam/issues/32080 is addressed - # and we bring back yandex to be run in case of upgrade boto - for provider in ["yandex"] - ] current_python_excluded_providers_folders = [ AIRFLOW_SOURCES_ROOT.joinpath(prefix, provider).as_posix() for prefix in PROVIDERS_PREFIXES @@ -146,18 +143,13 @@ def example_not_excluded_dags(xfail_db_exception: bool = False): if candidate.startswith(tuple(suspended_providers_folders)): param_marks.append(pytest.mark.skip(reason="Suspended provider")) - if os.environ.get("UPGRADE_BOTO", "false") == "true" and candidate.startswith( - tuple(temporary_excluded_upgrade_boto_providers_folders) - ): - param_marks.append(pytest.mark.skip(reason="Temporary excluded upgrade boto provider")) - if candidate.startswith(tuple(current_python_excluded_providers_folders)): param_marks.append( pytest.mark.skip(reason=f"Not supported for Python {CURRENT_PYTHON_VERSION}") ) for optional, dependencies in OPTIONAL_PROVIDERS_DEPENDENCIES.items(): - if candidate.endswith(optional): + if re.match(optional, candidate): for distribution_name, specifier in dependencies.items(): result, reason = match_optional_dependencies(distribution_name, specifier) if not result: diff --git a/tests/cli/commands/test_internal_api_command.py b/tests/cli/commands/test_internal_api_command.py index 99992e6266861..a1aaf2daca604 100644 --- a/tests/cli/commands/test_internal_api_command.py +++ b/tests/cli/commands/test_internal_api_command.py @@ -83,6 +83,9 @@ def test_ready_prefix_on_cmdline_dead_process(self): assert self.monitor._get_num_ready_workers_running() == 0 +# Those tests are skipped in isolation mode because they interfere with the internal API +# server already running in the background in the isolation mode. +@pytest.mark.skip_if_database_isolation_mode @pytest.mark.db_test @pytest.mark.skipif(not _ENABLE_AIP_44, reason="AIP-44 is disabled") class TestCliInternalAPI(_ComonCLIGunicornTestClass): diff --git a/tests/cli/commands/test_webserver_command.py b/tests/cli/commands/test_webserver_command.py index 07d95a9e5f75a..fa2e58af9efe4 100644 --- a/tests/cli/commands/test_webserver_command.py +++ b/tests/cli/commands/test_webserver_command.py @@ -226,6 +226,9 @@ def test_ready_prefix_on_cmdline_dead_process(self): assert self.monitor._get_num_ready_workers_running() == 0 +# Those tests are skipped in isolation mode because they interfere with the internal API +# server already running in the background in the isolation mode. +@pytest.mark.skip_if_database_isolation_mode @pytest.mark.db_test class TestCliWebServer(_ComonCLIGunicornTestClass): main_process_regexp = r"airflow webserver" diff --git a/tests/decorators/test_bash.py b/tests/decorators/test_bash.py index ba8948936eda1..9fa7999e83476 100644 --- a/tests/decorators/test_bash.py +++ b/tests/decorators/test_bash.py @@ -33,6 +33,9 @@ DEFAULT_DATE = timezone.datetime(2023, 1, 1) +# TODO(potiuk) see why this test hangs in DB isolation mode +pytestmark = pytest.mark.skip_if_database_isolation_mode + @pytest.mark.db_test class TestBashDecorator: diff --git a/tests/decorators/test_branch_external_python.py b/tests/decorators/test_branch_external_python.py index d991f22cd55e4..d2466365bef8d 100644 --- a/tests/decorators/test_branch_external_python.py +++ b/tests/decorators/test_branch_external_python.py @@ -24,7 +24,8 @@ from airflow.decorators import task from airflow.utils.state import State -pytestmark = pytest.mark.db_test +# TODO: (potiuk) - AIP-44 - check why this test hangs +pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] class Test_BranchExternalPythonDecoratedOperator: diff --git a/tests/decorators/test_branch_python.py b/tests/decorators/test_branch_python.py index 58bb216246049..3cd95b8d2a4ad 100644 --- a/tests/decorators/test_branch_python.py +++ b/tests/decorators/test_branch_python.py @@ -22,7 +22,8 @@ from airflow.decorators import task from airflow.utils.state import State -pytestmark = pytest.mark.db_test +# TODO: (potiuk) - AIP-44 - check why this test hangs +pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] class Test_BranchPythonDecoratedOperator: diff --git a/tests/decorators/test_branch_virtualenv.py b/tests/decorators/test_branch_virtualenv.py index a5c23de392de3..6cdfa1ddff25e 100644 --- a/tests/decorators/test_branch_virtualenv.py +++ b/tests/decorators/test_branch_virtualenv.py @@ -22,7 +22,8 @@ from airflow.decorators import task from airflow.utils.state import State -pytestmark = pytest.mark.db_test +# TODO: (potiuk) - AIP-44 - check why this test hangs +pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] class TestBranchPythonVirtualenvDecoratedOperator: diff --git a/tests/decorators/test_condition.py b/tests/decorators/test_condition.py index 315db6bfe0d12..28e0f0bf8fee0 100644 --- a/tests/decorators/test_condition.py +++ b/tests/decorators/test_condition.py @@ -28,7 +28,8 @@ from airflow.models.taskinstance import TaskInstance from airflow.utils.context import Context -pytestmark = pytest.mark.db_test +# TODO(potiuk) see why this test hangs in DB isolation mode +pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] @pytest.mark.skip_if_database_isolation_mode # Test is broken in db isolation mode diff --git a/tests/decorators/test_external_python.py b/tests/decorators/test_external_python.py index 5ed5874e3a55a..0d9a439aa2371 100644 --- a/tests/decorators/test_external_python.py +++ b/tests/decorators/test_external_python.py @@ -29,7 +29,7 @@ from airflow.decorators import setup, task, teardown from airflow.utils import timezone -pytestmark = pytest.mark.db_test +pytestmark = [pytest.mark.db_test, pytest.mark.need_serialized_dag] DEFAULT_DATE = timezone.datetime(2016, 1, 1) diff --git a/tests/decorators/test_python.py b/tests/decorators/test_python.py index 9d2b9a14c82b4..067beff3abbff 100644 --- a/tests/decorators/test_python.py +++ b/tests/decorators/test_python.py @@ -41,7 +41,7 @@ from airflow.utils.xcom import XCOM_RETURN_KEY from tests.operators.test_python import BasePythonTest -pytestmark = pytest.mark.db_test +pytestmark = [pytest.mark.db_test, pytest.mark.need_serialized_dag] if TYPE_CHECKING: @@ -281,6 +281,8 @@ class Test: def add_number(self, num: int) -> int: return self.num + num + # TODO(potiuk) see why this test hangs in DB isolation mode + @pytest.mark.skip_if_database_isolation_mode def test_fail_multiple_outputs_key_type(self): @task_decorator(multiple_outputs=True) def add_number(num: int): @@ -293,6 +295,8 @@ def add_number(num: int): with pytest.raises(AirflowException): ret.operator.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE) + # TODO(potiuk) see why this test hangs in DB isolation mode + @pytest.mark.skip_if_database_isolation_mode def test_fail_multiple_outputs_no_dict(self): @task_decorator(multiple_outputs=True) def add_number(num: int): @@ -541,6 +545,8 @@ def add_2(number: int): assert "add_2" in self.dag_non_serialized.task_ids + # TODO(potiuk) see why this test hangs in DB isolation mode + @pytest.mark.skip_if_database_isolation_mode def test_dag_task_multiple_outputs(self): """Tests dag.task property to generate task with multiple outputs""" @@ -863,6 +869,7 @@ def org_test_func(): assert decorated_test_func.__wrapped__ is org_test_func, "__wrapped__ attr is not the original function" +@pytest.mark.need_serialized_dag(False) @pytest.mark.skip_if_database_isolation_mode # Test is broken in db isolation mode def test_upstream_exception_produces_none_xcom(dag_maker, session): from airflow.exceptions import AirflowSkipException @@ -900,6 +907,7 @@ def down(a, b): assert result == "'example' None" +@pytest.mark.need_serialized_dag(False) @pytest.mark.skip_if_database_isolation_mode # Test is broken in db isolation mode @pytest.mark.parametrize("multiple_outputs", [True, False]) def test_multiple_outputs_produces_none_xcom_when_task_is_skipped(dag_maker, session, multiple_outputs): @@ -958,6 +966,7 @@ def other(x): ... assert caplog.messages == [] +@pytest.mark.need_serialized_dag(False) @pytest.mark.skip_if_database_isolation_mode # Test is broken in db isolation mode def test_task_decorator_dataset(dag_maker, session): from airflow.datasets import Dataset diff --git a/tests/decorators/test_python_virtualenv.py b/tests/decorators/test_python_virtualenv.py index 554b33ceb9b77..57a096ef192c7 100644 --- a/tests/decorators/test_python_virtualenv.py +++ b/tests/decorators/test_python_virtualenv.py @@ -30,7 +30,7 @@ from airflow.utils import timezone from airflow.utils.state import TaskInstanceState -pytestmark = pytest.mark.db_test +pytestmark = [pytest.mark.db_test, pytest.mark.need_serialized_dag] DEFAULT_DATE = timezone.datetime(2016, 1, 1) PYTHON_VERSION = f"{sys.version_info.major}{sys.version_info.minor}" @@ -373,6 +373,8 @@ def f(): assert teardown_task.on_failure_fail_dagrun is on_failure_fail_dagrun ret.operator.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE) + # TODO(potiuk) see why this test hangs in DB isolation mode + @pytest.mark.skip_if_database_isolation_mode def test_invalid_annotation(self, dag_maker): import uuid diff --git a/tests/decorators/test_sensor.py b/tests/decorators/test_sensor.py index e970894a38ed8..a283ed871ba1d 100644 --- a/tests/decorators/test_sensor.py +++ b/tests/decorators/test_sensor.py @@ -26,7 +26,7 @@ from airflow.sensors.base import PokeReturnValue from airflow.utils.state import State -pytestmark = pytest.mark.db_test +pytestmark = [pytest.mark.db_test, pytest.mark.need_serialized_dag] class TestSensorDecorator: @@ -52,6 +52,7 @@ def dummy_f(): sf >> df dr = dag_maker.create_dagrun() + sf.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) tis = dr.get_task_instances() assert len(tis) == 2 diff --git a/tests/decorators/test_short_circuit.py b/tests/decorators/test_short_circuit.py index 1d43de68421f9..1c8349b6c9c86 100644 --- a/tests/decorators/test_short_circuit.py +++ b/tests/decorators/test_short_circuit.py @@ -24,7 +24,7 @@ from airflow.utils.state import State from airflow.utils.trigger_rule import TriggerRule -pytestmark = pytest.mark.db_test +pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] DEFAULT_DATE = datetime(2022, 8, 17) diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index c2993e9ce8d95..4516686a671f7 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -1443,7 +1443,10 @@ def test_check_task_dependencies( # Parameterized tests to check for the correct firing # of the trigger_rule under various circumstances of mapped task # Numeric fields are in order: - # successes, skipped, failed, upstream_failed, done,removed + # successes, skipped, failed, upstream_failed, done,remove + # Does not work for database isolation mode because there is local test monkeypatching of upstream_failed + # That never gets propagated to internal_api + @pytest.mark.skip_if_database_isolation_mode @pytest.mark.parametrize( "trigger_rule, upstream_states, flag_upstream_failed, expect_state, expect_completed", [ @@ -1539,8 +1542,10 @@ def do_something_else(i): monkeypatch.setattr(_UpstreamTIStates, "calculate", lambda *_: upstream_states) ti = dr.get_task_instance("do_something_else", session=session) ti.map_index = 0 + base_task = ti.task + for map_index in range(1, 5): - ti = TaskInstance(dr.task_instances[-1].task, run_id=dr.run_id, map_index=map_index) + ti = TaskInstance(base_task, run_id=dr.run_id, map_index=map_index) session.add(ti) ti.dag_run = dr session.flush() diff --git a/tests/models/test_variable.py b/tests/models/test_variable.py index 3ec2691e5af95..6fb6fa15f214c 100644 --- a/tests/models/test_variable.py +++ b/tests/models/test_variable.py @@ -30,7 +30,7 @@ from tests.test_utils import db from tests.test_utils.config import conf_vars -pytestmark = pytest.mark.db_test +pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] class TestVariable: @@ -47,6 +47,7 @@ def setup_test_cases(self): db.clear_db_variables() crypto._fernet = None + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode, internal API has other fernet @conf_vars({("core", "fernet_key"): "", ("core", "unit_test_mode"): "True"}) def test_variable_no_encryption(self, session): """ @@ -60,6 +61,7 @@ def test_variable_no_encryption(self, session): # should mask anything. That logic is tested in test_secrets_masker.py self.mask_secret.assert_called_once_with("value", "key") + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode, internal API has other fernet @conf_vars({("core", "fernet_key"): Fernet.generate_key().decode()}) def test_variable_with_encryption(self, session): """ @@ -70,6 +72,7 @@ def test_variable_with_encryption(self, session): assert test_var.is_encrypted assert test_var.val == "value" + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode, internal API has other fernet @pytest.mark.parametrize("test_value", ["value", ""]) def test_var_with_encryption_rotate_fernet_key(self, test_value, session): """ @@ -152,6 +155,7 @@ def test_variable_update(self, session): Variable.update(key="test_key", value="value2", session=session) assert "value2" == Variable.get("test_key") + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode, API server has other ENV def test_variable_update_fails_on_non_metastore_variable(self, session): with mock.patch.dict("os.environ", AIRFLOW_VAR_KEY="env-value"): with pytest.raises(AttributeError): @@ -281,6 +285,7 @@ def test_caching_caches(self, mock_ensure_secrets: mock.Mock): mock_backend.get_variable.assert_called_once() # second call was not made because of cache assert first == second + @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode, internal API has other env def test_cache_invalidation_on_set(self, session): with mock.patch.dict("os.environ", AIRFLOW_VAR_KEY="from_env"): a = Variable.get("key") # value is saved in cache @@ -316,7 +321,7 @@ def test_masking_only_secret_values(variable_value, deserialize_json, expected_m val=variable_value, ) session.add(var) - session.flush() + session.commit() # Make sure we re-load it, not just get the cached object back session.expunge(var) _secrets_masker().patterns = set() @@ -326,5 +331,4 @@ def test_masking_only_secret_values(variable_value, deserialize_json, expected_m for expected_masked_value in expected_masked_values: assert expected_masked_value in _secrets_masker().patterns finally: - session.rollback() db.clear_db_variables() diff --git a/tests/operators/test_python.py b/tests/operators/test_python.py index 993d70cad3340..f24281275126d 100644 --- a/tests/operators/test_python.py +++ b/tests/operators/test_python.py @@ -1306,6 +1306,9 @@ def f(a): "AssertRewritingHook including captured stdout and we need to run " "it with `--assert=plain` pytest option and PYTEST_PLAIN_ASSERTS=true .", ) + # TODO(potiuk) check if this can be fixed in the future - for now we are skipping tests with venv + # and airflow context in DB isolation mode as they are passing None as DAG. + @pytest.mark.skip_if_database_isolation_mode def test_airflow_context(self, serializer): def f( # basic diff --git a/tests/providers/opensearch/conftest.py b/tests/providers/opensearch/conftest.py index 47a447188ecdd..934bbd642ad58 100644 --- a/tests/providers/opensearch/conftest.py +++ b/tests/providers/opensearch/conftest.py @@ -19,12 +19,19 @@ from typing import Any import pytest -from opensearchpy import OpenSearch +from airflow.hooks.base import BaseHook from airflow.models import Connection -from airflow.providers.opensearch.hooks.opensearch import OpenSearchHook from airflow.utils import db +try: + from opensearchpy import OpenSearch + + from airflow.providers.opensearch.hooks.opensearch import OpenSearchHook +except ImportError: + OpenSearch = None # type: ignore[assignment, misc] + OpenSearchHook = BaseHook # type: ignore[assignment,misc] + # TODO: FIXME - those Mocks have overrides that are not used but they also do not make Mypy Happy # mypy: disable-error-code="override" diff --git a/tests/providers/opensearch/hooks/test_opensearch.py b/tests/providers/opensearch/hooks/test_opensearch.py index 84360ae73f46a..43075e8532210 100644 --- a/tests/providers/opensearch/hooks/test_opensearch.py +++ b/tests/providers/opensearch/hooks/test_opensearch.py @@ -18,8 +18,9 @@ from unittest import mock -import opensearchpy import pytest + +opensearchpy = pytest.importorskip("opensearchpy") from opensearchpy import Urllib3HttpConnection from airflow.exceptions import AirflowException diff --git a/tests/providers/opensearch/operators/test_opensearch.py b/tests/providers/opensearch/operators/test_opensearch.py index 706112fef65b3..63ad7eafe48de 100644 --- a/tests/providers/opensearch/operators/test_opensearch.py +++ b/tests/providers/opensearch/operators/test_opensearch.py @@ -17,6 +17,9 @@ from __future__ import annotations import pytest + +opensearchpy = pytest.importorskip("opensearchpy") + from opensearchpy import Document, Keyword, Text from airflow.models import DAG diff --git a/tests/sensors/test_external_task_sensor.py b/tests/sensors/test_external_task_sensor.py index fbebd3d120156..e7a5991963e47 100644 --- a/tests/sensors/test_external_task_sensor.py +++ b/tests/sensors/test_external_task_sensor.py @@ -809,6 +809,7 @@ def test_catch_invalid_allowed_states(self): dag=self.dag, ) + @pytest.mark.skip_if_database_isolation_mode # Test is broken in db isolation mode def test_external_task_sensor_waits_for_task_check_existence(self): op = ExternalTaskSensor( task_id="test_external_task_sensor_check", @@ -821,6 +822,7 @@ def test_external_task_sensor_waits_for_task_check_existence(self): with pytest.raises(AirflowException): op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) + @pytest.mark.skip_if_database_isolation_mode # Test is broken in db isolation mode def test_external_task_sensor_waits_for_dag_check_existence(self): op = ExternalTaskSensor( task_id="test_external_task_sensor_check", diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index e9c8ceaf03979..d1c6787db39b6 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -399,6 +399,8 @@ def timetable_plugin(monkeypatch): ) +# TODO: (potiuk) - AIP-44 - check why this test hangs +@pytest.mark.skip_if_database_isolation_mode class TestStringifiedDAGs: """Unit tests for stringified DAGs.""" diff --git a/tests/www/views/test_views_trigger_dag.py b/tests/www/views/test_views_trigger_dag.py index c53213c3e68ea..9b2b2971982af 100644 --- a/tests/www/views/test_views_trigger_dag.py +++ b/tests/www/views/test_views_trigger_dag.py @@ -19,6 +19,7 @@ import datetime import json +from decimal import Decimal from urllib.parse import quote import pytest @@ -28,6 +29,7 @@ from airflow.operators.empty import EmptyOperator from airflow.security import permissions from airflow.utils import timezone +from airflow.utils.json import WebEncoder from airflow.utils.session import create_session from airflow.utils.types import DagRunType from tests.test_utils.api_connexion_utils import create_test_client @@ -92,6 +94,32 @@ def test_trigger_dag_conf(admin_client): assert run.conf == conf_dict +def test_trigger_dag_conf_serializable_fields(admin_client): + test_dag_id = "example_bash_operator" + time_now = timezone.utcnow() + conf_dict = { + "string": "Hello, World!", + "date_str": "2024-08-08T09:57:35.300858", + "datetime": time_now, + "decimal": Decimal(10.465), + } + expected_conf = { + "string": "Hello, World!", + "date_str": "2024-08-08T09:57:35.300858", + "datetime": time_now.isoformat(), + "decimal": 10.465, + } + + admin_client.post(f"dags/{test_dag_id}/trigger", data={"conf": json.dumps(conf_dict, cls=WebEncoder)}) + + with create_session() as session: + run = session.query(DagRun).filter(DagRun.dag_id == test_dag_id).first() + assert run is not None + assert DagRunType.MANUAL in run.run_id + assert run.run_type == DagRunType.MANUAL + assert run.conf == expected_conf + + def test_trigger_dag_conf_malformed(admin_client): test_dag_id = "example_bash_operator" From 6f2121a32f642f6be8e3d7dfe184cb9194921a7e Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Mon, 19 Aug 2024 09:01:49 -0400 Subject: [PATCH 021/161] Fix try selector refresh (#41483) (#41503) --- .../dag/details/taskInstance/TrySelector.tsx | 92 ++++++++----------- 1 file changed, 37 insertions(+), 55 deletions(-) diff --git a/airflow/www/static/js/dag/details/taskInstance/TrySelector.tsx b/airflow/www/static/js/dag/details/taskInstance/TrySelector.tsx index 5fc96803c85de..31d894fcaaa42 100644 --- a/airflow/www/static/js/dag/details/taskInstance/TrySelector.tsx +++ b/airflow/www/static/js/dag/details/taskInstance/TrySelector.tsx @@ -61,63 +61,10 @@ const TrySelector = ({ const logAttemptDropdownLimit = 10; const showDropdown = finalTryNumber > logAttemptDropdownLimit; - const tries = (tiHistory?.taskInstances || []).filter( - (t) => t?.startDate !== taskInstance?.startDate - ); - tries?.push(taskInstance); - return ( Task Tries - {!showDropdown && ( - - {/* Even without try history showing up we should still show all try numbers */} - {Array.from({ length: finalTryNumber }, (_, i) => i + 1).map( - (tryNumber, i) => { - let attempt; - if (tries.length) { - attempt = tries[i]; - } - return ( - - Status: {attempt.state} - - Duration:{" "} - {formatDuration( - getDuration(attempt.startDate, attempt.endDate) - )} - - - ) - } - hasArrow - portalProps={{ containerRef }} - placement="top" - isDisabled={!attempt} - > - - - ); - } - )} - - )} - {showDropdown && ( + {showDropdown ? ( + ) : ( + + {tiHistory?.taskInstances?.map((ti) => ( + + Status: {ti.state} + + Duration:{" "} + {formatDuration(getDuration(ti.startDate, ti.endDate))} + + + } + hasArrow + portalProps={{ containerRef }} + placement="top" + isDisabled={!ti} + > + + + ))} + )} ); From 29270afefa5911b1053b30e04544c3ad3bf3d735 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 19 Aug 2024 15:32:48 +0200 Subject: [PATCH 022/161] Remove debian bullseye support (#41568) (#41569) (cherry picked from commit 5c323a9c562c6e886dbba460fc733a6f8590bf8b) --- .github/workflows/ci-image-build.yml | 2 +- .github/workflows/prod-image-build.yml | 2 +- .github/workflows/prod-image-extra-checks.yml | 21 --- .github/workflows/push-image-cache.yml | 2 +- Dockerfile | 19 +-- Dockerfile.ci | 19 +-- README.md | 12 +- dev/breeze/doc/ci/05_workflows.md | 1 - dev/breeze/doc/ci/06_diagrams.md | 1 - .../doc/images/output_ci-image_build.svg | 116 +++++++------- .../doc/images/output_ci-image_build.txt | 2 +- .../doc/images/output_prod-image_build.svg | 146 +++++++++--------- .../doc/images/output_prod-image_build.txt | 2 +- .../src/airflow_breeze/global_constants.py | 2 +- .../howto/docker-compose/index.rst | 2 +- .../installation/dependencies.rst | 20 --- .../installation/prerequisites.rst | 3 +- docs/docker-stack/build.rst | 21 --- docs/docker-stack/changelog.rst | 4 + .../customizing/debian-bullseye.sh | 37 ----- generated/PYPI_README.md | 4 +- scripts/docker/install_os_dependencies.sh | 19 +-- 22 files changed, 146 insertions(+), 311 deletions(-) delete mode 100755 docs/docker-stack/docker-examples/customizing/debian-bullseye.sh diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 07ba028cf7139..1c4b31b55a604 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -60,7 +60,7 @@ on: # yamllint disable-line rule:truthy default: "true" type: string debian-version: - description: "Base Debian distribution to use for the build (bookworm/bullseye)" + description: "Base Debian distribution to use for the build (bookworm)" type: string default: "bookworm" install-mysql-client-type: diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index c75701c4567de..75d9d0054ec78 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -63,7 +63,7 @@ on: # yamllint disable-line rule:truthy required: true type: string debian-version: - description: "Base Debian distribution to use for the build (bookworm/bullseye)" + description: "Base Debian distribution to use for the build (bookworm)" type: string default: "bookworm" install-mysql-client-type: diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml index 380ecb5a67e63..82d327ba2f16d 100644 --- a/.github/workflows/prod-image-extra-checks.yml +++ b/.github/workflows/prod-image-extra-checks.yml @@ -64,27 +64,6 @@ on: # yamllint disable-line rule:truthy required: true type: string jobs: - bullseye-image: - uses: ./.github/workflows/prod-image-build.yml - with: - runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} - build-type: "Bullseye" - upload-package-artifact: "false" - image-tag: bullseye-${{ inputs.image-tag }} - debian-version: "bullseye" - python-versions: ${{ inputs.python-versions }} - default-python-version: ${{ inputs.default-python-version }} - platform: "linux/amd64" - branch: ${{ inputs.branch }} - # Always build images during the extra checks and never push them - push-image: "false" - use-uv: ${{ inputs.use-uv }} - build-provider-packages: ${{ inputs.build-provider-packages }} - upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} - chicken-egg-providers: ${{ inputs.chicken-egg-providers }} - constraints-branch: ${{ inputs.constraints-branch }} - docker-cache: ${{ inputs.docker-cache }} - myssql-client-image: uses: ./.github/workflows/prod-image-build.yml with: diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index 1cdb5861e43a7..0dc83a3fd66ea 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -41,7 +41,7 @@ on: # yamllint disable-line rule:truthy required: true type: string debian-version: - description: "Base Debian distribution to use for the build (bookworm/bullseye)" + description: "Base Debian distribution to use for the build (bookworm)" type: string default: "bookworm" install-mysql-client-type: diff --git a/Dockerfile b/Dockerfile index 5eb2b3355695f..aff4094603553 100644 --- a/Dockerfile +++ b/Dockerfile @@ -124,11 +124,7 @@ function get_runtime_apt_deps() { echo echo "DEBIAN CODENAME: ${debian_version}" echo - if [[ "${debian_version}" == "bullseye" ]]; then - debian_version_apt_deps="libffi7 libldap-2.4-2 libssl1.1 netcat" - else - debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd" - fi + debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd" echo echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION: ${debian_version}" echo @@ -177,19 +173,6 @@ function install_debian_dev_dependencies() { echo echo "DEBIAN CODENAME: ${debian_version}" echo - if [[ "${debian_version}" == "bullseye" ]]; then - echo - echo "Bullseye detected - replacing dependencies in additional dev apt deps" - echo - # Replace dependencies in additional dev apt deps to be compatible with Bullseye - ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS//libgcc-11-dev/libgcc-10-dev} - ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS//netcat-openbsd/netcat} - echo - echo "Replaced bullseye dev apt dependencies" - echo "${ADDITIONAL_DEV_APT_COMMAND}" - echo - fi - # shellcheck disable=SC2086 apt-get install -y --no-install-recommends ${DEV_APT_DEPS} ${ADDITIONAL_DEV_APT_DEPS} } diff --git a/Dockerfile.ci b/Dockerfile.ci index f46890cd8145c..736656795bfa5 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -70,11 +70,7 @@ function get_runtime_apt_deps() { echo echo "DEBIAN CODENAME: ${debian_version}" echo - if [[ "${debian_version}" == "bullseye" ]]; then - debian_version_apt_deps="libffi7 libldap-2.4-2 libssl1.1 netcat" - else - debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd" - fi + debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd" echo echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION: ${debian_version}" echo @@ -123,19 +119,6 @@ function install_debian_dev_dependencies() { echo echo "DEBIAN CODENAME: ${debian_version}" echo - if [[ "${debian_version}" == "bullseye" ]]; then - echo - echo "Bullseye detected - replacing dependencies in additional dev apt deps" - echo - # Replace dependencies in additional dev apt deps to be compatible with Bullseye - ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS//libgcc-11-dev/libgcc-10-dev} - ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS//netcat-openbsd/netcat} - echo - echo "Replaced bullseye dev apt dependencies" - echo "${ADDITIONAL_DEV_APT_COMMAND}" - echo - fi - # shellcheck disable=SC2086 apt-get install -y --no-install-recommends ${DEV_APT_DEPS} ${ADDITIONAL_DEV_APT_DEPS} } diff --git a/README.md b/README.md index 78ecf1c39fe44..71dbfac4910c6 100644 --- a/README.md +++ b/README.md @@ -122,9 +122,7 @@ The work to add Windows support is tracked via [#10388](https://github.com/apach it is not a high priority. You should only use Linux-based distros as "Production" execution environment as this is the only environment that is supported. The only distro that is used in our CI tests and that is used in the [Community managed DockerHub image](https://hub.docker.com/p/apache/airflow) is -`Debian Bookworm`. We also have support for legacy ``Debian Bullseye`` base image if you want to build a -custom image but it is deprecated and option to do it will be removed in the Dockerfile that -will accompany Airflow 2.9.3 so you are advised to switch to ``Debian Bookworm`` for your custom images. +`Debian Bookworm`. @@ -347,13 +345,9 @@ building and testing the OS version. Approximately 6 months before the end-of-re previous stable version of the OS, Airflow switches the images released to use the latest supported version of the OS. -For example since ``Debian Buster`` end-of-life was August 2022, Airflow switched the images in `main` branch -to use ``Debian Bullseye`` in February/March 2022. The version was used in the next MINOR release after -the switch happened. In case of the Bullseye switch - 2.3.0 version used ``Debian Bullseye``. -The images released in the previous MINOR version continue to use the version that all other releases -for the MINOR version used. Similar switch from ``Debian Bullseye`` to ``Debian Bookworm`` has been implemented +For example switch from ``Debian Bullseye`` to ``Debian Bookworm`` has been implemented before 2.8.0 release in October 2023 and ``Debian Bookworm`` will be the only option supported as of -Airflow 2.9.0. +Airflow 2.10.0. Users will continue to be able to build their images using stable Debian releases until the end of regular support and building and verifying of the images happens in our CI but no unit tests were executed using diff --git a/dev/breeze/doc/ci/05_workflows.md b/dev/breeze/doc/ci/05_workflows.md index 9ea39709c9439..b70a81ef64ce2 100644 --- a/dev/breeze/doc/ci/05_workflows.md +++ b/dev/breeze/doc/ci/05_workflows.md @@ -227,7 +227,6 @@ code. | Build CI images | Builds images in-workflow (not in the build images) | | Yes | Yes (1) | Yes (4) | | Generate constraints/CI verify | Generate constraints for the build and verify CI image | Yes (2) | Yes (2) | Yes (2) | Yes (2) | | Build PROD images | Builds images in-workflow (not in the build images) | | Yes | Yes (1) | Yes (4) | -| Build Bullseye PROD images | Builds images based on Bullseye debian | | Yes | Yes | Yes | | Run breeze tests | Run unit tests for Breeze | Yes | Yes | Yes | Yes | | Test OpenAPI client gen | Tests if OpenAPIClient continues to generate | Yes | Yes | Yes | Yes | | React WWW tests | React UI tests for new Airflow UI | Yes | Yes | Yes | Yes | diff --git a/dev/breeze/doc/ci/06_diagrams.md b/dev/breeze/doc/ci/06_diagrams.md index 89d6fc772c86e..afe51a309e8eb 100644 --- a/dev/breeze/doc/ci/06_diagrams.md +++ b/dev/breeze/doc/ci/06_diagrams.md @@ -379,7 +379,6 @@ sequenceDiagram Tests ->> GitHub Registry: Push PROD Images
[COMMIT_SHA] and Artifacts ->> Tests: Download source constraints - Note over Tests: Build Bullseye PROD Images
[COMMIT_SHA] and GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] Note over Tests: Run static checks diff --git a/dev/breeze/doc/images/output_ci-image_build.svg b/dev/breeze/doc/images/output_ci-image_build.svg index ee363b3c9ac7b..131b618e403ce 100644 --- a/dev/breeze/doc/images/output_ci-image_build.svg +++ b/dev/breeze/doc/images/output_ci-image_build.svg @@ -1,4 +1,4 @@ - +

Wait a minute

+

Please confirm

{{ message }}

{% if details %} diff --git a/tests/www/views/test_views_acl.py b/tests/www/views/test_views_acl.py index 17700749f6e32..9586d81aa6516 100644 --- a/tests/www/views/test_views_acl.py +++ b/tests/www/views/test_views_acl.py @@ -788,7 +788,7 @@ def test_success_fail_for_read_only_task_instance_access(client_only_dags_tis): past="false", ) resp = client_only_dags_tis.post("success", data=form) - check_content_not_in_response("Wait a minute", resp, resp_code=302) + check_content_not_in_response("Please confirm", resp, resp_code=302) GET_LOGS_WITH_METADATA_URL = ( diff --git a/tests/www/views/test_views_decorators.py b/tests/www/views/test_views_decorators.py index f10b3d66847f2..84492a2d52a56 100644 --- a/tests/www/views/test_views_decorators.py +++ b/tests/www/views/test_views_decorators.py @@ -116,7 +116,7 @@ def test_action_logging_post(session, admin_client): only_failed="false", ) resp = admin_client.post("clear", data=form) - check_content_in_response(["example_bash_operator", "Wait a minute"], resp) + check_content_in_response(["example_bash_operator", "Please confirm"], resp) # In mysql backend, this commit() is needed to write down the logs session.commit() _check_last_log( diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index 3eb258f4cd136..0c947c7553228 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -320,12 +320,12 @@ def client_ti_without_dag_edit(app): pytest.param( f"confirm?task_id=runme_0&dag_id=example_bash_operator&state=success" f"&dag_run_id={DEFAULT_DAGRUN}", - ["Wait a minute"], + ["Please confirm"], id="confirm-success", ), pytest.param( f"confirm?task_id=runme_0&dag_id=example_bash_operator&state=failed&dag_run_id={DEFAULT_DAGRUN}", - ["Wait a minute"], + ["Please confirm"], id="confirm-failed", ), pytest.param( From ceb605191bf7eb28ec5c85a91787a998cf5b70b1 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Thu, 22 Aug 2024 21:34:27 +0530 Subject: [PATCH 029/161] Adding url sanitisation for extra links (#41665) (#41680) (cherry picked from commit 6c463b3c37fd9aef56ceace7e7b141d892b57054) --- .../static/js/dag/details/taskInstance/ExtraLinks.tsx | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/airflow/www/static/js/dag/details/taskInstance/ExtraLinks.tsx b/airflow/www/static/js/dag/details/taskInstance/ExtraLinks.tsx index 1cd4c450c3680..06528eab6e7a1 100644 --- a/airflow/www/static/js/dag/details/taskInstance/ExtraLinks.tsx +++ b/airflow/www/static/js/dag/details/taskInstance/ExtraLinks.tsx @@ -53,6 +53,14 @@ const ExtraLinks = ({ const isExternal = (url: string | null) => url && /^(?:[a-z]+:)?\/\//.test(url); + const isSanitised = (url: string | null) => { + if (!url) { + return true; + } + const urlRegex = /^(https?:)/i; + return urlRegex.test(url); + }; + return ( Extra Links @@ -63,7 +71,7 @@ const ExtraLinks = ({ as={Link} colorScheme="blue" href={url} - isDisabled={!url} + isDisabled={!isSanitised(url)} target={isExternal(url) ? "_blank" : undefined} mr={2} > From 03e01e76d2203d37aa645096df195b4328665f6d Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 23 Aug 2024 19:53:38 +0800 Subject: [PATCH 030/161] Splitting syspath preparation into stages (#41672) (#41694) Co-authored-by: Amogh Desai --- airflow/settings.py | 17 ++++++++++------- tests/core/test_settings.py | 33 +++++++++++++++++++++++++++------ 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/airflow/settings.py b/airflow/settings.py index 175a63f69d2f4..dc24a2c5acc5a 100644 --- a/airflow/settings.py +++ b/airflow/settings.py @@ -675,11 +675,8 @@ def configure_action_logging() -> None: """Any additional configuration (register callback) for airflow.utils.action_loggers module.""" -def prepare_syspath(): - """Ensure certain subfolders of AIRFLOW_HOME are on the classpath.""" - if DAGS_FOLDER not in sys.path: - sys.path.append(DAGS_FOLDER) - +def prepare_syspath_for_config_and_plugins(): + """Update sys.path for the config and plugins directories.""" # Add ./config/ for loading custom log parsers etc, or # airflow_local_settings etc. config_path = os.path.join(AIRFLOW_HOME, "config") @@ -690,6 +687,12 @@ def prepare_syspath(): sys.path.append(PLUGINS_FOLDER) +def prepare_syspath_for_dags_folder(): + """Update sys.path to include the DAGs folder.""" + if DAGS_FOLDER not in sys.path: + sys.path.append(DAGS_FOLDER) + + def get_session_lifetime_config(): """Get session timeout configs and handle outdated configs gracefully.""" session_lifetime_minutes = conf.get("webserver", "session_lifetime_minutes", fallback=None) @@ -771,12 +774,13 @@ def import_local_settings(): def initialize(): """Initialize Airflow with all the settings from this file.""" configure_vars() - prepare_syspath() + prepare_syspath_for_config_and_plugins() configure_policy_plugin_manager() # Load policy plugins _before_ importing airflow_local_settings, as Pluggy uses LIFO and we want anything # in airflow_local_settings to take precendec load_policy_plugins(POLICY_PLUGIN_MANAGER) import_local_settings() + prepare_syspath_for_dags_folder() global LOGGING_CLASS_PATH LOGGING_CLASS_PATH = configure_logging() State.state_color.update(STATE_COLORS) @@ -806,7 +810,6 @@ def is_usage_data_collection_enabled() -> bool: MEGABYTE = KILOBYTE * KILOBYTE WEB_COLORS = {"LIGHTBLUE": "#4d9de0", "LIGHTORANGE": "#FF9933"} - # Updating serialized DAG can not be faster than a minimum interval to reduce database # write rate. MIN_SERIALIZED_DAG_UPDATE_INTERVAL = conf.getint("core", "min_serialized_dag_update_interval", fallback=30) diff --git a/tests/core/test_settings.py b/tests/core/test_settings.py index d05344bfa91d8..483ef24e25f7f 100644 --- a/tests/core/test_settings.py +++ b/tests/core/test_settings.py @@ -115,21 +115,42 @@ def teardown_method(self): for mod in [m for m in sys.modules if m not in self.old_modules]: del sys.modules[mod] + @mock.patch("airflow.settings.prepare_syspath_for_config_and_plugins") @mock.patch("airflow.settings.import_local_settings") - @mock.patch("airflow.settings.prepare_syspath") - def test_initialize_order(self, prepare_syspath, import_local_settings): + @mock.patch("airflow.settings.prepare_syspath_for_dags_folder") + def test_initialize_order( + self, + mock_prepare_syspath_for_dags_folder, + mock_import_local_settings, + mock_prepare_syspath_for_config_and_plugins, + ): """ - Tests that import_local_settings is called after prepare_classpath + Tests that import_local_settings is called between prepare_syspath_for_config_and_plugins + and prepare_syspath_for_dags_folder """ mock_local_settings = mock.Mock() - mock_local_settings.attach_mock(prepare_syspath, "prepare_syspath") - mock_local_settings.attach_mock(import_local_settings, "import_local_settings") + + mock_local_settings.attach_mock( + mock_prepare_syspath_for_config_and_plugins, "prepare_syspath_for_config_and_plugins" + ) + mock_local_settings.attach_mock(mock_import_local_settings, "import_local_settings") + mock_local_settings.attach_mock( + mock_prepare_syspath_for_dags_folder, "prepare_syspath_for_dags_folder" + ) import airflow.settings airflow.settings.initialize() - mock_local_settings.assert_has_calls([call.prepare_syspath(), call.import_local_settings()]) + expected_calls = [ + call.prepare_syspath_for_config_and_plugins(), + call.import_local_settings(), + call.prepare_syspath_for_dags_folder(), + ] + + mock_local_settings.assert_has_calls(expected_calls) + + assert mock_local_settings.mock_calls == expected_calls def test_import_with_dunder_all_not_specified(self): """ From 9deba7b9a2d602b6c7d304cf43e1c615628b7a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=90=98=E7=BF=8A=E4=BF=AE?= Date: Fri, 23 Aug 2024 22:27:53 +0800 Subject: [PATCH 031/161] fix log for notifier(instance) without __name__ (#41591) (#41699) Co-authored-by: obarisk Co-authored-by: Tzu-ping Chung --- airflow/models/taskinstance.py | 15 +++++++++++--- tests/models/test_taskinstance.py | 33 ++++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index baa4e3eed0c20..0c34d350247fb 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -1550,12 +1550,21 @@ def _run_finished_callback( """ if callbacks: callbacks = callbacks if isinstance(callbacks, list) else [callbacks] - for callback in callbacks: - log.info("Executing %s callback", callback.__name__) + + def get_callback_representation(callback: TaskStateChangeCallback) -> Any: + with contextlib.suppress(AttributeError): + return callback.__name__ + with contextlib.suppress(AttributeError): + return callback.__class__.__name__ + return callback + + for idx, callback in enumerate(callbacks): + callback_repr = get_callback_representation(callback) + log.info("Executing callback at index %d: %s", idx, callback_repr) try: callback(context) except Exception: - log.exception("Error when executing %s callback", callback.__name__) # type: ignore[attr-defined] + log.exception("Error in callback at index %d: %s", idx, callback_repr) def _log_state(*, task_instance: TaskInstance | TaskInstancePydantic, lead_msg: str = "") -> None: diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index 86357bad8fe90..cbd38e9b390c8 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -76,6 +76,7 @@ from airflow.models.taskreschedule import TaskReschedule from airflow.models.variable import Variable from airflow.models.xcom import LazyXComSelectSequence, XCom +from airflow.notifications.basenotifier import BaseNotifier from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator @@ -3421,7 +3422,9 @@ def on_execute_callable(context): ti.refresh_from_db() assert ti.state == State.SUCCESS - def test_finished_callbacks_handle_and_log_exception(self, caplog): + def test_finished_callbacks_callable_handle_and_log_exception(self, caplog): + called = completed = False + def on_finish_callable(context): nonlocal called, completed called = True @@ -3437,8 +3440,32 @@ def on_finish_callable(context): assert not completed callback_name = callback_input[0] if isinstance(callback_input, list) else callback_input callback_name = qualname(callback_name).split(".")[-1] - assert "Executing on_finish_callable callback" in caplog.text - assert "Error when executing on_finish_callable callback" in caplog.text + assert "Executing callback at index 0: on_finish_callable" in caplog.text + assert "Error in callback at index 0: on_finish_callable" in caplog.text + + def test_finished_callbacks_notifier_handle_and_log_exception(self, caplog): + class OnFinishNotifier(BaseNotifier): + """ + error captured by BaseNotifier + """ + + def __init__(self, error: bool): + super().__init__() + self.raise_error = error + + def notify(self, context): + self.execute() + + def execute(self) -> None: + if self.raise_error: + raise KeyError + + caplog.clear() + callbacks = [OnFinishNotifier(error=False), OnFinishNotifier(error=True)] + _run_finished_callback(callbacks=callbacks, context={}) + assert "Executing callback at index 0: OnFinishNotifier" in caplog.text + assert "Executing callback at index 1: OnFinishNotifier" in caplog.text + assert "KeyError" in caplog.text @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @provide_session From e8f981131bb82036d79cf7d0cade7d9cd7c03fd5 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Tue, 27 Aug 2024 13:42:30 +0530 Subject: [PATCH 032/161] Adding rel property to hyperlinks in logs (#41696) (#41783) * Adding rel property to hyperlinks in logs * fixing tests (cherry picked from commit 79db243d03cc4406290597ad400ab0f514975c79) --- .../static/js/dag/details/taskInstance/Logs/utils.test.tsx | 4 ++-- airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/www/static/js/dag/details/taskInstance/Logs/utils.test.tsx b/airflow/www/static/js/dag/details/taskInstance/Logs/utils.test.tsx index bcebb22ec9a05..ce6b082cb1999 100644 --- a/airflow/www/static/js/dag/details/taskInstance/Logs/utils.test.tsx +++ b/airflow/www/static/js/dag/details/taskInstance/Logs/utils.test.tsx @@ -147,10 +147,10 @@ describe("Test Logs Utils.", () => { const lines = parsedLogs!.split("\n"); expect(lines[lines.length - 1]).toContain( - 'https://apple.com' + 'https://apple.com' ); expect(lines[lines.length - 1]).toContain( - 'https://google.com' + 'https://google.com' ); }); }); diff --git a/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts b/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts index 2123f4800186d..4be81a6d0cb19 100644 --- a/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts +++ b/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts @@ -137,7 +137,7 @@ export const parseLogs = ( const lineWithHyperlinks = coloredLine .replace( urlRegex, - '$1' + '$1' ) .replace(logGroupStart, (textLine) => { const unfoldIdSuffix = "_unfold"; From da1820b1f7b1c0ecd9af5f50cdd31db847fb269a Mon Sep 17 00:00:00 2001 From: Jens Scheffler <95105677+jscheffl@users.noreply.github.com> Date: Tue, 27 Aug 2024 15:11:08 +0200 Subject: [PATCH 033/161] Bump micromatch from 4.0.5 to 4.0.8 in /airflow/www (#41755) Bumps [micromatch](https://github.com/micromatch/micromatch) from 4.0.5 to 4.0.8. - [Release notes](https://github.com/micromatch/micromatch/releases) - [Changelog](https://github.com/micromatch/micromatch/blob/4.0.8/CHANGELOG.md) - [Commits](https://github.com/micromatch/micromatch/compare/4.0.5...4.0.8) --- updated-dependencies: - dependency-name: micromatch dependency-type: indirect ... Signed-off-by: dependabot[bot] (cherry picked from commit e02052b07787e747a78938e60c206d324ef57e7e) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- airflow/www/yarn.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/airflow/www/yarn.lock b/airflow/www/yarn.lock index 20d9ce752095f..9995fa5213a6f 100644 --- a/airflow/www/yarn.lock +++ b/airflow/www/yarn.lock @@ -4477,7 +4477,7 @@ brace-expansion@^2.0.1: dependencies: balanced-match "^1.0.0" -braces@^3.0.2, braces@^3.0.3: +braces@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== @@ -8853,11 +8853,11 @@ micromark@^3.0.0: uvu "^0.5.0" micromatch@^4.0.4, micromatch@^4.0.5: - version "4.0.5" - resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6" - integrity sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA== + version "4.0.8" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202" + integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA== dependencies: - braces "^3.0.2" + braces "^3.0.3" picomatch "^2.3.1" microseconds@0.2.0: From af3566f907d5d9ba168902ee64b64a3f2dca0e26 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 27 Aug 2024 16:02:41 +0200 Subject: [PATCH 034/161] chore(docs): add an example for auth with keycloak (#41687) (#41791) * chore(docs): add an example for auth with keycloak Added a new section in the authentication documentation that provides a code of configuring Airflow to work with Keycloak. * chore(docs): add an example for auth with keycloak Fix spelling and styling * chore(docs): add an example for auth with keycloak Fix static checks Co-authored-by: Natsu <34879762+hoalongnatsu@users.noreply.github.com> --- .../auth-manager/webserver-authentication.rst | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/docs/apache-airflow-providers-fab/auth-manager/webserver-authentication.rst b/docs/apache-airflow-providers-fab/auth-manager/webserver-authentication.rst index feabad33a806f..48c8c8f1b1f25 100644 --- a/docs/apache-airflow-providers-fab/auth-manager/webserver-authentication.rst +++ b/docs/apache-airflow-providers-fab/auth-manager/webserver-authentication.rst @@ -229,3 +229,94 @@ webserver_config.py itself if you wish. roles = map_roles(teams) log.debug(f"User info from Github: {user_data}\nTeam info from Github: {teams}") return {"username": "github_" + user_data.get("login"), "role_keys": roles} + +Example using team based Authorization with KeyCloak +'''''''''''''''''''''''''''''''''''''''''''''''''''''''' +Here is an example of what you might have in your webserver_config.py: + +.. code-block:: python + + import os + import jwt + import requests + import logging + from base64 import b64decode + from cryptography.hazmat.primitives import serialization + from flask_appbuilder.security.manager import AUTH_DB, AUTH_OAUTH + from airflow import configuration as conf + from airflow.www.security import AirflowSecurityManager + + log = logging.getLogger(__name__) + + AUTH_TYPE = AUTH_OAUTH + AUTH_USER_REGISTRATION = True + AUTH_ROLES_SYNC_AT_LOGIN = True + AUTH_USER_REGISTRATION_ROLE = "Viewer" + OIDC_ISSUER = "https://sso.keycloak.me/realms/airflow" + + # Make sure you create these role on Keycloak + AUTH_ROLES_MAPPING = { + "Viewer": ["Viewer"], + "Admin": ["Admin"], + "User": ["User"], + "Public": ["Public"], + "Op": ["Op"], + } + + OAUTH_PROVIDERS = [ + { + "name": "keycloak", + "icon": "fa-key", + "token_key": "access_token", + "remote_app": { + "client_id": "airflow", + "client_secret": "xxx", + "server_metadata_url": "https://sso.keycloak.me/realms/airflow/.well-known/openid-configuration", + "api_base_url": "https://sso.keycloak.me/realms/airflow/protocol/openid-connect", + "client_kwargs": {"scope": "email profile"}, + "access_token_url": "https://sso.keycloak.me/realms/airflow/protocol/openid-connect/token", + "authorize_url": "https://sso.keycloak.me/realms/airflow/protocol/openid-connect/auth", + "request_token_url": None, + }, + } + ] + + # Fetch public key + req = requests.get(OIDC_ISSUER) + key_der_base64 = req.json()["public_key"] + key_der = b64decode(key_der_base64.encode()) + public_key = serialization.load_der_public_key(key_der) + + + class CustomSecurityManager(AirflowSecurityManager): + def oauth_user_info(self, provider, response): + if provider == "keycloak": + token = response["access_token"] + me = jwt.decode(token, public_key, algorithms=["HS256", "RS256"]) + + # Extract roles from resource access + realm_access = me.get("realm_access", {}) + groups = realm_access.get("roles", []) + + log.info("groups: {0}".format(groups)) + + if not groups: + groups = ["Viewer"] + + userinfo = { + "username": me.get("preferred_username"), + "email": me.get("email"), + "first_name": me.get("given_name"), + "last_name": me.get("family_name"), + "role_keys": groups, + } + + log.info("user info: {0}".format(userinfo)) + + return userinfo + else: + return {} + + + # Make sure to replace this with your own implementation of AirflowSecurityManager class + SECURITY_MANAGER_CLASS = CustomSecurityManager From 991906c2d0f0fc73931118192dbb4dba9e4315b7 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 27 Aug 2024 16:44:00 +0200 Subject: [PATCH 035/161] Remove deprecation warning for cgitb in Plugins Manager (#41732) (#41793) (cherry picked from commit 83ba17f41ee1f88c041289e8b88cf815bc61de7e) Co-authored-by: Jens Scheffler <95105677+jscheffl@users.noreply.github.com> --- airflow/plugins_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py index 76d72a4585009..8ccdef2c6390c 100644 --- a/airflow/plugins_manager.py +++ b/airflow/plugins_manager.py @@ -27,7 +27,6 @@ import os import sys import types -from cgitb import Hook from pathlib import Path from typing import TYPE_CHECKING, Any, Iterable @@ -78,7 +77,7 @@ registered_operator_link_classes: dict[str, type] | None = None registered_ti_dep_classes: dict[str, type] | None = None timetable_classes: dict[str, type[Timetable]] | None = None -hook_lineage_reader_classes: list[type[Hook]] | None = None +hook_lineage_reader_classes: list[type[HookLineageReader]] | None = None priority_weight_strategy_classes: dict[str, type[PriorityWeightStrategy]] | None = None """ Mapping of class names to class of OperatorLinks registered by plugins. From 4a346a07aeb326a5a588e437dc9e3173fa8782c3 Mon Sep 17 00:00:00 2001 From: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> Date: Wed, 28 Aug 2024 00:34:13 -0600 Subject: [PATCH 036/161] Don't Fail LocalTaskJob on heartbeat (#41704) (#41810) * Never fail an ltj over a heartbeat * Log a warning on failed heartbeat * Avoid using f-string in log * Remove unnecessary pass statement (cherry picked from commit 6647610a8e8e3de4d2bfb701e16d1c7b42edd3f8) Co-authored-by: Collin McNulty --- airflow/jobs/local_task_job_runner.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/airflow/jobs/local_task_job_runner.py b/airflow/jobs/local_task_job_runner.py index 48eb547a19383..95a471f239a66 100644 --- a/airflow/jobs/local_task_job_runner.py +++ b/airflow/jobs/local_task_job_runner.py @@ -208,9 +208,14 @@ def sigusr2_debug_handler(signum, frame): if span.is_recording(): span.add_event(name="perform_heartbeat") - perform_heartbeat( - job=self.job, heartbeat_callback=self.heartbeat_callback, only_if_necessary=False - ) + try: + perform_heartbeat( + job=self.job, heartbeat_callback=self.heartbeat_callback, only_if_necessary=False + ) + except Exception as e: + # Failing the heartbeat should never kill the localtaskjob + # If it repeatedly can't heartbeat, it will be marked as a zombie anyhow + self.log.warning("Heartbeat failed with Exception: %s", e) # If it's been too long since we've heartbeat, then it's possible that # the scheduler rescheduled this task, so kill launched processes. From 556552940c582a70b462133d83476baa708ebde7 Mon Sep 17 00:00:00 2001 From: Joao Amaral <7281460+joaopamaral@users.noreply.github.com> Date: Wed, 28 Aug 2024 07:41:52 -0300 Subject: [PATCH 037/161] Keep FAB compatibility for versions before 1.3.0 in 2.10 (#41549) (#41809) * Fix: Keep compatibility with old FAB versions (#41549) * Fix: Tests after #41549 (Keep compatibility with old FAB versions) * Fix test_dag and test_dagbag --- airflow/models/dag.py | 30 ++++--- tests/models/test_dag.py | 80 ++++++++++++++++++- tests/models/test_dagbag.py | 12 ++- tests/serialization/test_dag_serialization.py | 7 ++ 4 files changed, 115 insertions(+), 14 deletions(-) diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 5011b3ebb2b35..ace9650f50f28 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -57,6 +57,7 @@ import re2 import sqlalchemy_jsonfield from dateutil.relativedelta import relativedelta +from packaging import version as packaging_version from sqlalchemy import ( Boolean, Column, @@ -116,6 +117,7 @@ clear_task_instances, ) from airflow.models.tasklog import LogTemplate +from airflow.providers.fab import __version__ as FAB_VERSION from airflow.secrets.local_filesystem import LocalFilesystemBackend from airflow.security import permissions from airflow.settings import json @@ -940,16 +942,26 @@ def update_old_perm(permission: str): updated_access_control = {} for role, perms in access_control.items(): - updated_access_control[role] = updated_access_control.get(role, {}) - if isinstance(perms, (set, list)): - # Support for old-style access_control where only the actions are specified - updated_access_control[role][permissions.RESOURCE_DAG] = set(perms) + if packaging_version.parse(FAB_VERSION) >= packaging_version.parse("1.3.0"): + updated_access_control[role] = updated_access_control.get(role, {}) + if isinstance(perms, (set, list)): + # Support for old-style access_control where only the actions are specified + updated_access_control[role][permissions.RESOURCE_DAG] = set(perms) + else: + updated_access_control[role] = perms + if permissions.RESOURCE_DAG in updated_access_control[role]: + updated_access_control[role][permissions.RESOURCE_DAG] = { + update_old_perm(perm) + for perm in updated_access_control[role][permissions.RESOURCE_DAG] + } + elif isinstance(perms, dict): + # Not allow new access control format with old FAB versions + raise AirflowException( + "Please upgrade the FAB provider to a version >= 1.3.0 to allow " + "use the Dag Level Access Control new format." + ) else: - updated_access_control[role] = perms - if permissions.RESOURCE_DAG in updated_access_control[role]: - updated_access_control[role][permissions.RESOURCE_DAG] = { - update_old_perm(perm) for perm in updated_access_control[role][permissions.RESOURCE_DAG] - } + updated_access_control[role] = {update_old_perm(perm) for perm in perms} return updated_access_control diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index d2b23c02654f3..6e7d2d3894a26 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -40,6 +40,7 @@ import pytest import time_machine from dateutil.relativedelta import relativedelta +from packaging import version as packaging_version from pendulum.tz.timezone import Timezone from sqlalchemy import inspect, select from sqlalchemy.exc import SAWarning @@ -85,6 +86,7 @@ from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator from airflow.operators.subdag import SubDagOperator +from airflow.providers.fab import __version__ as FAB_VERSION from airflow.security import permissions from airflow.templates import NativeEnvironment, SandboxedEnvironment from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction, Timetable @@ -2767,12 +2769,20 @@ def test_replace_outdated_access_control_actions(self): outdated_permissions = { "role1": {permissions.ACTION_CAN_READ, permissions.ACTION_CAN_EDIT}, "role2": {permissions.DEPRECATED_ACTION_CAN_DAG_READ, permissions.DEPRECATED_ACTION_CAN_DAG_EDIT}, - "role3": {permissions.RESOURCE_DAG_RUN: {permissions.ACTION_CAN_CREATE}}, + "role3": self._get_compatible_access_control( + {permissions.RESOURCE_DAG_RUN: {permissions.ACTION_CAN_CREATE}} + ), } updated_permissions = { - "role1": {permissions.RESOURCE_DAG: {permissions.ACTION_CAN_READ, permissions.ACTION_CAN_EDIT}}, - "role2": {permissions.RESOURCE_DAG: {permissions.ACTION_CAN_READ, permissions.ACTION_CAN_EDIT}}, - "role3": {permissions.RESOURCE_DAG_RUN: {permissions.ACTION_CAN_CREATE}}, + "role1": self._get_compatible_access_control( + {permissions.RESOURCE_DAG: {permissions.ACTION_CAN_READ, permissions.ACTION_CAN_EDIT}} + ), + "role2": self._get_compatible_access_control( + {permissions.RESOURCE_DAG: {permissions.ACTION_CAN_READ, permissions.ACTION_CAN_EDIT}} + ), + "role3": self._get_compatible_access_control( + {permissions.RESOURCE_DAG_RUN: {permissions.ACTION_CAN_CREATE}} + ), } with pytest.warns(DeprecationWarning) as deprecation_warnings: @@ -2789,6 +2799,68 @@ def test_replace_outdated_access_control_actions(self): assert "permission is deprecated" in str(deprecation_warnings[0].message) assert "permission is deprecated" in str(deprecation_warnings[1].message) + def _get_compatible_access_control(self, perms): + if packaging_version.parse(FAB_VERSION) >= packaging_version.parse("1.3.0"): + return perms + return perms.get(permissions.RESOURCE_DAG, set()) + + @pytest.mark.parametrize( + "fab_version, perms, expected_exception, expected_perms", + [ + pytest.param( + "1.2.0", + { + "role1": {permissions.ACTION_CAN_READ, permissions.ACTION_CAN_EDIT}, + "role3": {permissions.RESOURCE_DAG_RUN: {permissions.ACTION_CAN_CREATE}}, + # will raise error in old FAB with new access control format + }, + AirflowException, + None, + id="old_fab_new_access_control_format", + ), + pytest.param( + "1.2.0", + { + "role1": [ + permissions.ACTION_CAN_READ, + permissions.ACTION_CAN_EDIT, + permissions.ACTION_CAN_READ, + ], + }, + None, + {"role1": {permissions.ACTION_CAN_READ, permissions.ACTION_CAN_EDIT}}, + id="old_fab_old_access_control_format", + ), + pytest.param( + "1.3.0", + { + "role1": {permissions.ACTION_CAN_READ, permissions.ACTION_CAN_EDIT}, # old format + "role3": {permissions.RESOURCE_DAG_RUN: {permissions.ACTION_CAN_CREATE}}, # new format + }, + None, + { + "role1": { + permissions.RESOURCE_DAG: {permissions.ACTION_CAN_READ, permissions.ACTION_CAN_EDIT} + }, + "role3": {permissions.RESOURCE_DAG_RUN: {permissions.ACTION_CAN_CREATE}}, + }, + id="new_fab_mixed_access_control_format", + ), + ], + ) + def test_access_control_format(self, fab_version, perms, expected_exception, expected_perms): + if expected_exception: + with patch("airflow.models.dag.FAB_VERSION", fab_version): + with pytest.raises( + expected_exception, + match="Please upgrade the FAB provider to a version >= 1.3.0 to allow use the Dag Level Access Control new format.", + ): + DAG(dag_id="dag_test", schedule=None, access_control=perms) + else: + with patch("airflow.models.dag.FAB_VERSION", fab_version): + dag = DAG(dag_id="dag_test", schedule=None, access_control=perms) + assert dag.access_control == expected_perms + def test_validate_executor_field_executor_not_configured(self): dag = DAG("test-dag", schedule=None) EmptyOperator(task_id="t1", dag=dag, executor="test.custom.executor") diff --git a/tests/models/test_dagbag.py b/tests/models/test_dagbag.py index 5e8f3c4af97fd..034e7e2b3a625 100644 --- a/tests/models/test_dagbag.py +++ b/tests/models/test_dagbag.py @@ -31,6 +31,7 @@ import pytest import time_machine +from packaging import version as packaging_version from sqlalchemy import func from sqlalchemy.exc import OperationalError @@ -40,6 +41,7 @@ from airflow.models.dag import DAG, DagModel from airflow.models.dagbag import DagBag from airflow.models.serialized_dag import SerializedDagModel +from airflow.providers.fab import __version__ as FAB_VERSION from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils.dates import timezone as tz from airflow.utils.session import create_session @@ -996,11 +998,19 @@ def _sync_perms(): dag.access_control = {"Public": {"can_read"}} _sync_perms() mock_sync_perm_for_dag.assert_called_once_with( - "test_example_bash_operator", {"Public": {"DAGs": {"can_read"}}} + "test_example_bash_operator", + { + "Public": {"DAGs": {"can_read"}} + if packaging_version.parse(FAB_VERSION) >= packaging_version.parse("1.3.0") + else {"can_read"} + }, ) @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @patch("airflow.www.security_appless.ApplessAirflowSecurityManager") + @pytest.mark.skipif( + packaging_version.parse(FAB_VERSION) < packaging_version.parse("1.3.0"), reason="Requires FAB 1.3.0+" + ) def test_sync_perm_for_dag_with_dict_access_control(self, mock_security_manager): """ Test that dagbag._sync_perm_for_dag will call ApplessAirflowSecurityManager.sync_perm_for_dag diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index 277923423e2c6..d7f09c20ff9d3 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -40,6 +40,7 @@ import pytest from dateutil.relativedelta import FR, relativedelta from kubernetes.client import models as k8s +from packaging import version as packaging_version import airflow from airflow.datasets import Dataset @@ -58,6 +59,7 @@ from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator +from airflow.providers.fab import __version__ as FAB_VERSION from airflow.security import permissions from airflow.sensors.bash import BashSensor from airflow.serialization.dag_dependency import DagDependency @@ -246,6 +248,11 @@ def detect_task_dependencies(task: Operator) -> DagDependency | None: # type: i } }, } + if packaging_version.parse(FAB_VERSION) >= packaging_version.parse("1.3.0") + else { + "__type": "set", + "__var": [permissions.ACTION_CAN_READ, permissions.ACTION_CAN_EDIT], + } }, }, "edge_info": {}, From f986fdb6b5502887f728cf966bbe1fdd8025f800 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 28 Aug 2024 13:21:07 +0200 Subject: [PATCH 038/161] Pin universal-pathlib to 0.2.2 as 0.2.3 generates static code check errors (#41715) (#41820) (cherry picked from commit 1c53961fd4cc1f9085680bf22fbe7f57e29948d4) Co-authored-by: Jens Scheffler <95105677+jscheffl@users.noreply.github.com> --- hatch_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hatch_build.py b/hatch_build.py index 110ebdb77251c..2f29d368667f3 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -498,7 +498,7 @@ # We should also remove "3rd-party-licenses/LICENSE-unicodecsv.txt" file when we remove this dependency "unicodecsv>=0.14.1", # The Universal Pathlib provides Pathlib-like interface for FSSPEC - "universal-pathlib>=0.2.2", + "universal-pathlib==0.2.2", # Temporarily pin to 0.2.2 as 0.2.3 generates mypy errors # Werkzug 3 breaks Flask-Login 0.6.2, also connexion needs to be updated to >= 3.0 # we should remove this limitation when FAB supports Flask 2.3 and we migrate connexion to 3+ "werkzeug>=2.0,<3", From 1bcf94bdc3b385c9a8d84d109d65b73060f8eb19 Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Wed, 28 Aug 2024 18:18:42 +0530 Subject: [PATCH 039/161] Fix: DAGs are not marked as stale if the dags folder change (#41433) (#41829) * Fix: DAGs are not marked as stale if the AIRFLOW__CORE__DAGS_FOLDER changes * Update airflow/dag_processing/manager.py * Add testcase * Add code comment * Update code comment * Update the logic for checking the current dag_directory * Update testcases * Remove unwanted code * Uncomment code * Add processor_subdir when creating processor_subdir * Fix test_retry_still_in_executor test * Remove config from test * Update airflow/dag_processing/manager.py Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> * Update if condition for readability --------- Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> (cherry picked from commit 9f30a41874454696ae2b215b2d86cb9a62968006) --- airflow/dag_processing/manager.py | 10 +++- tests/dag_processing/test_job_runner.py | 71 +++++++++++++++++++++++-- tests/jobs/test_scheduler_job.py | 1 + 3 files changed, 76 insertions(+), 6 deletions(-) diff --git a/airflow/dag_processing/manager.py b/airflow/dag_processing/manager.py index c03bc074d0abd..819da5d7e1d74 100644 --- a/airflow/dag_processing/manager.py +++ b/airflow/dag_processing/manager.py @@ -526,14 +526,20 @@ def deactivate_stale_dags( dags_parsed = session.execute(query) for dag in dags_parsed: + # When the DAG processor runs as part of the scheduler, and the user changes the DAGs folder, + # DAGs from the previous DAGs folder will be marked as stale. Note that this change has no impact + # on standalone DAG processors. + dag_not_in_current_dag_folder = os.path.commonpath([dag.fileloc, dag_directory]) != dag_directory # The largest valid difference between a DagFileStat's last_finished_time and a DAG's # last_parsed_time is the processor_timeout. Longer than that indicates that the DAG is # no longer present in the file. We have a stale_dag_threshold configured to prevent a # significant delay in deactivation of stale dags when a large timeout is configured - if ( + dag_removed_from_dag_folder_or_file = ( dag.fileloc in last_parsed and (dag.last_parsed_time + timedelta(seconds=stale_dag_threshold)) < last_parsed[dag.fileloc] - ): + ) + + if dag_not_in_current_dag_folder or dag_removed_from_dag_folder_or_file: cls.logger().info("DAG %s is missing and will be deactivated.", dag.dag_id) to_deactivate.add(dag.dag_id) diff --git a/tests/dag_processing/test_job_runner.py b/tests/dag_processing/test_job_runner.py index 8112b7222a697..b5d0b35580b4a 100644 --- a/tests/dag_processing/test_job_runner.py +++ b/tests/dag_processing/test_job_runner.py @@ -26,6 +26,7 @@ import random import socket import sys +import tempfile import textwrap import threading import time @@ -638,7 +639,7 @@ def test_scan_stale_dags(self): manager = DagProcessorJobRunner( job=Job(), processor=DagFileProcessorManager( - dag_directory="directory", + dag_directory=str(TEST_DAG_FOLDER), max_runs=1, processor_timeout=timedelta(minutes=10), signal_conn=MagicMock(), @@ -712,11 +713,11 @@ def test_scan_stale_dags_standalone_mode(self): """ Ensure only dags from current dag_directory are updated """ - dag_directory = "directory" + dag_directory = str(TEST_DAG_FOLDER) manager = DagProcessorJobRunner( job=Job(), processor=DagFileProcessorManager( - dag_directory=dag_directory, + dag_directory=TEST_DAG_FOLDER, max_runs=1, processor_timeout=timedelta(minutes=10), signal_conn=MagicMock(), @@ -740,7 +741,7 @@ def test_scan_stale_dags_standalone_mode(self): # Add stale DAG to the DB other_dag = other_dagbag.get_dag("test_start_date_scheduling") other_dag.last_parsed_time = timezone.utcnow() - other_dag.sync_to_db(processor_subdir="other") + other_dag.sync_to_db(processor_subdir="/other") # Add DAG to the file_parsing_stats stat = DagFileStat( @@ -762,6 +763,68 @@ def test_scan_stale_dags_standalone_mode(self): active_dag_count = session.query(func.count(DagModel.dag_id)).filter(DagModel.is_active).scalar() assert active_dag_count == 1 + def test_scan_stale_dags_when_dag_folder_change(self): + """ + Ensure dags from old dag_folder is marked as stale when dag processor + is running as part of scheduler. + """ + + def get_dag_string(filename) -> str: + return open(TEST_DAG_FOLDER / filename).read() + + with tempfile.TemporaryDirectory() as tmpdir: + old_dag_home = tempfile.mkdtemp(dir=tmpdir) + old_dag_file = tempfile.NamedTemporaryFile(dir=old_dag_home, suffix=".py") + old_dag_file.write(get_dag_string("test_example_bash_operator.py").encode()) + old_dag_file.flush() + new_dag_home = tempfile.mkdtemp(dir=tmpdir) + new_dag_file = tempfile.NamedTemporaryFile(dir=new_dag_home, suffix=".py") + new_dag_file.write(get_dag_string("test_scheduler_dags.py").encode()) + new_dag_file.flush() + + manager = DagProcessorJobRunner( + job=Job(), + processor=DagFileProcessorManager( + dag_directory=new_dag_home, + max_runs=1, + processor_timeout=timedelta(minutes=10), + signal_conn=MagicMock(), + dag_ids=[], + pickle_dags=False, + async_mode=True, + ), + ) + + dagbag = DagBag(old_dag_file.name, read_dags_from_db=False) + other_dagbag = DagBag(new_dag_file.name, read_dags_from_db=False) + + with create_session() as session: + # Add DAG from old dah home to the DB + dag = dagbag.get_dag("test_example_bash_operator") + dag.fileloc = old_dag_file.name + dag.last_parsed_time = timezone.utcnow() + dag.sync_to_db(processor_subdir=old_dag_home) + + # Add DAG from new DAG home to the DB + other_dag = other_dagbag.get_dag("test_start_date_scheduling") + other_dag.fileloc = new_dag_file.name + other_dag.last_parsed_time = timezone.utcnow() + other_dag.sync_to_db(processor_subdir=new_dag_home) + + manager.processor._file_paths = [new_dag_file] + + active_dag_count = ( + session.query(func.count(DagModel.dag_id)).filter(DagModel.is_active).scalar() + ) + assert active_dag_count == 2 + + manager.processor._scan_stale_dags() + + active_dag_count = ( + session.query(func.count(DagModel.dag_id)).filter(DagModel.is_active).scalar() + ) + assert active_dag_count == 1 + @mock.patch( "airflow.dag_processing.processor.DagFileProcessorProcess.waitable_handle", new_callable=PropertyMock ) diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index 2e96728d5ecae..38bde8bf26565 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -3568,6 +3568,7 @@ def test_retry_still_in_executor(self, dag_maker): dag_id="test_retry_still_in_executor", schedule="@once", session=session, + fileloc=os.devnull + "/test_retry_still_in_executor.py", ): dag_task1 = BashOperator( task_id="test_retry_handling_op", From 56990073a50e513a76ad67c0e7c4ac02394b551f Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Wed, 28 Aug 2024 21:54:56 +0800 Subject: [PATCH 040/161] Set end_date and duration for triggers completed with end_from_trigger as True. (#41834) Co-authored-by: Karthikeyan Singaravelan --- airflow/triggers/base.py | 2 +- tests/models/test_trigger.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/airflow/triggers/base.py b/airflow/triggers/base.py index 7b5338ad2fa19..bc1da861f3c2d 100644 --- a/airflow/triggers/base.py +++ b/airflow/triggers/base.py @@ -203,7 +203,7 @@ def handle_submit(self, *, task_instance: TaskInstance, session: Session = NEW_S """ # Mark the task with terminal state and prevent it from resuming on worker task_instance.trigger_id = None - task_instance.state = self.task_instance_state + task_instance.set_state(self.task_instance_state, session=session) self._submit_callback_if_necessary(task_instance=task_instance, session=session) self._push_xcoms_if_necessary(task_instance=task_instance) diff --git a/tests/models/test_trigger.py b/tests/models/test_trigger.py index 4aa5b8b581b8f..407d6edd753a8 100644 --- a/tests/models/test_trigger.py +++ b/tests/models/test_trigger.py @@ -19,7 +19,9 @@ import datetime import json from typing import Any, AsyncIterator +from unittest.mock import patch +import pendulum import pytest import pytz from cryptography.fernet import Fernet @@ -161,11 +163,15 @@ def test_submit_failure(session, create_task_instance): (TaskSkippedEvent, "skipped"), ], ) -def test_submit_event_task_end(session, create_task_instance, event_cls, expected): +@patch("airflow.utils.timezone.utcnow") +def test_submit_event_task_end(mock_utcnow, session, create_task_instance, event_cls, expected): """ Tests that events inheriting BaseTaskEndEvent *don't* re-wake their dependent but mark them in the appropriate terminal state and send xcom """ + now = pendulum.now("UTC") + mock_utcnow.return_value = now + # Make a trigger trigger = Trigger(classpath="does.not.matter", kwargs={}) trigger.id = 1 @@ -199,6 +205,8 @@ def get_xcoms(ti): ti = session.query(TaskInstance).one() assert ti.state == expected assert ti.next_kwargs is None + assert ti.end_date == now + assert ti.duration is not None actual_xcoms = {x.key: x.value for x in get_xcoms(ti)} assert actual_xcoms == {"return_value": "xcomret", "a": "b", "c": "d"} From bf2efaa646fcfa821a5b0e214a1b5cb120a62071 Mon Sep 17 00:00:00 2001 From: Shahar Epstein <60007259+shahar1@users.noreply.github.com> Date: Thu, 29 Aug 2024 07:50:34 +0300 Subject: [PATCH 041/161] logout link in no roles error page fix (#41813) (#41845) * logout link in no roles error page fix * review comments (cherry picked from commit 032ac87b1d93abc53d3281313c957708017e21d4) Co-authored-by: Gagan Bhullar --- airflow/www/templates/airflow/no_roles_permissions.html | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/airflow/www/templates/airflow/no_roles_permissions.html b/airflow/www/templates/airflow/no_roles_permissions.html index eaa54c06a2268..fa619c403c030 100644 --- a/airflow/www/templates/airflow/no_roles_permissions.html +++ b/airflow/www/templates/airflow/no_roles_permissions.html @@ -30,7 +30,12 @@

Your user has no roles and/or permissions!

Unfortunately your user has no roles, and therefore you cannot use Airflow.

Please contact your Airflow administrator (authentication - may be misconfigured) or log out to try again.

+ may be misconfigured) or +
+ + log out to try again. +
+

{{ hostname }}

From 6fb6fdb80ac7a2daf559f9635bec4d05a0c8d90d Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Fri, 30 Aug 2024 12:45:19 +0530 Subject: [PATCH 042/161] Handle Example dags case when checking for missing files (#41856) (#41874) Earlier PR create to address the issue was not handling the case for the Example Dags, due to which the example dags were marked as stale since they are not present in the dag_directory. This PR handles that scenarios and update the testcase accordingly. related: #41432 (cherry picked from commit 435e9687b0c56499bc29c21d3cada8ae9e0a8c53) --- airflow/dag_processing/manager.py | 11 ++- tests/dag_processing/test_job_runner.py | 89 ++++++++++++------------- 2 files changed, 52 insertions(+), 48 deletions(-) diff --git a/airflow/dag_processing/manager.py b/airflow/dag_processing/manager.py index 819da5d7e1d74..54ea721d5fb6d 100644 --- a/airflow/dag_processing/manager.py +++ b/airflow/dag_processing/manager.py @@ -41,6 +41,7 @@ from tabulate import tabulate import airflow.models +from airflow import example_dags from airflow.api_internal.internal_api_call import internal_api_call from airflow.callbacks.callback_requests import CallbackRequest, SlaCallbackRequest from airflow.configuration import conf @@ -69,6 +70,8 @@ from airflow.utils.session import NEW_SESSION, provide_session from airflow.utils.sqlalchemy import prohibit_commit, with_row_locks +example_dag_folder = next(iter(example_dags.__path__)) + if TYPE_CHECKING: from multiprocessing.connection import Connection as MultiprocessingConnection @@ -527,9 +530,11 @@ def deactivate_stale_dags( for dag in dags_parsed: # When the DAG processor runs as part of the scheduler, and the user changes the DAGs folder, - # DAGs from the previous DAGs folder will be marked as stale. Note that this change has no impact - # on standalone DAG processors. - dag_not_in_current_dag_folder = os.path.commonpath([dag.fileloc, dag_directory]) != dag_directory + # DAGs from the previous DAGs folder will be marked as stale. We also need to handle example dags + # differently. Note that this change has no impact on standalone DAG processors. + dag_not_in_current_dag_folder = ( + not os.path.commonpath([dag.fileloc, example_dag_folder]) == example_dag_folder + ) and (os.path.commonpath([dag.fileloc, dag_directory]) != dag_directory) # The largest valid difference between a DagFileStat's last_finished_time and a DAG's # last_parsed_time is the processor_timeout. Longer than that indicates that the DAG is # no longer present in the file. We have a stale_dag_threshold configured to prevent a diff --git a/tests/dag_processing/test_job_runner.py b/tests/dag_processing/test_job_runner.py index b5d0b35580b4a..4f79436d143e6 100644 --- a/tests/dag_processing/test_job_runner.py +++ b/tests/dag_processing/test_job_runner.py @@ -772,58 +772,57 @@ def test_scan_stale_dags_when_dag_folder_change(self): def get_dag_string(filename) -> str: return open(TEST_DAG_FOLDER / filename).read() - with tempfile.TemporaryDirectory() as tmpdir: - old_dag_home = tempfile.mkdtemp(dir=tmpdir) - old_dag_file = tempfile.NamedTemporaryFile(dir=old_dag_home, suffix=".py") - old_dag_file.write(get_dag_string("test_example_bash_operator.py").encode()) - old_dag_file.flush() - new_dag_home = tempfile.mkdtemp(dir=tmpdir) - new_dag_file = tempfile.NamedTemporaryFile(dir=new_dag_home, suffix=".py") - new_dag_file.write(get_dag_string("test_scheduler_dags.py").encode()) - new_dag_file.flush() - - manager = DagProcessorJobRunner( - job=Job(), - processor=DagFileProcessorManager( - dag_directory=new_dag_home, - max_runs=1, - processor_timeout=timedelta(minutes=10), - signal_conn=MagicMock(), - dag_ids=[], - pickle_dags=False, - async_mode=True, - ), - ) + def add_dag_to_db(file_path, dag_id, processor_subdir): + dagbag = DagBag(file_path, read_dags_from_db=False) + dag = dagbag.get_dag(dag_id) + dag.fileloc = file_path + dag.last_parsed_time = timezone.utcnow() + dag.sync_to_db(processor_subdir=processor_subdir) - dagbag = DagBag(old_dag_file.name, read_dags_from_db=False) - other_dagbag = DagBag(new_dag_file.name, read_dags_from_db=False) + def create_dag_folder(dag_id): + dag_home = tempfile.mkdtemp(dir=tmpdir) + dag_file = tempfile.NamedTemporaryFile(dir=dag_home, suffix=".py") + dag_file.write(get_dag_string(dag_id).encode()) + dag_file.flush() + return dag_home, dag_file - with create_session() as session: - # Add DAG from old dah home to the DB - dag = dagbag.get_dag("test_example_bash_operator") - dag.fileloc = old_dag_file.name - dag.last_parsed_time = timezone.utcnow() - dag.sync_to_db(processor_subdir=old_dag_home) + with tempfile.TemporaryDirectory() as tmpdir: + old_dag_home, old_dag_file = create_dag_folder("test_example_bash_operator.py") + new_dag_home, new_dag_file = create_dag_folder("test_scheduler_dags.py") + example_dag_home, example_dag_file = create_dag_folder("test_dag_warnings.py") + + with mock.patch("airflow.dag_processing.manager.example_dag_folder", example_dag_home): + manager = DagProcessorJobRunner( + job=Job(), + processor=DagFileProcessorManager( + dag_directory=new_dag_home, + max_runs=1, + processor_timeout=timedelta(minutes=10), + signal_conn=MagicMock(), + dag_ids=[], + pickle_dags=False, + async_mode=True, + ), + ) - # Add DAG from new DAG home to the DB - other_dag = other_dagbag.get_dag("test_start_date_scheduling") - other_dag.fileloc = new_dag_file.name - other_dag.last_parsed_time = timezone.utcnow() - other_dag.sync_to_db(processor_subdir=new_dag_home) + with create_session() as session: + add_dag_to_db(old_dag_file.name, "test_example_bash_operator", old_dag_home) + add_dag_to_db(new_dag_file.name, "test_start_date_scheduling", new_dag_home) + add_dag_to_db(example_dag_file.name, "test_dag_warnings", example_dag_home) - manager.processor._file_paths = [new_dag_file] + manager.processor._file_paths = [new_dag_file, example_dag_file] - active_dag_count = ( - session.query(func.count(DagModel.dag_id)).filter(DagModel.is_active).scalar() - ) - assert active_dag_count == 2 + active_dag_count = ( + session.query(func.count(DagModel.dag_id)).filter(DagModel.is_active).scalar() + ) + assert active_dag_count == 3 - manager.processor._scan_stale_dags() + manager.processor._scan_stale_dags() - active_dag_count = ( - session.query(func.count(DagModel.dag_id)).filter(DagModel.is_active).scalar() - ) - assert active_dag_count == 1 + active_dag_count = ( + session.query(func.count(DagModel.dag_id)).filter(DagModel.is_active).scalar() + ) + assert active_dag_count == 2 @mock.patch( "airflow.dag_processing.processor.DagFileProcessorProcess.waitable_handle", new_callable=PropertyMock From a6dfae313ff3ec8f6cef2fd51b4273699d598dc6 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 30 Aug 2024 11:04:56 +0200 Subject: [PATCH 043/161] Bump webpack from 5.76.0 to 5.94.0 in /airflow/www (#41864) (#41879) Bumps [webpack](https://github.com/webpack/webpack) from 5.76.0 to 5.94.0. - [Release notes](https://github.com/webpack/webpack/releases) - [Commits](https://github.com/webpack/webpack/compare/v5.76.0...v5.94.0) --- updated-dependencies: - dependency-name: webpack dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit e8888fe055ac8c341e2fa6631ff6ac5f089d54c5) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- airflow/www/package.json | 2 +- airflow/www/yarn.lock | 449 +++++++++++++++++++-------------------- 2 files changed, 215 insertions(+), 236 deletions(-) diff --git a/airflow/www/package.json b/airflow/www/package.json index 7e494cadd4cfb..1371d0cc08a89 100644 --- a/airflow/www/package.json +++ b/airflow/www/package.json @@ -93,7 +93,7 @@ "typescript": "^4.6.3", "url-loader": "4.1.0", "web-worker": "^1.2.0", - "webpack": "^5.76.0", + "webpack": "^5.94.0", "webpack-cli": "^4.0.0", "webpack-license-plugin": "^4.2.1", "webpack-manifest-plugin": "^4.0.0" diff --git a/airflow/www/yarn.lock b/airflow/www/yarn.lock index 9995fa5213a6f..58bf650ea3f47 100644 --- a/airflow/www/yarn.lock +++ b/airflow/www/yarn.lock @@ -2824,6 +2824,14 @@ "@jridgewell/gen-mapping" "^0.3.0" "@jridgewell/trace-mapping" "^0.3.9" +"@jridgewell/source-map@^0.3.3": + version "0.3.6" + resolved "https://registry.yarnpkg.com/@jridgewell/source-map/-/source-map-0.3.6.tgz#9d71ca886e32502eb9362c9a74a46787c36df81a" + integrity sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ== + dependencies: + "@jridgewell/gen-mapping" "^0.3.5" + "@jridgewell/trace-mapping" "^0.3.25" + "@jridgewell/sourcemap-codec@^1.4.10": version "1.4.14" resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" @@ -2850,7 +2858,7 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" -"@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25": +"@jridgewell/trace-mapping@^0.3.20", "@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25": version "0.3.25" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz#15f190e98895f3fc23276ee14bc76b675c2e50f0" integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ== @@ -2858,14 +2866,6 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" -"@jridgewell/trace-mapping@^0.3.7": - version "0.3.13" - resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.13.tgz#dcfe3e95f224c8fe97a87a5235defec999aa92ea" - integrity sha512-o1xbKhp9qnIAoHJSWd6KlCZfqslL4valSF81H8ImioOAxluWYWOpWkpyktY2vnt4tbrX9XYaxovq6cgowaJp2w== - dependencies: - "@jridgewell/resolve-uri" "^3.0.3" - "@jridgewell/sourcemap-codec" "^1.4.10" - "@jridgewell/trace-mapping@^0.3.9": version "0.3.14" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.14.tgz#b231a081d8f66796e475ad588a1ef473112701ed" @@ -3391,26 +3391,10 @@ dependencies: "@types/ms" "*" -"@types/eslint-scope@^3.7.3": - version "3.7.3" - resolved "https://registry.yarnpkg.com/@types/eslint-scope/-/eslint-scope-3.7.3.tgz#125b88504b61e3c8bc6f870882003253005c3224" - integrity sha512-PB3ldyrcnAicT35TWPs5IcwKD8S333HMaa2VVv4+wdvebJkjWuW/xESoB8IwRcog8HYVYamb1g/R31Qv5Bx03g== - dependencies: - "@types/eslint" "*" - "@types/estree" "*" - -"@types/eslint@*": - version "8.4.3" - resolved "https://registry.yarnpkg.com/@types/eslint/-/eslint-8.4.3.tgz#5c92815a3838b1985c90034cd85f26f59d9d0ece" - integrity sha512-YP1S7YJRMPs+7KZKDb9G63n8YejIwW9BALq7a5j2+H4yl6iOv9CB29edho+cuFRrvmJbbaH2yiVChKLJVysDGw== - dependencies: - "@types/estree" "*" - "@types/json-schema" "*" - -"@types/estree@*", "@types/estree@^0.0.51": - version "0.0.51" - resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.51.tgz#cfd70924a25a3fd32b218e5e420e6897e1ac4f40" - integrity sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ== +"@types/estree@^1.0.5": + version "1.0.5" + resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4" + integrity sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw== "@types/geojson@*": version "7946.0.10" @@ -3466,16 +3450,16 @@ jest-diff "^27.0.0" pretty-format "^27.0.0" -"@types/json-schema@*", "@types/json-schema@^7.0.8", "@types/json-schema@^7.0.9": - version "7.0.11" - resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3" - integrity sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ== - "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.7": version "7.0.7" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad" integrity sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA== +"@types/json-schema@^7.0.8", "@types/json-schema@^7.0.9": + version "7.0.11" + resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3" + integrity sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ== + "@types/json-to-pretty-yaml@^1.2.1": version "1.2.1" resolved "https://registry.yarnpkg.com/@types/json-to-pretty-yaml/-/json-to-pretty-yaml-1.2.1.tgz#bf193455477295d83c78f73c08d956f74321193e" @@ -3805,125 +3789,125 @@ lodash "^4.17.21" prop-types "^15.5.10" -"@webassemblyjs/ast@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.11.1.tgz#2bfd767eae1a6996f432ff7e8d7fc75679c0b6a7" - integrity sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw== +"@webassemblyjs/ast@1.12.1", "@webassemblyjs/ast@^1.12.1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.12.1.tgz#bb16a0e8b1914f979f45864c23819cc3e3f0d4bb" + integrity sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg== dependencies: - "@webassemblyjs/helper-numbers" "1.11.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.1" + "@webassemblyjs/helper-numbers" "1.11.6" + "@webassemblyjs/helper-wasm-bytecode" "1.11.6" -"@webassemblyjs/floating-point-hex-parser@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.1.tgz#f6c61a705f0fd7a6aecaa4e8198f23d9dc179e4f" - integrity sha512-iGRfyc5Bq+NnNuX8b5hwBrRjzf0ocrJPI6GWFodBFzmFnyvrQ83SHKhmilCU/8Jv67i4GJZBMhEzltxzcNagtQ== +"@webassemblyjs/floating-point-hex-parser@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz#dacbcb95aff135c8260f77fa3b4c5fea600a6431" + integrity sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw== -"@webassemblyjs/helper-api-error@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.1.tgz#1a63192d8788e5c012800ba6a7a46c705288fd16" - integrity sha512-RlhS8CBCXfRUR/cwo2ho9bkheSXG0+NwooXcc3PAILALf2QLdFyj7KGsKRbVc95hZnhnERon4kW/D3SZpp6Tcg== +"@webassemblyjs/helper-api-error@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz#6132f68c4acd59dcd141c44b18cbebbd9f2fa768" + integrity sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q== -"@webassemblyjs/helper-buffer@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.1.tgz#832a900eb444884cde9a7cad467f81500f5e5ab5" - integrity sha512-gwikF65aDNeeXa8JxXa2BAk+REjSyhrNC9ZwdT0f8jc4dQQeDQ7G4m0f2QCLPJiMTTO6wfDmRmj/pW0PsUvIcA== +"@webassemblyjs/helper-buffer@1.12.1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-buffer/-/helper-buffer-1.12.1.tgz#6df20d272ea5439bf20ab3492b7fb70e9bfcb3f6" + integrity sha512-nzJwQw99DNDKr9BVCOZcLuJJUlqkJh+kVzVl6Fmq/tI5ZtEyWT1KZMyOXltXLZJmDtvLCDgwsyrkohEtopTXCw== -"@webassemblyjs/helper-numbers@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.1.tgz#64d81da219fbbba1e3bd1bfc74f6e8c4e10a62ae" - integrity sha512-vDkbxiB8zfnPdNK9Rajcey5C0w+QJugEglN0of+kmO8l7lDb77AnlKYQF7aarZuCrv+l0UvqL+68gSDr3k9LPQ== +"@webassemblyjs/helper-numbers@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz#cbce5e7e0c1bd32cf4905ae444ef64cea919f1b5" + integrity sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g== dependencies: - "@webassemblyjs/floating-point-hex-parser" "1.11.1" - "@webassemblyjs/helper-api-error" "1.11.1" + "@webassemblyjs/floating-point-hex-parser" "1.11.6" + "@webassemblyjs/helper-api-error" "1.11.6" "@xtuc/long" "4.2.2" -"@webassemblyjs/helper-wasm-bytecode@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.1.tgz#f328241e41e7b199d0b20c18e88429c4433295e1" - integrity sha512-PvpoOGiJwXeTrSf/qfudJhwlvDQxFgelbMqtq52WWiXC6Xgg1IREdngmPN3bs4RoO83PnL/nFrxucXj1+BX62Q== +"@webassemblyjs/helper-wasm-bytecode@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz#bb2ebdb3b83aa26d9baad4c46d4315283acd51e9" + integrity sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA== -"@webassemblyjs/helper-wasm-section@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.1.tgz#21ee065a7b635f319e738f0dd73bfbda281c097a" - integrity sha512-10P9No29rYX1j7F3EVPX3JvGPQPae+AomuSTPiF9eBQeChHI6iqjMIwR9JmOJXwpnn/oVGDk7I5IlskuMwU/pg== +"@webassemblyjs/helper-wasm-section@1.12.1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.12.1.tgz#3da623233ae1a60409b509a52ade9bc22a37f7bf" + integrity sha512-Jif4vfB6FJlUlSbgEMHUyk1j234GTNG9dBJ4XJdOySoj518Xj0oGsNi59cUQF4RRMS9ouBUxDDdyBVfPTypa5g== dependencies: - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/helper-buffer" "1.11.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.1" - "@webassemblyjs/wasm-gen" "1.11.1" + "@webassemblyjs/ast" "1.12.1" + "@webassemblyjs/helper-buffer" "1.12.1" + "@webassemblyjs/helper-wasm-bytecode" "1.11.6" + "@webassemblyjs/wasm-gen" "1.12.1" -"@webassemblyjs/ieee754@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/ieee754/-/ieee754-1.11.1.tgz#963929e9bbd05709e7e12243a099180812992614" - integrity sha512-hJ87QIPtAMKbFq6CGTkZYJivEwZDbQUgYd3qKSadTNOhVY7p+gfP6Sr0lLRVTaG1JjFj+r3YchoqRYxNH3M0GQ== +"@webassemblyjs/ieee754@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/ieee754/-/ieee754-1.11.6.tgz#bb665c91d0b14fffceb0e38298c329af043c6e3a" + integrity sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg== dependencies: "@xtuc/ieee754" "^1.2.0" -"@webassemblyjs/leb128@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/leb128/-/leb128-1.11.1.tgz#ce814b45574e93d76bae1fb2644ab9cdd9527aa5" - integrity sha512-BJ2P0hNZ0u+Th1YZXJpzW6miwqQUGcIHT1G/sf72gLVD9DZ5AdYTqPNbHZh6K1M5VmKvFXwGSWZADz+qBWxeRw== +"@webassemblyjs/leb128@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/leb128/-/leb128-1.11.6.tgz#70e60e5e82f9ac81118bc25381a0b283893240d7" + integrity sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ== dependencies: "@xtuc/long" "4.2.2" -"@webassemblyjs/utf8@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/utf8/-/utf8-1.11.1.tgz#d1f8b764369e7c6e6bae350e854dec9a59f0a3ff" - integrity sha512-9kqcxAEdMhiwQkHpkNiorZzqpGrodQQ2IGrHHxCy+Ozng0ofyMA0lTqiLkVs1uzTRejX+/O0EOT7KxqVPuXosQ== - -"@webassemblyjs/wasm-edit@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.1.tgz#ad206ebf4bf95a058ce9880a8c092c5dec8193d6" - integrity sha512-g+RsupUC1aTHfR8CDgnsVRVZFJqdkFHpsHMfJuWQzWU3tvnLC07UqHICfP+4XyL2tnr1amvl1Sdp06TnYCmVkA== - dependencies: - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/helper-buffer" "1.11.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.1" - "@webassemblyjs/helper-wasm-section" "1.11.1" - "@webassemblyjs/wasm-gen" "1.11.1" - "@webassemblyjs/wasm-opt" "1.11.1" - "@webassemblyjs/wasm-parser" "1.11.1" - "@webassemblyjs/wast-printer" "1.11.1" - -"@webassemblyjs/wasm-gen@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.1.tgz#86c5ea304849759b7d88c47a32f4f039ae3c8f76" - integrity sha512-F7QqKXwwNlMmsulj6+O7r4mmtAlCWfO/0HdgOxSklZfQcDu0TpLiD1mRt/zF25Bk59FIjEuGAIyn5ei4yMfLhA== - dependencies: - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.1" - "@webassemblyjs/ieee754" "1.11.1" - "@webassemblyjs/leb128" "1.11.1" - "@webassemblyjs/utf8" "1.11.1" - -"@webassemblyjs/wasm-opt@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz#657b4c2202f4cf3b345f8a4c6461c8c2418985f2" - integrity sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw== - dependencies: - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/helper-buffer" "1.11.1" - "@webassemblyjs/wasm-gen" "1.11.1" - "@webassemblyjs/wasm-parser" "1.11.1" - -"@webassemblyjs/wasm-parser@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.1.tgz#86ca734534f417e9bd3c67c7a1c75d8be41fb199" - integrity sha512-rrBujw+dJu32gYB7/Lup6UhdkPx9S9SnobZzRVL7VcBH9Bt9bCBLEuX/YXOOtBsOZ4NQrRykKhffRWHvigQvOA== - dependencies: - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/helper-api-error" "1.11.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.1" - "@webassemblyjs/ieee754" "1.11.1" - "@webassemblyjs/leb128" "1.11.1" - "@webassemblyjs/utf8" "1.11.1" - -"@webassemblyjs/wast-printer@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wast-printer/-/wast-printer-1.11.1.tgz#d0c73beda8eec5426f10ae8ef55cee5e7084c2f0" - integrity sha512-IQboUWM4eKzWW+N/jij2sRatKMh99QEelo3Eb2q0qXkvPRISAj8Qxtmw5itwqK+TTkBuUIE45AxYPToqPtL5gg== - dependencies: - "@webassemblyjs/ast" "1.11.1" +"@webassemblyjs/utf8@1.11.6": + version "1.11.6" + resolved "https://registry.yarnpkg.com/@webassemblyjs/utf8/-/utf8-1.11.6.tgz#90f8bc34c561595fe156603be7253cdbcd0fab5a" + integrity sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA== + +"@webassemblyjs/wasm-edit@^1.12.1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-edit/-/wasm-edit-1.12.1.tgz#9f9f3ff52a14c980939be0ef9d5df9ebc678ae3b" + integrity sha512-1DuwbVvADvS5mGnXbE+c9NfA8QRcZ6iKquqjjmR10k6o+zzsRVesil54DKexiowcFCPdr/Q0qaMgB01+SQ1u6g== + dependencies: + "@webassemblyjs/ast" "1.12.1" + "@webassemblyjs/helper-buffer" "1.12.1" + "@webassemblyjs/helper-wasm-bytecode" "1.11.6" + "@webassemblyjs/helper-wasm-section" "1.12.1" + "@webassemblyjs/wasm-gen" "1.12.1" + "@webassemblyjs/wasm-opt" "1.12.1" + "@webassemblyjs/wasm-parser" "1.12.1" + "@webassemblyjs/wast-printer" "1.12.1" + +"@webassemblyjs/wasm-gen@1.12.1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-gen/-/wasm-gen-1.12.1.tgz#a6520601da1b5700448273666a71ad0a45d78547" + integrity sha512-TDq4Ojh9fcohAw6OIMXqiIcTq5KUXTGRkVxbSo1hQnSy6lAM5GSdfwWeSxpAo0YzgsgF182E/U0mDNhuA0tW7w== + dependencies: + "@webassemblyjs/ast" "1.12.1" + "@webassemblyjs/helper-wasm-bytecode" "1.11.6" + "@webassemblyjs/ieee754" "1.11.6" + "@webassemblyjs/leb128" "1.11.6" + "@webassemblyjs/utf8" "1.11.6" + +"@webassemblyjs/wasm-opt@1.12.1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.12.1.tgz#9e6e81475dfcfb62dab574ac2dda38226c232bc5" + integrity sha512-Jg99j/2gG2iaz3hijw857AVYekZe2SAskcqlWIZXjji5WStnOpVoat3gQfT/Q5tb2djnCjBtMocY/Su1GfxPBg== + dependencies: + "@webassemblyjs/ast" "1.12.1" + "@webassemblyjs/helper-buffer" "1.12.1" + "@webassemblyjs/wasm-gen" "1.12.1" + "@webassemblyjs/wasm-parser" "1.12.1" + +"@webassemblyjs/wasm-parser@1.12.1", "@webassemblyjs/wasm-parser@^1.12.1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-parser/-/wasm-parser-1.12.1.tgz#c47acb90e6f083391e3fa61d113650eea1e95937" + integrity sha512-xikIi7c2FHXysxXe3COrVUPSheuBtpcfhbpFj4gmu7KRLYOzANztwUU0IbsqvMqzuNK2+glRGWCEqZo1WCLyAQ== + dependencies: + "@webassemblyjs/ast" "1.12.1" + "@webassemblyjs/helper-api-error" "1.11.6" + "@webassemblyjs/helper-wasm-bytecode" "1.11.6" + "@webassemblyjs/ieee754" "1.11.6" + "@webassemblyjs/leb128" "1.11.6" + "@webassemblyjs/utf8" "1.11.6" + +"@webassemblyjs/wast-printer@1.12.1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wast-printer/-/wast-printer-1.12.1.tgz#bcecf661d7d1abdaf989d8341a4833e33e2b31ac" + integrity sha512-+X4WAlOisVWQMikjbcvY2e0rwPsKQ9F688lksZhBcPycBBuii3O7m8FACbDMWDojpAqvjIncrG8J0XHKyQfVeA== + dependencies: + "@webassemblyjs/ast" "1.12.1" "@xtuc/long" "4.2.2" "@webpack-cli/configtest@^1.2.0": @@ -3976,10 +3960,10 @@ acorn-globals@^6.0.0: acorn "^7.1.1" acorn-walk "^7.1.1" -acorn-import-assertions@^1.7.6: - version "1.8.0" - resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz#ba2b5939ce62c238db6d93d81c9b111b29b855e9" - integrity sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw== +acorn-import-attributes@^1.9.5: + version "1.9.5" + resolved "https://registry.yarnpkg.com/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz#7eb1557b1ba05ef18b5ed0ec67591bfab04688ef" + integrity sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ== acorn-jsx@^5.3.2: version "5.3.2" @@ -3996,10 +3980,10 @@ acorn@^7.1.1: resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== -acorn@^8.2.4, acorn@^8.5.0, acorn@^8.7.1: - version "8.7.1" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.1.tgz#0197122c843d1bf6d0a5e83220a788f278f63c30" - integrity sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A== +acorn@^8.2.4, acorn@^8.5.0, acorn@^8.7.1, acorn@^8.8.2: + version "8.12.1" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.12.1.tgz#71616bdccbe25e27a54439e0046e89ca76df2248" + integrity sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg== agent-base@6: version "6.0.2" @@ -4503,26 +4487,15 @@ browser-process-hrtime@^1.0.0: resolved "https://registry.yarnpkg.com/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz#3c9b4b7d782c8121e56f10106d84c0d0ffc94626" integrity sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow== -browserslist@^4.0.0, browserslist@^4.14.5, browserslist@^4.16.6, browserslist@^4.17.5, browserslist@^4.20.3: - version "4.20.4" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.20.4.tgz#98096c9042af689ee1e0271333dbc564b8ce4477" - integrity sha512-ok1d+1WpnU24XYN7oC3QWgTyMhY/avPJ/r9T00xxvUOIparA/gc+UPUMaod3i+G6s+nI2nUb9xZ5k794uIwShw== - dependencies: - caniuse-lite "^1.0.30001349" - electron-to-chromium "^1.4.147" - escalade "^3.1.1" - node-releases "^2.0.5" - picocolors "^1.0.0" - -browserslist@^4.22.2, browserslist@^4.23.0: - version "4.23.1" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.23.1.tgz#ce4af0534b3d37db5c1a4ca98b9080f985041e96" - integrity sha512-TUfofFo/KsK/bWZ9TWQ5O26tsWW4Uhmt8IYklbnUa70udB6P2wA7w7o4PY4muaEPBQaAX+CEnmmIA41NVHtPVw== +browserslist@^4.0.0, browserslist@^4.16.6, browserslist@^4.17.5, browserslist@^4.20.3, browserslist@^4.21.10, browserslist@^4.22.2, browserslist@^4.23.0: + version "4.23.3" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.23.3.tgz#debb029d3c93ebc97ffbc8d9cbb03403e227c800" + integrity sha512-btwCFJVjI4YWDNfau8RhZ+B1Q/VLoUITrm3RlP6y1tYGWIOa+InuYiRGXUBXo8nA1qKmHMyLB/iVQg5TT4eFoA== dependencies: - caniuse-lite "^1.0.30001629" - electron-to-chromium "^1.4.796" - node-releases "^2.0.14" - update-browserslist-db "^1.0.16" + caniuse-lite "^1.0.30001646" + electron-to-chromium "^1.5.4" + node-releases "^2.0.18" + update-browserslist-db "^1.1.0" bser@2.1.1: version "2.1.1" @@ -4607,15 +4580,15 @@ caniuse-api@^3.0.0: lodash.memoize "^4.1.2" lodash.uniq "^4.5.0" -caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001349: +caniuse-lite@^1.0.0: version "1.0.30001589" resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001589.tgz" integrity sha512-vNQWS6kI+q6sBlHbh71IIeC+sRwK2N3EDySc/updIGhIee2x5z00J4c1242/5/d6EpEMdOnk/m+6tuk4/tcsqg== -caniuse-lite@^1.0.30001629: - version "1.0.30001632" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001632.tgz#964207b7cba5851701afb4c8afaf1448db3884b6" - integrity sha512-udx3o7yHJfUxMLkGohMlVHCvFvWmirKh9JAH/d7WOLPetlH+LTL5cocMZ0t7oZx/mdlOWXti97xLZWc8uURRHg== +caniuse-lite@^1.0.30001646: + version "1.0.30001653" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001653.tgz#b8af452f8f33b1c77f122780a4aecebea0caca56" + integrity sha512-XGWQVB8wFQ2+9NZwZ10GxTYC5hk0Fa+q8cSkr0tgvMhYhMHP/QC+WTgrePMDBWiWc/pV+1ik82Al20XOK25Gcw== ccount@^2.0.0: version "2.0.1" @@ -5851,15 +5824,10 @@ echarts@^5.4.2: tslib "2.3.0" zrender "5.4.3" -electron-to-chromium@^1.4.147: - version "1.4.156" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.156.tgz#fc398e1bfbe586135351ebfaf198473a82923af5" - integrity sha512-/Wj5NC7E0wHaMCdqxWz9B0lv7CcycDTiHyXCtbbu3pXM9TV2AOp8BtMqkVuqvJNdEvltBG6LxT2Q+BxY4LUCIA== - -electron-to-chromium@^1.4.796: - version "1.4.798" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.798.tgz#6a3fcab2edc1e66e3883466f6b4b8944323c0164" - integrity sha512-by9J2CiM9KPGj9qfp5U4FcPSbXJG7FNzqnYaY4WLzX+v2PHieVGmnsA4dxfpGE3QEC7JofpPZmn7Vn1B9NR2+Q== +electron-to-chromium@^1.5.4: + version "1.5.13" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.13.tgz#1abf0410c5344b2b829b7247e031f02810d442e6" + integrity sha512-lbBcvtIJ4J6sS4tb5TLp1b4LyfCdMkwStzXPyAgVgTRAsep4bvrAGaBOP7ZJtQMNJpSQ9SqG4brWOroNaQtm7Q== elkjs@^0.7.1: version "0.7.1" @@ -5886,10 +5854,10 @@ emojis-list@^3.0.0: resolved "https://registry.yarnpkg.com/emojis-list/-/emojis-list-3.0.0.tgz#5570662046ad29e2e916e71aae260abdff4f6a78" integrity sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q== -enhanced-resolve@^5.10.0: - version "5.12.0" - resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.12.0.tgz#300e1c90228f5b570c4d35babf263f6da7155634" - integrity sha512-QHTXI/sZQmko1cbDoNAa3mJ5qhWUUNAq3vR0/YiD379fWQrcfuoX1+HW2S0MTt7XmoPLapdaDKUtelUSPic7hQ== +enhanced-resolve@^5.17.1: + version "5.17.1" + resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz#67bfbbcc2f81d511be77d686a90267ef7f898a15" + integrity sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg== dependencies: graceful-fs "^4.2.4" tapable "^2.2.0" @@ -6017,10 +5985,10 @@ es-abstract@^1.20.4: unbox-primitive "^1.0.2" which-typed-array "^1.1.9" -es-module-lexer@^0.9.0: - version "0.9.3" - resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-0.9.3.tgz#6f13db00cc38417137daf74366f535c8eb438f19" - integrity sha512-1HQ2M2sPtxwnvOvT1ZClHyQDiggdNjURWpY2we6aMKCQiUVxTmVs2UYPLIrD84sS+kMdUwfBSylbJPwNnBrnHQ== +es-module-lexer@^1.2.1: + version "1.5.4" + resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-1.5.4.tgz#a8efec3a3da991e60efa6b633a7cad6ab8d26b78" + integrity sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw== es-set-tostringtag@^2.0.1: version "2.0.1" @@ -6882,10 +6850,10 @@ gopd@^1.0.1: dependencies: get-intrinsic "^1.1.3" -graceful-fs@^4.1.2, graceful-fs@^4.2.4, graceful-fs@^4.2.9: - version "4.2.9" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.9.tgz#041b05df45755e587a24942279b9d113146e1c96" - integrity sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ== +graceful-fs@^4.1.2, graceful-fs@^4.2.11, graceful-fs@^4.2.4, graceful-fs@^4.2.9: + version "4.2.11" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" + integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== graphlib@^2.1.8: version "2.1.8" @@ -9104,15 +9072,10 @@ node-readfiles@^0.2.0: dependencies: es6-promise "^3.2.1" -node-releases@^2.0.14: - version "2.0.14" - resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.14.tgz#2ffb053bceb8b2be8495ece1ab6ce600c4461b0b" - integrity sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw== - -node-releases@^2.0.5: - version "2.0.5" - resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.5.tgz#280ed5bc3eba0d96ce44897d8aee478bfb3d9666" - integrity sha512-U9h1NLROZTq9uE1SNffn6WuPDg8icmi3ns4rEl/oTfIle4iLjTliCzgTsbaIFMq/Xn078/lfY/BL0GWZ+psK4Q== +node-releases@^2.0.18: + version "2.0.18" + resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.18.tgz#f010e8d35e2fe8d6b2944f03f70213ecedc4ca3f" + integrity sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g== normalize-package-data@^3.0.2: version "3.0.3" @@ -10547,10 +10510,10 @@ schema-utils@^2.6.5, schema-utils@^2.7.0: ajv "^6.12.4" ajv-keywords "^3.5.2" -schema-utils@^3.0.0, schema-utils@^3.1.0, schema-utils@^3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.1.1.tgz#bc74c4b6b6995c1d88f76a8b77bea7219e0c8281" - integrity sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw== +schema-utils@^3.0.0, schema-utils@^3.1.1, schema-utils@^3.2.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.3.0.tgz#f50a88877c3c01652a15b622ae9e9795df7a60fe" + integrity sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg== dependencies: "@types/json-schema" "^7.0.8" ajv "^6.12.5" @@ -10590,6 +10553,13 @@ serialize-javascript@^6.0.0: dependencies: randombytes "^2.1.0" +serialize-javascript@^6.0.1: + version "6.0.2" + resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-6.0.2.tgz#defa1e055c83bf6d59ea805d8da862254eb6a6c2" + integrity sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g== + dependencies: + randombytes "^2.1.0" + setimmediate@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285" @@ -11226,18 +11196,28 @@ terser-webpack-plugin@<5.0.0: terser "^5.3.4" webpack-sources "^1.4.3" -terser-webpack-plugin@^5.1.3: - version "5.3.3" - resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.3.tgz#8033db876dd5875487213e87c627bca323e5ed90" - integrity sha512-Fx60G5HNYknNTNQnzQ1VePRuu89ZVYWfjRAeT5rITuCY/1b08s49e5kSQwHDirKZWuoKOBRFS98EUUoZ9kLEwQ== +terser-webpack-plugin@^5.3.10: + version "5.3.10" + resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.10.tgz#904f4c9193c6fd2a03f693a2150c62a92f40d199" + integrity sha512-BKFPWlPDndPs+NGGCr1U59t0XScL5317Y0UReNrHaw9/FwhPENlq6bfgs+4yPfyP51vqC1bQ4rp1EfXW5ZSH9w== dependencies: - "@jridgewell/trace-mapping" "^0.3.7" + "@jridgewell/trace-mapping" "^0.3.20" jest-worker "^27.4.5" schema-utils "^3.1.1" - serialize-javascript "^6.0.0" - terser "^5.7.2" + serialize-javascript "^6.0.1" + terser "^5.26.0" -terser@^5.3.4, terser@^5.7.2: +terser@^5.26.0: + version "5.31.6" + resolved "https://registry.yarnpkg.com/terser/-/terser-5.31.6.tgz#c63858a0f0703988d0266a82fcbf2d7ba76422b1" + integrity sha512-PQ4DAriWzKj+qgehQ7LK5bQqCFNMmlhjR2PFFLuqGCpuCAauxemVBWwWOxo3UIwWQx8+Pr61Df++r76wDmkQBg== + dependencies: + "@jridgewell/source-map" "^0.3.3" + acorn "^8.8.2" + commander "^2.20.0" + source-map-support "~0.5.20" + +terser@^5.3.4: version "5.14.2" resolved "https://registry.yarnpkg.com/terser/-/terser-5.14.2.tgz#9ac9f22b06994d736174f4091aa368db896f1c10" integrity sha512-oL0rGeM/WFQCUd0y2QrWxYnq7tfSuKBiqTjRPWrRgB46WD/kiwHwF8T23z78H6Q6kGCuuHcPB+KULHRdxvVGQA== @@ -11578,10 +11558,10 @@ unload@2.2.0: "@babel/runtime" "^7.6.2" detect-node "^2.0.4" -update-browserslist-db@^1.0.16: - version "1.0.16" - resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.16.tgz#f6d489ed90fb2f07d67784eb3f53d7891f736356" - integrity sha512-KVbTxlBYlckhF5wgfyZXTWnMn7MMZjMu9XG8bPlliUOP9ThaF4QnhP8qrjrH7DRzHfSk0oQv1wToW+iA5GajEQ== +update-browserslist-db@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz#7ca61c0d8650766090728046e416a8cde682859e" + integrity sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ== dependencies: escalade "^3.1.2" picocolors "^1.0.1" @@ -11738,10 +11718,10 @@ walker@^1.0.7: dependencies: makeerror "1.0.12" -watchpack@^2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.4.0.tgz#fa33032374962c78113f93c7f2fb4c54c9862a5d" - integrity sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg== +watchpack@^2.4.1: + version "2.4.2" + resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.4.2.tgz#2feeaed67412e7c33184e5a79ca738fbd38564da" + integrity sha512-TnbFSbcOCcDgjZ4piURLCbJ3nJhznVh9kw6F6iokjiFPl8ONxe9A6nMDVXDiNbrSfLILs6vB07F7wLBrwPYzJw== dependencies: glob-to-regexp "^0.4.1" graceful-fs "^4.1.2" @@ -11833,34 +11813,33 @@ webpack-sources@^3.2.1, webpack-sources@^3.2.3: resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-3.2.3.tgz#2d4daab8451fd4b240cc27055ff6a0c2ccea0cde" integrity sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w== -webpack@^5.76.0: - version "5.76.0" - resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.76.0.tgz#f9fb9fb8c4a7dbdcd0d56a98e56b8a942ee2692c" - integrity sha512-l5sOdYBDunyf72HW8dF23rFtWq/7Zgvt/9ftMof71E/yUb1YLOBmTgA2K4vQthB3kotMrSj609txVE0dnr2fjA== +webpack@^5.94.0: + version "5.94.0" + resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.94.0.tgz#77a6089c716e7ab90c1c67574a28da518a20970f" + integrity sha512-KcsGn50VT+06JH/iunZJedYGUJS5FGjow8wb9c0v5n1Om8O1g4L6LjtfxwlXIATopoQu+vOXXa7gYisWxCoPyg== dependencies: - "@types/eslint-scope" "^3.7.3" - "@types/estree" "^0.0.51" - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/wasm-edit" "1.11.1" - "@webassemblyjs/wasm-parser" "1.11.1" + "@types/estree" "^1.0.5" + "@webassemblyjs/ast" "^1.12.1" + "@webassemblyjs/wasm-edit" "^1.12.1" + "@webassemblyjs/wasm-parser" "^1.12.1" acorn "^8.7.1" - acorn-import-assertions "^1.7.6" - browserslist "^4.14.5" + acorn-import-attributes "^1.9.5" + browserslist "^4.21.10" chrome-trace-event "^1.0.2" - enhanced-resolve "^5.10.0" - es-module-lexer "^0.9.0" + enhanced-resolve "^5.17.1" + es-module-lexer "^1.2.1" eslint-scope "5.1.1" events "^3.2.0" glob-to-regexp "^0.4.1" - graceful-fs "^4.2.9" + graceful-fs "^4.2.11" json-parse-even-better-errors "^2.3.1" loader-runner "^4.2.0" mime-types "^2.1.27" neo-async "^2.6.2" - schema-utils "^3.1.0" + schema-utils "^3.2.0" tapable "^2.1.1" - terser-webpack-plugin "^5.1.3" - watchpack "^2.4.0" + terser-webpack-plugin "^5.3.10" + watchpack "^2.4.1" webpack-sources "^3.2.3" whatwg-encoding@^1.0.5: From 09ec2616568f8a18e0d5fe408110fae06ddf748f Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 30 Aug 2024 14:49:01 +0200 Subject: [PATCH 044/161] Adding tojson filter to example_inlet_event_extra example dag (#41873) (#41890) (cherry picked from commit 87a4a5137519448cfff85fc506b9267ec3e18364) Co-authored-by: Amogh Desai --- airflow/example_dags/example_inlet_event_extra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/example_dags/example_inlet_event_extra.py b/airflow/example_dags/example_inlet_event_extra.py index 471a215532b47..4b7567fc2f87e 100644 --- a/airflow/example_dags/example_inlet_event_extra.py +++ b/airflow/example_dags/example_inlet_event_extra.py @@ -57,5 +57,5 @@ def read_dataset_event(*, inlet_events=None): BashOperator( task_id="read_dataset_event_from_classic", inlets=[ds], - bash_command="echo {{ inlet_events['s3://output/1.txt'][-1].extra }}", + bash_command="echo '{{ inlet_events['s3://output/1.txt'][-1].extra | tojson }}'", ) From fa03a3212019760c696fc2ea34a03c38ec0b72dd Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Fri, 30 Aug 2024 19:42:54 +0530 Subject: [PATCH 045/161] Skip test_scan_stale_dags_when_dag_folder_change in DB isolation mode (#41893) (#41895) Since the similar test(test_scan_stale_dags_standalone_mode) are skipped in DB isolation mode (cherry picked from commit 07af14ae75820a98b60ecffa2949ef7ad70bacab) --- tests/dag_processing/test_job_runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/dag_processing/test_job_runner.py b/tests/dag_processing/test_job_runner.py index 4f79436d143e6..0e15a2d1f6690 100644 --- a/tests/dag_processing/test_job_runner.py +++ b/tests/dag_processing/test_job_runner.py @@ -763,6 +763,7 @@ def test_scan_stale_dags_standalone_mode(self): active_dag_count = session.query(func.count(DagModel.dag_id)).filter(DagModel.is_active).scalar() assert active_dag_count == 1 + @pytest.mark.skip_if_database_isolation_mode # Test is broken in db isolation mode def test_scan_stale_dags_when_dag_folder_change(self): """ Ensure dags from old dag_folder is marked as stale when dag processor From aafad2aeee926dd1a621e67bb49de0a6bf62fc03 Mon Sep 17 00:00:00 2001 From: Jens Scheffler <95105677+jscheffl@users.noreply.github.com> Date: Fri, 30 Aug 2024 20:18:31 +0200 Subject: [PATCH 046/161] Make Scarf usage reporting in major+minor versions and counters in buckets (#41900) (cherry picked from commit 5a045196704943ef3195f3eeaacf2adae30e5ec1) --- airflow/utils/usage_data_collection.py | 18 ++++++++-- airflow/www/views.py | 7 ++-- tests/utils/test_usage_data_collection.py | 43 ++++++++++++++++++++--- tests/www/views/test_views.py | 6 ++-- 4 files changed, 62 insertions(+), 12 deletions(-) diff --git a/airflow/utils/usage_data_collection.py b/airflow/utils/usage_data_collection.py index 389b239adac54..def8ce983aae1 100644 --- a/airflow/utils/usage_data_collection.py +++ b/airflow/utils/usage_data_collection.py @@ -80,8 +80,8 @@ def get_database_version() -> str: return "None" version_info = settings.engine.dialect.server_version_info - # Example: (1, 2, 3) -> "1.2.3" - return ".".join(map(str, version_info)) if version_info else "None" + # Example: (1, 2, 3) -> "1.2" (cut only major+minor w/o patch) + return ".".join(map(str, version_info[0:2])) if version_info else "None" def get_database_name() -> str: @@ -95,7 +95,8 @@ def get_executor() -> str: def get_python_version() -> str: - return platform.python_version() + # Cut only major+minor from the python version string (e.g. 3.10.12 --> 3.10) + return ".".join(platform.python_version().split(".")[0:2]) def get_plugin_counts() -> dict[str, int]: @@ -108,3 +109,14 @@ def get_plugin_counts() -> dict[str, int]: "appbuilder_menu_items": sum(len(x["appbuilder_menu_items"]) for x in plugin_info), "timetables": sum(len(x["timetables"]) for x in plugin_info), } + + +def to_bucket(counter: int) -> str: + """As we don't want to have preceise numbers, make number into a bucket.""" + if counter == 0: + return "0" + buckets = [0, 5, 10, 20, 50, 100, 200, 500, 1000, 2000] + for idx, val in enumerate(buckets[1:]): + if buckets[idx] < counter and counter <= val: + return f"{buckets[idx] + 1}-{val}" + return f"{buckets[-1]}+" diff --git a/airflow/www/views.py b/airflow/www/views.py index a485f84ed4b1c..cd470807a992b 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -240,6 +240,9 @@ def build_scarf_url(dags_count: int) -> str: appbuilder_views_count = plugin_counts["appbuilder_views"] appbuilder_menu_items_count = plugin_counts["appbuilder_menu_items"] timetables_count = plugin_counts["timetables"] + dag_bucket = usage_data_collection.to_bucket(dags_count) + plugins_bucket = usage_data_collection.to_bucket(plugins_count) + timetable_bucket = usage_data_collection.to_bucket(timetables_count) # Path Format: # /{version}/{python_version}/{platform}/{arch}/{database}/{db_version}/{executor}/{num_dags}/{plugin_count}/{flask_blueprint_count}/{appbuilder_view_count}/{appbuilder_menu_item_count}/{timetables} @@ -248,8 +251,8 @@ def build_scarf_url(dags_count: int) -> str: scarf_url = ( f"{scarf_domain}/webserver" f"/{version}/{python_version}" - f"/{platform_sys}/{platform_arch}/{db_name}/{db_version}/{executor}/{dags_count}" - f"/{plugins_count}/{flask_blueprints_count}/{appbuilder_views_count}/{appbuilder_menu_items_count}/{timetables_count}" + f"/{platform_sys}/{platform_arch}/{db_name}/{db_version}/{executor}/{dag_bucket}" + f"/{plugins_bucket}/{flask_blueprints_count}/{appbuilder_views_count}/{appbuilder_menu_items_count}/{timetable_bucket}" ) return scarf_url diff --git a/tests/utils/test_usage_data_collection.py b/tests/utils/test_usage_data_collection.py index 5244de1a58ea5..b104d1bfe365a 100644 --- a/tests/utils/test_usage_data_collection.py +++ b/tests/utils/test_usage_data_collection.py @@ -24,7 +24,12 @@ from airflow import __version__ as airflow_version from airflow.configuration import conf -from airflow.utils.usage_data_collection import get_database_version, usage_data_collection +from airflow.utils.usage_data_collection import ( + get_database_version, + get_python_version, + to_bucket, + usage_data_collection, +) @pytest.mark.parametrize("is_enabled, is_prerelease", [(False, True), (True, True)]) @@ -51,7 +56,7 @@ def test_scarf_analytics( ): platform_sys = platform.system() platform_machine = platform.machine() - python_version = platform.python_version() + python_version = get_python_version() executor = conf.get("core", "EXECUTOR") scarf_endpoint = "https://apacheairflow.gateway.scarf.sh/scheduler" usage_data_collection() @@ -74,12 +79,42 @@ def test_scarf_analytics( @pytest.mark.parametrize( "version_info, expected_version", [ - ((1, 2, 3), "1.2.3"), # Normal version tuple + ((1, 2, 3), "1.2"), # Normal version tuple (None, "None"), # No version info available ((1,), "1"), # Single element version tuple - ((1, 2, 3, "beta", 4), "1.2.3.beta.4"), # Complex version tuple with strings + ((1, 2, 3, "beta", 4), "1.2"), # Complex version tuple with strings ], ) def test_get_database_version(version_info, expected_version): with mock.patch("airflow.settings.engine.dialect.server_version_info", new=version_info): assert get_database_version() == expected_version + + +@pytest.mark.parametrize( + "version_info, expected_version", + [ + ("1.2.3", "1.2"), # Normal version + ("4", "4"), # Single element version + ("1.2.3.beta4", "1.2"), # Complex version tuple with strings + ], +) +def test_get_python_version(version_info, expected_version): + with mock.patch("platform.python_version", return_value=version_info): + assert get_python_version() == expected_version + + +@pytest.mark.parametrize( + "counter, expected_bucket", + [ + (0, "0"), + (1, "1-5"), + (5, "1-5"), + (6, "6-10"), + (11, "11-20"), + (20, "11-20"), + (21, "21-50"), + (10000, "2000+"), + ], +) +def test_to_bucket(counter, expected_bucket): + assert to_bucket(counter) == expected_bucket diff --git a/tests/www/views/test_views.py b/tests/www/views/test_views.py index 76f71c7f386c0..3cc6a875713e6 100644 --- a/tests/www/views/test_views.py +++ b/tests/www/views/test_views.py @@ -604,7 +604,7 @@ def test_invalid_dates(app, admin_client, url, content): @patch("airflow.utils.usage_data_collection.get_database_version", return_value="12.3") @patch("airflow.utils.usage_data_collection.get_database_name", return_value="postgres") @patch("airflow.utils.usage_data_collection.get_executor", return_value="SequentialExecutor") -@patch("airflow.utils.usage_data_collection.get_python_version", return_value="3.8.5") +@patch("airflow.utils.usage_data_collection.get_python_version", return_value="3.8") @patch("airflow.utils.usage_data_collection.get_plugin_counts") def test_build_scarf_url( get_plugin_counts, @@ -626,8 +626,8 @@ def test_build_scarf_url( result = build_scarf_url(5) expected_url = ( "https://apacheairflow.gateway.scarf.sh/webserver/" - f"{airflow_version}/3.8.5/Linux/x86_64/postgres/12.3/SequentialExecutor/5" - f"/10/15/20/25/30" + f"{airflow_version}/3.8/Linux/x86_64/postgres/12.3/SequentialExecutor/1-5" + f"/6-10/15/20/25/21-50" ) if enabled: assert result == expected_url From d4c2dd0f416417eb9eb5cba65279722bdc278fbf Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Sun, 1 Sep 2024 08:10:01 +0100 Subject: [PATCH 047/161] Update release command for Airflow 2 (#41907) * Update release command for Airflow 2 For airflow 2, we create a branch out of v2-10-test and sync that branch with v2-10-stable to make a release. * Update CI check --- .github/workflows/basic-tests.yml | 3 +- dev/README_RELEASE_AIRFLOW.md | 49 ++++++++----------- ...ut_release-management_start-rc-process.svg | 26 +++++----- ...ut_release-management_start-rc-process.txt | 2 +- .../commands/release_candidate_command.py | 10 ++-- .../release_management_commands_config.py | 1 + 6 files changed, 46 insertions(+), 45 deletions(-) diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 9828a14993581..ae968103dbd15 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -372,7 +372,8 @@ jobs: run: breeze release-management create-minor-branch --version-branch 2-8 --answer yes - name: "Check Airflow RC process command" run: > - breeze release-management start-rc-process --version 2.8.3rc1 --previous-version 2.8.0 --answer yes + breeze release-management start-rc-process --version 2.8.3rc1 --previous-version 2.8.0 + --sync-branch sync_v2_8_test --answer yes - name: "Check Airflow release process command" run: > breeze release-management start-release --release-candidate 2.8.3rc1 --previous-release 2.8.0 diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 19d2bcec04be8..2d63edf30119f 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -22,8 +22,8 @@ - [Selecting what to put into the release](#selecting-what-to-put-into-the-release) - [Selecting what to cherry-pick](#selecting-what-to-cherry-pick) - - [Making the cherry picking](#making-the-cherry-picking) - - [Reviewing cherry-picked PRs and assigning labels](#reviewing-cherry-picked-prs-and-assigning-labels) + - [Backporting the PRs](#backporting-the-prs) + - [Reviewing Backported PRs and assigning labels](#reviewing-backported-prs-and-assigning-labels) - [Prepare the Apache Airflow Package RC](#prepare-the-apache-airflow-package-rc) - [Update the milestone](#update-the-milestone) - [Build RC artifacts](#build-rc-artifacts) @@ -73,9 +73,6 @@ The first step of a release is to work out what is being included. This differs ## Selecting what to cherry-pick -For obvious reasons, you can't cherry-pick every change from `main` into the release branch - -some are incompatible without a large set of other changes, some are brand-new features, and some just don't need to be in a release. - In general only security fixes, data-loss bugs and regression fixes are essential to bring into a patch release; also changes in dependencies (pyproject.toml) resulting from releasing newer versions of packages that Airflow depends on. Other bugfixes can be added on a best-effort basis, but if something is going to be very difficult to backport @@ -93,7 +90,7 @@ and mark those as well. You can accomplish this by running the following command ./dev/airflow-github needs-categorization 2.3.2 HEAD ``` -Often you also want to cherry-pick changes related to CI and development tools, to include the latest +Often you also want to backport changes related to CI and development tools, to include the latest stability fixes in CI and improvements in development tools. Usually you can see the list of such changes via (this will exclude already merged changes): @@ -105,9 +102,9 @@ git log --oneline --decorate apache/v2-2-stable..apache/main -- Dockerfile* scri Most of those PRs should be marked with `changelog:skip` label, so that they are excluded from the user-facing changelog as they only matter for developers of Airflow. We have a tool -that allows to easily review the cherry-picked PRs and mark them with the right label - see below. +that allows to easily review the backported PRs and mark them with the right label - see below. -You also likely want to cherry-pick some of the latest doc changes in order to bring clarification and +You also likely want to backport some of the latest doc changes in order to bring clarification and explanations added to the documentation. Usually you can see the list of such changes via: ```shell @@ -119,29 +116,24 @@ git log --oneline --decorate apache/v2-2-stable..apache/main -- docs/apache-airf Those changes that are "doc-only" changes should be marked with `type:doc-only` label so that they land in documentation part of the changelog. The tool to review and assign the labels is described below. -## Making the cherry picking - -It is recommended to clone Airflow upstream (not your fork) and run the commands on -the relevant test branch in this clone. That way origin points to the upstream repo. +## Backporting the PRs -To see cherry picking candidates (unmerged PR with the appropriate milestone), from the test -branch you can run: +If a PR needs to be backported, checkout v2-10-test and make a new branch for the backport: ```shell -./dev/airflow-github compare 2.1.2 --unmerged +git checkout v2-10-test +git pull && git checkout -b ``` -You can start cherry picking from the bottom of the list. (older commits first) - -When you cherry-pick, pick in chronological order onto the `vX-Y-test` release branch. -You'll move them over to be on `vX-Y-stable` once the release is cut. Use the `-x` option -to keep a reference to the original commit we cherry picked from. ("cherry picked from commit ...") +Then cherry-pick the commit from main: ```shell git cherry-pick -x ``` -## Reviewing cherry-picked PRs and assigning labels +Make your PR and wait for reviews and approval + +## Reviewing Backported PRs and assigning labels We have the tool that allows to review cherry-picked PRs and assign the labels [./assign_cherry_picked_prs_with_milestone.py](./assign_cherry_picked_prs_with_milestone.py) @@ -152,7 +144,7 @@ It allows to manually review and assign milestones and labels to cherry-picked P ./dev/assign_cherry_picked_prs_with_milestone.py assign-prs --previous-release v2-2-stable --current-release apache/v2-2-test --milestone-number 48 ``` -It summarises the state of each cherry-picked PR including information whether it is going to be +It summarises the state of each Backported PR including information whether it is going to be excluded or included in changelog or included in doc-only part of it. It also allows to re-assign the PRs to the target milestone and apply the `changelog:skip` or `type:doc-only` label. @@ -160,7 +152,7 @@ You can also add `--skip-assigned` flag if you want to automatically skip the qu for the PRs that are already correctly assigned to the milestone. You can also avoid the "Are you OK?" question with `--assume-yes` flag. -You can review the list of PRs cherry-picked and produce a nice summary with `--print-summary` (this flag +You can review the list of PRs backported and produce a nice summary with `--print-summary` (this flag assumes the `--skip-assigned` flag, so that the summary can be produced without questions: ```shell @@ -169,7 +161,7 @@ assumes the `--skip-assigned` flag, so that the summary can be produced without --output-folder /tmp ``` -This will produce summary output with nice links that you can use to review the cherry-picked changes, +This will produce summary output with nice links that you can use to review the backported changes, but it also produces files with list of commits separated by type in the folder specified. In the case above, it will produce three files that you can use in the next step: @@ -225,6 +217,7 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag export VERSION_SUFFIX=rc3 export VERSION_BRANCH=2-1 export VERSION_WITHOUT_RC=${VERSION/rc?/} + export SYNC_BRANCH=sync_v2_10_test # Set AIRFLOW_REPO_ROOT to the path of your git repo export AIRFLOW_REPO_ROOT=$(pwd) @@ -253,8 +246,8 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag - Check out the 'test' branch ```shell script - git checkout v${VERSION_BRANCH}-test - git reset --hard origin/v${VERSION_BRANCH}-test + git checkout ${SYNC_BRANCH} + git reset --hard origin/${SYNC_BRANCH} ``` - Set your version in `airflow/__init__.py`, `airflow/api_connexion/openapi/v1.yaml` (without the RC tag). @@ -284,7 +277,7 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag create a fragment to document its change, to generate the body of the release note based on the cherry picked commits: ``` - ./dev/airflow-github changelog v2-3-stable v2-3-test + ./dev/airflow-github changelog v2-3-stable ${SYNC_BRANCH} ``` - Commit the release note change. @@ -310,7 +303,7 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag ```shell script git checkout main git pull # Ensure that the script is up-to-date - breeze release-management start-rc-process --version ${VERSION} --previous-version + breeze release-management start-rc-process --version ${VERSION} --previous-version --sync-branch ${SYNC_BRANCH} ``` - Create issue in github for testing the release using this subject: diff --git a/dev/breeze/doc/images/output_release-management_start-rc-process.svg b/dev/breeze/doc/images/output_release-management_start-rc-process.svg index a91c45583c55e..7d8bd3725991d 100644 --- a/dev/breeze/doc/images/output_release-management_start-rc-process.svg +++ b/dev/breeze/doc/images/output_release-management_start-rc-process.svg @@ -1,4 +1,4 @@ - +

{% endif %}