From 8805f37a7fa42a5edb33bfc87973053536cacb62 Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Mon, 23 Jan 2023 12:44:48 +0100 Subject: [PATCH] Update Airflow version to 2.5.1 (#29074) --- .github/ISSUE_TEMPLATE/airflow_bug_report.yml | 2 +- Dockerfile | 2 +- README.md | 14 +-- RELEASE_NOTES.rst | 113 ++++++++++++++++++ airflow/api_connexion/openapi/v1.yaml | 2 +- airflow/utils/db.py | 3 +- chart/Chart.yaml | 22 ++-- chart/newsfragments/28074.significant.rst | 3 - chart/newsfragments/29074.significant.rst | 3 + chart/values.schema.json | 4 +- chart/values.yaml | 4 +- .../installation/supported-versions.rst | 2 +- newsfragments/08212.misc.rst | 1 - .../pre_commit_supported_versions.py | 2 +- 14 files changed, 144 insertions(+), 33 deletions(-) delete mode 100644 chart/newsfragments/28074.significant.rst create mode 100644 chart/newsfragments/29074.significant.rst delete mode 100644 newsfragments/08212.misc.rst diff --git a/.github/ISSUE_TEMPLATE/airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_bug_report.yml index 92aba93901be9..9634f12cede36 100644 --- a/.github/ISSUE_TEMPLATE/airflow_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_bug_report.yml @@ -25,7 +25,7 @@ body: the latest release or main to see if the issue is fixed before reporting it. multiple: false options: - - "2.5.0" + - "2.5.1" - "main (development)" - "Other Airflow 2 version (please specify below)" validations: diff --git a/Dockerfile b/Dockerfile index 74634887a97ce..f57d25e285fb1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,7 @@ ARG AIRFLOW_UID="50000" ARG AIRFLOW_USER_HOME_DIR=/home/airflow # latest released version here -ARG AIRFLOW_VERSION="2.5.0" +ARG AIRFLOW_VERSION="2.5.1" ARG PYTHON_BASE_IMAGE="python:3.7-slim-bullseye" diff --git a/README.md b/README.md index 839775e589928..cff982bd9cd2e 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ Airflow is not a streaming solution, but it is often used to process real-time d Apache Airflow is tested with: -| | Main version (dev) | Stable version (2.5.0) | +| | Main version (dev) | Stable version (2.5.1) | |---------------------|------------------------------|------------------------------| | Python | 3.7, 3.8, 3.9, 3.10 | 3.7, 3.8, 3.9, 3.10 | | Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | @@ -158,15 +158,15 @@ them to the appropriate format and workflow that your tool requires. ```bash -pip install 'apache-airflow==2.5.0' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.0/constraints-3.7.txt" +pip install 'apache-airflow==2.5.1' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-3.7.txt" ``` 2. Installing with extras (i.e., postgres, google) ```bash -pip install 'apache-airflow[postgres,google]==2.5.0' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.0/constraints-3.7.txt" +pip install 'apache-airflow[postgres,google]==2.5.1' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-3.7.txt" ``` For information on installing provider packages, check @@ -275,7 +275,7 @@ Apache Airflow version life cycle: | Version | Current Patch/Minor | State | First Release | Limited Support | EOL/Terminated | |-----------|-----------------------|-----------|-----------------|-------------------|------------------| -| 2 | 2.5.0 | Supported | Dec 17, 2020 | TBD | TBD | +| 2 | 2.5.1 | Supported | Dec 17, 2020 | TBD | TBD | | 1.10 | 1.10.15 | EOL | Aug 27, 2018 | Dec 17, 2020 | June 17, 2021 | | 1.9 | 1.9.0 | EOL | Jan 03, 2018 | Aug 27, 2018 | Aug 27, 2018 | | 1.8 | 1.8.2 | EOL | Mar 19, 2017 | Jan 03, 2018 | Jan 03, 2018 | @@ -305,7 +305,7 @@ They are based on the official release schedule of Python and Kubernetes, nicely 2. The "oldest" supported version of Python/Kubernetes is the default one until we decide to switch to later version. "Default" is only meaningful in terms of "smoke tests" in CI PRs, which are run using this default version and the default reference image available. Currently `apache/airflow:latest` - and `apache/airflow:2.5.0` images are Python 3.7 images. This means that default reference image will + and `apache/airflow:2.5.1` images are Python 3.7 images. This means that default reference image will become the default at the time when we start preparing for dropping 3.7 support which is few months before the end of life for Python 3.7. diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 9c2cf3c25adc1..20936bb620cc0 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -21,6 +21,119 @@ .. towncrier release notes start +Airflow 2.5.1 (2023-01-20) +-------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +Trigger gevent ``monkeypatching`` via environment variable (#28283) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + + If you are using gevent for your webserver deployment and used local settings to ``monkeypatch`` gevent, + you might want to replace local settings patching with an ``_AIRFLOW_PATCH_GEVENT`` environment variable + set to 1 in your webserver. This ensures gevent patching is done as early as possible. (#8212) + +Bug Fixes +^^^^^^^^^ +- Fix masking of non-sensitive environment variables (#28802) +- Remove swagger-ui extra from connexion and install ``swagger-ui-dist`` via npm package (#28788) +- Fix ``UIAlert`` should_show when ``AUTH_ROLE_PUBLIC`` set (#28781) +- Only patch single label when adopting pod (#28776) +- Update CSRF token to expire with session (#28730) +- Fix "airflow tasks render" cli command for mapped task instances (#28698) +- Allow XComArgs for ``external_task_ids`` of ExternalTaskSensor (#28692) +- Row-lock TIs to be removed during mapped task expansion (#28689) +- Handle ConnectionReset exception in Executor cleanup (#28685) +- Fix description of output redirection for access_log for gunicorn (#28672) +- Add back join to zombie query that was dropped in #28198 (#28544) +- Fix calendar view for CronTriggerTimeTable dags (#28411) +- After running the DAG the employees table is empty. (#28353) +- Fix ``DetachedInstanceError`` when finding zombies in Dag Parsing process (#28198) +- Nest header blocks in ``divs`` to fix ``dagid`` copy nit on dag.html (#28643) +- Fix UI caret direction (#28624) +- Guard not-yet-expanded ti in trigger rule dep (#28592) +- Move TI ``setNote`` endpoints under TaskInstance in OpenAPI (#28566) +- Consider previous run in ``CronTriggerTimetable`` (#28532) +- Ensure correct log dir in file task handler (#28477) +- Fix bad pods pickled in executor_config (#28454) +- Add ``ensure_ascii=False`` in trigger dag run API (#28451) +- Add setters to MappedOperator on_*_callbacks (#28313) +- Fix ``ti._try_number`` for deferred and up_for_reschedule tasks (#26993) +- separate ``callModal`` from dag.js (#28410) +- A manual run can't look like a scheduled one (#28397) +- Dont show task/run durations when there is no start_date (#28395) +- Maintain manual scroll position in task logs (#28386) +- Correctly select a mapped task's "previous" task (#28379) +- Trigger gevent ``monkeypatching`` via environment variable (#28283) +- Fix db clean warnings (#28243) +- Make arguments 'offset' and 'length' not required (#28234) +- Make live logs reading work for "other" k8s executors (#28213) +- Add custom pickling hooks to ``LazyXComAccess`` (#28191) +- fix next run datasets error (#28165) +- Ensure that warnings from ``@dag`` decorator are reported in dag file (#28153) +- Do not warn when airflow dags tests command is used (#28138) +- Ensure the ``dagbag_size`` metric decreases when files are deleted (#28135) +- Improve run/task grid view actions (#28130) +- Make BaseJob.most_recent_job favor "running" jobs (#28119) +- Don't emit FutureWarning when code not calling old key (#28109) +- Add ``airflow.api.auth.backend.session`` to backend sessions in compose (#28094) +- Resolve false warning about calling conf.get on moved item (#28075) +- Return list of tasks that will be changed (#28066) +- Handle bad zip files nicely when parsing DAGs. (#28011) +- Prevent double loading of providers from local paths (#27988) +- Fix deadlock when chaining multiple empty mapped tasks (#27964) +- fix: current_state method on TaskInstance doesn't filter by map_index (#27898) +- Don't log CLI actions if db not initialized (#27851) +- Make sure we can get out of a faulty scheduler state (#27834) +- dagrun, ``next_dagruns_to_examine``, add MySQL index hint (#27821) +- Handle DAG disappearing mid-flight when dag verification happens (#27720) +- fix: continue checking sla (#26968) +- Allow generation of connection URI to work when no conn type (#26765) + +Misc/Internal +^^^^^^^^^^^^^ +- Remove limit for ``dnspython`` after eventlet got fixed (#29004) +- Limit ``dnspython`` to < ``2.3.0`` until eventlet incompatibility is solved (#28962) +- Add automated version replacement in example dag indexes (#28090) +- Cleanup and do housekeeping with plugin examples (#28537) +- Limit ``SQLAlchemy`` to below ``2.0`` (#28725) +- Bump ``json5`` from ``1.0.1`` to ``1.0.2`` in ``/airflow/www`` (#28715) +- Fix some docs on using sensors with taskflow (#28708) +- Change Architecture and OperatingSystem classes into ``Enums`` (#28627) +- Add doc-strings and small improvement to email util (#28634) +- Fix ``Connection.get_extra`` type (#28594) +- navbar, cap dropdown size, and add scroll bar (#28561) +- Emit warnings for ``conf.get*`` from the right source location (#28543) +- Move MyPY plugins of ours to dev folder (#28498) +- Add retry to ``purge_inactive_dag_warnings`` (#28481) +- Re-enable Plyvel on ARM as it now builds cleanly (#28443) +- Add SIGUSR2 handler for LocalTaskJob and workers to aid debugging (#28309) +- Convert ``test_task_command`` to Pytest and ``unquarantine`` tests in it (#28247) +- Make invalid characters exception more readable (#28181) +- Bump decode-uri-component from ``0.2.0`` to ``0.2.2`` in ``/airflow/www`` (#28080) +- Use asserts instead of exceptions for executor not started (#28019) +- Simplify dataset ``subgraph`` logic (#27987) +- Order TIs by ``map_index`` (#27904) +- Additional info about Segmentation Fault in ``LocalTaskJob`` (#27381) + +Doc Only Changes +^^^^^^^^^^^^^^^^ +- Mention mapped operator in cluster policy doc (#28885) +- Slightly improve description of Dynamic DAG generation preamble (#28650) +- Restructure Docs (#27235) +- Update scheduler docs about low priority tasks (#28831) +- Clarify that versioned constraints are fixed at release time (#28762) +- Clarify about docker compose (#28729) +- Adding an example dag for dynamic task mapping (#28325) +- Use docker compose v2 command (#28605) +- Add AIRFLOW_PROJ_DIR to docker-compose example (#28517) +- Remove outdated Optional Provider Feature outdated documentation (#28506) +- Add documentation for [core] mp_start_method config (#27993) +- Documentation for the LocalTaskJob return code counter (#27972) +- Note which versions of Python are supported (#27798) + + Airflow 2.5.0 (2022-12-02) -------------------------- diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index f2cc9b3af20d8..9e4ec7e1d4a8e 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -229,7 +229,7 @@ info: This means that the server encountered an unexpected condition that prevented it from fulfilling the request. - version: '2.4.0' + version: '2.5.1' license: name: Apache 2.0 url: http://www.apache.org/licenses/LICENSE-2.0.html diff --git a/airflow/utils/db.py b/airflow/utils/db.py index 49600c143bf18..182bd38759be1 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -76,6 +76,7 @@ "2.4.2": "b0d31815b5a6", "2.4.3": "e07f49787c9d", "2.5.0": "290244fb8b83", + "2.5.1": "290244fb8b83", } @@ -780,7 +781,6 @@ def _configured_alembic_environment() -> Generator[EnvironmentContext, None, Non config, script, ) as env, settings.engine.connect() as connection: - alembic_logger = logging.getLogger("alembic") level = alembic_logger.level alembic_logger.setLevel(logging.WARNING) @@ -1179,7 +1179,6 @@ def _create_table_as( def _move_dangling_data_to_new_table( session, source_table: Table, source_query: Query, target_table_name: str ): - bind = session.get_bind() dialect_name = bind.dialect.name diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 756602bd72ac4..028837cfa9757 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -20,7 +20,7 @@ apiVersion: v2 name: airflow version: 1.8.0-dev -appVersion: 2.5.0 +appVersion: 2.5.1 description: The official Helm chart to deploy Apache Airflow, a platform to programmatically author, schedule, and monitor workflows home: https://airflow.apache.org/ @@ -47,25 +47,25 @@ annotations: url: https://airflow.apache.org/docs/helm-chart/1.6.0/ artifacthub.io/screenshots: | - title: DAGs View - url: https://airflow.apache.org/docs/apache-airflow/2.5.0/_images/dags.png + url: https://airflow.apache.org/docs/apache-airflow/2.5.1/_images/dags.png - title: Datasets View - url: https://airflow.apache.org/docs/apache-airflow/2.5.0/_images/datasets.png + url: https://airflow.apache.org/docs/apache-airflow/2.5.1/_images/datasets.png - title: Grid View - url: https://airflow.apache.org/docs/apache-airflow/2.5.0/_images/grid.png + url: https://airflow.apache.org/docs/apache-airflow/2.5.1/_images/grid.png - title: Graph View - url: https://airflow.apache.org/docs/apache-airflow/2.5.0/_images/graph.png + url: https://airflow.apache.org/docs/apache-airflow/2.5.1/_images/graph.png - title: Calendar View - url: https://airflow.apache.org/docs/apache-airflow/2.5.0/_images/calendar.png + url: https://airflow.apache.org/docs/apache-airflow/2.5.1/_images/calendar.png - title: Variable View - url: https://airflow.apache.org/docs/apache-airflow/2.5.0/_images/variable_hidden.png + url: https://airflow.apache.org/docs/apache-airflow/2.5.1/_images/variable_hidden.png - title: Gantt Chart - url: https://airflow.apache.org/docs/apache-airflow/2.5.0/_images/gantt.png + url: https://airflow.apache.org/docs/apache-airflow/2.5.1/_images/gantt.png - title: Task Duration - url: https://airflow.apache.org/docs/apache-airflow/2.5.0/_images/duration.png + url: https://airflow.apache.org/docs/apache-airflow/2.5.1/_images/duration.png - title: Code View - url: https://airflow.apache.org/docs/apache-airflow/2.5.0/_images/code.png + url: https://airflow.apache.org/docs/apache-airflow/2.5.1/_images/code.png - title: Task Instance Context Menu - url: https://airflow.apache.org/docs/apache-airflow/2.5.0/_images/context.png + url: https://airflow.apache.org/docs/apache-airflow/2.5.1/_images/context.png artifacthub.io/changes: | - description: Make cleanup job history configurable kind: added diff --git a/chart/newsfragments/28074.significant.rst b/chart/newsfragments/28074.significant.rst deleted file mode 100644 index 12d6d33129fb7..0000000000000 --- a/chart/newsfragments/28074.significant.rst +++ /dev/null @@ -1,3 +0,0 @@ -Default Airflow image is updated to ``2.5.0`` - -The default Airflow image that is used with the Chart is now ``2.5.0``, previously it was ``2.4.1``. diff --git a/chart/newsfragments/29074.significant.rst b/chart/newsfragments/29074.significant.rst new file mode 100644 index 0000000000000..64c855a5edfb4 --- /dev/null +++ b/chart/newsfragments/29074.significant.rst @@ -0,0 +1,3 @@ +Default Airflow image is updated to ``2.5.1`` + +The default Airflow image that is used with the Chart is now ``2.5.1``, previously it was ``2.4.1``. diff --git a/chart/values.schema.json b/chart/values.schema.json index 6a3b83aa46055..68ebf5d70fd36 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -77,13 +77,13 @@ "defaultAirflowTag": { "description": "Default airflow tag to deploy.", "type": "string", - "default": "2.5.0", + "default": "2.5.1", "x-docsSection": "Common" }, "airflowVersion": { "description": "Airflow version (Used to make some decisions based on Airflow Version being deployed).", "type": "string", - "default": "2.5.0", + "default": "2.5.1", "x-docsSection": "Common" }, "securityContext": { diff --git a/chart/values.yaml b/chart/values.yaml index 247d807acd3b9..d2aa91cb8a454 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -49,10 +49,10 @@ airflowHome: /opt/airflow defaultAirflowRepository: apache/airflow # Default airflow tag to deploy -defaultAirflowTag: "2.5.0" +defaultAirflowTag: "2.5.1" # Airflow version (Used to make some decisions based on Airflow Version being deployed) -airflowVersion: "2.5.0" +airflowVersion: "2.5.1" # Images images: diff --git a/docs/apache-airflow/installation/supported-versions.rst b/docs/apache-airflow/installation/supported-versions.rst index 56385ebfd8a12..38ace709db585 100644 --- a/docs/apache-airflow/installation/supported-versions.rst +++ b/docs/apache-airflow/installation/supported-versions.rst @@ -29,7 +29,7 @@ Apache Airflow version life cycle: ========= ===================== ========= =============== ================= ================ Version Current Patch/Minor State First Release Limited Support EOL/Terminated ========= ===================== ========= =============== ================= ================ -2 2.5.0 Supported Dec 17, 2020 TBD TBD +2 2.5.1 Supported Dec 17, 2020 TBD TBD 1.10 1.10.15 EOL Aug 27, 2018 Dec 17, 2020 June 17, 2021 1.9 1.9.0 EOL Jan 03, 2018 Aug 27, 2018 Aug 27, 2018 1.8 1.8.2 EOL Mar 19, 2017 Jan 03, 2018 Jan 03, 2018 diff --git a/newsfragments/08212.misc.rst b/newsfragments/08212.misc.rst deleted file mode 100644 index acce074f103ec..0000000000000 --- a/newsfragments/08212.misc.rst +++ /dev/null @@ -1 +0,0 @@ -If you are using gevent for your webserver deployment and used local settings to monkeypatch gevent, you might want to replace local settings patching with an ``_AIRFLOW_PATCH_GEVENT`` environment variable set to 1 in your webserver. This ensures gevent patching is done as early as possible. diff --git a/scripts/ci/pre_commit/pre_commit_supported_versions.py b/scripts/ci/pre_commit/pre_commit_supported_versions.py index 5692e27d7d160..4c28a71f0d07b 100755 --- a/scripts/ci/pre_commit/pre_commit_supported_versions.py +++ b/scripts/ci/pre_commit/pre_commit_supported_versions.py @@ -27,7 +27,7 @@ HEADERS = ("Version", "Current Patch/Minor", "State", "First Release", "Limited Support", "EOL/Terminated") SUPPORTED_VERSIONS = ( - ("2", "2.5.0", "Supported", "Dec 17, 2020", "TBD", "TBD"), + ("2", "2.5.1", "Supported", "Dec 17, 2020", "TBD", "TBD"), ("1.10", "1.10.15", "EOL", "Aug 27, 2018", "Dec 17, 2020", "June 17, 2021"), ("1.9", "1.9.0", "EOL", "Jan 03, 2018", "Aug 27, 2018", "Aug 27, 2018"), ("1.8", "1.8.2", "EOL", "Mar 19, 2017", "Jan 03, 2018", "Jan 03, 2018"),