Mark daskexecutor provider as removed (#35965)

Following the discussion in airflow devlist we mark daskexecutor as removed. See: https://lists.apache.org/thread/fxv44cqqljrrhll3fdpdgc9h9fz5ghcy (cherry picked from commit 9c1c9f4)
apache · Dec 5, 2023 · 48d7ac4 · 48d7ac4
1 parent 2a008cf
commit 48d7ac4
Show file tree

Hide file tree

Showing 35 changed files with 93 additions and 128 deletions.
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
@@ -672,16 +672,16 @@ aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam,
 apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kafka,
 apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs,
 apprise, arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups,
-cloudant, cncf.kubernetes, cohere, common.io, common.sql, crypto, dask, daskexecutor, databricks,
-datadog, dbt.cloud, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord,
-doc, doc_gen, docker, druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github,
-github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc,
-jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp,
-microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openai, openfaas, openlineage, opensearch,
-opsgenie, oracle, otel, pagerduty, pandas, papermill, password, pgvector, pinecone, pinot, plexus,
-postgres, presto, rabbitmq, redis, s3, s3fs, salesforce, samba, saml, segment, sendgrid, sentry,
-sftp, singularity, slack, smtp, snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram,
-trino, vertica, virtualenv, weaviate, webhdfs, winrm, yandex, zendesk
+cloudant, cncf.kubernetes, cohere, common.io, common.sql, crypto, databricks, datadog, dbt.cloud,
+deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker,
+druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google,
+google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes,
+ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql,
+mysql, neo4j, odbc, openai, openfaas, openlineage, opensearch, opsgenie, oracle, otel, pagerduty,
+pandas, papermill, password, pgvector, pinecone, pinot, plexus, postgres, presto, rabbitmq, redis,
+s3, s3fs, salesforce, samba, saml, segment, sendgrid, sentry, sftp, singularity, slack, smtp,
+snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv,
+weaviate, webhdfs, winrm, yandex, zendesk
   .. END EXTRAS HERE
 
 Provider packages

diff --git a/Dockerfile b/Dockerfile
@@ -35,7 +35,7 @@
 #                        much smaller.
 #
 # Use the same builder frontend version for everyone
-ARG AIRFLOW_EXTRAS="aiobotocore,amazon,async,celery,cncf.kubernetes,daskexecutor,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,openlineage,pandas,postgres,redis,sendgrid,sftp,slack,snowflake,ssh,statsd,virtualenv"
+ARG AIRFLOW_EXTRAS="aiobotocore,amazon,async,celery,cncf.kubernetes,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,openlineage,pandas,postgres,redis,sendgrid,sftp,slack,snowflake,ssh,statsd,virtualenv"
 ARG ADDITIONAL_AIRFLOW_EXTRAS=""
 ARG ADDITIONAL_PYTHON_DEPS=""
 

diff --git a/IMAGES.rst b/IMAGES.rst
@@ -135,7 +135,7 @@ This will build the image using command similar to:
 .. code-block:: bash
 
     pip install \
-      apache-airflow[async,amazon,celery,cncf.kubernetes,docker,daskexecutor,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv]==2.0.0 \
+      apache-airflow[async,amazon,celery,cncf.kubernetes,docker,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv]==2.0.0 \
       --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.0.0/constraints-3.8.txt"
 
 .. note::

diff --git a/INSTALL b/INSTALL
@@ -98,16 +98,16 @@ aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam,
 apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kafka,
 apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs,
 apprise, arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups,
-cloudant, cncf.kubernetes, cohere, common.io, common.sql, crypto, dask, daskexecutor, databricks,
-datadog, dbt.cloud, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord,
-doc, doc_gen, docker, druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github,
-github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc,
-jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp,
-microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openai, openfaas, openlineage, opensearch,
-opsgenie, oracle, otel, pagerduty, pandas, papermill, password, pgvector, pinecone, pinot, plexus,
-postgres, presto, rabbitmq, redis, s3, s3fs, salesforce, samba, saml, segment, sendgrid, sentry,
-sftp, singularity, slack, smtp, snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram,
-trino, vertica, virtualenv, weaviate, webhdfs, winrm, yandex, zendesk
+cloudant, cncf.kubernetes, cohere, common.io, common.sql, crypto, databricks, datadog, dbt.cloud,
+deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker,
+druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google,
+google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes,
+ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql,
+mysql, neo4j, odbc, openai, openfaas, openlineage, opensearch, opsgenie, oracle, otel, pagerduty,
+pandas, papermill, password, pgvector, pinecone, pinot, plexus, postgres, presto, rabbitmq, redis,
+s3, s3fs, salesforce, samba, saml, segment, sendgrid, sentry, sftp, singularity, slack, smtp,
+snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv,
+weaviate, webhdfs, winrm, yandex, zendesk
 # END EXTRAS HERE
 
 # For installing Airflow in development environments - see CONTRIBUTING.rst

diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml
@@ -63,7 +63,7 @@ core:
     executor:
       description: |
         The executor class that airflow should use. Choices include
-        ``SequentialExecutor``, ``LocalExecutor``, ``CeleryExecutor``, ``DaskExecutor``,
+        ``SequentialExecutor``, ``LocalExecutor``, ``CeleryExecutor``,
         ``KubernetesExecutor``, ``CeleryKubernetesExecutor``, ``LocalKubernetesExecutor`` or the
         full import path to the class when using a custom executor.
       version_added: ~

diff --git a/airflow/executors/executor_constants.py b/airflow/executors/executor_constants.py
@@ -21,7 +21,6 @@
 SEQUENTIAL_EXECUTOR = "SequentialExecutor"
 CELERY_EXECUTOR = "CeleryExecutor"
 CELERY_KUBERNETES_EXECUTOR = "CeleryKubernetesExecutor"
-DASK_EXECUTOR = "DaskExecutor"
 KUBERNETES_EXECUTOR = "KubernetesExecutor"
 DEBUG_EXECUTOR = "DebugExecutor"
 MOCK_EXECUTOR = "MockExecutor"
diff --git a/airflow/executors/executor_loader.py b/airflow/executors/executor_loader.py
@@ -28,7 +28,6 @@
 from airflow.executors.executor_constants import (
     CELERY_EXECUTOR,
     CELERY_KUBERNETES_EXECUTOR,
-    DASK_EXECUTOR,
     DEBUG_EXECUTOR,
     KUBERNETES_EXECUTOR,
     LOCAL_EXECUTOR,
@@ -64,7 +63,6 @@ class ExecutorLoader:
         CELERY_EXECUTOR: "airflow.providers.celery.executors.celery_executor.CeleryExecutor",
         CELERY_KUBERNETES_EXECUTOR: "airflow.providers.celery."
         "executors.celery_kubernetes_executor.CeleryKubernetesExecutor",
-        DASK_EXECUTOR: "airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor",
         KUBERNETES_EXECUTOR: "airflow.providers.cncf.kubernetes."
         "executors.kubernetes_executor.KubernetesExecutor",
         DEBUG_EXECUTOR: "airflow.executors.debug_executor.DebugExecutor",
@@ -218,5 +216,4 @@ def __load_local_kubernetes_executor(cls) -> BaseExecutor:
 UNPICKLEABLE_EXECUTORS = (
     LOCAL_EXECUTOR,
     SEQUENTIAL_EXECUTOR,
-    DASK_EXECUTOR,
 )
diff --git a/airflow/providers/daskexecutor/CHANGELOG.rst b/airflow/providers/daskexecutor/CHANGELOG.rst
@@ -27,6 +27,12 @@
 Changelog
 ---------
 
+1.1.1
+.....
+
+* Mark daskexecutor provider as removed according to `consensus on the dev@airflow.apache.org <https://lists.apache.org/thread/fxv44cqqljrrhll3fdpdgc9h9fz5ghcy>`_
+
+
 1.1.0
 .....
 

diff --git a/airflow/providers/daskexecutor/provider.yaml b/airflow/providers/daskexecutor/provider.yaml
@@ -21,9 +21,11 @@ name: Dask Executor
 description: |
     `Dask <https://www.dask.org/>`__
 
-suspended: false
+removed: true
+suspended: true
 source-date-epoch: 1700827453
 versions:
+  - 1.1.1
   - 1.1.0
   - 1.0.1
   - 1.0.0

diff --git a/airflow/providers/installed_providers.txt b/airflow/providers/installed_providers.txt
@@ -2,7 +2,6 @@ amazon
 celery
 cncf.kubernetes
 common.sql
-daskexecutor
 docker
 elasticsearch
 ftp

diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -423,7 +423,6 @@ def get_airflow_extras():
     "async",
     "celery",
     "cncf.kubernetes",
-    "daskexecutor",
     "docker",
     "elasticsearch",
     "ftp",

diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py
@@ -109,17 +109,17 @@ def test_get_provider_requirements():
 
 def test_get_removed_providers():
     # Modify it every time we schedule provider for removal or remove it
-    assert [] == get_removed_provider_ids()
+    assert ["daskexecutor"] == get_removed_provider_ids()
 
 
 def test_get_suspended_provider_ids():
     # Modify it every time we suspend/resume provider
-    assert [] == get_suspended_provider_ids()
+    assert ["daskexecutor"] == get_suspended_provider_ids()
 
 
 def test_get_suspended_provider_folders():
     # Modify it every time we suspend/resume provider
-    assert [] == get_suspended_provider_folders()
+    assert ["daskexecutor"] == get_suspended_provider_folders()
 
 
 @pytest.mark.parametrize(

diff --git a/docker_tests/test_prod_image.py b/docker_tests/test_prod_image.py
@@ -122,7 +122,6 @@ def test_pip_dependencies_conflict(self):
         ],
         "celery": ["celery", "flower", "vine"],
         "cncf.kubernetes": ["kubernetes", "cryptography"],
-        "dask": ["cloudpickle", "distributed"],
         "docker": ["docker"],
         "elasticsearch": ["elasticsearch"],
         "google": [

diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst
@@ -247,8 +247,6 @@ Some of those enable Airflow to use executors to run tasks with them - other tha
 +---------------------+-----------------------------------------------------+-----------------------------------------------------------------+----------------------------------------------+
 | cncf.kubernetes     | ``pip install 'apache-airflow[cncf.kubernetes]'``   | Kubernetes client libraries, KubernetesPodOperator & friends    | KubernetesExecutor, LocalKubernetesExecutor  |
 +---------------------+-----------------------------------------------------+-----------------------------------------------------------------+----------------------------------------------+
-| daskexecutor        | ``pip install 'apache-airflow[daskexecutor]'``      | Dask libraries including ``distributed`` execution library      | DaskExecutor                                 |
-+---------------------+-----------------------------------------------------+-----------------------------------------------------------------+----------------------------------------------+
 | docker              | ``pip install 'apache-airflow[docker]'``            | Docker hooks and operators                                      |                                              |
 +---------------------+-----------------------------------------------------+-----------------------------------------------------------------+----------------------------------------------+
 | elasticsearch       | ``pip install 'apache-airflow[elasticsearch]'``     | Elasticsearch hooks and Log Handler                             |                                              |
@@ -392,8 +390,6 @@ so there is no replacement for ``crypto`` extra.
 +---------------------+-----------------------------+
 | crypto              |                             |
 +---------------------+-----------------------------+
-| dask                | daskexecutor                |
-+---------------------+-----------------------------+
 | druid               | apache.druid                |
 +---------------------+-----------------------------+
 | gcp                 | google                      |

diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst
@@ -85,7 +85,6 @@ List of default extras in the production Dockerfile:
 * async
 * celery
 * cncf.kubernetes
-* daskexecutor
 * docker
 * elasticsearch
 * ftp

diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json
@@ -295,16 +295,6 @@
     ],
     "excluded-python-versions": []
   },
-  "daskexecutor": {
-    "deps": [
-      "apache-airflow>=2.5.0",
-      "cloudpickle>=1.4.1",
-      "dask>=2.9.0,!=2022.10.1,!=2023.5.0",
-      "distributed>=2.11.1,!=2023.5.0"
-    ],
-    "cross-providers-deps": [],
-    "excluded-python-versions": []
-  },
   "databricks": {
     "deps": [
       "aiohttp>=3.6.3, <4",