Merge pull request #1461 from opendatacube/deprecate_executor_api

Mark executors as deprecated

.github/workflows/main.yml

@@ -70,7 +70,6 @@ jobs:
         password: ${{ secrets.GADOCKERSVC_PASSWORD }}
 
     - name: Build Docker
-      if: steps.changes.outputs.docker == 'true'
       uses: docker/build-push-action@v4
       with:
         file: docker/Dockerfile

datacube/execution/worker.py

@@ -7,6 +7,7 @@
 """
 
 import click
+from deprecat import deprecat
 
 KNOWN_WORKER_TYPES = ['distributed', 'dask', 'celery']
 
@@ -25,11 +26,7 @@ def parse_executor_opt(ctx, param, value):
     return ex_type, host, port
 
 
-def launch_celery_worker(host, port, nprocs, password=''):
-    from datacube import _celery_runner as cr
-    cr.launch_worker(host, port, password=password, nprocs=nprocs)
-
-
+@deprecat(reason="Executors have been deprecated and will be removed in v1.9", version='1.8.14')
 def launch_distributed_worker(host, port, nprocs, nthreads=1):
     import subprocess
 
@@ -45,13 +42,13 @@ def launch_distributed_worker(host, port, nprocs, nthreads=1):
 
 @click.command(name='worker')
 @click.option('--executor', type=(click.Choice(KNOWN_WORKER_TYPES), str),  # type: ignore
-              help="(distributed|dask(alias for distributed)|celery) host:port",
+              help="WARNING: executors have been deprecated in v1.8.14, and will be removed in v1.9.\n"
+              "(distributed|dask(alias for distributed)) host:port",
               default=(None, None),
               callback=parse_executor_opt)
 @click.option('--nprocs', type=int, default=0, help='Number of worker processes to launch')
 def main(executor, nprocs):
-    launchers = dict(celery=launch_celery_worker,
-                     dask=launch_distributed_worker,
+    launchers = dict(dask=launch_distributed_worker,
                      distributed=launch_distributed_worker)
     ex_type, host, port = executor
     return launchers[ex_type](host, port, nprocs)

datacube/executor.py

@@ -5,10 +5,12 @@
 #
 # type: ignore
 import sys
+from deprecat import deprecat
 
 _REMOTE_LOG_FORMAT_STRING = '%(asctime)s {} %(process)d %(name)s %(levelname)s %(message)s'
 
 
+@deprecat(reason="Executors have been deprecated and will be removed in v1.9", version='1.8.14')
 class SerialExecutor(object):
     def __repr__(self):
         return 'SerialExecutor'
@@ -73,6 +75,7 @@ def setup_logging():
     logging.root.handlers = [handler]
 
 
+@deprecat(reason="Executors have been deprecated and will be removed in v1.9", version='1.8.14')
 def _get_distributed_executor(scheduler):
     """
     :param scheduler: Address of a scheduler
@@ -144,6 +147,7 @@ def _run_cloud_pickled_function(f_data, *args, **kwargs):
     return func(*args, **kwargs)
 
 
+@deprecat(reason="Executors have been deprecated and will be removed in v1.9", version='1.8.14')
 def _get_concurrent_executor(workers, use_cloud_pickle=False):
     try:
         from concurrent.futures import ProcessPoolExecutor, as_completed
@@ -221,6 +225,7 @@ def release(future):
     return MultiprocessingExecutor(ProcessPoolExecutor(workers), use_cloud_pickle)
 
 
+@deprecat(reason="Executors have been deprecated and will be removed in v1.9", version='1.8.14')
 def get_executor(scheduler, workers, use_cloud_pickle=True):
     """
     Return a task executor based on input parameters. Falling back as required.

datacube/ui/click.py

@@ -285,7 +285,9 @@ def _setup_executor(ctx, param, value):
 executor_cli_options = click.option('--executor',  # type: ignore
                                     type=(click.Choice(list(EXECUTOR_TYPES)), str),
                                     default=['serial', None],
-                                    help="Run parallelized, either locally or distributed. eg:\n"
+                                    help="WARNING: executors have been deprecated in v1.8.14, "
+                                    "and will be removed in v1.9.\n"
+                                    "Run parallelized, either locally or distributed. eg:\n"
                                          "--executor multiproc 4 (OR)\n"
                                          "--executor distributed 10.0.0.8:8888",
                                     callback=_setup_executor)

docker/constraints.in

@@ -57,3 +57,5 @@ setuptools_scm>=3.4
 toml
 wheel
 twine
+
+deprecat

docker/constraints.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
 #
 #    pip-compile --strip-extras constraints.in
 #
@@ -103,6 +103,8 @@ dask==2023.2.0
     #   distributed
 decorator==5.1.1
     # via validators
+deprecat==2.1.1
+    # via -r constraints.in
 distributed==2023.2.0
     # via -r constraints.in
 docutils==0.18.1
@@ -112,10 +114,6 @@ docutils==0.18.1
     #   sphinx
     #   sphinx-click
     #   sphinx-rtd-theme
-exceptiongroup==1.1.0
-    # via
-    #   hypothesis
-    #   pytest
 fiona==1.9.1
     # via -r constraints.in
 fonttools==4.38.0
@@ -328,8 +326,6 @@ rich==13.3.1
     # via twine
 ruamel-yaml==0.17.21
     # via -r constraints.in
-ruamel-yaml-clib==0.2.7
-    # via ruamel-yaml
 s3transfer==0.6.0
     # via boto3
 secretstorage==3.3.3
@@ -390,11 +386,6 @@ toml==0.10.2
     # via
     #   -r constraints.in
     #   responses
-tomli==2.0.1
-    # via
-    #   coverage
-    #   pytest
-    #   setuptools-scm
 toolz==0.12.0
     # via
     #   -r constraints.in
@@ -429,7 +420,9 @@ werkzeug==2.2.2
 wheel==0.38.4
     # via -r constraints.in
 wrapt==1.11.2
-    # via astroid
+    # via
+    #   astroid
+    #   deprecat
 xarray==2023.2.0
     # via -r constraints.in
 xmltodict==0.13.0

docs/about/whats_new.rst

@@ -14,6 +14,7 @@ v1.8.next
 - Documentation improvements (:pull:`1455`)
 - Increase default maturity leniency to +-500ms (:pull:`1458`)
 - Add option to specify maturity timedelta when using ``--archive-less-mature`` option (:pull:`1460`)
+- Mark executors as deprecated (:pull:`1461`)
 
 
 v1.8.13 (6th June 2023)

setup.py

@@ -111,6 +111,7 @@
         'toolz',
         'xarray>=0.9',  # >0.9 fixes most problems with `crs` attributes being lost
         'packaging',
+        'deprecat',
     ],
     extras_require=extras_require,
     tests_require=tests_require,