Add Joblib Launcher for parallel execution (multi-process and multi-threaded)

.circleci/config.yml

@@ -15,7 +15,7 @@ jobs:
             ~/miniconda3/bin/conda init bash
       - run:
           name: "Testing Hydra"
-          no_output_timeout: 60m
+          no_output_timeout: 30m
           command: |
             export NOX_PYTHON_VERSIONS=3.6
             ~/miniconda3/bin/conda create -n hydra python=3.6 -q -y
@@ -35,7 +35,7 @@ jobs:
             ~/miniconda3/bin/conda init bash
       - run:
           name: "Testing Hydra"
-          no_output_timeout: 60m
+          no_output_timeout: 30m
           command: |
             export NOX_PYTHON_VERSIONS=3.7
             ~/miniconda3/bin/conda create -n hydra python=3.7 -q -y
@@ -55,7 +55,7 @@ jobs:
             ~/miniconda3/bin/conda init bash
       - run:
           name: "Testing Hydra"
-          no_output_timeout: 60m
+          no_output_timeout: 30m
           command: |
             export NOX_PYTHON_VERSIONS=3.8
             ~/miniconda3/bin/conda create -n hydra python=3.8 -q -y
@@ -66,13 +66,17 @@ jobs:
   # Linux
   py36_linux:
     docker:
-      - image: circleci/python:3.6
+#      - image: circleci/python:3.6
+      # Temporary workaround around 3.6 image
+      - image: circleci/python@sha256:19a446501d3e0ca273ed3161eeca3780dadbc9e3f3a788c8fd2bc92f5ed9bed0
     steps:
       - checkout
       - run: sudo apt-get install -y expect
       - run: echo 'export NOX_PYTHON_VERSIONS=3.6' >> $BASH_ENV
       - run: sudo pip install nox
-      - run: nox
+      - run:
+          name: "Testing Hydra"
+          command: nox
 
   py37_linux:
     docker:
@@ -82,7 +86,9 @@ jobs:
       - run: sudo apt-get install -y expect
       - run: echo 'export NOX_PYTHON_VERSIONS=3.7' >> $BASH_ENV
       - run: sudo pip install nox
-      - run: nox
+      - run:
+          name: "Testing Hydra"
+          command: nox
 
   py38_linux:
     docker:
@@ -92,7 +98,9 @@ jobs:
       - run: sudo apt-get install -y expect
       - run: echo 'export NOX_PYTHON_VERSIONS=3.8' >> $BASH_ENV
       - run: sudo pip install nox
-      - run: nox
+      - run:
+          name: "Testing Hydra"
+          command: nox
 
   # windows
   py36_win:
@@ -111,6 +119,7 @@ jobs:
             conda run -n hydra pip install nox
       - run:
           name: Testing Hydra
+          no_output_timeout: 30m
           command: |
             $env:NOX_PYTHON_VERSIONS=3.6
             $env:PYTHONIOENCODING="utf_8"
@@ -134,14 +143,15 @@ jobs:
             conda run -n hydra pip install nox
       - run:
           name: Testing Hydra
+          no_output_timeout: 30m
           command: |
             $env:NOX_PYTHON_VERSIONS=3.7
             $env:PYTHONIOENCODING="utf_8"
             conda activate hydra
             nox
             exit $LASTEXITCODE
 
-# TODO: test re-enable in a few months
+  # TODO: Test later, for now low priority
 #  py38_win:
 #    executor: win/default
 #    steps:
@@ -212,7 +222,7 @@ workflows:
       # windows
       - py36_win
       - py37_win
-      # - py38_win
+#      - py38_win
       - deploy-website:
           filters:
             branches:

.coveragerc

@@ -11,4 +11,5 @@ exclude_lines =
     raise NotImplementedError
     raise TypeError
     @deprecated
-    assert False
+    assert False
+

.isort.cfg

@@ -5,5 +5,5 @@ force_grid_wrap=0
 use_parentheses=True
 line_length=88
 ensure_newline_before_comments=True
-known_third_party=omegaconf,ray,pytest,typing_extensions
+known_third_party=joblib,omegaconf,ray,pytest,typing_extensions
 known_first_party=hydra,hydra_plugins

news/392.plugin

@@ -0,0 +1 @@
+Add Joblib Launcher plugin.

noxfile.py

@@ -75,6 +75,33 @@ def test_example_app(session, install_cmd):
     )
 
 
+@nox.session
+def lint(session):
+    session.install("--upgrade", "setuptools", "pip")
+    # needed for isort to properly identify pytest as third party
+    session.install("pytest")
+    session.install(
+        "mypy",
+        "flake8",
+        "flake8-copyright",
+        "git+git://github.com/timothycrosley/isort.git@c54b3dd4620f9b3e7ac127c583ecb029fb90f1b7",
+    )
+    session.run("pip", "install", "-e", ".", silent=SILENT)
+    session.run("flake8", "--config", ".circleci/flake8_py3.cfg")
+
+    session.install("black")
+    # if this fails you need to format your code with black
+    session.run("black", "--check", ".")
+
+    session.run("isort", "--check", ".")
+
+    # Mypy
+    session.run("mypy", ".", "--strict")
+    # Mypy for plugins
+    for plugin in get_all_plugins():
+        session.run("mypy", os.path.join("plugins", plugin["path"]), "--strict")
+
+
 @nox.session(python=PYTHON_VERSIONS)
 @nox.parametrize(
     "install_cmd",
@@ -189,33 +216,6 @@ def coverage(session):
     session.run("coverage", "erase")
 
 
-@nox.session
-def lint(session):
-    session.install("--upgrade", "setuptools", "pip")
-    # needed for isort to properly identify pytest as third party
-    session.install("pytest")
-    session.install(
-        "mypy",
-        "flake8",
-        "flake8-copyright",
-        "git+git://github.com/timothycrosley/isort.git@c54b3dd4620f9b3e7ac127c583ecb029fb90f1b7",
-    )
-    session.run("pip", "install", "-e", ".", silent=SILENT)
-    session.run("flake8", "--config", ".circleci/flake8_py3.cfg")
-
-    session.install("black")
-    # if this fails you need to format your code with black
-    session.run("black", "--check", ".")
-
-    session.run("isort", "--check", ".")
-
-    # Mypy
-    session.run("mypy", ".", "--strict")
-    # Mypy for plugins
-    for plugin in get_all_plugins():
-        session.run("mypy", os.path.join("plugins", plugin["path"]), "--strict")
-
-
 @nox.session(python=PYTHON_VERSIONS)
 def test_jupyter_notebook(session):
     versions = copy.copy(DEFAULT_PYTHON_VERSIONS)

plugins/hydra_joblib_launcher/MANIFEST.in

@@ -0,0 +1,3 @@
+global-exclude *.pyc
+global-exclude __pycache__
+recursive-include hydra_plugins/* *.yaml

plugins/hydra_joblib_launcher/README.md

@@ -0,0 +1,2 @@
+# Hydra Joblib Launcher
+See [website](https://hydra.cc/docs/plugins/joblib_launcher) for more information

plugins/hydra_joblib_launcher/example/config.yaml

@@ -0,0 +1,4 @@
+defaults:
+  - hydra/launcher: joblib
+
+task: 1

plugins/hydra_joblib_launcher/example/my_app.py

@@ -0,0 +1,19 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import logging
+import os
+import time
+
+import hydra
+
+log = logging.getLogger(__name__)
+
+
+@hydra.main(config_path="config.yaml")
+def my_app(cfg):
+    log.info(f"Process ID {os.getpid()} executing task {cfg.task} ...")
+
+    time.sleep(1)
+
+
+if __name__ == "__main__":
+    my_app()

plugins/hydra_joblib_launcher/hydra_plugins/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# type: ignore
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

plugins/hydra_joblib_launcher/hydra_plugins/hydra_joblib_launcher/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from .joblib_launcher import JoblibLauncher
+
+__all__ = ["JoblibLauncher"]

plugins/hydra_joblib_launcher/hydra_plugins/hydra_joblib_launcher/conf/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved

plugins/hydra_joblib_launcher/hydra_plugins/hydra_joblib_launcher/conf/hydra/launcher/joblib.yaml

@@ -0,0 +1,39 @@
+hydra:
+  launcher:
+    class: hydra_plugins.hydra_joblib_launcher.JoblibLauncher
+    params:
+      joblib: ${hydra.joblib}
+
+  joblib:
+    # maximum number of concurrently running jobs. if -1, all CPUs are used
+    n_jobs: -1
+
+    # allows to hard-code backend, otherwise inferred based on prefer and require
+    backend: null
+
+    # processes or threads, soft hint to choose backend
+    prefer: processes
+
+    # null or sharedmem, sharedmem will select thread-based backend
+    require: null
+
+    # if greater than zero, prints progress messages
+    verbose: 0
+
+    # timeout limit for each task
+    timeout: null
+
+    # number of batches to be pre-dispatched
+    pre_dispatch: 2*n_jobs
+
+    # number of atomic tasks to dispatch at once to each worker
+    batch_size: auto
+
+    # folder used for memmapping large arrays for sharing memory with workers
+    temp_folder: null
+
+    # thresholds size of arrays that triggers automated memmapping
+    max_nbytes: 1M
+
+    # memmapping mode for numpy arrays passed to workers
+    mmap_mode: r