Add a simple conda cluster manager.

This makes it super easy to create remotes with conda environments.
Updated the docs and fixed a minor but in the automator when using this.

automan/api.py

@@ -8,3 +8,4 @@
 from .automation import compare_runs, filter_by_name, filter_cases # noqa
 
 from .cluster_manager import ClusterManager # noqa
+from .conda_cluster_manager import CondaClusterManager # noqa

automan/automation.py

@@ -783,6 +783,8 @@ def __init__(self, simulation_dir, output_dir, all_problems,
         if cluster_manager_factory is None:
             from automan.cluster_manager import ClusterManager
             self.cluster_manager_factory = ClusterManager
+        else:
+            self.cluster_manager_factory = cluster_manager_factory
         self._setup_argparse()
 
     # #### Public Protocol ########################################

automan/conda_cluster_manager.py

@@ -0,0 +1,93 @@
+import os
+import sys
+from textwrap import dedent
+import subprocess
+
+from .cluster_manager import ClusterManager
+
+
+class CondaClusterManager(ClusterManager):  # pragma: no cover
+
+    # The path to conda root on the remote, this is a relative path
+    # and is relative to the home directory.
+    CONDA_ROOT = 'miniconda3'
+
+    BOOTSTRAP = dedent("""\
+    #!/bin/bash
+
+    set -e
+    CONDA_ROOT=%s
+    ENV_FILE="{project_name}/environments.yml"
+    if [ -f $ENV_FILE ] ; then
+        ~/$CONDA_ROOT/bin/conda env create -q -f $ENV_FILE -n {project_name}
+    else
+        ~/$CONDA_ROOT/bin/conda create -y -q -n {project_name} psutil execnet
+    fi
+
+    source ~/$CONDA_ROOT/bin/activate {project_name}
+    pip install automan
+
+    cd {project_name}
+    if [ -f "requirements.txt" ] ; then
+        pip install -r requirements.txt
+    fi
+    """ % CONDA_ROOT)
+
+    UPDATE = dedent("""\
+    #!/bin/bash
+
+    set -e
+    CONDA_ROOT=%s
+    ENV_FILE="{project_name}/environments.yml"
+    if [ -f $ENV_FILE ] ; then
+        ~/$CONDA_ROOT/bin/conda env update -q -f $ENV_FILE -n {project_name}
+    fi
+
+    source ~/$CONDA_ROOT/bin/activate {project_name}
+
+    cd {project_name}
+    if [ -f "requirements.txt" ] ; then
+        pip install -r requirements.txt
+    fi
+
+    """ % CONDA_ROOT)
+
+    def _get_virtualenv(self):
+        return None
+
+    def _get_conda_env_root(self, host):
+        cmd = [
+            'ssh', host,
+            '~/{conda_root}/bin/conda info --base'.format(
+                conda_root=self.CONDA_ROOT
+            )
+        ]
+        path = subprocess.check_output(cmd).strip()
+        if type(path) is bytes:
+            path = path.decode('utf-8')
+        return path
+
+    def add_worker(self, host, home, nfs):
+        if host == 'localhost':
+            self.workers.append(dict(host=host, home=home, nfs=nfs))
+        else:
+            root = self.root
+            curdir = os.path.basename(os.getcwd())
+            if nfs:
+                python = sys.executable
+                chdir = curdir
+            else:
+                conda_root = self._get_conda_env_root(host)
+                python = os.path.join(
+                    conda_root, 'envs/{project_name}/bin/python'.format(
+                        project_name=self.project_name
+                    )
+                )
+                chdir = os.path.join(home, root, curdir)
+            self.workers.append(
+                dict(host=host, home=home, nfs=nfs, python=python, chdir=chdir)
+            )
+
+        self._write_config()
+        if host != 'localhost' and not nfs:
+            self._bootstrap(host, home)

automan/tests/test_automation.py

@@ -20,6 +20,8 @@
 except ImportError:
     raise unittest.SkipTest('test_jobs requires psutil')
 
+from automan.cluster_manager import ClusterManager
+
 from automan.tests.test_jobs import wait_until, safe_rmtree
 
 
@@ -417,6 +419,14 @@ class TestAutomator(TestAutomationBase):
     def setUp(self):
         super(TestAutomator, self).setUp()
 
+    def test_automator_accepts_cluster_manager(self):
+        # Given/When
+        a = Automator('sim', 'output', [EllipticalDrop],
+                      cluster_manager_factory=ClusterManager)
+
+        # Then
+        self.assertEqual(a.cluster_manager_factory, ClusterManager)
+
     @mock.patch.object(TaskRunner, 'run')
     def test_automator(self, mock_run):
         # Given

docs/source/reference/automan.rst

@@ -19,3 +19,7 @@ Cluster management module
 .. automodule:: automan.cluster_manager
    :members:
    :undoc-members:
+
+.. automodule:: automan.conda_cluster_manager
+   :members:
+   :undoc-members:

docs/source/tutorial.rst

@@ -599,6 +599,28 @@ to the :py:class:`automan.automation.Automator` class as the
 you wish to use conda or some other tool to manage the Python environment on
 the remote computer.
 
+We also provide a simple
+:py:class:`automan.conda_cluster_manager.CondaClusterManager` which will setup
+a remote computer so long as it has conda_ on it. If your project directory
+has an ``environments.yml`` and/or a ``requirements.txt`` it will use those to
+setup the environment. This is really a prototype and you may feel free to
+customize this. To use the conda cluster manager you could do the following in
+the tutorial example::
+
+    from automan.api import CondaClusterManager
+
+    automator = Automator(
+        simulation_dir='outputs',
+        output_dir='manuscript/figures',
+        all_problems=[Squares, Powers],
+        cluster_manager_factory=CondaClusterManager
+    )
+    automator.run()
+
+You may also subclass this or customize the bootstrap code and use that.
+
+.. _conda: https://conda.io/
+
 
 Using docker
 ------------