Finish hooking the pieces together for Conda resolution.

With a test case.

Author: jmchilton

Makefile

@@ -83,7 +83,7 @@ test-install-pypi:
 	bash install_test/test_install.bash
 
 test-install-wheel: dist
-	PULSAR_INSTALL_TARGET=$(shell pwd)/dist/pulsar_app*.whl bash install_test/test_install.bash
+	PULSAR_INSTALL_TARGET=$(shell pwd)/dist/pulsar_app*.whl bash install_test/test_install_conda.bash
 
 coverage:
 	coverage run --source $(SOURCE_DIR) setup.py $(TEST_DIR)

install_test/common_functions.bash

@@ -2,6 +2,7 @@ set -e
 
 shopt -s nullglob
 
+PULSAR_TARGET_POORT="${PULSAR_TARGET_POORT:-8913}"
 PULSAR_INSTALL_TARGET="${PULSAR_INSTALL_TARGET:-pulsar-app}"
 PLANEMO_INSTALL_TARGET="${PLANEMO_INSTALL_TARGET:-planemo==0.36.1}"
 
@@ -38,9 +39,20 @@ stop_pulsar() {
 check_pulsar() {
     cd pulsar
 
+    if curl -s "http://localhost:$PULSAR_TARGET_POORT"
+    then
+        echo "Port $PULSAR_TARGET_POORT already bound, Pulsar will fail to start."
+        exit 1;
+    fi
+
     echo "Starting Pulsar in daemon mode."
     pulsar --daemon
     echo "Waiting for Pulsar to start."
+    while ! curl -s "http://localhost:$PULSAR_TARGET_POORT";
+    do
+        printf "."
+        sleep 1;
+    done
     sleep 2
     echo "Running a standalone Pulsar job."
     pulsar-check # runs a test job

install_test/test_install_conda.bash

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+SCRIPT_DIR="$( cd "$(dirname "$0")" ; pwd )"
+. "$SCRIPT_DIR/common_functions.bash"
+
+init_temp_dir
+
+init_pulsar
+
+cd pulsar
+echo "Running pulsar-config with --auto_conda"
+pulsar-config --auto_conda
+
+cd ..
+
+check_pulsar
+
+init_planemo "conda_testing"
+
+run_planemo --job_config_file "$SCRIPT_DIR/galaxy_job_conf.xml" test_tools/bwa.xml
+
+stop_pulsar

pulsar/core.py

@@ -124,7 +124,11 @@ def __setup_object_store(self, conf):
     def __setup_dependency_manager(self, conf):
         dependencies_dir = conf.get("tool_dependency_dir", "dependencies")
         resolvers_config_file = conf.get("dependency_resolvers_config_file", "dependency_resolvers_conf.xml")
-        self.dependency_manager = DependencyManager(dependencies_dir, resolvers_config_file)
+        conda_config = {}
+        for key, value in conf.items():
+            if key.startswith("conda_"):
+                conda_config[key] = value
+        self.dependency_manager = DependencyManager(dependencies_dir, resolvers_config_file, **conda_config)
 
     def __setup_job_metrics(self, conf):
         job_metrics = conf.get("job_metrics", None)

pulsar/managers/base/__init__.py

@@ -176,15 +176,16 @@ def _list_dir(self, directory_or_none):
         else:
             return listdir(directory_or_none)
 
-    def _expand_command_line(self, command_line, dependencies_description):
+    def _expand_command_line(self, command_line, dependencies_description, job_directory=None):
         if dependencies_description is None:
             return command_line
 
         requirements = dependencies_description.requirements
         installed_tool_dependencies = dependencies_description.installed_tool_dependencies
         dependency_commands = self.dependency_manager.dependency_shell_commands(
             requirements=requirements,
-            installed_tool_dependencies=installed_tool_dependencies
+            installed_tool_dependencies=installed_tool_dependencies,
+            job_directory=job_directory,
         )
         if dependency_commands:
             command_line = "%s; %s" % ("; ".join(dependency_commands), command_line)

pulsar/managers/base/directory.py

@@ -105,7 +105,7 @@ def _tool_id(self, job_id):
 
     # Helpers methods related to setting up job script files.
     def _setup_job_file(self, job_id, command_line, dependencies_description=None, env=[]):
-        command_line = self._expand_command_line(command_line, dependencies_description)
+        command_line = self._expand_command_line(command_line, dependencies_description, job_directory=self.job_directory(job_id).job_directory)
         script_env = self._job_template_env(job_id, command_line=command_line, env=env)
         script = job_script(**script_env)
         return self._write_job_script(job_id, script)

pulsar/managers/queued_cli.py

@@ -26,7 +26,7 @@ def launch(self, job_id, command_line, submit_params={}, dependencies_descriptio
         stdout_path = self._stdout_path(job_id)
         stderr_path = self._stderr_path(job_id)
         job_name = self._job_name(job_id)
-        command_line = self._expand_command_line(command_line, dependencies_description)
+        command_line = self._expand_command_line(command_line, dependencies_description, job_directory=self.job_directory(job_id).job_directory)
         job_script_kwargs = self._job_template_env(job_id, command_line=command_line, env=env)
         extra_kwargs = job_interface.job_script_kwargs(stdout_path, stderr_path, job_name)
         job_script_kwargs.update(extra_kwargs)

pulsar/managers/unqueued.py

@@ -152,7 +152,7 @@ def _prepare_run(self, job_id, command_line, dependencies_description, env):
         if platform.system().lower() == "windows":
             # TODO: Don't ignore requirements and env without warning. Ideally
             # process them or at least warn about them being ignored.
-            command_line = self._expand_command_line(command_line, dependencies_description)
+            command_line = self._expand_command_line(command_line, dependencies_description, job_directory=self.job_directory(job_id).job_directory)
         else:
             command_line = self._setup_job_file(job_id, command_line, dependencies_description=dependencies_description, env=env)
         return command_line

pulsar/scripts/config.py

@@ -29,6 +29,8 @@
 HELP_DIRECTORY = "Directory containing the configuration files for Pulsar."
 HELP_MQ = ("Write configuration files for message queue server deployment "
            "instead of more traditional RESTful web based pulsar.")
+HELP_AUTO_CONDA = ("Auto initialize Conda for tool resolution and auto install "
+                   "dependencies on demand.")
 HELP_NO_LOGGING = ("Do not write Pulsar's default logging configuration to server.ini "
                    "and if uwsgi is configured do not configure its logging either.")
 HELP_SUPERVISOR = ("Write a supervisord configuration file for "
@@ -145,6 +147,10 @@ def main(argv=None):
     arg_parser.add_argument("--directory",
                             default=".",
                             help=HELP_DIRECTORY)
+    arg_parser.add_argument("--auto_conda",
+                            action="store_true",
+                            default=False,
+                            help=HELP_AUTO_CONDA)
     arg_parser.add_argument("--mq",
                             action="store_true",
                             default=False,
@@ -198,6 +204,10 @@ def main(argv=None):
     if not os.path.exists(directory):
         os.makedirs(directory)
 
+    default_dependencies_dir = os.path.join(directory, "dependencies")
+    if not os.path.exists(default_dependencies_dir):
+        os.makedirs(default_dependencies_dir)
+
     print("Bootstrapping pulsar configuration into directory %s" % relative_directory)
     _handle_app_yaml(args, directory)
     _handle_server_ini(args, directory)
@@ -298,6 +308,9 @@ def _handle_app_yaml(args, directory):
         contents += 'private_token: %s\n' % args.private_token
     if args.mq:
         contents += 'message_queue_url: "amqp://guest:guest@localhost:5672//"\n'
+    if args.auto_conda:
+        contents += 'conda_auto_init: true\n'
+        contents += 'conda_auto_install: true\n'
     else:
         if not IS_WINDOWS and args.libdrmaa_path:
             contents += 'manager:\n  type: queued_drmaa\n'