Remove repetition in Galaxy env setup in the job script and fix issues

with remote metadata on Pulsar.

lib/galaxy/dependencies/pinned-requirements.txt

@@ -57,7 +57,7 @@ anyjson==0.3.3
 
 # Pulsar requirements
 psutil==4.1.0
-pulsar-galaxy-lib==0.7.0.dev5
+pulsar-galaxy-lib==0.7.5
 
 # sqlalchemy-migrate and dependencies
 sqlalchemy-migrate==0.10.0

lib/galaxy/jobs/command_factory.py

@@ -17,19 +17,7 @@
 CAPTURE_RETURN_CODE = "return_code=$?"
 YIELD_CAPTURED_CODE = 'sh -c "exit $return_code"'
 SETUP_GALAXY_FOR_METADATA = """
-if [ "$GALAXY_LIB" != "None" ]; then
-    if [ -n "$PYTHONPATH" ]; then
-        PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"
-    else
-        PYTHONPATH="$GALAXY_LIB"
-    fi
-    export PYTHONPATH
-fi
-if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \
-     -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then
-    . "$GALAXY_VIRTUAL_ENV/bin/activate"
-fi
-GALAXY_PYTHON=`command -v python`
+_galaxy_setup_environment True
 """
 
 

lib/galaxy/jobs/runners/pulsar.py

@@ -51,15 +51,18 @@
     'PulsarEmbeddedJobRunner',
 )
 
-MINIMUM_PULSAR_VERSION = LooseVersion("0.7.0.dev3")
+MINIMUM_PULSAR_VERSIONS = {
+    '_default_': LooseVersion("0.7.0.dev3"),
+    'remote_metadata': LooseVersion("0.7.5"),
+}
 
 NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE = "Pulsar misconfiguration - Pulsar client configured to set metadata remotely, but remote Pulsar isn't properly configured with a galaxy_home directory."
 NO_REMOTE_DATATYPES_CONFIG = "Pulsar client is configured to use remote datatypes configuration when setting metadata externally, but Pulsar is not configured with this information. Defaulting to datatypes_conf.xml."
 GENERIC_REMOTE_ERROR = "Failed to communicate with remote job server."
 FAILED_REMOTE_ERROR = "Remote job server indicated a problem running or monitoring this job."
 LOST_REMOTE_ERROR = "Remote job server could not determine this job's state."
 
-UPGRADE_PULSAR_ERROR = "Galaxy is misconfigured, please contact administrator. The target Pulsar server is unsupported, this version of Galaxy requires Pulsar version %s or newer." % MINIMUM_PULSAR_VERSION
+UPGRADE_PULSAR_ERROR = "Galaxy is misconfigured, please contact administrator. The target Pulsar server is unsupported, this version of Galaxy requires Pulsar version %s or newer."
 
 # Is there a good way to infer some default for this? Can only use
 # url_for from web threads. https://gist.github.com/jmchilton/9098762
@@ -310,6 +313,11 @@ def queue_job(self, job_wrapper):
         pulsar_job_state.job_destination = job_destination
         self.monitor_job(pulsar_job_state)
 
+    def __needed_features(self, client):
+            return {
+                'remote_metadata': PulsarJobRunner.__remote_metadata(client),
+            }
+
     def __prepare_job(self, job_wrapper, job_destination):
         """Build command-line and Pulsar client for this job."""
         command_line = None
@@ -319,8 +327,9 @@ def __prepare_job(self, job_wrapper, job_destination):
         try:
             client = self.get_client_from_wrapper(job_wrapper)
             tool = job_wrapper.tool
-            remote_job_config = client.setup(tool.id, tool.version)
-            PulsarJobRunner.check_job_config(remote_job_config)
+            remote_job_config = client.setup(tool.id, tool.version, tool.requires_galaxy_python_environment)
+            needed_features = self.__needed_features(client)
+            PulsarJobRunner.check_job_config(remote_job_config, check_features=needed_features)
             rewrite_parameters = PulsarJobRunner.__rewrite_parameters(client)
             prepare_kwds = {}
             if rewrite_parameters:
@@ -593,12 +602,17 @@ def __client_outputs(self, client, job_wrapper):
         return client_outputs
 
     @staticmethod
-    def check_job_config(remote_job_config):
+    def check_job_config(remote_job_config, check_features=None):
+        check_features = check_features or {}
         # 0.6.0 was newest Pulsar version that did not report it's version.
         pulsar_version = LooseVersion(remote_job_config.get('pulsar_version', "0.6.0"))
+        needed_version = LooseVersion("0.0.0")
         log.info("pulsar_version is %s" % pulsar_version)
-        if pulsar_version < MINIMUM_PULSAR_VERSION:
-            raise UnsupportedPulsarException()
+        for feature in list(check_features.keys()) + ['_default_']:
+            if pulsar_version < MINIMUM_PULSAR_VERSIONS[feature]:
+                needed_version = max(needed_version, MINIMUM_PULSAR_VERSIONS[feature])
+        if pulsar_version < needed_version:
+            raise UnsupportedPulsarException(needed_version)
 
     @staticmethod
     def __dependencies_description(pulsar_client, job_wrapper):
@@ -854,5 +868,5 @@ def tool_directory(self):
 
 class UnsupportedPulsarException(Exception):
 
-    def __init__(self):
-        super(UnsupportedPulsarException, self).__init__(UPGRADE_PULSAR_ERROR)
+    def __init__(self, needed):
+        super(UnsupportedPulsarException, self).__init__(UPGRADE_PULSAR_ERROR % needed)

lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh

@@ -1,25 +1,32 @@
 #!$shell
 
+_galaxy_setup_environment() {
+    local _use_framework_galaxy="$1"
+    if [ "$GALAXY_LIB" != "None" -a "$_use_framework_galaxy" = "True" ]; then
+        if [ -n "$PYTHONPATH" ]; then
+            PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"
+        else
+            PYTHONPATH="$GALAXY_LIB"
+        fi
+        export PYTHONPATH
+    fi
+    $env_setup_commands
+    if [ "$GALAXY_VIRTUAL_ENV" != "None" -a  "$_use_framework_galaxy" = "True" \
+         -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" \
+         -a "`command -v python`" != "$GALAXY_VIRTUAL_ENV/bin/python" ]; then
+        . "$GALAXY_VIRTUAL_ENV/bin/activate"
+    fi
+}
+
 $headers
 $integrity_injection
 $slots_statement
 export GALAXY_SLOTS
+GALAXY_VIRTUAL_ENV="$galaxy_virtual_env"
 PRESERVE_GALAXY_ENVIRONMENT="$preserve_python_environment"
 GALAXY_LIB="$galaxy_lib"
-if [ "$GALAXY_LIB" != "None" -a "$PRESERVE_GALAXY_ENVIRONMENT" = "True" ]; then
-    if [ -n "$PYTHONPATH" ]; then
-        PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"
-    else
-        PYTHONPATH="$GALAXY_LIB"
-    fi
-    export PYTHONPATH
-fi
-$env_setup_commands
-GALAXY_VIRTUAL_ENV="$galaxy_virtual_env"
-if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \
-     -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" -a "$PRESERVE_GALAXY_ENVIRONMENT" = "True" ]; then
-    . "$GALAXY_VIRTUAL_ENV/bin/activate"
-fi
+_galaxy_setup_environment "$PRESERVE_GALAXY_ENVIRONMENT"
+GALAXY_PYTHON=`command -v python`
 $instrument_pre_commands
 cd $working_directory
 $command