Merge pull request #79 from natefoo/env-list

Convert env to a list compatible with Galaxy job conf

docs/topics/tpv_by_example.rst

@@ -405,7 +405,7 @@ multiple TPV config files, again based on order of appearance.
        params:
          nativeSpecification: "--nodes=1 --ntasks={cores} --ntasks-per-node={cores} --mem={mem*1024}"
 
-     toolshed.g2.bx.psu.edu/repos/iuc/hisat2/hisat2/*:
+     toolshed.g2.bx.psu.edu/repos/iuc/hisat2/hisat2/.*:
        mem: cores * 4
        gpus: 1
 
@@ -417,6 +417,42 @@ multiple TPV config files, again based on order of appearance.
 In this example, dispatching a hisat2 job would result in a mem value of 8, with 1 gpu. However, dispatching
 the specific version of `2.1.0+galaxy7` would result in the additional env variable, with mem remaining at 8.
 
+Job Environment
+---------------
+
+As seen in the previous example, it is possible to specify environment variables that will be set in the job's executing
+environment. It is also possible to source environment files and execute commands, using the same syntax as in Galaxy's
+job_conf.yml, by specifying ``env`` as a list instead of a dictionary.
+
+.. code-block:: yaml
+   :linenos:
+
+   tools:
+     default:
+       cores: 2
+       mem: 4
+       params:
+         nativeSpecification: "--nodes=1 --ntasks={cores} --ntasks-per-node={cores} --mem={mem*1024}"
+       env:
+         - execute: echo "Don't Panic!"
+
+     toolshed.g2.bx.psu.edu/repos/iuc/hisat2/hisat2/.*:
+       mem: cores * 4
+       gpus: 1
+       env:
+         - name: MY_ADDITIONAL_FLAG
+           value: "arthur"
+         - file: /galaxy/tools/hisat2.env
+
+     toolshed.g2.bx.psu.edu/repos/iuc/hisat2/hisat2/2.1.0+galaxy7:
+       inherits: toolshed.g2.bx.psu.edu/repos/iuc/hisat2/hisat2/.*:
+       env:
+         MY_ADDITIONAL_FLAG: "zaphod"
+
+In this example, all jobs will execute the command ``echo "Don't Panic!"``. All versions of hisat2 will have
+``$MY_ADDITIONAL_FLAG`` set and will source the file ``/galaxy/tools/hisat2.env``, but version ``2.1.0+galaxy7`` will
+have the value ``zaphod`` set for ``$MY_ADDITIONAL_FLAG`` instead of the hisat2 default of ``arthur``.
+
 Job Resubmission
 ----------------
 TPV has explict support for job resubmissions, so that advanced control over job resubmission is possible.

tpv/core/entities.py

@@ -190,7 +190,7 @@ def __init__(self, loader, id=None, abstract=False, cores=None, mem=None, gpus=N
         self.max_cores = max_cores
         self.max_mem = max_mem
         self.max_gpus = max_gpus
-        self.env = env
+        self.env = self.convert_env(env)
         self.params = params
         self.resubmit = resubmit
         self.tpv_tags = TagSetManager.from_dict(tpv_tags or {})
@@ -234,6 +234,11 @@ def evaluate_complex_property(self, prop, context, stringify=False):
         return self.process_complex_property(
             prop, context, lambda p, c: self.loader.eval_code_block(p, c, as_f_string=True), stringify=stringify)
 
+    def convert_env(self, env):
+        if isinstance(env, dict):
+            env = [dict(name=k, value=v) for (k, v) in env.items()]
+        return env
+
     def validate(self):
         """
         Validates each code block and makes sure the code can be compiled.
@@ -275,6 +280,16 @@ def __repr__(self):
                f"tags={self.tpv_tags}, rank={self.rank[:10] if self.rank else ''}, inherits={self.inherits}, "\
                f"context={self.context}"
 
+    def merge_env_list(self, original, replace):
+        for i, original_elem in enumerate(original):
+            for j, replace_elem in enumerate(replace):
+                if (("name" in replace_elem and original_elem.get("name") == replace_elem["name"])
+                        or original_elem == replace_elem):
+                    original[i] = replace.pop(j)
+                    break
+        original.extend(replace)
+        return original
+
     def override(self, entity):
         if entity.merge_order <= self.merge_order:
             # Use the broader class as a base when copying. Useful in particular for Rules
@@ -292,8 +307,7 @@ def override(self, entity):
         new_entity.max_cores = self.max_cores if self.max_cores is not None else entity.max_cores
         new_entity.max_mem = self.max_mem if self.max_mem is not None else entity.max_mem
         new_entity.max_gpus = self.max_gpus if self.max_gpus is not None else entity.max_gpus
-        new_entity.env = copy.copy(entity.env) or {}
-        new_entity.env.update(self.env or {})
+        new_entity.env = self.merge_env_list(copy.deepcopy(entity.env) or [], copy.deepcopy(self.env) or [])
         new_entity.params = copy.copy(entity.params) or {}
         new_entity.params.update(self.params or {})
         new_entity.resubmit = copy.copy(entity.resubmit) or {}

tpv/core/mapper.py

@@ -75,7 +75,7 @@ def to_galaxy_destination(self, destination):
             tags=destination.handler_tags,
             runner=destination.runner,
             params=destination.params,
-            env=[dict(name=k, value=v) for (k, v) in destination.env.items()],
+            env=destination.env,
             resubmit=list(destination.resubmit.values()),
         )