Merge pull request #1658 from einon/pass_lists_as_file

Optionally pass dataset lists as a single file within tool wrappers

lib/galaxy/tools/evaluation.py

@@ -130,7 +130,7 @@ def input():
         param_dict.update( incoming )
 
         input_dataset_paths = dataset_path_rewrites( input_paths )
-        self.__populate_wrappers(param_dict, input_dataset_paths)
+        self.__populate_wrappers(param_dict, input_dataset_paths, job_working_directory)
         self.__populate_input_dataset_wrappers(param_dict, input_datasets, input_dataset_paths)
         self.__populate_output_dataset_wrappers(param_dict, output_datasets, output_paths, job_working_directory)
         self.__populate_output_collection_wrappers(param_dict, output_collections, output_paths, job_working_directory)
@@ -165,18 +165,20 @@ def do_walk( inputs, input_values ):
 
         do_walk( inputs, input_values )
 
-    def __populate_wrappers(self, param_dict, input_dataset_paths):
+    def __populate_wrappers(self, param_dict, input_dataset_paths, job_working_directory):
 
         def wrap_input( input_values, input ):
             if isinstance( input, DataToolParameter ) and input.multiple:
                 value = input_values[ input.name ]
                 dataset_instances = DatasetListWrapper.to_dataset_instances( value )
                 input_values[ input.name ] = \
-                    DatasetListWrapper( dataset_instances,
+                    DatasetListWrapper( job_working_directory,
+                                        dataset_instances,
                                         dataset_paths=input_dataset_paths,
                                         datatypes_registry=self.app.datatypes_registry,
                                         tool=self.tool,
                                         name=input.name )
+
             elif isinstance( input, DataToolParameter ):
                 # FIXME: We're populating param_dict with conversions when
                 #        wrapping values, this should happen as a separate
@@ -234,6 +236,7 @@ def wrap_input( input_values, input ):
                     name=input.name
                 )
                 wrapper = DatasetCollectionWrapper(
+                    job_working_directory,
                     dataset_collection,
                     **wrapper_kwds
                 )
@@ -306,6 +309,7 @@ def __populate_output_collection_wrappers(self, param_dict, output_collections,
                 name=name
             )
             wrapper = DatasetCollectionWrapper(
+                job_working_directory,
                 out_collection,
                 **wrapper_kwds
             )

lib/galaxy/tools/parameters/basic.py

@@ -1114,7 +1114,7 @@ def to_dict( self, trans, view='collection', value_mapper=None, other_values={}
             'options'   : options,
             'value'     : value,
             'display'   : self.display,
-            'multiple'  : self.multiple
+            'multiple'  : self.multiple,
         })
 
         return d

lib/galaxy/tools/parameters/wrapped.py

@@ -58,7 +58,8 @@ def wrap_values( self, inputs, input_values, skip_missing_values=False ):
                 value = input_values[ input.name ]
                 dataset_instances = DatasetListWrapper.to_dataset_instances( value )
                 input_values[ input.name ] = \
-                    DatasetListWrapper( dataset_instances,
+                    DatasetListWrapper( None,
+                                        dataset_instances,
                                         datatypes_registry=trans.app.datatypes_registry,
                                         tool=tool,
                                         name=input.name )
@@ -72,6 +73,7 @@ def wrap_values( self, inputs, input_values, skip_missing_values=False ):
                 input_values[ input.name ] = SelectToolParameterWrapper( input, input_values[ input.name ], tool.app, other_values=incoming )
             elif isinstance( input, DataCollectionToolParameter ):
                 input_values[ input.name ] = DatasetCollectionWrapper(
+                    None,
                     input_values[ input.name ],
                     datatypes_registry=trans.app.datatypes_registry,
                     tool=tool,

lib/galaxy/tools/wrappers.py

@@ -1,4 +1,6 @@
+import os
 import pipes
+import tempfile
 from galaxy import exceptions
 from galaxy.util.none_like import NoneDataset
 from galaxy.util import odict
@@ -263,11 +265,18 @@ def _dataset_wrapper( self, dataset, dataset_paths, **kwargs ):
                 wrapper_kwds[ "dataset_path" ] = dataset_paths[ real_path ]
         return DatasetFilenameWrapper( dataset, **wrapper_kwds )
 
+    def paths_as_file(self, sep="\n"):
+        handle, filepath = tempfile.mkstemp(prefix="gx_file_list", dir=self.job_working_directory)
+        contents = sep.join(map(str, self))
+        os.write(handle, contents)
+        os.close(handle)
+        return filepath
+
 
 class DatasetListWrapper( list, ToolParameterValueWrapper, HasDatasets ):
     """
     """
-    def __init__( self, datasets, dataset_paths=[], **kwargs ):
+    def __init__( self, job_working_directory, datasets, dataset_paths=[], **kwargs ):
         if not isinstance(datasets, list):
             datasets = [datasets]
 
@@ -279,6 +288,7 @@ def to_wrapper( dataset ):
             return self._dataset_wrapper( dataset, dataset_paths, **kwargs )
 
         list.__init__( self, map( to_wrapper, datasets ) )
+        self.job_working_directory = job_working_directory
 
     @staticmethod
     def to_dataset_instances( dataset_instance_sources ):
@@ -300,8 +310,9 @@ def __str__( self ):
 
 class DatasetCollectionWrapper( ToolParameterValueWrapper, HasDatasets ):
 
-    def __init__( self, has_collection, dataset_paths=[], **kwargs ):
+    def __init__( self, job_working_directory, has_collection, dataset_paths=[], **kwargs ):
         super(DatasetCollectionWrapper, self).__init__()
+        self.job_working_directory = job_working_directory
 
         if has_collection is None:
             self.__input_supplied = False
@@ -330,7 +341,7 @@ def __init__( self, has_collection, dataset_paths=[], **kwargs ):
             element_identifier = dataset_collection_element.element_identifier
 
             if dataset_collection_element.is_collection:
-                element_wrapper = DatasetCollectionWrapper( dataset_collection_element, dataset_paths, **kwargs )
+                element_wrapper = DatasetCollectionWrapper(job_working_directory, dataset_collection_element, dataset_paths, **kwargs )
             else:
                 element_wrapper = self._dataset_wrapper( element_object, dataset_paths, **kwargs)
 

test/functional/tools/paths_as_file.xml

@@ -0,0 +1,37 @@
+<tool id="paths_as_file" name="paths_as_file" version="0.1.0">
+    <configfiles>
+        <configfile name="check_paths_file"><![CDATA[
+import sys
+paths_file = sys.argv[1]
+sep = sys.argv[2]
+if sep == "NEWLINE":
+    sep = "\n"
+with open(paths_file, "r") as f:
+    paths = f.read()
+assert paths == sep.join(sys.argv[3:])
+]]></configfile>
+    </configfiles>
+    <command detect_errors="exit_code"><![CDATA[
+      python $check_paths_file $inputs.paths_as_file NEWLINE #for $f in $inputs# ${f} #end for#
+      &&
+      python $check_paths_file $inputs.paths_as_file(sep=',') ',' #for $f in $inputs# ${f} #end for#
+      &&
+      printf 'All Done' > $out1
+    ]]></command>
+    <inputs>
+        <param name="inputs" type="data" format="txt" multiple="true" label="Data 1" />
+    </inputs>
+    <outputs>
+        <data format="txt" name="out1" />
+    </outputs>
+    <tests>
+    <test>
+      <param name="inputs" value="simple_line.txt,simple_line_alternative.txt" />
+      <output name="out1">
+          <assert_contents>
+              <has_line line="All Done" />
+          </assert_contents>
+      </output>
+    </test>
+  </tests>
+</tool>

test/functional/tools/samples_tool_conf.xml

@@ -44,6 +44,7 @@
   <tool file="parallelism_optional.xml" />
   <tool file="implicit_default_conds.xml" />
   <tool file="multi_data_param.xml" />
+  <tool file="paths_as_file.xml" />
   <tool file="column_param.xml" />
   <tool file="column_multi_param.xml" />
   <tool file="special_params.xml" />