Test and clarifications for composite uploads.

- Add an API test for datatype-defined composite uploads - including exercising newline conversion and the space_to_tab parameter.
- Add a test decorator skip_without_datatype to mirror skip_without_tool for this test, improve both decorators.
- The ftype parameter in the composite test tools does nothing - drop it and drop it from the XSD spec.
- Slightly improve the documentation for these composite_data elements in the XSD.

lib/galaxy/tools/xsd/galaxy.xsd

@@ -1062,18 +1062,18 @@ of ``type`` ``data``.</xs:documentation>
   </xs:complexType>
   <xs:complexType name="TestCompositeData">
     <xs:annotation>
-      <xs:documentation xml:lang="en">Define extra composite input files for test input.</xs:documentation>
+      <xs:documentation xml:lang="en">Define extra composite input files for test
+input. The specified ``ftype`` on the parent ``param`` should specify a composite
+datatype with defined static composite files. The order of the defined composite
+files on the datatype must match the order specified with these elements and All
+non-optional composite inputs must be specified as part of the ``param``.
+</xs:documentation>
     </xs:annotation>
     <xs:attribute name="value" type="xs:string" use="required">
       <xs:annotation>
         <xs:documentation xml:lang="en">Path relative to test-data of composite file.</xs:documentation>
       </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="ftype" type="xs:string">
-      <xs:annotation>
-        <xs:documentation xml:lang="en">Optional datatype of composite file for test input.</xs:documentation>
-      </xs:annotation>
-    </xs:attribute>
   </xs:complexType>
   <xs:complexType name="TestCollection">
     <xs:annotation>

test/api/test_tools.py

@@ -6,7 +6,8 @@
     DatasetCollectionPopulator,
     DatasetPopulator,
     LibraryPopulator,
-    skip_without_tool
+    skip_without_tool,
+    skip_without_datatype,
 )
 from galaxy.tools.verify.test_data import TestDataResolver
 
@@ -119,6 +120,71 @@ def test_rdata_not_decompressed(self):
         rdata_metadata = self._upload_and_get_details(open(rdata_path, "rb"), file_type="auto")
         self.assertEquals(rdata_metadata["file_ext"], "rdata")
 
+    @skip_without_datatype("velvet")
+    def test_composite_datatype(self):
+        with self.dataset_populator.test_history() as history_id:
+            dataset = self._velvet_upload(history_id, extra_inputs={
+                "files_1|url_paste": "roadmaps content",
+                "files_1|type": "upload_dataset",
+                "files_2|url_paste": "log content",
+                "files_2|type": "upload_dataset",
+            })
+
+            roadmaps_content = self._get_roadmaps_content(history_id, dataset)
+            assert roadmaps_content.strip() == "roadmaps content", roadmaps_content
+
+    @skip_without_datatype("velvet")
+    def test_composite_datatype_space_to_tab(self):
+        # Like previous test but set one upload with space_to_tab to True to
+        # verify that works.
+        with self.dataset_populator.test_history() as history_id:
+            dataset = self._velvet_upload(history_id, extra_inputs={
+                "files_1|url_paste": "roadmaps content",
+                "files_1|type": "upload_dataset",
+                "files_1|space_to_tab": "Yes",
+                "files_2|url_paste": "log content",
+                "files_2|type": "upload_dataset",
+            })
+
+            roadmaps_content = self._get_roadmaps_content(history_id, dataset)
+            assert roadmaps_content.strip() == "roadmaps\tcontent", roadmaps_content
+
+    @skip_without_datatype("velvet")
+    def test_composite_datatype_posix_lines(self):
+        # Like previous test but set one upload with space_to_tab to True to
+        # verify that works.
+        with self.dataset_populator.test_history() as history_id:
+            dataset = self._velvet_upload(history_id, extra_inputs={
+                "files_1|url_paste": "roadmaps\rcontent",
+                "files_1|type": "upload_dataset",
+                "files_1|space_to_tab": "Yes",
+                "files_2|url_paste": "log\rcontent",
+                "files_2|type": "upload_dataset",
+            })
+
+            roadmaps_content = self._get_roadmaps_content(history_id, dataset)
+            assert roadmaps_content.strip() == "roadmaps\ncontent", roadmaps_content
+
+    def _velvet_upload(self, history_id, extra_inputs):
+        payload = self.dataset_populator.upload_payload(
+            history_id,
+            "sequences content",
+            file_type="velvet",
+            extra_inputs=extra_inputs,
+        )
+        run_response = self.dataset_populator.tools_post(payload)
+        self.dataset_populator.wait_for_tool_run(history_id, run_response)
+        datasets = run_response.json()["outputs"]
+
+        assert len(datasets) == 1
+        dataset = datasets[0]
+
+        return dataset
+
+    def _get_roadmaps_content(self, history_id, dataset):
+        roadmaps_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=dataset, filename="Roadmaps")
+        return roadmaps_content
+
     def test_unzip_collection(self):
         with self.dataset_populator.test_history() as history_id:
             hdca_id = self.__build_pair(history_id, ["123", "456"])

test/base/interactor.py

@@ -188,10 +188,7 @@ def stage_data_async(self, test_data, history_id, shed_tool_id, async=True):
                 file_name = self.functional_test_case.get_filename(composite_file.get('value'), shed_tool_id=shed_tool_id)
                 files["files_%s|file_data" % i] = open(file_name, 'rb')
                 tool_input.update({
-                    # "files_%d|NAME" % i: name,
                     "files_%d|type" % i: "upload_dataset",
-                    # TODO:
-                    # "files_%d|space_to_tab" % i: composite_file.get( 'space_to_tab', False )
                 })
             name = test_data['name']
         else:

test/base/populators.py

@@ -2,6 +2,7 @@
 import json
 import time
 
+from functools import wraps
 from operator import itemgetter
 
 import requests
@@ -26,8 +27,9 @@
 
 
 def skip_without_tool(tool_id):
-    """ Decorate an API test method as requiring a specific tool,
-    have nose skip the test case is the tool is unavailable.
+    """Decorate an API test method as requiring a specific tool.
+
+    Have test framework skip the test case is the tool is unavailable.
     """
 
     def method_wrapper(method):
@@ -39,21 +41,46 @@ def get_tool_ids(api_test_case):
             tool_ids = [itemgetter("id")(_) for _ in tools]
             return tool_ids
 
+        @wraps(method)
         def wrapped_method(api_test_case, *args, **kwargs):
-            if tool_id not in get_tool_ids(api_test_case):
-                from nose.plugins.skip import SkipTest
-                raise SkipTest()
-
+            _raise_skip_if(tool_id not in get_tool_ids(api_test_case))
             return method(api_test_case, *args, **kwargs)
 
-        # Must preserve method name so nose can detect and report tests by
-        # name.
-        wrapped_method.__name__ = method.__name__
         return wrapped_method
 
     return method_wrapper
 
 
+def skip_without_datatype(extension):
+    """Decorate an API test method as requiring a specific datatype.
+
+    Have test framework skip the test case is the tool is unavailable.
+    """
+
+    def has_datatype(api_test_case):
+        index_response = api_test_case.galaxy_interactor.get("datatypes")
+        assert index_response.status_code == 200, "Failed to fetch datatypes for target Galaxy."
+        datatypes = index_response.json()
+        assert isinstance(datatypes, list)
+        return extension in datatypes
+
+    def method_wrapper(method):
+        @wraps(method)
+        def wrapped_method(api_test_case, *args, **kwargs):
+            _raise_skip_if(not has_datatype(api_test_case))
+            method(api_test_case, *args, **kwargs)
+
+        return wrapped_method
+
+    return method_wrapper
+
+
+def _raise_skip_if(check):
+    if check:
+        from nose.plugins.skip import SkipTest
+        raise SkipTest()
+
+
 # Deprecated mixin, use dataset populator instead.
 # TODO: Rework existing tests to target DatasetPopulator in a setup method instead.
 class TestsDatasets:

test/functional/tools/composite.xml

@@ -10,9 +10,9 @@
   <tests>
     <test>
       <param name="input" value="velveth_test1/output.html" ftype="velvet" >
-        <composite_data value='velveth_test1/Sequences' ftype="Sequences"/>
-        <composite_data value='velveth_test1/Roadmaps' ftype="Roadmaps"/>
-        <composite_data value='velveth_test1/Log'/>
+        <composite_data value="velveth_test1/Sequences"/>
+        <composite_data value="velveth_test1/Roadmaps"/>
+        <composite_data value="velveth_test1/Log"/>
       </param>
       <output name="unused_reads_fasta" file="velveth_test1/Sequences" compare="diff"/>
     </test>

test/functional/tools/composite_output.xml

@@ -10,9 +10,9 @@
   <tests>
     <test>
       <param name="input" value="velveth_test1/output.html" ftype="velvet" >
-        <composite_data value='velveth_test1/Sequences' ftype="Sequences"/>
-        <composite_data value='velveth_test1/Roadmaps' ftype="Roadmaps"/>
-        <composite_data value='velveth_test1/Log'/>
+        <composite_data value="velveth_test1/Sequences" />
+        <composite_data value="velveth_test1/Roadmaps" />
+        <composite_data value="velveth_test1/Log" />
       </param>
       <output name="output" file="velveth_test1/output.html">
         <extra_files type="file" name="Sequences" value="velveth_test1/Sequences" />

test/functional/tools/composite_output_tests.xml

@@ -14,15 +14,15 @@
   <tests>
     <test>
       <param name="input" value="velveth_test1/output.html" ftype="velvet" >
-        <composite_data value='velveth_test1/Sequences' ftype="Sequences"/>
-        <composite_data value='velveth_test1/Roadmaps' ftype="Roadmaps"/>
-        <composite_data value='velveth_test1/Log'/>
+        <composite_data value="velveth_test1/Sequences" />
+        <composite_data value="velveth_test1/Roadmaps" />
+        <composite_data value="velveth_test1/Log"/>
       </param>
       <output name="output" file="velveth_test1/output.html">
         <extra_files type="file" name="Sequences" value="velveth_test1/Sequences" />
         <extra_files type="file" name="Roadmaps" value="velveth_test1/Roadmaps" />
         <extra_files type="file" name="Log" value="composite_output_expected_log" />
-        <extra_files type="file" name="md5out" md5="f2b33fb7b3d0eb95090a16060e6a24f9" /><!-- md5sum or "1 2 3" -->
+        <extra_files type="file" name="md5out" md5="f2b33fb7b3d0eb95090a16060e6a24f9" /><!-- md5sum of "1 2 3" -->
       </output>
     </test>
   </tests>

test/functional/tools/metadata.xml

@@ -10,9 +10,9 @@
   <tests>
     <test>
       <param name="input" value="velveth_test1/output.html" ftype="velvet" >
-        <composite_data value='velveth_test1/Sequences' ftype="Sequences"/>
-        <composite_data value='velveth_test1/Roadmaps' ftype="Roadmaps"/>
-        <composite_data value='velveth_test1/Log'/>
+        <composite_data value="velveth_test1/Sequences" />
+        <composite_data value="velveth_test1/Roadmaps" />
+        <composite_data value="velveth_test1/Log"/>
         <metadata name="base_name" value="Example Metadata" />
       </param>
       <!-- This ouptut tests setting input metadata above -->

tools/sr_assembly/velvetg.xml

@@ -189,9 +189,9 @@
   <tests>
     <test>
       <param name="input" value="velveth_test1/output.html" ftype="velvet" >
-        <composite_data value='velveth_test1/Sequences' ftype="Sequences"/>
-        <composite_data value='velveth_test1/Roadmaps' ftype="Roadmaps"/>
-        <composite_data value='velveth_test1/Log'/>
+        <composite_data value="velveth_test1/Sequences"/>
+        <composite_data value="velveth_test1/Roadmaps"/>
+        <composite_data value="velveth_test1/Log"/>
       </param>
       <param name="afg" value="yes" />
       <param name="generate_unused" value="yes" />