Adds scattering by multiple fields for WDL and Cromwell

janis_core/__meta__.py

@@ -1,4 +1,4 @@
-__version__ = "v0.6.1"
+__version__ = "v0.6.2"
 
 GITHUB_URL = "https://github.com/PMCC-BioinformaticsCore/janis"
 DOCS_URL = "https://janis.readthedocs.io/en/latest/"

janis_core/tests/test_translation_wdl.py

@@ -2,6 +2,7 @@
 from typing import List, Dict, Any
 
 import wdlgen
+from janis_core.utils.scatter import ScatterDescription, ScatterMethod, ScatterMethods
 
 import janis_core.translations.wdl as wdl
 from janis_core import (
@@ -20,10 +21,52 @@
     Boolean,
     Int,
 )
+from janis_core.tests.testtools import SingleTestTool
 from janis_core.translations import WdlTranslator
 from janis_core.types import CpuSelector, StringFormatter
 
 
+class MultipleEcho(CommandTool):
+    @staticmethod
+    def tool():
+        return "TestStepTool"
+
+    @staticmethod
+    def base_command():
+        return "echo"
+
+    def inputs(self):
+        return [
+            ToolInput("input1", TxtSecondary, position=0),
+            ToolInput("input2", String(optional=True), position=1),
+            ToolInput("input3", String(optional=True), position=2),
+            ToolInput("input4", String(optional=True), position=3),
+        ]
+
+    def friendly_name(self):
+        return None
+
+    def outputs(self):
+        return [ToolOutput("out", Stdout)]
+
+    @staticmethod
+    def container():
+        return None
+
+    @staticmethod
+    def version():
+        return None
+
+
+class TxtSecondary(File):
+    def __init__(self, optional=False):
+        super().__init__(optional=optional, extension=".txt")
+
+    @staticmethod
+    def secondary_files():
+        return [".qt"]
+
+
 class TestTool(CommandTool):
     @staticmethod
     def tool():
@@ -158,7 +201,7 @@ def test_translate_single_to_array_edge(self):
             "stp2", TestTool(arrayInp=stp1.std, testtool=w.inp), ignore_missing=True
         )
 
-        outp = wdl.translate_step_node(stp2, "stp1", stp2.id(), {})
+        outp = wdl.translate_step_node(stp2, stp2.id(), {}, set())
         self.assertEqual(
             outp.get_string().split("\n")[3].strip(), f"arrayInp=[{stp1.id()}.std]"
         )
@@ -672,3 +715,155 @@ def test_max_memory(self):
         self.assertEqual(
             1, resources["TestTranslationtoolWf.testtranslationtool_runtime_memory"]
         )
+
+
+class TestWdlScatterByMultipleFields(unittest.TestCase):
+    def test_scatter_single(self):
+        w = WorkflowBuilder("sbmf")
+        w.input("inp", Array(str))
+        w.input("inp2", str)
+
+        step = w.step(
+            "dotTool",
+            SingleTestTool(inputs=w.inp, input2=w.inp2),
+            scatter=ScatterDescription(fields=["inputs"], method=ScatterMethods.dot),
+        )
+
+        outp = wdl.translate_step_node(step, "A.SingleTestTool", {}, {"inp", "inp2"})
+        expected = """\
+scatter (i in inp) {
+   call A.SingleTestTool as dotTool {
+    input:
+      inputs=i,
+      input2=inp2
+  }
+}"""
+        self.assertEqual(expected, outp.get_string(indent=0))
+
+    def test_scatter_single_no_description(self):
+        w = WorkflowBuilder("sbmf")
+        w.input("inp", Array(str))
+        w.input("inp2", str)
+
+        step = w.step(
+            "dotTool", SingleTestTool(inputs=w.inp, input2=w.inp2), scatter="inputs"
+        )
+
+        outp = wdl.translate_step_node(step, "A.SingleTestTool", {}, {"inp", "inp2"})
+        expected = """\
+scatter (i in inp) {
+   call A.SingleTestTool as dotTool {
+    input:
+      inputs=i,
+      input2=inp2
+  }
+}"""
+        self.assertEqual(expected, outp.get_string(indent=0))
+
+    def test_dot_2(self):
+        w = WorkflowBuilder("sbmf")
+        w.input("inp", Array(str))
+        w.input("inp2", Array(str))
+
+        step = w.step(
+            "dotTool",
+            SingleTestTool(inputs=w.inp, input2=w.inp2),
+            scatter=ScatterDescription(
+                fields=["inputs", "input2"], method=ScatterMethods.dot
+            ),
+        )
+
+        outp = wdl.translate_step_node(step, "A.SingleTestTool", {}, {"inp", "inp2"})
+        expected = """\
+scatter (Q in zip(inp, inp2)) {
+   call A.SingleTestTool as dotTool {
+    input:
+      inputs=Q.left,
+      input2=Q.right
+  }
+}"""
+        self.assertEqual(expected, outp.get_string(indent=0))
+
+    def test_dot_3(self):
+        w = WorkflowBuilder("sbmf")
+        w.input("inp", Array(str))
+        w.input("inp2", Array(str))
+        w.input("inp3", Array(str))
+
+        step = w.step(
+            "dotTool",
+            SingleTestTool(inputs=w.inp, input2=w.inp2, input3=w.inp3),
+            scatter=ScatterDescription(
+                fields=["inputs", "input2", "input3"], method=ScatterMethods.dot
+            ),
+        )
+
+        outp = wdl.translate_step_node(
+            step, "A.SingleTestTool", {}, {"inp", "inp2", "inp3"}
+        )
+        expected = """\
+scatter (Q in zip(inp, zip(inp2, inp3))) {
+   call A.SingleTestTool as dotTool {
+    input:
+      inputs=Q.left,
+      input2=Q.right.left,
+      input3=Q.right.right
+  }
+}"""
+        self.assertEqual(expected, outp.get_string(indent=0))
+
+    def test_dot_4(self):
+        w = WorkflowBuilder("sbmf")
+        w.input("inp", Array(str))
+        w.input("inp2", Array(str))
+        w.input("inp3", Array(str))
+        w.input("inp4", Array(str))
+
+        step = w.step(
+            "dotTool",
+            SingleTestTool(inputs=w.inp, input2=w.inp2, input3=w.inp3, input4=w.inp4),
+            scatter=ScatterDescription(
+                fields=["inputs", "input2", "input3", "input4"],
+                method=ScatterMethods.dot,
+            ),
+        )
+
+        outp = wdl.translate_step_node(
+            step, "A.SingleTestTool", {}, {"inp", "inp2", "inp3", "inp4"}
+        )
+        expected = """\
+scatter (Q in zip(inp, zip(inp2, zip(inp3, inp4)))) {
+   call A.SingleTestTool as dotTool {
+    input:
+      inputs=Q.left,
+      input2=Q.right.left,
+      input3=Q.right.right.left,
+      input4=Q.right.right.right
+  }
+}"""
+        self.assertEqual(expected, outp.get_string(indent=0))
+
+    def test_dot_2_secondary(self):
+        w = WorkflowBuilder("sbmf")
+        w.input("inp", Array(TxtSecondary))
+        w.input("inp2", Array(str))
+
+        step = w.step(
+            "dotTool",
+            MultipleEcho(input1=w.inp, input2=w.inp2),
+            scatter=ScatterDescription(
+                fields=["input1", "input2"], method=ScatterMethods.dot
+            ),
+        )
+
+        outp = wdl.translate_step_node(step, "A.SingleTestTool", {}, {"inp", "inp2"})
+        expected = """\
+scatter (Q in zip(transpose([inp, inp_qt]), inp2)) {
+   call A.SingleTestTool as dotTool {
+    input:
+      input1=Q.left[0],
+      input1_qt=Q.left[1],
+      input2=Q.right
+  }
+}"""
+        self.assertEqual(expected, outp.get_string(indent=0))

janis_core/tests/tests_util.py

@@ -1,5 +1,8 @@
 from unittest import TestCase
-from janis_core.utils import get_value_for_hints_and_ordered_resource_tuple
+from janis_core.utils import (
+    get_value_for_hints_and_ordered_resource_tuple,
+    recursive_2param_wrap,
+)
 
 
 class TestOrderedHints(TestCase):
@@ -50,3 +53,30 @@ def test_hint_doesnt_exist(self):
         hints = {"hinttype3": "ht3h1"}
         val = get_value_for_hints_and_ordered_resource_tuple(hints, self.ordered_hints)
         self.assertIsNone(val)
+
+
+class Test2ItemMethodWrap(TestCase):
+    def test_0_items(self):
+        items = []
+        self.assertRaises(Exception, recursive_2param_wrap, "zip", items)
+
+    def test_1_item(self):
+        items = ["item"]
+        self.assertRaises(Exception, recursive_2param_wrap, "zip", items)
+
+    def test_2_items(self):
+        items = ["item1", "item2"]
+        self.assertEqual("zip(item1, item2)", recursive_2param_wrap("zip", items))
+
+    def test_3_items(self):
+        items = ["item1", "item2", "item3"]
+        self.assertEqual(
+            "zip(item1, zip(item2, item3))", recursive_2param_wrap("zip", items)
+        )
+
+    def test_4_items(self):
+        items = ["item1", "item2", "item3", "item4"]
+        self.assertEqual(
+            "zip(item1, zip(item2, zip(item3, item4)))",
+            recursive_2param_wrap("zip", items),
+        )

janis_core/tests/testtools.py

@@ -24,7 +24,12 @@ def base_command():
         return "echo"
 
     def inputs(self):
-        return [ToolInput("inputs", String())]
+        return [
+            ToolInput("inputs", String(), position=0),
+            ToolInput("input2", String(optional=True), position=1),
+            ToolInput("input3", String(optional=True), position=2),
+            ToolInput("input4", String(optional=True), position=3),
+        ]
 
     def friendly_name(self):
         return None