compose workflow with workflow

giannisdoukas · Jun 7, 2020 · 2af1110 · 2af1110
1 parent 980b0f2
commit 2af1110
Show file tree

Hide file tree

Showing 8 changed files with 456 additions and 19 deletions.
diff --git a/cwlkernel/cwlrepository/CWLComponent.py b/cwlkernel/cwlrepository/CWLComponent.py
@@ -40,6 +40,10 @@ def inputs(self) -> List[Dict]:
     def outputs(self) -> List[Dict]:
         pass
 
+    @abstractmethod
+    def compose_requirements(self) -> Dict:
+        pass
+
     @classmethod
     def _convert_inputs_from_dict_to_list(cls, inputs: Dict) -> List[Dict]:
         return [{'id': id, **cwl_input} for id, cwl_input in inputs.items()]
@@ -84,6 +88,9 @@ def _packed_steps(self) -> Dict:
         to_return.pop('cwlVersion')
         return to_return
 
+    def compose_requirements(self) -> Dict:
+        return {}
+
 
 class CWLWorkflow(WorkflowComponent):
 
@@ -122,6 +129,7 @@ def add(self, component: WorkflowComponent, step_name: str) -> None:
             'in': {},
             'out': []
         }
+        self._requirements = {**self._requirements, **component.compose_requirements()}
 
     def remove(self, component: WorkflowComponent) -> None:
         raise NotImplementedError()
@@ -130,6 +138,15 @@ def add_input(self, workflow_input: Dict, step_id: str, in_step_id: str):
         self._inputs.append(workflow_input)
         self._steps[step_id]['in'][in_step_id] = workflow_input['id']
 
+    def add_output_source(self, output_ref: str, type_of: str):
+        references = output_ref.split('/')
+        output_id = references[-1]
+        references = references[:-1]
+        self._outputs.append(
+            {'id': output_id, 'type': type_of, 'outputSource': output_ref}
+        )
+        self._steps[references[0]]['out'].append(output_id)
+
     def to_yaml(self) -> str:
         yaml_text = StringIO()
         result = self.to_dict()
@@ -165,6 +182,9 @@ def inputs(self) -> List[Dict]:
     def outputs(self) -> List[Dict]:
         return deepcopy(self._outputs)
 
+    def compose_requirements(self) -> Dict:
+        return {'SubworkflowFeatureRequirement': {}}
+
 
 class WorkflowComponentFactory:
     def get_workflow_component(self, yaml_string: str) -> WorkflowComponent:

diff --git a/cwlkernel/kernel_magics.py b/cwlkernel/kernel_magics.py
@@ -44,6 +44,12 @@ def newWorkflowAddStep(kernel: CWLKernel, ids: str):
     kernel._workflow_composer.add(tool, step_id)
 
 
+@CWLKernel.register_magic
+def newWorkflowAddOutputSource(kernel: CWLKernel, args: str):
+    reference, type_of = args.split()
+    kernel._workflow_composer.add_output_source(reference, type_of)
+
+
 @CWLKernel.register_magic
 def newWorkflow(kernel: CWLKernel, id: str):
     kernel._workflow_composer = CWLWorkflow(id)

diff --git a/examples/Compose.ipynb b/examples/Compose.ipynb
@@ -81,25 +81,95 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/json": {
+       "class": "Workflow",
+       "cwlVersion": "v1.0",
+       "id": "main",
+       "inputs": [
+        {
+         "id": "inputfile",
+         "type": "File"
+        }
+       ],
+       "outputs": [],
+       "requirements": {},
+       "steps": {
+        "headstepid": {
+         "in": {
+          "headinput": "inputfile"
+         },
+         "out": [
+          "headoutput"
+         ],
+         "run": "head.cwl"
+        },
+        "tailstepid": {
+         "in": {
+          "tailinput": "headstepid/headoutput"
+         },
+         "out": [],
+         "run": "tail.cwl"
+        }
+       }
+      },
+      "text/plain": [
+       "{\"cwlVersion\": \"v1.0\", \"class\": \"Workflow\", \"id\": \"main\", \"inputs\": [{\"id\": \"inputfile\", \"type\": \"File\"}], \"outputs\": [], \"steps\": {\"tailstepid\": {\"run\": \"tail.cwl\", \"in\": {\"tailinput\": \"headstepid/headoutput\"}, \"out\": []}, \"headstepid\": {\"run\": \"head.cwl\", \"in\": {\"headinput\": \"inputfile\"}, \"out\": [\"headoutput\"]}}, \"requirements\": {}}"
+      ]
+     },
+     "metadata": {
+      "application/json": {
+       "expanded": false,
+       "root": "root"
+      }
+     },
+     "output_type": "display_data"
+    }
+   ],
    "source": [
-    "% new-workflow main\n",
-    "% new-workflow-add-step tail tailstepid\n",
-    "% new-workflow-add-step head headstepid\n",
-    "% new-workflow-add-input head headinput\n",
+    "% newWorkflow main\n",
+    "% newWorkflowAddStep tail tailstepid\n",
+    "% newWorkflowAddStep head headstepid\n",
+    "% newWorkflowAddInput headstepid headinput\n",
     "id: inputfile\n",
     "type: File\n",
-    "% new-workflow-add-step-in tail tailinput head/headoutput\n",
-    "% new-workflow-build main"
+    "% newWorkflowAddStepIn tailstepid headstepid headoutput\n",
+    "tailinput: headstepid/headoutput\n",
+    "% newWorkflowBuild"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Add data in memory"
+     ]
+    },
+    {
+     "data": {
+      "application/json": {},
+      "text/plain": [
+       "{}"
+      ]
+     },
+     "metadata": {
+      "application/json": {
+       "expanded": false,
+       "root": "root"
+      }
+     },
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "% execute main\n",
     "inputfile: \n",