enable linking data between multiple steps

giannisdoukas · Jun 30, 2020 · bd1e2e0 · bd1e2e0
1 parent a429a15
commit bd1e2e0
Show file tree

Hide file tree

Showing 3 changed files with 69 additions and 34 deletions.
diff --git a/cwlkernel/CWLKernel.py b/cwlkernel/CWLKernel.py
@@ -3,6 +3,7 @@
 import os
 import re
 import traceback
+from io import StringIO
 from pathlib import Path
 from typing import List, Dict, Optional, Tuple, Union, Callable, NoReturn
 
@@ -155,11 +156,12 @@ def _send_json_response(self, json_data: Union[Dict, List]):
 
     def _set_data(self, code: str) -> NoReturn:
         if len(code.split()) > 0:
-            cwd = self._cwl_executor.file_manager.get_files_uri().path
+            cwd = Path(self._cwl_executor.file_manager.get_files_uri().path)
             data = self._preprocess_data(yaml.load(code, Loader=yaml.Loader))
-
             self._cwl_executor.validate_input_files(data, cwd)
-            self._yaml_input_data = code
+            code_stream = StringIO()
+            yaml.safe_dump(data, code_stream)
+            self._yaml_input_data = code_stream.getvalue()
             self.send_response(self.iopub_socket, 'stream', {'name': 'stdout', 'text': 'Add data in memory'})
 
     def _preprocess_data(self, data: Dict) -> Dict:
@@ -170,13 +172,19 @@ def _preprocess_data(self, data: Dict) -> Dict:
         @param data: the actual data
         @return the data after the transformation
         """
+
+        has_change = False
         for key_id in data:
             if isinstance(data[key_id], dict) and \
                     'class' in data[key_id] and \
                     data[key_id]['class'] == 'File' and \
                     '$data' in data[key_id]:
-                data[key_id]['location'] = self._results_manager.get_last_result_by_id(data['headinput']['$data'])
+                has_change = True
+                data[key_id]['location'] = self._results_manager.get_last_result_by_id(data[key_id]["$data"])
                 data[key_id].pop('$data')
+        if has_change is True:
+            self.send_text_to_stdout('set data to:\n')
+            self._send_json_response(data)
         return data
 
     def _clear_data(self):

diff --git a/cwlkernel/CoreExecutor.py b/cwlkernel/CoreExecutor.py
@@ -68,7 +68,6 @@ def execute(self) -> Tuple[UUID, Dict, Optional[Exception]]:
     def validate_input_files(cls, yaml_input: Dict, cwd: Path) -> NoReturn:
         for arg in yaml_input:
             if isinstance(yaml_input[arg], dict) and 'class' in yaml_input[arg] and yaml_input[arg]['class'] == 'File':
-                # TODO: check about path vs location
                 selector = 'location' if 'location' in yaml_input[arg] else 'path'
                 file_path = Path(yaml_input[arg][selector])
                 if not file_path.is_absolute():

diff --git a/examples/multipleSteps.ipynb b/examples/multipleSteps.ipynb
@@ -22,7 +22,7 @@
    ],
    "source": [
     "class: CommandLineTool\n",
-    "cwlVersion: v1.0\n",
+    "cwlVersion: v1.1\n",
     "id: head\n",
     "baseCommand:\n",
     "  - head\n",
@@ -58,7 +58,7 @@
    ],
    "source": [
     "class: CommandLineTool\n",
-    "cwlVersion: v1.0\n",
+    "cwlVersion: v1.1\n",
     "id: tail\n",
     "baseCommand:\n",
     "  - tail\n",
@@ -96,19 +96,19 @@
       "application/json": {
        "headoutput": {
         "basename": "head.out",
-        "checksum": "sha1$728aa6b07e4a63f29b8ab55af49609d84c8482e1",
+        "checksum": "sha1$e186f07099395040cf9d83ff1eb0a5dad4801937",
         "class": "File",
         "http://commonwl.org/cwltool#generation": 0,
         "id": "headoutput",
-        "location": "file:///private/tmp/CWLKERNEL_DATA/a2f1fc37-4811-4c97-896b-12b0d7885ed8/runtime_data/head.out",
+        "location": "file:///private/tmp/CWLKERNEL_DATA/959ee3d6-c5ec-4fa3-9250-72882bbdead3/runtime_data/head.out",
         "nameext": ".out",
         "nameroot": "head",
         "result_counter": 0,
-        "size": 656
+        "size": 688
        }
       },
       "text/plain": [
-       "{\"headoutput\": {\"location\": \"file:///private/tmp/CWLKERNEL_DATA/a2f1fc37-4811-4c97-896b-12b0d7885ed8/runtime_data/head.out\", \"basename\": \"head.out\", \"nameroot\": \"head\", \"nameext\": \".out\", \"class\": \"File\", \"checksum\": \"sha1$728aa6b07e4a63f29b8ab55af49609d84c8482e1\", \"size\": 656, \"http://commonwl.org/cwltool#generation\": 0, \"id\": \"headoutput\", \"result_counter\": 0}}"
+       "{\"headoutput\": {\"location\": \"file:///private/tmp/CWLKERNEL_DATA/959ee3d6-c5ec-4fa3-9250-72882bbdead3/runtime_data/head.out\", \"basename\": \"head.out\", \"nameroot\": \"head\", \"nameext\": \".out\", \"class\": \"File\", \"checksum\": \"sha1$e186f07099395040cf9d83ff1eb0a5dad4801937\", \"size\": 688, \"http://commonwl.org/cwltool#generation\": 0, \"id\": \"headoutput\", \"result_counter\": 0}}"
       ]
      },
      "metadata": {
@@ -124,7 +124,7 @@
     "% execute head\n",
     "headinput:\n",
     "    class: File\n",
-    "    location: /Users/dks/Desktop/data.csv\n",
+    "    location: /Users/dks/Workspaces/CWLKernel/tests/input_data/data.csv\n",
     "number_of_lines: 10"
    ]
   },
@@ -137,16 +137,16 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "dateRep,day,month,year,cases,deaths,countriesAndTerritories,geoId,countryterritoryCode,popData2018,continentExp\n",
-      "11/05/2020,11,5,2020,369,5,Afghanistan,AF,AFG,37172386,Asia\n",
-      "10/05/2020,10,5,2020,255,6,Afghanistan,AF,AFG,37172386,Asia\n",
-      "09/05/2020,9,5,2020,215,3,Afghanistan,AF,AFG,37172386,Asia\n",
-      "08/05/2020,8,5,2020,171,2,Afghanistan,AF,AFG,37172386,Asia\n",
-      "07/05/2020,7,5,2020,168,9,Afghanistan,AF,AFG,37172386,Asia\n",
-      "06/05/2020,6,5,2020,330,5,Afghanistan,AF,AFG,37172386,Asia\n",
-      "05/05/2020,5,5,2020,190,5,Afghanistan,AF,AFG,37172386,Asia\n",
-      "04/05/2020,4,5,2020,235,13,Afghanistan,AF,AFG,37172386,Asia\n",
-      "03/05/2020,3,5,2020,134,4,Afghanistan,AF,AFG,37172386,Asia\n"
+      "-54,-85,-5,47,39,20,-58,24,12,13,4,-22,-1,-70,44,-30,91,-6,40,-24\n",
+      "29,35,10,-83,48,47,-99,-69,-47,23,22,100,72,-81,32,30,-90,57,61,81\n",
+      "-28,31,-71,-67,40,33,-8,12,-76,12,94,-71,-70,-51,95,-59,58,-30,57,-16\n",
+      "-24,-34,-84,-56,-27,-45,-64,-9,-18,72,18,21,33,74,94,22,-31,-72,94,60\n",
+      "17,45,-68,-76,-32,72,-10,-9,35,88,31,41,66,32,97,-15,-76,-20,-27,-84\n",
+      "53,76,-98,12,-47,14,-34,54,72,-12,-20,56,81,-24,93,-61,-81,68,-68,-5\n",
+      "49,51,-38,-84,26,41,90,59,22,-24,80,73,52,-75,53,-92,25,-50,76,-44\n",
+      "-12,-17,53,31,15,55,-52,90,-93,-64,82,-12,-98,-70,-2,64,67,-87,-58,69\n",
+      "28,11,-73,37,24,100,-94,-34,-64,52,-76,-41,-67,41,-50,-38,-25,85,42,62\n",
+      "-49,-55,76,92,81,-4,66,49,-70,48,-46,-19,-22,-48,-80,50,74,-46,73,33\n"
      ]
     }
    ],
@@ -156,9 +156,37 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "set data to:\n"
+     ]
+    },
+    {
+     "data": {
+      "application/json": {
+       "number_of_lines": 5,
+       "tailinput": {
+        "class": "File",
+        "location": "/private/tmp/CWLKERNEL_DATA/959ee3d6-c5ec-4fa3-9250-72882bbdead3/results/f8e1f99f-bfa1-4671-af4b-78a7b4320ab0/head.out"
+       }
+      },
+      "text/plain": [
+       "{\"tailinput\": {\"class\": \"File\", \"location\": \"/private/tmp/CWLKERNEL_DATA/959ee3d6-c5ec-4fa3-9250-72882bbdead3/results/f8e1f99f-bfa1-4671-af4b-78a7b4320ab0/head.out\"}, \"number_of_lines\": 5}"
+      ]
+     },
+     "metadata": {
+      "application/json": {
+       "expanded": false,
+       "root": "root"
+      }
+     },
+     "output_type": "display_data"
+    },
     {
      "name": "stdout",
      "output_type": "stream",
@@ -171,19 +199,19 @@
       "application/json": {
        "tailoutput": {
         "basename": "tail.out",
-        "checksum": "sha1$1bd1dc77a1b65db7766212208e2f8f360a53a724",
+        "checksum": "sha1$dce7238df9eab877ea503f7c4b5b9ef71e0fd65f",
         "class": "File",
         "http://commonwl.org/cwltool#generation": 0,
         "id": "tailoutput",
-        "location": "file:///private/tmp/CWLKERNEL_DATA/a2f1fc37-4811-4c97-896b-12b0d7885ed8/runtime_data/tail.out",
+        "location": "file:///private/tmp/CWLKERNEL_DATA/959ee3d6-c5ec-4fa3-9250-72882bbdead3/runtime_data/tail.out",
         "nameext": ".out",
         "nameroot": "tail",
         "result_counter": 1,
-        "size": 301
+        "size": 346
        }
       },
       "text/plain": [
-       "{\"tailoutput\": {\"location\": \"file:///private/tmp/CWLKERNEL_DATA/a2f1fc37-4811-4c97-896b-12b0d7885ed8/runtime_data/tail.out\", \"basename\": \"tail.out\", \"nameroot\": \"tail\", \"nameext\": \".out\", \"class\": \"File\", \"checksum\": \"sha1$1bd1dc77a1b65db7766212208e2f8f360a53a724\", \"size\": 301, \"http://commonwl.org/cwltool#generation\": 0, \"id\": \"tailoutput\", \"result_counter\": 1}}"
+       "{\"tailoutput\": {\"location\": \"file:///private/tmp/CWLKERNEL_DATA/959ee3d6-c5ec-4fa3-9250-72882bbdead3/runtime_data/tail.out\", \"basename\": \"tail.out\", \"nameroot\": \"tail\", \"nameext\": \".out\", \"class\": \"File\", \"checksum\": \"sha1$dce7238df9eab877ea503f7c4b5b9ef71e0fd65f\", \"size\": 346, \"http://commonwl.org/cwltool#generation\": 0, \"id\": \"tailoutput\", \"result_counter\": 1}}"
       ]
      },
      "metadata": {
@@ -199,24 +227,24 @@
     "% execute tail\n",
     "tailinput:\n",
     "    class: File\n",
-    "    location: /private/tmp/CWLKERNEL_DATA/a2f1fc37-4811-4c97-896b-12b0d7885ed8/runtime_data/head.out\n",
+    "    $data: headoutput\n",
     "number_of_lines: 5"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "07/05/2020,7,5,2020,168,9,Afghanistan,AF,AFG,37172386,Asia\n",
-      "06/05/2020,6,5,2020,330,5,Afghanistan,AF,AFG,37172386,Asia\n",
-      "05/05/2020,5,5,2020,190,5,Afghanistan,AF,AFG,37172386,Asia\n",
-      "04/05/2020,4,5,2020,235,13,Afghanistan,AF,AFG,37172386,Asia\n",
-      "03/05/2020,3,5,2020,134,4,Afghanistan,AF,AFG,37172386,Asia\n"
+      "53,76,-98,12,-47,14,-34,54,72,-12,-20,56,81,-24,93,-61,-81,68,-68,-5\n",
+      "49,51,-38,-84,26,41,90,59,22,-24,80,73,52,-75,53,-92,25,-50,76,-44\n",
+      "-12,-17,53,31,15,55,-52,90,-93,-64,82,-12,-98,-70,-2,64,67,-87,-58,69\n",
+      "28,11,-73,37,24,100,-94,-34,-64,52,-76,-41,-67,41,-50,-38,-25,85,42,62\n",
+      "-49,-55,76,92,81,-4,66,49,-70,48,-46,-19,-22,-48,-80,50,74,-46,73,33\n"
      ]
     }
    ],