Merge dfdc647 into 4ce7018

daliuge-common/dlg/common/__init__.py

@@ -71,13 +71,18 @@ def b2s(b, enc='utf8'):
         return b.decode(enc)
     def u2s(u):
         return u
+    def s2b(s, enc='utf-8'):
+        return s.encode(enc)
 else:
     def b2s(b, enc='utf8'):
         return b
     def u2s(u, enc='utf-8'):
         return u.encode(enc)
+    def s2b(s, enc='utf-8'):
+        return s
 b2s.__doc__ = "Converts bytes into a string"
 u2s.__doc__ = 'Converts text into a string'
+s2b.__doc__ = 'Converts a string into bytes'
 
 
 class dropdict(dict):

daliuge-translator/dlg/dropmake/cwl.py

@@ -23,6 +23,7 @@
 import logging
 import os
 from zipfile import ZipFile
+import io
 
 import cwlgen
 
@@ -33,7 +34,7 @@
 logger = logging.getLogger(__name__)
 
 
-def create_workflow(drops, pgt_path, cwl_path, zip_path):
+def create_workflow(drops, cwl_filename):
     """
     Create a CWL workflow from a given Physical Graph Template
 
@@ -76,9 +77,10 @@ def create_workflow(drops, pgt_path, cwl_path, zip_path):
 
             # create command line tool description
             filename = "step" + str(index) + ".cwl"
-            filename_with_path = os.path.join(pgt_path, filename)
-            create_command_line_tool(node, filename_with_path)
-            step_files.append(filename_with_path)
+            contents = create_command_line_tool(node)
+
+            # add contents of command line tool description to list of step files
+            step_files.append({"filename":filename, "contents": contents})
 
             # create step
             step = cwlgen.WorkflowStep("step" + str(index), run=filename)
@@ -94,19 +96,20 @@ def create_workflow(drops, pgt_path, cwl_path, zip_path):
             # add step to workflow
             cwl_workflow.steps.append(step)
 
-    # save CWL to path
-    with open(cwl_path, "w") as f:
-        f.write(cwl_workflow.export_string())
-
     # put workflow and command line tool description files all together in a zip
-    zipObj = ZipFile(zip_path, 'w')
+    zipBuffer = io.BytesIO()
+    zipObj = ZipFile(zipBuffer, 'w')
     for step_file in step_files:
-        zipObj.write(step_file, os.path.basename(step_file))
-    zipObj.write(cwl_path, os.path.basename(cwl_path))
+        step_file_buffer = io.BytesIO(common.s2b(step_file["contents"]))
+        zipObj.writestr(step_file["filename"], step_file_buffer.getvalue())
+    workflow_file_buffer = io.BytesIO(common.s2b(cwl_workflow.export_string()))
+    zipObj.writestr(cwl_filename, workflow_file_buffer.getvalue())
     zipObj.close()
 
+    return zipBuffer.getvalue()
+
 
-def create_command_line_tool(node, filename):
+def create_command_line_tool(node):
     """
     Create a command line tool description file for a single step in a CWL
     workflow.
@@ -144,6 +147,4 @@ def create_command_line_tool(node, filename):
         output_file = cwlgen.CommandOutputParameter('output_file_' + str(index), param_type='stdout', output_binding=file_binding, doc='output file ' + str(index))
         cwl_tool.outputs.append(output_file)
 
-    # write to file
-    with open(filename, "w") as f:
-        f.write(cwl_tool.export_string())
+    return cwl_tool.export_string()

daliuge-translator/dlg/dropmake/web/lg_web.py

@@ -218,11 +218,14 @@ def pgtcwl_get():
 
         # get paths used while creating the CWL files
         root_path = pgt_path("")
-        cwl_path = pgt_path(cwl_filename)
         zip_path = pgt_path(zip_filename)
 
         # create the CWL workflow
-        create_workflow(pgtp.drops, root_path, cwl_path, zip_path);
+        zip_contents = create_workflow(pgtp.drops, cwl_filename);
+
+        # maybe eventually we don't need to write to file
+        with open(zip_path, 'wb') as f:
+            f.write(zip_contents)
 
         # respond with download of ZIP file
         return static_file(zip_filename, root=root_path, download=True)

daliuge-translator/dlg/translator/tool_commands.py

@@ -299,7 +299,11 @@ def cwl(parser, args):
 
     # create the CWL workflow
     from ..dropmake.cwl import create_workflow
-    create_workflow(pgt, "", "workflow.cwl", opts.output)
+    zip_contents = create_workflow(pgt, "workflow.cwl")
+
+    # write output
+    with _open_o(opts.output, "wb") as f:
+        f.write(zip_contents)
 
 def register_commands():
     tool.cmdwrap('lgweb', 'A Web server for the Logical Graph Editor', 'dlg.dropmake.web.lg_web:run')

daliuge-translator/test/dropmake/test_pg_gen.py

@@ -205,7 +205,11 @@ def test_cwl_translate(self):
                 cwl_out_zip = cwl_output_dir + '/workflow.zip'
                 output_list.append((cwl_out, cwl_out_zip))
 
-                create_workflow(pgt, "", cwl_out, cwl_out_zip)
+                cwl_contents = create_workflow(pgt, 'workflow.cwl')
+
+                # write output
+                with open(cwl_out_zip, 'wb') as f:
+                    f.write(cwl_contents)
 
         for out, zip in output_list:
             zip_ref = zipfile.ZipFile(zip)