diff --git a/daliuge-common/dlg/common/__init__.py b/daliuge-common/dlg/common/__init__.py index 87ce97d1d..d5bb59740 100644 --- a/daliuge-common/dlg/common/__init__.py +++ b/daliuge-common/dlg/common/__init__.py @@ -71,13 +71,18 @@ def b2s(b, enc='utf8'): return b.decode(enc) def u2s(u): return u + def s2b(s, enc='utf-8'): + return s.encode(enc) else: def b2s(b, enc='utf8'): return b def u2s(u, enc='utf-8'): return u.encode(enc) + def s2b(s, enc='utf-8'): + return s b2s.__doc__ = "Converts bytes into a string" u2s.__doc__ = 'Converts text into a string' +s2b.__doc__ = 'Converts a string into bytes' class dropdict(dict): diff --git a/daliuge-translator/dlg/dropmake/cwl.py b/daliuge-translator/dlg/dropmake/cwl.py index 3d0fb9741..d043fa444 100644 --- a/daliuge-translator/dlg/dropmake/cwl.py +++ b/daliuge-translator/dlg/dropmake/cwl.py @@ -23,6 +23,7 @@ import logging import os from zipfile import ZipFile +import io import cwlgen @@ -33,7 +34,7 @@ logger = logging.getLogger(__name__) -def create_workflow(drops, pgt_path, cwl_path, zip_path): +def create_workflow(drops, cwl_filename): """ Create a CWL workflow from a given Physical Graph Template @@ -76,9 +77,10 @@ def create_workflow(drops, pgt_path, cwl_path, zip_path): # create command line tool description filename = "step" + str(index) + ".cwl" - filename_with_path = os.path.join(pgt_path, filename) - create_command_line_tool(node, filename_with_path) - step_files.append(filename_with_path) + contents = create_command_line_tool(node) + + # add contents of command line tool description to list of step files + step_files.append({"filename":filename, "contents": contents}) # create step step = cwlgen.WorkflowStep("step" + str(index), run=filename) @@ -94,19 +96,20 @@ def create_workflow(drops, pgt_path, cwl_path, zip_path): # add step to workflow cwl_workflow.steps.append(step) - # save CWL to path - with open(cwl_path, "w") as f: - f.write(cwl_workflow.export_string()) - # put workflow and command line tool description files all together in a zip - zipObj = ZipFile(zip_path, 'w') + zipBuffer = io.BytesIO() + zipObj = ZipFile(zipBuffer, 'w') for step_file in step_files: - zipObj.write(step_file, os.path.basename(step_file)) - zipObj.write(cwl_path, os.path.basename(cwl_path)) + step_file_buffer = io.BytesIO(common.s2b(step_file["contents"])) + zipObj.writestr(step_file["filename"], step_file_buffer.getvalue()) + workflow_file_buffer = io.BytesIO(common.s2b(cwl_workflow.export_string())) + zipObj.writestr(cwl_filename, workflow_file_buffer.getvalue()) zipObj.close() + return zipBuffer.getvalue() + -def create_command_line_tool(node, filename): +def create_command_line_tool(node): """ Create a command line tool description file for a single step in a CWL workflow. @@ -144,6 +147,4 @@ def create_command_line_tool(node, filename): output_file = cwlgen.CommandOutputParameter('output_file_' + str(index), param_type='stdout', output_binding=file_binding, doc='output file ' + str(index)) cwl_tool.outputs.append(output_file) - # write to file - with open(filename, "w") as f: - f.write(cwl_tool.export_string()) + return cwl_tool.export_string() diff --git a/daliuge-translator/dlg/dropmake/web/lg_web.py b/daliuge-translator/dlg/dropmake/web/lg_web.py index 57cea91a4..197fcf955 100644 --- a/daliuge-translator/dlg/dropmake/web/lg_web.py +++ b/daliuge-translator/dlg/dropmake/web/lg_web.py @@ -218,11 +218,14 @@ def pgtcwl_get(): # get paths used while creating the CWL files root_path = pgt_path("") - cwl_path = pgt_path(cwl_filename) zip_path = pgt_path(zip_filename) # create the CWL workflow - create_workflow(pgtp.drops, root_path, cwl_path, zip_path); + zip_contents = create_workflow(pgtp.drops, cwl_filename); + + # maybe eventually we don't need to write to file + with open(zip_path, 'wb') as f: + f.write(zip_contents) # respond with download of ZIP file return static_file(zip_filename, root=root_path, download=True) diff --git a/daliuge-translator/dlg/translator/tool_commands.py b/daliuge-translator/dlg/translator/tool_commands.py index 7baf55df0..ee092359a 100644 --- a/daliuge-translator/dlg/translator/tool_commands.py +++ b/daliuge-translator/dlg/translator/tool_commands.py @@ -299,7 +299,11 @@ def cwl(parser, args): # create the CWL workflow from ..dropmake.cwl import create_workflow - create_workflow(pgt, "", "workflow.cwl", opts.output) + zip_contents = create_workflow(pgt, "workflow.cwl") + + # write output + with _open_o(opts.output, "wb") as f: + f.write(zip_contents) def register_commands(): tool.cmdwrap('lgweb', 'A Web server for the Logical Graph Editor', 'dlg.dropmake.web.lg_web:run') diff --git a/daliuge-translator/test/dropmake/test_pg_gen.py b/daliuge-translator/test/dropmake/test_pg_gen.py index 1fa046d81..efa20506f 100644 --- a/daliuge-translator/test/dropmake/test_pg_gen.py +++ b/daliuge-translator/test/dropmake/test_pg_gen.py @@ -205,7 +205,11 @@ def test_cwl_translate(self): cwl_out_zip = cwl_output_dir + '/workflow.zip' output_list.append((cwl_out, cwl_out_zip)) - create_workflow(pgt, "", cwl_out, cwl_out_zip) + cwl_contents = create_workflow(pgt, 'workflow.cwl') + + # write output + with open(cwl_out_zip, 'wb') as f: + f.write(cwl_contents) for out, zip in output_list: zip_ref = zipfile.ZipFile(zip)