Removed redundant SimpleManagerClient. Remove unroll and partition st…

…eps from 'dlg cwl' tool so that 'unroll' and 'cwl' steps become composable.

daliuge-common/dlg/clients.py

@@ -185,10 +185,3 @@ def __init__(self, host='localhost', port=constants.MASTER_DEFAULT_REST_PORT, ti
 
     def create_island(self, island_host, nodes):
         self._post_json('/managers/%s/dataisland' % (urllib.quote(island_host)), {'nodes': nodes})
-
-class SimpleManagerClient():
-    """
-    A dummy manager client that only responds with a simple node list
-    """
-    def nodes(self):
-        return ['localhost', 'localhost']

daliuge-translator/dlg/dropmake/cwl.py

@@ -33,7 +33,7 @@
 logger = logging.getLogger(__name__)
 
 
-def create_workflow(pgt, pgt_path, cwl_path, zip_path):
+def create_workflow(drops, pgt_path, cwl_path, zip_path):
     """
     Create a CWL workflow from a given Physical Graph Template
 
@@ -56,7 +56,7 @@ def create_workflow(pgt, pgt_path, cwl_path, zip_path):
     files = {}
 
     # look for input and output files in the pg_spec
-    for index, node in enumerate(pgt):
+    for index, node in enumerate(drops):
         command = node.get('command', None)
         dataType = node.get('dt', None)
         outputId = node.get('oid', None)
@@ -66,7 +66,7 @@ def create_workflow(pgt, pgt_path, cwl_path, zip_path):
             files[outputs[0]] = "step" + str(index) + "/output_file_0"
 
     # add steps to the workflow
-    for index, node in enumerate(pgt):
+    for index, node in enumerate(drops):
         dataType = node.get('dt', '')
 
         if dataType == 'BashShellApp':
@@ -124,9 +124,10 @@ def create_command_line_tool(node, filename):
 
     # TODO: find a better way of specifying command line program + arguments
     base_command = base_command[:base_command.index(" ")]
+    base_command = common.u2s(base_command)
 
     # cwlgen's Serializer class doesn't support python 2.7's unicode types
-    cwl_tool = cwlgen.CommandLineTool(tool_id=node['app'], label=common.u2s(node['nm']), base_command=base_command, cwl_version='v1.0')
+    cwl_tool = cwlgen.CommandLineTool(tool_id=common.u2s(node['app']), label=common.u2s(node['nm']), base_command=base_command, cwl_version='v1.0')
 
     # add inputs
     for index, input in enumerate(inputs):

daliuge-translator/dlg/dropmake/web/lg_web.py

@@ -47,7 +47,7 @@
 import pkg_resources
 
 from ... import common, restutils
-from ...clients import CompositeManagerClient, SimpleManagerClient
+from ...clients import CompositeManagerClient
 from ..pg_generator import unroll, partition, GraphException
 from ..pg_manager import PGManager
 from ..scheduler import SchedulerException
@@ -209,26 +209,23 @@ def pgtcwl_get():
         # get PGT from manager
         pgtp = pg_mgr.get_pgt(pgt_name)
 
-        # build node list
-        mgr_client = SimpleManagerClient()
-        node_list = mgr_client.nodes()
-
-        # mapping PGTP to resources (node list)
-        pg_spec = pgtp.to_pg_spec(node_list, ret_str=False)
+        # debug
+        print("pgtp:" + str(pgtp) + ":" + str(dir(pgtp)))
 
         # build filename for CWL file from PGT filename
         cwl_filename = pgt_name[:-6] + ".cwl"
         zip_filename = pgt_name[:-6] + ".zip"
 
-        # build path for CWL file
+        # get paths used while creating the CWL files
+        root_path = pgt_path("")
         cwl_path = pgt_path(cwl_filename)
         zip_path = pgt_path(zip_filename)
 
         # create the CWL workflow
-        create_workflow(pg_spec, pgt_path(""), cwl_path, zip_path);
+        create_workflow(pgtp.drops, root_path, cwl_path, zip_path);
 
         # respond with download of ZIP file
-        return static_file(zip_filename, root=pgt_path(""), download=True)
+        return static_file(zip_filename, root=root_path, download=True)
     else:
         response.status = 404
         return "{0}: JSON graph {1} not found\n".format(err_prefix, pgt_name)

daliuge-translator/dlg/translator/tool_commands.py

@@ -286,18 +286,16 @@ def dlg_submit(parser, args):
 
 
 def cwl(parser, args):
-    print("cwl tool")
-
     tool.add_logging_options(parser)
     _add_output_options(parser)
-    apps = _add_unroll_options(parser)
-    _add_partition_options(parser)
+    parser.add_option('-P', '--physical-graph-template', action='store', dest='pgt_path', type='string',
+                      help='Path to the Physical Graph Template (default: stdin)', default='-')
     (opts, args) = parser.parse_args(args)
     tool.setup_logging(opts)
 
-    # get the pgt
-    pgt = unroll(opts.lg_path, opts.oid_prefix, zerorun=opts.zerorun, app=apps[opts.app])
-    partition(pgt, opts)
+    # load the pgt
+    with _open_i(opts.pgt_path) as fi:
+        pgt = json.load(fi)
 
     # create the CWL workflow
     from ..dropmake.cwl import create_workflow