Skip to content

Commit

Permalink
Factored out code common to two translation requests (into unroll_and…
Browse files Browse the repository at this point in the history 
…_partition_with_params()). Fixed bug where json.loads() was called twice on incoming JSON because the string was not unescaped. Added unit test for new POST route.
  • Loading branch information
@james-strauss-uwa
james-strauss-uwa committed Sep 6, 2019
1 parent b83c1c2 commit 0e85c1c
Show file tree
Hide file tree
Showing 2 changed files with 65 additions and 45 deletions.
80 changes: 37 additions & 43 deletions dlg/dropmake/web/lg_web.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -316,24 +316,10 @@ def gen_pgt():
return "{0}: logical graph {1} not found\n".format(err_prefix, lg_name)

try:

# LG -> PGT
pgt = unroll(lg_path(lg_name))

# Read parameters from request
algo = query.get('algo', 'none')
num_partitions = query.get('num_par', default=1, type=int)
num_islands = query.get('num_islands', default=0, type=int)
par_label = query.get('par_label', 'Partition')
algo_params = {}
for name, typ in ALGO_PARAMS:
if name in query:
algo_params[name] = query.get(name, type=typ)

# Partition the PGT
pgt = partition(pgt, algo=algo, num_partitions=num_partitions,
num_islands=num_islands, partition_label=par_label,
show_gojs=True, **algo_params)
pgt = unroll_and_partition_with_params(lg_path(lg_name), query)

num_partitions = pgt._num_parts;

pgt_id = pg_mgr.add_pgt(pgt, lg_name)

Expand Down Expand Up @@ -364,50 +350,58 @@ def gen_pgt_post():

# Retrieve json data.
json_string = request.forms.get('json_data')
logical_graph = json.loads(json.loads(json_string))
logical_graph = json.loads(json_string.decode('string-escape').strip('"'))

try:
# Save graph
new_path = save(lg_name, logical_graph)

# Unrolling LG to PGT.
pgt = unroll(new_path)

# Define partitioning parameters.
algo = request.forms.get('algo', 'none')
num_partitions = request.forms.get('num_par', default=1, type=int)
num_islands = request.forms.get('num_islands', default=0, type=int)
par_label = request.forms.get('par_label', 'Partition')

# Build a map with extra parameters, more specific to some algorithms.
algo_params = {}
for name, typ in ALGO_PARAMS:
if name in request.forms:
algo_params[name] = request.forms.get(name, type=typ)
# LG -> PGT
pgt = unroll_and_partition_with_params(new_path, request.forms)

# Partition the PGT
pgt = partition(pgt, algo=algo, num_partitions=num_partitions,
num_islands=num_islands, partition_label=par_label,
show_gojs=True,
**algo_params)
num_partitions = pgt._num_parts;

pgt_id = pg_mgr.add_pgt(pgt, lg_name)

part_info = ' - '.join(['{0}:{1}'.format(k, v) for k, v in pgt.result().items()])
tpl = file_as_string('pg_viewer.html')
return template(tpl, pgt_view_json_name=pgt_id,
partition_info=part_info,
title="Physical Graph Template %s" %
('' if num_partitions == 0 else 'Partitioning'))
return template(tpl, pgt_view_json_name=pgt_id, partition_info=part_info, title="Physical Graph Template %s" % ('' if num_partitions == 0 else 'Partitioning'))
except GraphException as ge:
return "Invalid Logical Graph {1}: {0}".format(str(ge), lg_name), 500
except SchedulerException as se:
return "Graph scheduling exception {1}: {0}".format(str(se),
lg_name), 500
return "Graph scheduling exception {1}: {0}".format(str(se), lg_name), 500
except Exception:
trace_msg = traceback.format_exc()
return "Graph partition exception {1}: {0}".format(trace_msg, lg_name), 500


def unroll_and_partition_with_params(lg_path, algo_params_source):
print("unroll_and_partition_with_params()")

# Unrolling LG to PGT.
pgt = unroll(lg_path)

# Define partitioning parameters.
algo = algo_params_source.get('algo', 'none')
num_partitions = algo_params_source.get('num_par', default=1, type=int)
num_islands = algo_params_source.get('num_islands', default=0, type=int)
par_label = algo_params_source.get('par_label', 'Partition')

# Build a map with extra parameters, more specific to some algorithms.
algo_params = {}
for name, typ in ALGO_PARAMS:
if name in algo_params_source:
algo_params[name] = algo_params_source.get(name, type=typ)

# Partition the PGT
pgt = partition(pgt, algo=algo, num_partitions=num_partitions,
num_islands=num_islands, partition_label=par_label,
show_gojs=True,
**algo_params)

return pgt


def save(lg_name, logical_graph):
"""
Saves graph.
Expand Down
30 changes: 28 additions & 2 deletions test/dropmake/test_lgweb.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@

import pkg_resources

import six.moves.urllib_parse as urllib # @UnresolvedImport

from dlg import tool, utils
from dlg.restutils import RestClient, RestClientException


lg_dir = pkg_resources.resource_filename(__name__, '.') # @UndefinedVariable
lgweb_port = 8000

Expand Down Expand Up @@ -114,6 +115,31 @@ def test_get_pgtjson(self):
# good!
c._get_json('/pgt_jsonbody?pgt_name=logical_graphs/chiles_simple1_pgt.json')

def test_get_pgt_post(self):

c = RestClient('localhost', lgweb_port, 10)

# load logical graph JSON from file
json_data = "{\"modelData\":{\"fileType\":3,\"repo\":\"james-strauss-uwa/eagle-test\",\"filePath\":\"test\",\"sha\":\"1c6b3f778c5dc72a80001e7f2d08f639be9ad8a6\",\"git_url\":\"https://api.github.com/repos/james-strauss-uwa/eagle-test/git/blobs/1c6b3f778c5dc72a80001e7f2d08f639be9ad8a6\"},\"nodeDataArray\":[{\"category\":\"memory\",\"categoryType\":\"DataDrop\",\"type\":\"memory\",\"isData\":true,\"isGroup\":false,\"canHaveInputs\":true,\"canHaveOutputs\":true,\"colour\":\" #394BB2\",\"key\":-1,\"text\":\"Enter label\",\"loc\":\"200 100\",\"inputPorts\":[],\"outputPorts\":[{\"Id\":\"ab5ada14-04d7-4b03-816d-c43428d4e2e4\",\"IdText\":\"event\"}],\"inputLocalPorts\":[],\"outputLocalPorts\":[],\"appFields\":[],\"fields\":[{\"text\":\"Data volume\",\"name\":\"data_volume\",\"value\":\"5\"},{\"text\":\"Group end\",\"name\":\"group_end\",\"value\":\"0\"}]},{\"category\":\"BashShellApp\",\"categoryType\":\"ApplicationDrop\",\"type\":\"BashShellApp\",\"isData\":false,\"isGroup\":false,\"canHaveInputs\":true,\"canHaveOutputs\":true,\"colour\":\" #1C2833\",\"key\":-2,\"text\":\"Enter label\",\"loc\":\"400 200\",\"inputPorts\":[{\"Id\":\"c12aa833-43a9-4c1e-abaa-c77396010a31\",\"IdText\":\"event\"}],\"outputPorts\":[{\"Id\":\"47c421b8-5cdc-4ff7-ab7e-b75140f2d951\",\"IdText\":\"event\"}],\"inputLocalPorts\":[],\"outputLocalPorts\":[],\"appFields\":[],\"fields\":[{\"text\":\"Execution time\",\"name\":\"execution_time\",\"value\":\"5\"},{\"text\":\"Num CPUs\",\"name\":\"num_cpus\",\"value\":\"1\"},{\"text\":\"Group start\",\"name\":\"group_start\",\"value\":\"0\"},{\"text\":\"Arg01\",\"name\":\"Arg01\",\"value\":\"\"},{\"text\":\"Arg02\",\"name\":\"Arg02\",\"value\":\"\"},{\"text\":\"Arg03\",\"name\":\"Arg03\",\"value\":\"\"},{\"text\":\"Arg04\",\"name\":\"Arg04\",\"value\":\"\"},{\"text\":\"Arg05\",\"name\":\"Arg05\",\"value\":\"\"},{\"text\":\"Arg06\",\"name\":\"Arg06\",\"value\":\"\"},{\"text\":\"Arg07\",\"name\":\"Arg07\",\"value\":\"\"},{\"text\":\"Arg08\",\"name\":\"Arg08\",\"value\":\"\"},{\"text\":\"Arg09\",\"name\":\"Arg09\",\"value\":\"\"},{\"text\":\"Arg10\",\"name\":\"Arg10\",\"value\":\"\"}]},{\"category\":\"Component\",\"categoryType\":\"ApplicationDrop\",\"type\":\"Component\",\"isData\":false,\"isGroup\":false,\"canHaveInputs\":true,\"canHaveOutputs\":true,\"colour\":\" #3498DB\",\"key\":-3,\"text\":\"Enter label\",\"loc\":\"600 300\",\"inputPorts\":[{\"Id\":\"0178b7ce-79ed-406d-9e4b-ee2a53c168ec\",\"IdText\":\"event\"}],\"outputPorts\":[{\"Id\":\"f487361b-f633-43ba-978d-c65d7a52c34d\",\"IdText\":\"event\"}],\"inputLocalPorts\":[],\"outputLocalPorts\":[],\"appFields\":[],\"fields\":[{\"text\":\"Execution time\",\"name\":\"execution_time\",\"value\":\"5\"},{\"text\":\"Num CPUs\",\"name\":\"num_cpus\",\"value\":\"1\"},{\"text\":\"Group start\",\"name\":\"group_start\",\"value\":\"0\"},{\"text\":\"Appclass\",\"name\":\"appclass\",\"value\":\"test.graphsRepository\"},{\"text\":\"Arg01\",\"name\":\"Arg01\",\"value\":\"\"},{\"text\":\"Arg02\",\"name\":\"Arg02\",\"value\":\"\"},{\"text\":\"Arg03\",\"name\":\"Arg03\",\"value\":\"\"},{\"text\":\"Arg04\",\"name\":\"Arg04\",\"value\":\"\"},{\"text\":\"Arg05\",\"name\":\"Arg05\",\"value\":\"\"},{\"text\":\"Arg06\",\"name\":\"Arg06\",\"value\":\"\"},{\"text\":\"Arg07\",\"name\":\"Arg07\",\"value\":\"\"},{\"text\":\"Arg08\",\"name\":\"Arg08\",\"value\":\"\"},{\"text\":\"Arg09\",\"name\":\"Arg09\",\"value\":\"\"},{\"text\":\"Arg10\",\"name\":\"Arg10\",\"value\":\"\"}]},{\"category\":\"file\",\"categoryType\":\"DataDrop\",\"type\":\"file\",\"isData\":true,\"isGroup\":false,\"canHaveInputs\":true,\"canHaveOutputs\":true,\"colour\":\" #394BB2\",\"key\":-4,\"text\":\"Enter label\",\"loc\":\"800 400\",\"inputPorts\":[{\"Id\":\"c05689f4-6c5a-47dc-b3b1-fcbdfa09e4df\",\"IdText\":\"event\"}],\"outputPorts\":[],\"inputLocalPorts\":[],\"outputLocalPorts\":[],\"appFields\":[],\"fields\":[{\"text\":\"Data volume\",\"name\":\"data_volume\",\"value\":\"5\"},{\"text\":\"Group end\",\"name\":\"group_end\",\"value\":\"0\"},{\"text\":\"Check file path exists\",\"name\":\"check_filepath_exists\",\"value\":\"1\"},{\"text\":\"File path\",\"name\":\"filepath\",\"value\":\"\"},{\"text\":\"Directory name\",\"name\":\"dirname\",\"value\":\"\"}]}],\"linkDataArray\":[{\"from\":-1,\"fromPort\":\"ab5ada14-04d7-4b03-816d-c43428d4e2e4\",\"to\":-2,\"toPort\":\"c12aa833-43a9-4c1e-abaa-c77396010a31\"},{\"from\":-2,\"fromPort\":\"47c421b8-5cdc-4ff7-ab7e-b75140f2d951\",\"to\":-3,\"toPort\":\"0178b7ce-79ed-406d-9e4b-ee2a53c168ec\"},{\"from\":-3,\"fromPort\":\"f487361b-f633-43ba-978d-c65d7a52c34d\",\"to\":-4,\"toPort\":\"c05689f4-6c5a-47dc-b3b1-fcbdfa09e4df\"}]}"

# new graphs cannot currently be added
form_data = {
'algo': 'metis',
'lg_name': 'metis.graph',
'json_data': json_data,
'num_islands': 0,
'num_par': 1,
'par_label': 'Partition',
'max_load_imb': 100,
'max_cpu': 8
}

try:
content = urllib.urlencode(form_data)
ret = c._POST('/gen_pgt', content, content_type='application/x-www-form-urlencoded')
except RestClientException as e:
self.fail(e)

def test_load_lgeditor(self):

c = RestClient('localhost', lgweb_port, 10)
Expand Down Expand Up @@ -159,4 +185,4 @@ def test_show_schedule_mat(self):
self._test_pgt_action('show_schedule_mat', False)

def test_get_gantt_chart(self):
self._test_pgt_action('pgt_gantt_chart', True)
self._test_pgt_action('pgt_gantt_chart', True)

0 comments on commit 0e85c1c

Please sign in to comment.