Skip to content

Commit

Permalink
Merge bee8b98 into 52c3dce
Browse files Browse the repository at this point in the history
  • Loading branch information
pritchardn committed May 13, 2022
2 parents 52c3dce + bee8b98 commit 91d9cee
Show file tree
Hide file tree
Showing 3 changed files with 78 additions and 12 deletions.
19 changes: 9 additions & 10 deletions daliuge-engine/dlg/manager/composite_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -349,16 +349,15 @@ def addGraphSpec(self, sessionId, graphSpec):
# attribute set
logger.info(f"Separating graph using partition attribute {self._partitionAttr}")
perPartition = collections.defaultdict(list)
try:
if graphSpec[-1]["rmode"] is not None:
init_pg_repro_data(graphSpec)
self._graph["reprodata"] = graphSpec.pop()
logger.debug(
"Composite manager found reprodata in dropspecList, rmode=%s",
self._graph["reprodata"]["rmode"],
)
except KeyError:
pass
if "rmode" in graphSpec[-1]:
init_pg_repro_data(graphSpec)
self._graph["reprodata"] = graphSpec.pop()
logger.debug(
"Composite manager found reprodata in dropspecList, rmode=%s",
self._graph["reprodata"]["rmode"],
)
if graphSpec[-1] == {}:
graphSpec.pop()
for dropSpec in graphSpec:
if self._partitionAttr not in dropSpec:
msg = "Drop %s doesn't specify a %s attribute" % (
Expand Down
67 changes: 67 additions & 0 deletions daliuge-engine/test/manager/test_dim.py
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,73 @@ def assertGraphStatus(sessionId, expectedStatus):
# a.setCompleted()
assertGraphStatus(sessionId, DROPStates.CANCELLED)

def test_submit_unreprodata(self):
"""
Need to ensure that the DIM can handle a graph with empty reprodata
(the default if nothing is provided at translation time)
"""
graphSpec = [
{
"oid": "A",
"type": "plain",
"storage": Categories.MEMORY,
"node": hostname,
"consumers": ["B"],
},
{
"oid": "B",
"type": "app",
"app": "dlg.apps.simple.SleepAndCopyApp",
"sleepTime": 1,
"outputs": ["C"],
"node": hostname,
},
{
"oid": "C",
"type": "plain",
"storage": Categories.MEMORY,
"node": hostname,
},
{} # A dummy empty reprodata (the default if absolutely nothing is specified)
]
self.dim.createSession('a')
self.assertEqual(0, self.dim.getGraphSize('a'))
self.dim.addGraphSpec('a', graphSpec)
self.assertEqual(len(graphSpec), self.dim.getGraphSize('a'))

def test_submit_noreprodata(self):
"""
Need to ensure that the DIM can handle a graph with no reprodata
(the default if nothing is provided at translation time)
"""
graphSpec = [
{
"oid": "A",
"type": "plain",
"storage": Categories.MEMORY,
"node": hostname,
"consumers": ["B"],
},
{
"oid": "B",
"type": "app",
"app": "dlg.apps.simple.SleepAndCopyApp",
"sleepTime": 1,
"outputs": ["C"],
"node": hostname,
},
{
"oid": "C",
"type": "plain",
"storage": Categories.MEMORY,
"node": hostname,
},
]
self.dim.createSession('a')
self.assertEqual(0, self.dim.getGraphSize('a'))
self.dim.addGraphSpec('a', graphSpec)
self.assertEqual(len(graphSpec), self.dim.getGraphSize('a'))


class TestREST(LocalDimStarter, unittest.TestCase):
def test_fullRound(self):
Expand Down
4 changes: 2 additions & 2 deletions daliuge-translator/dlg/dropmake/web/lg_web.py
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,7 @@ def gen_pg():
mgr_client.create_session(ssid)
# print "session created"
completed_uids = common.get_roots(pg_spec)
# pg_spec.append(reprodata)
pg_spec.append(reprodata)
mgr_client.append_graph(ssid, pg_spec)
# print "graph appended"
mgr_client.deploy_session(ssid, completed_uids=completed_uids)
Expand Down Expand Up @@ -567,7 +567,6 @@ def gen_pgt_post():

# Retrieve rmode value
rmode = reqform.get("rmode", str(REPRO_DEFAULT.value))

# Retrieve json data.
json_string = reqform.get("json_data")
try:
Expand All @@ -586,6 +585,7 @@ def gen_pgt_post():
validate(logical_graph, lg_schema)
except ValidationError as ve:
error = "Validation Error {1}: {0}".format(str(ve), lg_name)
logical_graph = prepare_lgt(logical_graph, rmode)
# LG -> PGT
pgt = unroll_and_partition_with_params(logical_graph, reqform)
par_algo = reqform.get("algo", "none")
Expand Down

0 comments on commit 91d9cee

Please sign in to comment.