diff --git a/daliuge-common/dlg/common/__init__.py b/daliuge-common/dlg/common/__init__.py index fa6868b43..4365a7884 100644 --- a/daliuge-common/dlg/common/__init__.py +++ b/daliuge-common/dlg/common/__init__.py @@ -41,9 +41,9 @@ class Categories: GATHER = 'Gather' GROUP_BY = 'GroupBy' LOOP = 'Loop' - BRANCH = 'Branch' VARIABLES = 'Variables' + BRANCH = 'Branch' DATA = 'Data' COMPONENT = 'Component' PYTHON_APP = 'PythonApp' @@ -67,6 +67,7 @@ class Categories: APP_DROP_TYPES = [ Categories.COMPONENT, Categories.PYTHON_APP, + Categories.BRANCH, Categories.BASH_SHELL_APP, Categories.MPI, Categories.DYNLIB_APP, diff --git a/daliuge-engine/build_engine.sh b/daliuge-engine/build_engine.sh index f53694f69..9f233bf18 100755 --- a/daliuge-engine/build_engine.sh +++ b/daliuge-engine/build_engine.sh @@ -11,9 +11,11 @@ case "$1" in echo "Build finished!" exit 1 ;; "dev") + C_TAG="master" + [[ ! -z $2 ]] && C_TAG=$2 export VCS_TAG=`git rev-parse --abbrev-ref HEAD | tr '[:upper:]' '[:lower:]'` - echo "Building daliuge-engine development version using daliuge-common:master" - docker build --build-arg VCS_TAG=master --no-cache -t icrar/daliuge-engine:${VCS_TAG} -f docker/Dockerfile . + echo "Building daliuge-engine development version using daliuge-common:${C_TAG}" + docker build --build-arg VCS_TAG=${C_TAG} --no-cache -t icrar/daliuge-engine:${VCS_TAG} -f docker/Dockerfile . echo "Build finished!" exit 1;; "casa") diff --git a/daliuge-engine/dlg/apps/simple.py b/daliuge-engine/dlg/apps/simple.py index 8324b57fb..3d2adac75 100644 --- a/daliuge-engine/dlg/apps/simple.py +++ b/daliuge-engine/dlg/apps/simple.py @@ -28,7 +28,7 @@ import numpy as np from .. import droputils, utils -from ..drop import BarrierAppDROP, ContainerDROP +from ..drop import BarrierAppDROP, BranchAppDrop, ContainerDROP from ..meta import dlg_float_param, dlg_string_param from ..meta import dlg_bool_param, dlg_int_param from ..meta import dlg_component, dlg_batch_input @@ -373,3 +373,16 @@ def run(self): for o in outs: o.len = len(content) o.write(content) # send content to all outputs + +class SimpleBranch(BranchAppDrop, NullBarrierApp): + """Simple branch app that is told the result of its condition""" + + def initialize(self, **kwargs): + self.result = self._getArg(kwargs, 'result', True) + BranchAppDrop.initialize(self, **kwargs) + + def run(self): + pass + + def condition(self): + return self.result \ No newline at end of file diff --git a/daliuge-engine/dlg/ddap_protocol.py b/daliuge-engine/dlg/ddap_protocol.py index 749629d06..dcab9b8c6 100644 --- a/daliuge-engine/dlg/ddap_protocol.py +++ b/daliuge-engine/dlg/ddap_protocol.py @@ -44,7 +44,7 @@ class DROPStates: COMPLETED. Later, they transition through EXPIRED, eventually arriving to DELETED. """ - INITIALIZED, WRITING, COMPLETED, ERROR, EXPIRED, DELETED, CANCELLED = range(7) + INITIALIZED, WRITING, COMPLETED, ERROR, EXPIRED, DELETED, CANCELLED, SKIPPED = range(8) class AppDROPStates: @@ -54,7 +54,7 @@ class AppDROPStates: are started. Depending on the execution result they eventually move to the FINISHED or ERROR state. """ - NOT_RUN, RUNNING, FINISHED, ERROR, CANCELLED = range(5) + NOT_RUN, RUNNING, FINISHED, ERROR, CANCELLED, SKIPPED = range(6) class DROPPhases: diff --git a/daliuge-engine/dlg/drop.py b/daliuge-engine/dlg/drop.py index 54a567710..956782a38 100644 --- a/daliuge-engine/dlg/drop.py +++ b/daliuge-engine/dlg/drop.py @@ -923,7 +923,7 @@ def setError(self): Moves this DROP to the ERROR state. ''' - if self.status == DROPStates.CANCELLED: + if self.status in (DROPStates.CANCELLED, DROPStates.SKIPPED): return self._closeWriters() @@ -945,7 +945,10 @@ def setCompleted(self): status = self.status if status == DROPStates.CANCELLED: return - if status not in [DROPStates.INITIALIZED, DROPStates.WRITING]: + elif status == DROPStates.SKIPPED: + self._fire('dropCompleted', status=status) + return + elif status not in [DROPStates.INITIALIZED, DROPStates.WRITING]: raise Exception("%r not in INITIALIZED or WRITING state (%s), cannot setComplete()" % (self, self.status)) self._closeWriters() @@ -969,6 +972,11 @@ def cancel(self): self._closeWriters() self.status = DROPStates.CANCELLED + def skip(self): + '''Moves this drop to the SKIPPED state closing any writers we opened''' + self._closeWriters() + self.status = DROPStates.SKIPPED + @property def node(self): return self._node @@ -1560,6 +1568,14 @@ def cancel(self): super(AppDROP, self).cancel() self.execStatus = AppDROPStates.CANCELLED + def skip(self): + '''Moves this application drop to its SKIPPED state''' + super().skip() + self.execStatus = AppDROPStates.SKIPPED + for o in self._outputs.values(): + o.skip() + self._fire('producerFinished', status=self.status, execStatus=self.execStatus) + class InputFiredAppDROP(AppDROP): """ @@ -1594,6 +1610,7 @@ def initialize(self, **kwargs): super(InputFiredAppDROP, self).initialize(**kwargs) self._completedInputs = [] self._errorInputs = [] + self._skippedInputs = [] # Error threshold must be within 0 and 100 if self.input_error_threshold < 0 or self.input_error_threshold > 100: @@ -1634,13 +1651,18 @@ def dropCompleted(self, uid, drop_state): self._errorInputs.append(uid) elif drop_state == DROPStates.COMPLETED: self._completedInputs.append(uid) + elif drop_state == DROPStates.SKIPPED: + self._skippedInputs.append(uid) else: raise Exception('Invalid DROP state in dropCompleted: %s' % drop_state) error_len = len(self._errorInputs) ok_len = len(self._completedInputs) + skipped_len = len(self._skippedInputs) + + # We have enough inputs to proceed + if (skipped_len + error_len + ok_len) == n_eff_inputs: - if (error_len + ok_len) == n_eff_inputs: # calculate the number of errors that have already occurred percent_failed = math.floor((error_len/float(n_eff_inputs)) * 100) @@ -1654,6 +1676,8 @@ def dropCompleted(self, uid, drop_state): self.execStatus = AppDROPStates.ERROR self.status = DROPStates.ERROR self._notifyAppIsFinished() + elif skipped_len == n_eff_inputs: + self.skip() else: self.async_execute() @@ -1668,7 +1692,7 @@ def async_execute(self): t.start() @track_current_drop - def execute(self): + def execute(self, _send_notifications=True): """ Manually trigger the execution of this application. @@ -1703,7 +1727,8 @@ def execute(self): drop_state = DROPStates.ERROR self.status = drop_state - self._notifyAppIsFinished() + if _send_notifications: + self._notifyAppIsFinished() def run(self): """ @@ -1726,6 +1751,22 @@ def initialize(self, **kwargs): super(BarrierAppDROP, self).initialize(**kwargs) +class BranchAppDrop(BarrierAppDROP): + """ + A special kind of application with exactly two outputs. After normal + execution, the application decides whether a certain condition is met. + If the condition is met, the first output is considered as COMPLETED, + while the other is moved to SKIPPED state, and vice-versa. + """ + + @track_current_drop + def execute(self, _send_notifications=True): + if len(self._outputs) != 2: + raise InvalidDropException(self, f'BranchAppDrops should have exactly 2 outputs, not {len(self._outputs)}') + BarrierAppDROP.execute(self, _send_notifications=False) + self.outputs[1 if self.condition() else 0].skip() + self._notifyAppIsFinished() + class PlasmaDROP(AbstractDROP): ''' A DROP that points to data stored in a Plasma Store diff --git a/daliuge-engine/dlg/manager/web/session.html b/daliuge-engine/dlg/manager/web/session.html index ab3d72ca1..ef1a863ad 100644 --- a/daliuge-engine/dlg/manager/web/session.html +++ b/daliuge-engine/dlg/manager/web/session.html @@ -97,11 +97,13 @@ var status_update_handler = function(statuses) { - + // This is the order in which blocks are drawn in the progress bar, + // so we want "done" states first and "nothing happened yet" states + // towards the end var states = ['completed', 'finished', 'running', 'writing', 'error', 'expired', 'deleted', - 'cancelled', + 'cancelled', 'skipped', 'not_run', 'initialized']; var states_idx = d3.scale.ordinal().domain(states).rangePoints([0, states.length - 1]); @@ -111,7 +113,7 @@ /* Get total and per-status counts, then normalize to 0-100% */ var total = statuses.length; - var status_counts = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + var status_counts = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; statuses.reduce(function(status_counts, s) { var idx = states_idx(get_status_name(s)); status_counts[idx] = status_counts[idx] + 1; diff --git a/daliuge-engine/test/test_drop.py b/daliuge-engine/test/test_drop.py index d0401d1b8..321541022 100644 --- a/daliuge-engine/test/test_drop.py +++ b/daliuge-engine/test/test_drop.py @@ -26,6 +26,7 @@ import random import shutil import sqlite3 +import string import tempfile from dlg import droputils @@ -35,6 +36,7 @@ DirectoryContainer, ContainerDROP, InputFiredAppDROP, RDBMSDrop from dlg.droputils import DROPWaiterCtx from dlg.exceptions import InvalidDropException +from dlg.apps.simple import NullBarrierApp, SimpleBranch try: @@ -780,5 +782,125 @@ def test_rdbms_drop(self): finally: os.unlink(dbfile) -if __name__ == '__main__': - unittest.main() +class BranchAppDropTests(unittest.TestCase): + """Tests for the BranchAppDrop class""" + + def _simple_branch_with_outputs(self, result, uids): + a = SimpleBranch(uids[0], uids[0], result=result) + b, c = (InMemoryDROP(x, x) for x in uids[1:]) + a.addOutput(b) + a.addOutput(c) + return a, b, c + + def _assert_drop_in_status(self, drop, status, execStatus): + self.assertEqual(drop.status, status, f'{drop} has status {drop.status} != {status}') + if isinstance(drop, AppDROP): + self.assertEqual(drop.execStatus, execStatus, f'{drop} has execStatus {drop.execStatus} != {execStatus}') + + def _assert_drop_complete_or_skipped(self, drop, complete_expected): + if complete_expected: + self._assert_drop_in_status(drop, DROPStates.COMPLETED, AppDROPStates.FINISHED) + else: + self._assert_drop_in_status(drop, DROPStates.SKIPPED, AppDROPStates.SKIPPED) + + def _test_single_branch_graph(self, result, levels): + """ + Test this graph, where each level appends one drop to each branch + + ,-- true --> B --> ... + A ---- false --> C --> ... + """ + a, b, c = self._simple_branch_with_outputs(result, 'abc') + last_true = b + last_false = c + all_drops = [a, b, c] + + # all_uids is ['de', 'fg', 'hi', ....] + all_uids = [string.ascii_lowercase[i: i + 2] + for i in range(3, len(string.ascii_lowercase), 2)] + + for level, uids in zip(range(levels), all_uids): + if level % 2: + x, y = (InMemoryDROP(uid, uid) for uid in uids) + last_true.addOutput(x) + last_false.addOutput(y) + else: + x, y = (NullBarrierApp(uid, uid) for uid in uids) + last_true.addConsumer(x) + last_false.addConsumer(y) + all_drops += [x, y] + last_true = x + last_false = y + + with DROPWaiterCtx(self, [last_true, last_false], 2, [DROPStates.COMPLETED, DROPStates.SKIPPED]): + a.async_execute() + + # Depending on "result", the "true" branch will be run or skipped + self._assert_drop_complete_or_skipped(last_true, result) + self._assert_drop_complete_or_skipped(last_false, not result) + + def test_simple_branch(self): + """Check that simple branch event transmission wroks""" + self._test_single_branch_graph(True, 0) + self._test_single_branch_graph(False, 0) + + def test_simple_branch_one_level(self): + """Like test_simple_branch_app, but events propagate downstream one level""" + self._test_single_branch_graph(True, 1) + self._test_single_branch_graph(False, 1) + + def test_simple_branch_two_levels(self): + """Like test_skipped_propagates, but events propagate more levels""" + self._test_single_branch_graph(True, 2) + self._test_single_branch_graph(False, 2) + + def test_simple_branch_more_levels(self): + for levels in (3, 4, 5, 6, 7): + self._test_single_branch_graph(True, levels) + self._test_single_branch_graph(False, levels) + + def _test_multi_branch_graph(self, *results): + """ + Test this graph, each level appends a new branch to the first output + of the previous branch. + + ,- false --> C ,- false --> F + A ----- true --> B --> D ----- true --> E --> ... + """ + + a, b, c = self._simple_branch_with_outputs(results[0], 'abc') + all_drops = [a, b, c] + last_first_output = b + + # all_uids is ['de', 'fg', 'hi', ....] + all_uids = [string.ascii_lowercase[i: i + 3] + for i in range(4, len(string.ascii_lowercase), 3)] + + for uids, result in zip(all_uids, results[1:]): + x, y, z = self._simple_branch_with_outputs(result, uids) + all_drops += [x, y, z] + last_first_output.addConsumer(x) + last_first_output = y + + with DROPWaiterCtx(self, all_drops, 2, [DROPStates.COMPLETED, DROPStates.SKIPPED]): + a.async_execute() + + # TODO: Checking each individual drop depending on "results" would be a + # tricky business, so we are skipping that check for now. + + def test_multi_branch_one_level(self): + """Check that simple branch event transmission wroks""" + self._test_multi_branch_graph(True) + self._test_multi_branch_graph(False) + + def test_multi_branch_two_levels(self): + """Like test_simple_branch_app, but events propagate downstream one level""" + self._test_multi_branch_graph(True, True) + self._test_multi_branch_graph(True, False) + self._test_multi_branch_graph(False, False) + + def test_multi_branch_more_levels(self): + """Like test_skipped_propagates, but events propagate more levels""" + for levels in (3, 4, 5, 6, 7): + self._test_multi_branch_graph(True, levels) + self._test_multi_branch_graph(False, levels) \ No newline at end of file diff --git a/daliuge-translator/build_translator.sh b/daliuge-translator/build_translator.sh index abe8b354e..ac6409cc7 100755 --- a/daliuge-translator/build_translator.sh +++ b/daliuge-translator/build_translator.sh @@ -11,11 +11,13 @@ case "$1" in echo "Build finished!" exit 1 ;; "dev") + C_TAG="master" + [[ ! -z "$2" ]] && C_TAG=$2 export VCS_TAG=`git rev-parse --abbrev-ref HEAD | tr '[:upper:]' '[:lower:]'` - echo "Building daliuge-translator development version using daliuge-common:master" + echo "Building daliuge-translator development version using daliuge-common:${C_TAG}" # The complete casa and arrow installation is only required for the Plasma streaming # and should not go much further. - docker build --build-arg VCS_TAG=master --no-cache -t icrar/daliuge-translator:${VCS_TAG} -f docker/Dockerfile . + docker build --build-arg VCS_TAG=${C_TAG} --no-cache -t icrar/daliuge-translator:${VCS_TAG} -f docker/Dockerfile . echo "Build finished!" exit 1;; "casa") diff --git a/daliuge-translator/dlg/dropmake/.DS_Store b/daliuge-translator/dlg/dropmake/.DS_Store new file mode 100644 index 000000000..1e7ef3fca Binary files /dev/null and b/daliuge-translator/dlg/dropmake/.DS_Store differ diff --git a/daliuge-translator/dlg/dropmake/branchPattern2.graph b/daliuge-translator/dlg/dropmake/branchPattern2.graph deleted file mode 100644 index 40975b5ac..000000000 --- a/daliuge-translator/dlg/dropmake/branchPattern2.graph +++ /dev/null @@ -1,708 +0,0 @@ -{ - "linkDataArray": [ - { - "from": -5, - "fromPort": "d6ea4d68-da83-43d8-a4b9-0e7a94065cae", - "loop_aware": "0", - "to": -6, - "toPort": "ff0a11a2-5969-4d5c-b10e-520d46d5d726" - }, - { - "from": -2, - "fromPort": "cd08b2a9-6627-40b8-9e80-770a24d992fd", - "loop_aware": "0", - "to": -7, - "toPort": "4903980e-e085-4647-84c6-c2993ed5915e" - }, - { - "from": -7, - "fromPort": "f0fefdfc-1f0d-453b-8ba6-edc856046406", - "loop_aware": "0", - "to": -4, - "toPort": "8e19799b-2639-4d6c-a142-64d8586aa42f" - }, - { - "from": -2, - "fromPort": "d36f0a39-bdcf-47f1-9e19-2c2ae32f19bf", - "loop_aware": "0", - "to": -9, - "toPort": "2b43bb09-ff68-4821-a2d8-cd3d0ce22889" - }, - { - "from": -9, - "fromPort": "e473ab2b-dbdc-4be4-b545-6e5054c4c4e7", - "loop_aware": "0", - "to": -3, - "toPort": "6338439e-38c0-4511-968b-8d46ca645e8d" - }, - { - "from": -6, - "fromPort": "1fe972ed-b688-49bb-b048-f8a0be69da1a", - "loop_aware": "0", - "to": -2, - "toPort": "07b7af88-bfc6-46f7-80c4-bd182c32e7b1" - } - ], - "modelData": { - "eagleCommitHash": "6180aad4b42c06bb1b61ea2f18f57bcb8384d786", - "eagleVersion": "v3.0.5", - "filePath": "branchPattern.graph", - "fileType": "graph", - "git_url": "", - "readonly": true, - "repo": "", - "repoBranch": "", - "repoService": "Unknown", - "schemaVersion": "OJS", - "sha": "" - }, - "nodeDataArray": [ - { - "canHaveInputs": false, - "canHaveOutputs": false, - "category": "Description", - "categoryType": "Other", - "collapsed": false, - "color": "#9B3065", - "description": "", - "drawOrderHint": 0, - "exitAppFields": [], - "exitApplicationKey": null, - "exitApplicationName": "", - "exitApplicationType": "None", - "expanded": false, - "fields": [ - { - "description": "", - "name": "", - "readonly": false, - "text": "", - "type": "Unknown", - "value": "This pattern shows a branch construct. NOTE: Branches are still work in progress and will not work as desired in the execution engine. This graph will also be used to guide and test the further implementation." - } - ], - "flipPorts": false, - "height": 138.7811321752724, - "inputAppFields": [], - "inputApplicationKey": null, - "inputApplicationName": "", - "inputApplicationType": "None", - "inputLocalPorts": [], - "inputPorts": [], - "isData": false, - "isGroup": false, - "key": -1, - "outputAppFields": [], - "outputApplicationKey": null, - "outputApplicationName": "", - "outputApplicationType": "None", - "outputLocalPorts": [], - "outputPorts": [], - "precious": false, - "readonly": false, - "selected": false, - "showPorts": false, - "streaming": false, - "subject": null, - "text": "Description", - "width": 203.32409704359478, - "x": 392.61464198071883, - "y": 25.11521521288031 - }, - { - "canHaveInputs": true, - "canHaveOutputs": true, - "category": "Branch", - "categoryType": "Control", - "collapsed": false, - "color": "#00BDA1", - "description": "A conditional branch to control flow.", - "drawOrderHint": 0, - "exitAppFields": [], - "exitApplicationKey": null, - "exitApplicationName": "", - "exitApplicationType": "None", - "expanded": false, - "fields": [ - { - "description": "the number of seconds to sleep", - "name": "sleepTime", - "readonly": false, - "text": "sleepTime", - "type": "Integer", - "value": 5 - }, - { - "description": "Application class", - "name": "appclass", - "readonly": true, - "text": "appclass", - "type": "String", - "value": "dlg.apps.simple.SleepApp" - }, - { - "description": "Estimated execution time", - "name": "execution_time", - "readonly": false, - "text": "Execution time", - "type": "Float", - "value": 5 - }, - { - "description": "Number of cores used", - "name": "num_cpus", - "readonly": false, - "text": "Num CPUs", - "type": "Integer", - "value": 1 - }, - { - "description": "Component is start of a group", - "name": "group_start", - "readonly": false, - "text": "Group start", - "type": "Boolean", - "value": false - } - ], - "flipPorts": false, - "height": 200, - "inputAppFields": [ - ], - "inputPorts": [ - { - "Id": "07b7af88-bfc6-46f7-80c4-bd182c32e7b1", - "IdText": "Input", - "event": false, - "type": "String" - } - ], - "isData": false, - "isGroup": false, - "key": -2, - "outputApplicationKey": null, - "outputApplicationName": "", - "outputApplicationType": "None", - "outputLocalPorts": [], - "outputPorts": [ - { - "Id": "cd08b2a9-6627-40b8-9e80-770a24d992fd", - "IdText": "Y", - "event": false, - "type": "String" - }, - { - "Id": "d36f0a39-bdcf-47f1-9e19-2c2ae32f19bf", - "IdText": "N", - "event": false, - "type": "String" - } - ], - "precious": false, - "readonly": false, - "selected": false, - "showPorts": false, - "streaming": false, - "subject": null, - "text": "Branch", - "width": 200, - "x": 696.215505295709, - "y": 470.54421905458094 - }, - { - "canHaveInputs": true, - "canHaveOutputs": true, - "category": "PythonApp", - "categoryType": "Application", - "collapsed": false, - "color": "#3498DB", - "description": "A simple APP that sleeps the specified amount of time (0 by default). This is mainly useful (and used) to test graph translation and structure without executing real algorithms. Very useful for debugging.", - "drawOrderHint": 0, - "exitAppFields": [], - "exitApplicationKey": null, - "exitApplicationName": "", - "exitApplicationType": "None", - "expanded": false, - "fields": [ - { - "description": "the number of seconds to sleep", - "name": "sleepTime", - "readonly": false, - "text": "sleepTime", - "type": "Integer", - "value": 5 - }, - { - "description": "Application class", - "name": "appclass", - "readonly": true, - "text": "appclass", - "type": "String", - "value": "dlg.apps.simple.SleepApp" - }, - { - "description": "Estimated execution time", - "name": "execution_time", - "readonly": false, - "text": "Execution time", - "type": "Float", - "value": 5 - }, - { - "description": "Number of cores used", - "name": "num_cpus", - "readonly": false, - "text": "Num CPUs", - "type": "Integer", - "value": 1 - }, - { - "description": "Component is start of a group", - "name": "group_start", - "readonly": false, - "text": "Group start", - "type": "Boolean", - "value": false - } - ], - "flipPorts": false, - "height": 200, - "inputAppFields": [], - "inputApplicationKey": null, - "inputApplicationName": "", - "inputApplicationType": "None", - "inputLocalPorts": [], - "inputPorts": [ - { - "Id": "6338439e-38c0-4511-968b-8d46ca645e8d", - "IdText": "N", - "event": false, - "type": "String" - } - ], - "isData": false, - "isGroup": false, - "key": -3, - "outputAppFields": [], - "outputApplicationKey": null, - "outputApplicationName": "", - "outputApplicationType": "None", - "outputLocalPorts": [], - "outputPorts": [], - "precious": false, - "readonly": false, - "selected": false, - "showPorts": false, - "streaming": false, - "subject": null, - "text": "NoApp", - "width": 200, - "x": 1041.4673658268866, - "y": 478.19241314177 - }, - { - "canHaveInputs": true, - "canHaveOutputs": true, - "category": "PythonApp", - "categoryType": "Application", - "collapsed": false, - "color": "#3498DB", - "description": "A simple APP that sleeps the specified amount of time (0 by default). This is mainly useful (and used) to test graph translation and structure without executing real algorithms. Very useful for debugging.", - "drawOrderHint": 0, - "exitAppFields": [], - "exitApplicationKey": null, - "exitApplicationName": "", - "exitApplicationType": "None", - "expanded": false, - "fields": [ - { - "description": "the number of seconds to sleep", - "name": "sleepTime", - "readonly": false, - "text": "sleepTime", - "type": "Integer", - "value": 5 - }, - { - "description": "Application class", - "name": "appclass", - "readonly": true, - "text": "appclass", - "type": "String", - "value": "dlg.apps.simple.SleepApp" - }, - { - "description": "Estimated execution time", - "name": "execution_time", - "readonly": false, - "text": "Execution time", - "type": "Float", - "value": 5 - }, - { - "description": "Number of cores used", - "name": "num_cpus", - "readonly": false, - "text": "Num CPUs", - "type": "Integer", - "value": 1 - }, - { - "description": "Component is start of a group", - "name": "group_start", - "readonly": false, - "text": "Group start", - "type": "Boolean", - "value": false - } - ], - "flipPorts": false, - "height": 200, - "inputAppFields": [], - "inputApplicationKey": null, - "inputApplicationName": "", - "inputApplicationType": "None", - "inputLocalPorts": [], - "inputPorts": [ - { - "Id": "8e19799b-2639-4d6c-a142-64d8586aa42f", - "IdText": "Y", - "event": false, - "type": "String" - } - ], - "isData": false, - "isGroup": false, - "key": -4, - "outputAppFields": [], - "outputApplicationKey": null, - "outputApplicationName": "", - "outputApplicationType": "None", - "outputLocalPorts": [], - "outputPorts": [], - "precious": false, - "readonly": false, - "selected": false, - "showPorts": false, - "streaming": false, - "subject": null, - "text": "YesApp", - "width": 200, - "x": 781.533996081309, - "y": 673.3141387138617 - }, - { - "canHaveInputs": true, - "canHaveOutputs": true, - "category": "PythonApp", - "categoryType": "Application", - "collapsed": false, - "color": "#3498DB", - "description": "A simple APP that sleeps the specified amount of time (0 by default). This is mainly useful (and used) to test graph translation and structure without executing real algorithms. Very useful for debugging.", - "drawOrderHint": 0, - "exitAppFields": [], - "exitApplicationKey": null, - "exitApplicationName": "", - "exitApplicationType": "None", - "expanded": false, - "fields": [ - { - "description": "the number of seconds to sleep", - "name": "sleepTime", - "readonly": false, - "text": "sleepTime", - "type": "Integer", - "value": 5 - }, - { - "description": "Application class", - "name": "appclass", - "readonly": true, - "text": "appclass", - "type": "String", - "value": "dlg.apps.simple.SleepApp" - }, - { - "description": "Estimated execution time", - "name": "execution_time", - "readonly": false, - "text": "Execution time", - "type": "Float", - "value": 5 - }, - { - "description": "Number of cores used", - "name": "num_cpus", - "readonly": false, - "text": "Num CPUs", - "type": "Integer", - "value": 1 - }, - { - "description": "Component is start of a group", - "name": "group_start", - "readonly": false, - "text": "Group start", - "type": "Boolean", - "value": false - } - ], - "flipPorts": false, - "height": 200, - "inputAppFields": [], - "inputApplicationKey": null, - "inputApplicationName": "", - "inputApplicationType": "None", - "inputLocalPorts": [], - "inputPorts": [], - "isData": false, - "isGroup": false, - "key": -5, - "outputAppFields": [], - "outputApplicationKey": null, - "outputApplicationName": "", - "outputApplicationType": "None", - "outputLocalPorts": [], - "outputPorts": [ - { - "Id": "d6ea4d68-da83-43d8-a4b9-0e7a94065cae", - "IdText": "Input", - "event": false, - "type": "String" - } - ], - "precious": false, - "readonly": false, - "selected": false, - "showPorts": false, - "streaming": false, - "subject": null, - "text": "StartApp", - "width": 200, - "x": 278.48731015063254, - "y": 305.44739922270634 - }, - { - "canHaveInputs": true, - "canHaveOutputs": true, - "category": "Memory", - "categoryType": "Data", - "collapsed": false, - "color": "#394BB2", - "description": "", - "drawOrderHint": 0, - "exitAppFields": [], - "exitApplicationKey": null, - "exitApplicationName": "", - "exitApplicationType": "None", - "expanded": false, - "fields": [ - { - "description": "Estimated size of the data contained in this node", - "name": "data_volume", - "readonly": false, - "text": "Data volume", - "type": "Float", - "value": 5 - }, - { - "description": "Is this node the end of a group?", - "name": "group_end", - "readonly": false, - "text": "Group end", - "type": "Boolean", - "value": false - } - ], - "flipPorts": false, - "height": 200, - "inputAppFields": [], - "inputApplicationKey": null, - "inputApplicationName": "", - "inputApplicationType": "None", - "inputLocalPorts": [], - "inputPorts": [ - { - "Id": "ff0a11a2-5969-4d5c-b10e-520d46d5d726", - "IdText": "Input", - "event": false, - "type": "String" - } - ], - "isData": true, - "isGroup": false, - "key": -6, - "outputAppFields": [], - "outputApplicationKey": null, - "outputApplicationName": "", - "outputApplicationType": "None", - "outputLocalPorts": [], - "outputPorts": [ - { - "Id": "1fe972ed-b688-49bb-b048-f8a0be69da1a", - "IdText": "Input", - "event": false, - "type": "String" - } - ], - "precious": false, - "readonly": false, - "selected": false, - "showPorts": false, - "streaming": false, - "subject": null, - "text": "Memory", - "width": 200, - "x": 579.8721087698915, - "y": 313.2036256577609 - }, - { - "canHaveInputs": true, - "canHaveOutputs": true, - "category": "Memory", - "categoryType": "Data", - "collapsed": false, - "color": "#394BB2", - "description": "", - "drawOrderHint": 0, - "exitAppFields": [], - "exitApplicationKey": null, - "exitApplicationName": "", - "exitApplicationType": "None", - "expanded": false, - "fields": [ - { - "description": "Estimated size of the data contained in this node", - "name": "data_volume", - "readonly": false, - "text": "Data volume", - "type": "Float", - "value": 5 - }, - { - "description": "Is this node the end of a group?", - "name": "group_end", - "readonly": false, - "text": "Group end", - "type": "Boolean", - "value": false - } - ], - "flipPorts": false, - "height": 200, - "inputAppFields": [], - "inputApplicationKey": null, - "inputApplicationName": "", - "inputApplicationType": "None", - "inputLocalPorts": [], - "inputPorts": [ - { - "Id": "4903980e-e085-4647-84c6-c2993ed5915e", - "IdText": "Y", - "event": false, - "type": "String" - } - ], - "isData": true, - "isGroup": false, - "key": -7, - "outputAppFields": [], - "outputApplicationKey": null, - "outputApplicationName": "", - "outputApplicationType": "None", - "outputLocalPorts": [], - "outputPorts": [ - { - "Id": "f0fefdfc-1f0d-453b-8ba6-edc856046406", - "IdText": "Y", - "event": false, - "type": "String" - } - ], - "precious": false, - "readonly": true, - "selected": false, - "showPorts": false, - "streaming": false, - "subject": null, - "text": "Y", - "width": 200, - "x": 738.874750688509, - "y": 571.9291788842213 - }, - { - "canHaveInputs": true, - "canHaveOutputs": true, - "category": "Memory", - "categoryType": "Data", - "collapsed": false, - "color": "#394BB2", - "description": "", - "drawOrderHint": 0, - "exitAppFields": [], - "exitApplicationKey": null, - "exitApplicationName": "", - "exitApplicationType": "None", - "expanded": false, - "fields": [ - { - "description": "Estimated size of the data contained in this node", - "name": "data_volume", - "readonly": false, - "text": "Data volume", - "type": "Float", - "value": 5 - }, - { - "description": "Is this node the end of a group?", - "name": "group_end", - "readonly": false, - "text": "Group end", - "type": "Boolean", - "value": false - } - ], - "flipPorts": false, - "height": 200, - "inputAppFields": [], - "inputApplicationKey": null, - "inputApplicationName": "", - "inputApplicationType": "None", - "inputLocalPorts": [], - "inputPorts": [ - { - "Id": "2b43bb09-ff68-4821-a2d8-cd3d0ce22889", - "IdText": "N", - "event": false, - "type": "String" - } - ], - "isData": true, - "isGroup": false, - "key": -9, - "outputAppFields": [], - "outputApplicationKey": null, - "outputApplicationName": "", - "outputApplicationType": "None", - "outputLocalPorts": [], - "outputPorts": [ - { - "Id": "e473ab2b-dbdc-4be4-b545-6e5054c4c4e7", - "IdText": "N", - "event": false, - "type": "String" - } - ], - "precious": false, - "readonly": true, - "selected": false, - "showPorts": false, - "streaming": false, - "subject": null, - "text": "N", - "width": 200, - "x": 865.8414355612979, - "y": 480.86831609817546 - } - ] -} \ No newline at end of file diff --git a/daliuge-translator/dlg/dropmake/branchPattern2.pg.graph b/daliuge-translator/dlg/dropmake/branchPattern2.pg.graph deleted file mode 100644 index de75793f1..000000000 --- a/daliuge-translator/dlg/dropmake/branchPattern2.pg.graph +++ /dev/null @@ -1,174 +0,0 @@ -[ - { - "oid": "1_-2_0", - "type": "app", - "app": "dlg.apps.simple.SleepApp", - "rank": [ - 0 - ], - "loop_cxt": null, - "sleepTime": 5, - "tw": 5, - "num_cpus": 1, - "appclass": "dlg.apps.simple.SleepApp", - "execution_time": 5, - "group_start": false, - "iid": "0", - "lg_key": -2, - "dt": "Branch", - "nm": "Branch", - "consumers": [ - "1_-7_0", - "1_-9_0" - ], - "inputs": [ - "1_-6_0" - ], - "node": "#0", - "island": "#0" - }, - { - "oid": "1_-3_0", - "type": "app", - "app": "dlg.apps.simple.SleepApp", - "rank": [ - 0 - ], - "loop_cxt": null, - "sleepTime": 5, - "tw": 5, - "num_cpus": 1, - "appclass": "dlg.apps.simple.SleepApp", - "execution_time": 5, - "group_start": false, - "iid": "0", - "lg_key": -3, - "dt": "PythonApp", - "nm": "NoApp", - "inputs": [ - "1_-9_0" - ], - "node": "#0", - "island": "#0" - }, - { - "oid": "1_-4_0", - "type": "app", - "app": "dlg.apps.simple.SleepApp", - "rank": [ - 0 - ], - "loop_cxt": null, - "sleepTime": 5, - "tw": 5, - "num_cpus": 1, - "appclass": "dlg.apps.simple.SleepApp", - "execution_time": 5, - "group_start": false, - "iid": "0", - "lg_key": -4, - "dt": "PythonApp", - "nm": "YesApp", - "inputs": [ - "1_-7_0" - ], - "node": "#0", - "island": "#0" - }, - { - "oid": "1_-5_0", - "type": "app", - "app": "dlg.apps.simple.SleepApp", - "rank": [ - 0 - ], - "loop_cxt": null, - "sleepTime": 5, - "tw": 5, - "num_cpus": 1, - "appclass": "dlg.apps.simple.SleepApp", - "execution_time": 5, - "group_start": false, - "iid": "0", - "lg_key": -5, - "dt": "PythonApp", - "nm": "StartApp", - "outputs": [ - "1_-6_0" - ], - "node": "#0", - "island": "#0" - }, - { - "oid": "1_-6_0", - "type": "plain", - "storage": "Memory", - "rank": [ - 0 - ], - "loop_cxt": null, - "dw": 5, - "data_volume": 5, - "group_end": false, - "iid": "0", - "lg_key": -6, - "dt": "Memory", - "nm": "Memory", - "producers": [ - "1_-5_0" - ], - "consumers": [ - "1_-2_0" - ], - "node": "#0", - "island": "#0" - }, - { - "oid": "1_-7_0", - "type": "plain", - "storage": "Memory", - "rank": [ - 0 - ], - "loop_cxt": null, - "dw": 5, - "data_volume": 5, - "group_end": false, - "iid": "0", - "lg_key": -7, - "dt": "Memory", - "nm": "Y", - "inputs": [ - "1_-2_0" - ], - "consumers": [ - "1_-4_0" - ], - "node": "#0", - "island": "#0" - }, - { - "oid": "1_-9_0", - "type": "plain", - "storage": "Memory", - "rank": [ - 0 - ], - "loop_cxt": null, - "dw": 5, - "data_volume": 5, - "group_end": false, - "iid": "0", - "lg_key": -9, - "dt": "Memory", - "nm": "N", - "inputs": [ - "1_-2_0" - ], - "consumers": [ - "1_-3_0" - ], - "node": "#0", - "island": "#0" - } -] \ No newline at end of file diff --git a/daliuge-translator/dlg/dropmake/pg_generator.py b/daliuge-translator/dlg/dropmake/pg_generator.py index 54db7a8a8..a2d434db6 100644 --- a/daliuge-translator/dlg/dropmake/pg_generator.py +++ b/daliuge-translator/dlg/dropmake/pg_generator.py @@ -300,19 +300,32 @@ def is_start_listener(self): ) def is_group_start(self): - return ( - self.has_group() - and "group_start" in self.jd - and 1 == int(self.jd["group_start"]) - ) + result = False + if self.has_group() \ + and "group_start" in self.jd: + gs = self.jd["group_start"] + if type(gs) == type(True): + result = gs + elif type(gs) in [type(1), type(1.)]: + result = (1 == gs) + elif type(gs) == type("s"): + result = gs.lower() in ("true", "1") + return result - def is_group_end(self): - return ( - self.has_group() - and "group_end" in self.jd - and 1 == int(self.jd["group_end"]) - ) + def is_group_end(self): + result = False + if self.has_group() \ + and "group_end" in self.jd: + ge = self.jd["group_end"] + if type(ge) == type(True): + result = ge + elif type(ge) in [type(1), type(1.)]: + result = (1 == ge) + elif type(ge) == type("s"): + result = ge.lower() in ("true", "1") + return result + def is_group(self): return self._isgrp diff --git a/daliuge-translator/test/dropmake/logical_graphs/lofar_std.graph b/daliuge-translator/test/dropmake/logical_graphs/lofar_std.graph index 0b954796f..370effb14 100644 --- a/daliuge-translator/test/dropmake/logical_graphs/lofar_std.graph +++ b/daliuge-translator/test/dropmake/logical_graphs/lofar_std.graph @@ -2356,7 +2356,7 @@ { "text": "Group Start", "name": "group_start", - "value": 1, + "value": true, "description": "" }, { @@ -2556,7 +2556,7 @@ { "text": "Group End", "name": "group_end", - "value": 1, + "value": true, "description": "" } ]