From 16e8b02f2c622cdc5adbb36cb46746b77a98862f Mon Sep 17 00:00:00 2001 From: Andreas Wicenec Date: Tue, 8 Aug 2023 14:01:31 +0800 Subject: [PATCH] Doc and simple changes --- daliuge-engine/dlg/apps/bash_shell_app.py | 6 +++--- daliuge-engine/dlg/apps/pyfunc.py | 7 ++----- daliuge-engine/dlg/apps/simple.py | 11 ++++++++--- daliuge-engine/dlg/named_port_utils.py | 4 ++-- daliuge-engine/test/graphs/ArrayLoopPG.graph | 6 ------ 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/daliuge-engine/dlg/apps/bash_shell_app.py b/daliuge-engine/dlg/apps/bash_shell_app.py index 7efb3a495..a02d6510f 100644 --- a/daliuge-engine/dlg/apps/bash_shell_app.py +++ b/daliuge-engine/dlg/apps/bash_shell_app.py @@ -57,7 +57,7 @@ logger = logging.getLogger(__name__) -def mesage_stdouts(prefix, stdout, stderr, enc="utf8"): +def message_stdouts(prefix, stdout, stderr, enc="utf8"): msg = prefix if not stdout and not stderr: return msg @@ -300,7 +300,7 @@ def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE): self._recompute_data["status"] = str(pcode) if pcode == 0 and logger.isEnabledFor(logging.DEBUG): logger.debug( - mesage_stdouts( + message_stdouts( "Command finished successfully", pstdout, pstderr ) ) @@ -308,7 +308,7 @@ def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE): message = "Command didn't finish successfully (exit code %d)" % ( pcode, ) - logger.error(mesage_stdouts(message, pstdout, pstderr)) + logger.error(message_stdouts(message, pstdout, pstderr)) raise Exception(message) def dataURL(self) -> str: diff --git a/daliuge-engine/dlg/apps/pyfunc.py b/daliuge-engine/dlg/apps/pyfunc.py index 1f774acf1..4039a4e44 100644 --- a/daliuge-engine/dlg/apps/pyfunc.py +++ b/daliuge-engine/dlg/apps/pyfunc.py @@ -414,7 +414,7 @@ def run(self): """ Function positional and keyword argument treatment: - Function arguments can be provided in four different ways: + Function arguments can be provided in three different ways: 1) Through an input port 2) By specifying ApplicationArgs (one for each argument) 3) Through defaults at the time of function definition @@ -429,13 +429,10 @@ def run(self): somewhat risky, since the order is relevant and in this code derived from the order defined in the graph (same order as defined in the component description). - Input ports will NOT be used by order (anymore), but by the name of the port. + Input ports will NOT be used by index (anymore), but by using the name of the port. Since each input port requires an associated data drop, this provides a unique mapping. This also allows to pass values to any function argument through a port. - Function argument values as well as the function code can be provided in - serialised (pickle) form by setting the 'pickle' flag. Note that this flag - is valid for all arguments and the code (if specified) in a global way. """ funcargs = {} # Inputs are un-pickled and treated as the arguments of the function diff --git a/daliuge-engine/dlg/apps/simple.py b/daliuge-engine/dlg/apps/simple.py index 25eeeba32..c25862a35 100644 --- a/daliuge-engine/dlg/apps/simple.py +++ b/daliuge-engine/dlg/apps/simple.py @@ -524,11 +524,12 @@ def reduce_gather_inputs(self): data = drop_loaders.load_numpy(input) # skip gather for the first input result = ( - reduce(data, axis=self.reduce_axes) + reduce(data, axis=self.reduce_axes, allow_pickle=True) if result is None else gather( result, - reduce(data, axis=self.reduce_axes), + reduce(data, axis=self.reduce_axes, allow_pickle=True), + allow_pickle=True, ) ) return result @@ -540,7 +541,11 @@ def gather_inputs(self): for input in self.inputs: data = drop_loaders.load_numpy(input) # assign instead of gather for the first input - result = data if result is None else gather(result, data) + result = ( + data + if result is None + else gather(result, data, allow_pickle=True) + ) return result diff --git a/daliuge-engine/dlg/named_port_utils.py b/daliuge-engine/dlg/named_port_utils.py index 0f822de43..e24146dad 100644 --- a/daliuge-engine/dlg/named_port_utils.py +++ b/daliuge-engine/dlg/named_port_utils.py @@ -170,7 +170,7 @@ def replace_named_ports( separator: str = " ", ) -> Tuple[str, str]: """ - Function attempts to identify component arguments that match port names. + Function attempts to identify CLI component arguments that match port names. Inputs: iitems: itemized input port dictionary @@ -213,13 +213,13 @@ def replace_named_ports( portPosargsDict = collections.OrderedDict( zip(posargs, [None] * len(posargs)) ) - portkeyargs = {} logger.debug( "posargs: %s; keyargs: %s, %s", posargs, keyargs, check_ports_dict(inport_names), ) + portkeyargs = {} ipkeyargs = {} opkeyargs = {} if check_ports_dict(inport_names): diff --git a/daliuge-engine/test/graphs/ArrayLoopPG.graph b/daliuge-engine/test/graphs/ArrayLoopPG.graph index 83f6a4c33..d43bf3b16 100644 --- a/daliuge-engine/test/graphs/ArrayLoopPG.graph +++ b/daliuge-engine/test/graphs/ArrayLoopPG.graph @@ -41,7 +41,6 @@ "integer": false, "low": "0", "high": "100", - "dropclass": "dlg.apps.simple.RandomArrayApp", "execution_time": 5, "group_start": false, "applicationArgs": {}, @@ -70,7 +69,6 @@ "num_cpus": 1, "execution_time": 5, "group_start": true, - "dropclass": "dlg.apps.simple.PickOne", "applicationArgs": {}, "iid": "0/0/0", "lg_key": -3, @@ -105,7 +103,6 @@ "num_cpus": 1, "execution_time": 5, "group_start": true, - "dropclass": "dlg.apps.simple.PickOne", "applicationArgs": {}, "iid": "0/0/1", "lg_key": -3, @@ -143,7 +140,6 @@ "num_cpus": 1, "execution_time": 5, "group_start": true, - "dropclass": "dlg.apps.simple.PickOne", "applicationArgs": {}, "iid": "0/1/0", "lg_key": -3, @@ -178,7 +174,6 @@ "num_cpus": 1, "execution_time": 5, "group_start": true, - "dropclass": "dlg.apps.simple.PickOne", "applicationArgs": {}, "iid": "0/1/1", "lg_key": -3, @@ -494,7 +489,6 @@ "loop_ctx": null, "weight": 5, "num_cpus": 1, - "dropclass": "dlg.apps.simple.GenericScatterApp", "execution_time": 5, "group_start": false, "array": "",