Doc and simple changes

daliuge-engine/dlg/apps/bash_shell_app.py

@@ -57,7 +57,7 @@
 logger = logging.getLogger(__name__)
 
 
-def mesage_stdouts(prefix, stdout, stderr, enc="utf8"):
+def message_stdouts(prefix, stdout, stderr, enc="utf8"):
     msg = prefix
     if not stdout and not stderr:
         return msg
@@ -300,15 +300,15 @@ def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE):
         self._recompute_data["status"] = str(pcode)
         if pcode == 0 and logger.isEnabledFor(logging.DEBUG):
             logger.debug(
-                mesage_stdouts(
+                message_stdouts(
                     "Command finished successfully", pstdout, pstderr
                 )
             )
         elif pcode != 0:
             message = "Command didn't finish successfully (exit code %d)" % (
                 pcode,
             )
-            logger.error(mesage_stdouts(message, pstdout, pstderr))
+            logger.error(message_stdouts(message, pstdout, pstderr))
             raise Exception(message)
 
     def dataURL(self) -> str:

daliuge-engine/dlg/apps/pyfunc.py

@@ -414,7 +414,7 @@ def run(self):
         """
         Function positional and keyword argument treatment:
 
-        Function arguments can be provided in four different ways:
+        Function arguments can be provided in three different ways:
         1) Through an input port
         2) By specifying ApplicationArgs (one for each argument)
         3) Through defaults at the time of function definition
@@ -429,13 +429,10 @@ def run(self):
         somewhat risky, since the order is relevant and in this code derived from the
         order defined in the graph (same order as defined in the component description).
 
-        Input ports will NOT be used by order (anymore), but by the name of the port.
+        Input ports will NOT be used by index (anymore), but by using the name of the port.
         Since each input port requires an associated data drop, this provides a unique
         mapping. This also allows to pass values to any function argument through a port.
 
-        Function argument values as well as the function code can be provided in
-        serialised (pickle) form by setting the 'pickle' flag. Note that this flag
-        is valid for all arguments and the code (if specified) in a global way.
         """
         funcargs = {}
         # Inputs are un-pickled and treated as the arguments of the function

daliuge-engine/dlg/apps/simple.py

@@ -524,11 +524,12 @@ def reduce_gather_inputs(self):
             data = drop_loaders.load_numpy(input)
             # skip gather for the first input
             result = (
-                reduce(data, axis=self.reduce_axes)
+                reduce(data, axis=self.reduce_axes, allow_pickle=True)
                 if result is None
                 else gather(
                     result,
-                    reduce(data, axis=self.reduce_axes),
+                    reduce(data, axis=self.reduce_axes, allow_pickle=True),
+                    allow_pickle=True,
                 )
             )
         return result
@@ -540,7 +541,11 @@ def gather_inputs(self):
         for input in self.inputs:
             data = drop_loaders.load_numpy(input)
             # assign instead of gather for the first input
-            result = data if result is None else gather(result, data)
+            result = (
+                data
+                if result is None
+                else gather(result, data, allow_pickle=True)
+            )
         return result
 
 

daliuge-engine/dlg/named_port_utils.py

@@ -170,7 +170,7 @@ def replace_named_ports(
     separator: str = " ",
 ) -> Tuple[str, str]:
     """
-    Function attempts to identify component arguments that match port names.
+    Function attempts to identify CLI component arguments that match port names.
 
     Inputs:
         iitems: itemized input port dictionary
@@ -213,13 +213,13 @@ def replace_named_ports(
     portPosargsDict = collections.OrderedDict(
         zip(posargs, [None] * len(posargs))
     )
-    portkeyargs = {}
     logger.debug(
         "posargs: %s; keyargs: %s, %s",
         posargs,
         keyargs,
         check_ports_dict(inport_names),
     )
+    portkeyargs = {}
     ipkeyargs = {}
     opkeyargs = {}
     if check_ports_dict(inport_names):

daliuge-engine/test/graphs/ArrayLoopPG.graph

@@ -41,7 +41,6 @@
         "integer": false,
         "low": "0",
         "high": "100",
-        "dropclass": "dlg.apps.simple.RandomArrayApp",
         "execution_time": 5,
         "group_start": false,
         "applicationArgs": {},
@@ -70,7 +69,6 @@
         "num_cpus": 1,
         "execution_time": 5,
         "group_start": true,
-        "dropclass": "dlg.apps.simple.PickOne",
         "applicationArgs": {},
         "iid": "0/0/0",
         "lg_key": -3,
@@ -105,7 +103,6 @@
         "num_cpus": 1,
         "execution_time": 5,
         "group_start": true,
-        "dropclass": "dlg.apps.simple.PickOne",
         "applicationArgs": {},
         "iid": "0/0/1",
         "lg_key": -3,
@@ -143,7 +140,6 @@
         "num_cpus": 1,
         "execution_time": 5,
         "group_start": true,
-        "dropclass": "dlg.apps.simple.PickOne",
         "applicationArgs": {},
         "iid": "0/1/0",
         "lg_key": -3,
@@ -178,7 +174,6 @@
         "num_cpus": 1,
         "execution_time": 5,
         "group_start": true,
-        "dropclass": "dlg.apps.simple.PickOne",
         "applicationArgs": {},
         "iid": "0/1/1",
         "lg_key": -3,
@@ -494,7 +489,6 @@
         "loop_ctx": null,
         "weight": 5,
         "num_cpus": 1,
-        "dropclass": "dlg.apps.simple.GenericScatterApp",
         "execution_time": 5,
         "group_start": false,
         "array": "",