Added PythonObjects to palettes

daliuge-engine/dlg/apps/pyfunc.py

@@ -83,15 +83,18 @@ def import_using_name(app, fname):
     logger.debug("Importing %s", fname)
     parts = fname.split(".")
     # If only one part check if builtin
+    b = globals()["__builtins__"]
+    logger.debug(f"Function: {parts[0]}: {hasattr(b, parts[0])}")
     if len(parts) < 2:
-        b = globals()["__builtins__"]
         logger.debug(f"Builtins: {type(b)}")
         logger.debug(f"Function {fname}: {hasattr(b, fname)}")
         if fname in b:
             return b[fname]
         else:
             msg = "%s is not builtin and does not contain a module name" % fname
             raise InvalidDropException(app, msg)
+    elif parts[0] in b.keys():
+        return b[parts[0]]
     else:
         if len(parts) > 1:
             if parts[-1] in ["__init__", "__class__"]:
@@ -143,6 +146,33 @@ class DropParser(Enum):
     DATAURL = "dataurl"  # input only
 
 
+##
+# @brief PythonMemberFunction
+# @details A placeholder APP to aid construction of new class member function applications.
+# This is mainly useful (and used) when starting a new workflow from scratch.
+# @par EAGLE_START
+# @param category PythonMemberFunction
+# @param tag daliuge
+# @param func_name object.__init__/String/ComponentParameter/NoPort/ReadWrite//False/False/Python function name
+# @param func_code /String/ComponentParameter/NoPort/ReadWrite//False/False/Python function code, e.g. 'def function_name(args): return args'
+# @param dropclass dlg.apps.pyfunc.PyFuncApp/String/ComponentParameter/NoPort/ReadOnly//False/False/Application class
+# @param object /Object/ApplicationArgument/InputOutput/ReadWrite//False/False/object port
+# @param execution_time 5/Float/ConstraintParameter/NoPort/ReadOnly//False/False/Estimated execution time
+# @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used
+# @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group?
+# @param input_error_threshold 0/Integer/ComponentParameter/NoPort/ReadWrite//False/False/The allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed
+# @param n_tries 1/Integer/ComponentParameter/NoPort/ReadWrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up
+# @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
+# @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
+#     \~English Mapping from argname to default value. Should match only the last part of the argnames list.
+#               Values are interpreted as Python code literals and that means string values need to be quoted.
+# @par EAGLE_END
+class PyMemberApp(BarrierAppDROP):
+    """A placeholder member function that just aids the generation of the palette component"""
+
+    pass
+
+
 ##
 # @brief PyFuncApp
 # @details An application that wraps a simple python function.

daliuge-engine/dlg/apps/simple.py

@@ -103,6 +103,8 @@ class PythonApp(BarrierAppDROP):
 # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
 # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
 # @par EAGLE_END
+
+
 class SleepApp(BarrierAppDROP):
     """A BarrierAppDrop that sleeps the specified amount of time (0 by default)"""
 
@@ -122,7 +124,7 @@ def run(self):
         self._run()
         try:
             # If data is coming through a named port we load it from there.
-            if isinstance(self.sleep_time, (InMemoryDROP, FileDROP, DropProxy, Any)):
+            if isinstance(self.sleep_time, (InMemoryDROP, FileDROP, DropProxy)):
                 logger.debug("Trying to read from %s", self.sleep_time)
                 self.sleep_time = drop_loaders.load_pickle(self.sleep_time)
             time.sleep(self.sleep_time)
@@ -238,7 +240,7 @@ def run(self):
 # @param execution_time 5/Float/ConstraintParameter/NoPort/ReadOnly//False/False/Estimated execution time
 # @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used
 # @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group?
-# @param array /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/Port carrying the averaged array
+# @param array /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/random array
 # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
 # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
 # @par EAGLE_END
@@ -464,7 +466,7 @@ def run(self):
 # @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used
 # @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group?
 # @param array_in /Object.Array/ApplicationArgument/InputPort/ReadWrite//False/False/Port for the input array(s)
-# @param array_out /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/Port carrying the reduced array
+# @param array_out /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/reduced array
 # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
 # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
 # @par EAGLE_END
@@ -564,7 +566,7 @@ def gather_inputs(self):
 # @param execution_time 5/Float/ConstraintParameter/NoPort/ReadOnly//False/False/Estimated execution time
 # @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used
 # @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group?
-# @param hello "world"/Object/ApplicationArgument/OutputPort/ReadWrite//False/False/The port carrying the message produced by the app.
+# @param hello "world"/Object/ApplicationArgument/OutputPort/ReadWrite//False/False/message
 # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
 # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
 # @par EAGLE_END
@@ -621,7 +623,7 @@ def run(self):
 # @param execution_time 5/Float/ConstraintParameter/NoPort/ReadOnly//False/False/Estimated execution time
 # @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used
 # @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group?
-# @param content /String/ApplicationArgument/OutputPort/ReadWrite//False/False/The port carrying the content read from the URL
+# @param content /String/ApplicationArgument/OutputPort/ReadWrite//False/False/content read from URL
 # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
 # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
 # @par EAGLE_END
@@ -678,7 +680,7 @@ def run(self):
 # @param execution_time 5/Float/ConstraintParameter/NoPort/ReadOnly//False/False/Estimated execution time
 # @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used
 # @param array_in /Object.Array/ApplicationArgument/InputPort/ReadWrite//False/False/A numpy array of arrays, where the first axis is of length <numSplit>
-# @param array_out /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/Port carrying the reduced array
+# @param array_out /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/reduced array
 # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
 # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
 # @par EAGLE_END
@@ -744,7 +746,7 @@ def run(self):
 # @param execution_time 5/Float/ConstraintParameter/NoPort/ReadOnly//False/False/Estimated execution time
 # @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used
 # @param array_in /Object.Array/ApplicationArgument/InputPort/ReadWrite//False/False/A numpy array of arrays
-# @param array_out /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/Port carrying the reduced array
+# @param array_out /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/reduced array
 # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
 # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
 # @par EAGLE_END
@@ -822,7 +824,7 @@ def condition(self):
 # @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used
 # @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group?
 # @param rest_array /Object.array/ApplicationArgument/InputOutput/ReadWrite//False/False/List of elements
-# @param element /Object.element/ApplicationArgument/OutputPort/ReadWrite//False/False/Port carrying the first element of input array
+# @param element /Object.element/ApplicationArgument/OutputPort/ReadWrite//False/False/first element
 # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
 # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
 # @par EAGLE_END
@@ -883,7 +885,7 @@ def run(self):
 # @param execution_time 5/Float/ConstraintParameter/NoPort/ReadOnly//False/False/Estimated execution time
 # @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used
 # @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group?
-# @param array /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/Port carrying the random array.
+# @param array /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/random array
 # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
 # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
 # @par EAGLE_END

daliuge-engine/dlg/data/drops/memory.py

@@ -156,6 +156,27 @@ def generate_reproduce_data(self):
         return {"data_hash": common_hash(data)}
 
 
+##
+# @brief PythonObject
+# @details A Python object stored in memory
+# @par EAGLE_START
+# @param category PythonObject
+# @param tag daliuge
+# @param object /Object/ApplicationArgument/InputOutput/ReadWrite//False/False/object
+# @param persist False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Object should be serialized
+# @param data_volume 5/Float/ConstraintParameter/NoPort/ReadWrite//False/False/Estimated size of the data contained in the object
+# @param dropclass dlg.data.drops.memory.InMemoryDROP/String/ComponentParameter/NoPort/ReadOnly//False/False/Drop class
+# @par EAGLE_END
+class PythonObjectDROP(InMemoryDROP):
+    """
+    A python object is an InMemoryDROP but has a special category of PythonObject.
+
+    This is only here to force the creation of the component for the palette.
+    """
+
+    pass
+
+
 ##
 # @brief SharedMemory
 # @details Data stored in shared memory

daliuge-engine/test/apps/test_pyfunc.py

@@ -107,6 +107,10 @@ def test_function_invalid_module(self):
             func_name="doesnt_exist.func1",
         )
 
+    def test_function_builtin_module(self):
+        # The function lives in an unknown module/package
+        pyfunc.PyFuncApp("a", "a", func_name="object.__init__")
+
     def test_function_invalid_fname(self):
         # The function lives in an unknown module/package
         self.assertRaises(