diff --git a/daliuge-engine/dlg/apps/pyfunc.py b/daliuge-engine/dlg/apps/pyfunc.py index f0b84e148..fcdd8c589 100644 --- a/daliuge-engine/dlg/apps/pyfunc.py +++ b/daliuge-engine/dlg/apps/pyfunc.py @@ -83,8 +83,9 @@ def import_using_name(app, fname): logger.debug("Importing %s", fname) parts = fname.split(".") # If only one part check if builtin + b = globals()["__builtins__"] + logger.debug(f"Function: {parts[0]}: {hasattr(b, parts[0])}") if len(parts) < 2: - b = globals()["__builtins__"] logger.debug(f"Builtins: {type(b)}") logger.debug(f"Function {fname}: {hasattr(b, fname)}") if fname in b: @@ -92,6 +93,8 @@ def import_using_name(app, fname): else: msg = "%s is not builtin and does not contain a module name" % fname raise InvalidDropException(app, msg) + elif parts[0] in b.keys(): + return b[parts[0]] else: if len(parts) > 1: if parts[-1] in ["__init__", "__class__"]: @@ -143,6 +146,33 @@ class DropParser(Enum): DATAURL = "dataurl" # input only +## +# @brief PythonMemberFunction +# @details A placeholder APP to aid construction of new class member function applications. +# This is mainly useful (and used) when starting a new workflow from scratch. +# @par EAGLE_START +# @param category PythonMemberFunction +# @param tag daliuge +# @param func_name object.__init__/String/ComponentParameter/NoPort/ReadWrite//False/False/Python function name +# @param func_code /String/ComponentParameter/NoPort/ReadWrite//False/False/Python function code, e.g. 'def function_name(args): return args' +# @param dropclass dlg.apps.pyfunc.PyFuncApp/String/ComponentParameter/NoPort/ReadOnly//False/False/Application class +# @param object /Object/ApplicationArgument/InputOutput/ReadWrite//False/False/object port +# @param execution_time 5/Float/ConstraintParameter/NoPort/ReadOnly//False/False/Estimated execution time +# @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used +# @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group? +# @param input_error_threshold 0/Integer/ComponentParameter/NoPort/ReadWrite//False/False/The allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries 1/Integer/ComponentParameter/NoPort/ReadWrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique +# @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique +# \~English Mapping from argname to default value. Should match only the last part of the argnames list. +# Values are interpreted as Python code literals and that means string values need to be quoted. +# @par EAGLE_END +class PyMemberApp(BarrierAppDROP): + """A placeholder member function that just aids the generation of the palette component""" + + pass + + ## # @brief PyFuncApp # @details An application that wraps a simple python function. diff --git a/daliuge-engine/dlg/apps/simple.py b/daliuge-engine/dlg/apps/simple.py index df6cae6ba..f27223101 100644 --- a/daliuge-engine/dlg/apps/simple.py +++ b/daliuge-engine/dlg/apps/simple.py @@ -103,6 +103,8 @@ class PythonApp(BarrierAppDROP): # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique # @par EAGLE_END + + class SleepApp(BarrierAppDROP): """A BarrierAppDrop that sleeps the specified amount of time (0 by default)""" @@ -122,7 +124,7 @@ def run(self): self._run() try: # If data is coming through a named port we load it from there. - if isinstance(self.sleep_time, (InMemoryDROP, FileDROP, DropProxy, Any)): + if isinstance(self.sleep_time, (InMemoryDROP, FileDROP, DropProxy)): logger.debug("Trying to read from %s", self.sleep_time) self.sleep_time = drop_loaders.load_pickle(self.sleep_time) time.sleep(self.sleep_time) @@ -238,7 +240,7 @@ def run(self): # @param execution_time 5/Float/ConstraintParameter/NoPort/ReadOnly//False/False/Estimated execution time # @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used # @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group? -# @param array /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/Port carrying the averaged array +# @param array /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/random array # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique # @par EAGLE_END @@ -464,7 +466,7 @@ def run(self): # @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used # @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group? # @param array_in /Object.Array/ApplicationArgument/InputPort/ReadWrite//False/False/Port for the input array(s) -# @param array_out /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/Port carrying the reduced array +# @param array_out /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/reduced array # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique # @par EAGLE_END @@ -564,7 +566,7 @@ def gather_inputs(self): # @param execution_time 5/Float/ConstraintParameter/NoPort/ReadOnly//False/False/Estimated execution time # @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used # @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group? -# @param hello "world"/Object/ApplicationArgument/OutputPort/ReadWrite//False/False/The port carrying the message produced by the app. +# @param hello "world"/Object/ApplicationArgument/OutputPort/ReadWrite//False/False/message # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique # @par EAGLE_END @@ -621,7 +623,7 @@ def run(self): # @param execution_time 5/Float/ConstraintParameter/NoPort/ReadOnly//False/False/Estimated execution time # @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used # @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group? -# @param content /String/ApplicationArgument/OutputPort/ReadWrite//False/False/The port carrying the content read from the URL +# @param content /String/ApplicationArgument/OutputPort/ReadWrite//False/False/content read from URL # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique # @par EAGLE_END @@ -678,7 +680,7 @@ def run(self): # @param execution_time 5/Float/ConstraintParameter/NoPort/ReadOnly//False/False/Estimated execution time # @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used # @param array_in /Object.Array/ApplicationArgument/InputPort/ReadWrite//False/False/A numpy array of arrays, where the first axis is of length -# @param array_out /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/Port carrying the reduced array +# @param array_out /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/reduced array # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique # @par EAGLE_END @@ -744,7 +746,7 @@ def run(self): # @param execution_time 5/Float/ConstraintParameter/NoPort/ReadOnly//False/False/Estimated execution time # @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used # @param array_in /Object.Array/ApplicationArgument/InputPort/ReadWrite//False/False/A numpy array of arrays -# @param array_out /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/Port carrying the reduced array +# @param array_out /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/reduced array # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique # @par EAGLE_END @@ -822,7 +824,7 @@ def condition(self): # @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used # @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group? # @param rest_array /Object.array/ApplicationArgument/InputOutput/ReadWrite//False/False/List of elements -# @param element /Object.element/ApplicationArgument/OutputPort/ReadWrite//False/False/Port carrying the first element of input array +# @param element /Object.element/ApplicationArgument/OutputPort/ReadWrite//False/False/first element # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique # @par EAGLE_END @@ -883,7 +885,7 @@ def run(self): # @param execution_time 5/Float/ConstraintParameter/NoPort/ReadOnly//False/False/Estimated execution time # @param num_cpus 1/Integer/ConstraintParameter/NoPort/ReadOnly//False/False/Number of cores used # @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group? -# @param array /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/Port carrying the random array. +# @param array /Object.Array/ApplicationArgument/OutputPort/ReadWrite//False/False/random array # @param input_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique # @param output_parser pickle/Select/ComponentParameter/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique # @par EAGLE_END diff --git a/daliuge-engine/dlg/data/drops/memory.py b/daliuge-engine/dlg/data/drops/memory.py index 6393038d1..b02ba97cc 100644 --- a/daliuge-engine/dlg/data/drops/memory.py +++ b/daliuge-engine/dlg/data/drops/memory.py @@ -156,6 +156,27 @@ def generate_reproduce_data(self): return {"data_hash": common_hash(data)} +## +# @brief PythonObject +# @details A Python object stored in memory +# @par EAGLE_START +# @param category PythonObject +# @param tag daliuge +# @param object /Object/ApplicationArgument/InputOutput/ReadWrite//False/False/object +# @param persist False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Object should be serialized +# @param data_volume 5/Float/ConstraintParameter/NoPort/ReadWrite//False/False/Estimated size of the data contained in the object +# @param dropclass dlg.data.drops.memory.InMemoryDROP/String/ComponentParameter/NoPort/ReadOnly//False/False/Drop class +# @par EAGLE_END +class PythonObjectDROP(InMemoryDROP): + """ + A python object is an InMemoryDROP but has a special category of PythonObject. + + This is only here to force the creation of the component for the palette. + """ + + pass + + ## # @brief SharedMemory # @details Data stored in shared memory diff --git a/daliuge-engine/test/apps/test_pyfunc.py b/daliuge-engine/test/apps/test_pyfunc.py index e7344b26d..b94464ad0 100644 --- a/daliuge-engine/test/apps/test_pyfunc.py +++ b/daliuge-engine/test/apps/test_pyfunc.py @@ -107,6 +107,10 @@ def test_function_invalid_module(self): func_name="doesnt_exist.func1", ) + def test_function_builtin_module(self): + # The function lives in an unknown module/package + pyfunc.PyFuncApp("a", "a", func_name="object.__init__") + def test_function_invalid_fname(self): # The function lives in an unknown module/package self.assertRaises(