From a310776c1bebb004a24e183b27f61bbc9b6d3892 Mon Sep 17 00:00:00 2001 From: Andreas Wicenec Date: Fri, 6 May 2022 18:49:49 +0800 Subject: [PATCH] additional test --- daliuge-engine/dlg/apps/pyfunc.py | 17 +- daliuge-engine/test/graphs/compilePG.graph | 154 ++++++++++++++++++ .../test/graphs/test_graphExecution.py | 22 +++ 3 files changed, 186 insertions(+), 7 deletions(-) create mode 100644 daliuge-engine/test/graphs/compilePG.graph diff --git a/daliuge-engine/dlg/apps/pyfunc.py b/daliuge-engine/dlg/apps/pyfunc.py index 219304a3d..ca389087c 100644 --- a/daliuge-engine/dlg/apps/pyfunc.py +++ b/daliuge-engine/dlg/apps/pyfunc.py @@ -393,7 +393,10 @@ def run(self): # if we have named ports use the inputs with # the correct UIDs logger.debug(f"Parameters found: {self.parameters}") + posargs = self.arguments.args[:self.fn_npos] kwargs = {} + self.pargs = [] + pargsDict = collections.OrderedDict(zip(posargs,[None]*len(posargs))) # Initialize pargs dictionary if ('inputs' in self.parameters and isinstance(self.parameters['inputs'][0], dict)): logger.debug(f"Using named ports to identify inputs: "+\ f"{self.parameters['inputs']}") @@ -403,21 +406,21 @@ def run(self): key = list(self.parameters['inputs'][i].values())[0] # value for final dict is value in inputs dict value = inputs[list(self.parameters['inputs'][i].keys())[0]] - kwargs.update({key:value}) + if key in posargs: + pargsDict.update({key:value}) + else: + kwargs.update({key:value}) else: for i in range(min(len(inputs),self.fn_nargs)): kwargs.update({self.arguments.args[i]: list(inputs.values())[i]}) - logger.debug(f"updating funcargs with {kwargs}") + logger.debug(f"updating funcargs with input ports {kwargs}") self.funcargs.update(kwargs) # Try to get values for still missing positional arguments from Application Args - self.pargs = [] if "applicationArgs" in self.parameters: appArgs = self.parameters["applicationArgs"] # we'll pop them _dum = [appArgs.pop(k) for k in self.func_def_keywords if k in appArgs] - kwargs = {} - posargs = self.arguments.args[:self.fn_npos] for pa in posargs: if pa not in self.funcargs: if pa in appArgs: @@ -429,14 +432,14 @@ def run(self): value = ast.literal_eval(value) except: pass - kwargs.update({ + pargsDict.update({ pa: value }) elif pa != 'self': logger.warning(f"Required positional argument '{pa}' not found!") logger.debug(f"updating posargs with {list(kwargs.values())}") - self.pargs.extend(list(kwargs.values())) + self.pargs.extend(list(pargsDict.values())) # Try to get values for still missing kwargs arguments from Application kws kwargs = {} diff --git a/daliuge-engine/test/graphs/compilePG.graph b/daliuge-engine/test/graphs/compilePG.graph new file mode 100644 index 000000000..23380ebb5 --- /dev/null +++ b/daliuge-engine/test/graphs/compilePG.graph @@ -0,0 +1,154 @@ +[ + { + "oid": "2022-05-06T08:43:26_-1_0", + "type": "app", + "app": "dlg.apps.pyfunc.PyFuncApp", + "rank": [ + 0 + ], + "loop_cxt": null, + "tw": 5, + "num_cpus": 1, + "appclass": "dlg.apps.pyfunc.PyFuncApp", + "execution_time": 5, + "group_start": false, + "input_error_threshold": 0, + "n_tries": 1, + "applicationArgs": { + "func_name": { + "text": "Function Name", + "value": "compile", + "defaultValue": "", + "description": "Python function name", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false + }, + "func_code": { + "text": "Function Code", + "value": "", + "defaultValue": "", + "description": "Python function code, e.g. 'def function_name(args): return args'", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false + }, + "pickle": { + "text": "Pickle", + "value": false, + "defaultValue": "false", + "description": "Whether the python arguments are pickled.", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false + }, + "func_defaults": { + "text": "Function Defaults", + "value": "", + "defaultValue": "", + "description": "Mapping from argname to default value. Should match only the last part of the argnames list. Values are interpreted as Python code literals and that means string values need to be quoted.", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false + }, + "func_arg_mapping": { + "text": "Function Arguments Mapping", + "value": "", + "defaultValue": "", + "description": "Mapping between argument name and input drop uids", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false + }, + "source": { + "text": "source", + "value": "print('Hello')", + "defaultValue": "", + "description": "", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false + }, + "filename": { + "text": "filename", + "value": "", + "defaultValue": "", + "description": "", + "readonly": false, + "type": "String", + "precious": true, + "options": [], + "positional": false + }, + "mode": { + "text": "mode", + "value": "exec", + "defaultValue": "", + "description": "", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false + }, + "optimize": { + "text": "optimize", + "value": true, + "defaultValue": "", + "description": "", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false + } + }, + "iid": "0", + "lg_key": -1, + "dt": "PythonApp", + "nm": "compile", + "inputs": [ + { + "2022-05-06T08:43:26_-2_0": "filename" + } + ], + "node": "localhost", + "island": "localhost" + }, + { + "oid": "2022-05-06T08:43:26_-2_0", + "type": "plain", + "storage": "Memory", + "rank": [ + 0 + ], + "loop_cxt": null, + "dw": 0, + "data_volume": 0, + "group_end": false, + "applicationArgs": {}, + "iid": "0", + "lg_key": -2, + "dt": "Memory", + "nm": "NULL", + "consumers": [ + { + "2022-05-06T08:43:26_-1_0": "filename" + } + ], + "node": "localhost", + "island": "localhost" + } +] \ No newline at end of file diff --git a/daliuge-engine/test/graphs/test_graphExecution.py b/daliuge-engine/test/graphs/test_graphExecution.py index 4a40bd008..befbae7fd 100644 --- a/daliuge-engine/test/graphs/test_graphExecution.py +++ b/daliuge-engine/test/graphs/test_graphExecution.py @@ -148,3 +148,25 @@ def test_namedPorts_with_kwonlyargs(self): with droputils.DROPWaiterCtx(self, init_drop, 3): [a.setCompleted() for a in start_drops] + def test_pos_only_args(self): + """ + Use a graph with compile function to test positional only arguments + """ + sessionId = "lalo" + with pkg_resources.resource_stream( + "test", "graphs/compilePG.graph" + ) as f: # @UndefinedVariable + graphSpec = json.load(f) + # dropSpecs = graph_loader.loadDropSpecs(graphSpec) + self.createSessionAndAddGraph(sessionId, graphSpec=graphSpec) + + # Deploy now and get OIDs + self.dim.deploySession(sessionId) + sd = self.dm._sessions[sessionId].drops["2022-05-06T08:43:26_-2_0"] + fd = self.dm._sessions[sessionId].drops["2022-05-06T08:43:26_-1_0"] + with droputils.DROPWaiterCtx(self, fd, 3): + sd.write("''".encode()) + sd.setCompleted() + + #logger.debug(f'PyfuncAPPDrop signature: {dir(fd)}') + logger.debug(f'PyfuncAPPDrop status: {fd.status}')