From 4a7cf463c0289dce43ff2faf18f4f568081ac595 Mon Sep 17 00:00:00 2001 From: Moritz Wicenec Date: Tue, 12 Jul 2022 15:08:43 +0800 Subject: [PATCH 1/7] potential fix in place --- daliuge-translator/dlg/dropmake/web/main.js | 41 ++++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/daliuge-translator/dlg/dropmake/web/main.js b/daliuge-translator/dlg/dropmake/web/main.js index e974bda5e..d46e5c0cd 100644 --- a/daliuge-translator/dlg/dropmake/web/main.js +++ b/daliuge-translator/dlg/dropmake/web/main.js @@ -62,9 +62,13 @@ function openSettingsModal(){ async function initiateDeploy(method, selected, name){ if (selected === false){ - changeSelectedDeployMethod(name) + console.log("new option deploying") + await changeSelectedDeployMethod(name) } + console.log("new option set") + var activeUrlReachable = await checkUrlStatus(window.localStorage.getItem("manager_url")) + console.log("new option verified") if(!activeUrlReachable){ $("#warning-alert").fadeTo(2000, 1000).slideUp(200, function() { @@ -85,18 +89,35 @@ async function initiateDeploy(method, selected, name){ } } -function changeSelectedDeployMethod(name) { - var deployMethodsArray = JSON.parse(localStorage.getItem("deployMethods")) - deployMethodsArray.forEach(element => { - element.active = "false" - if(element.name === name){ - element.active = "true" - } +async function changeSelectedDeployMethod(name) { + return new Promise((resolve, reject) => { + var deployMethodsArray = JSON.parse(localStorage.getItem("deployMethods")) + deployMethodsArray.forEach(element => { + element.active = "false" + if(element.name === name){ + element.active = "true" + } + }) + asyncLocalStorage.setItem('deployMethods', JSON.stringify(deployMethodsArray)).then(function(){ + updateDeployOptionsDropdown() + console.log("changing deploymethod") + resolve(true) + }) + }) - localStorage.setItem('deployMethods', JSON.stringify(deployMethodsArray)) - updateDeployOptionsDropdown() } +const asyncLocalStorage = { + setItem: async function (key, value) { + await null; + return localStorage.setItem(key, value); + }, + getItem: async function (key) { + await null; + return localStorage.getItem(key); + } +}; + function updateDeployOptionsDropdown() { //remove old options $(".deployMethodMenuItem").remove() From 728b883d06777883a0c7197308c69fe9f053298b Mon Sep 17 00:00:00 2001 From: Andreas Wicenec Date: Tue, 12 Jul 2022 16:24:39 +0800 Subject: [PATCH 2/7] Trial for manager_url setting --- daliuge-translator/dlg/dropmake/web/main.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/daliuge-translator/dlg/dropmake/web/main.js b/daliuge-translator/dlg/dropmake/web/main.js index d46e5c0cd..47aa275c4 100644 --- a/daliuge-translator/dlg/dropmake/web/main.js +++ b/daliuge-translator/dlg/dropmake/web/main.js @@ -68,7 +68,7 @@ async function initiateDeploy(method, selected, name){ console.log("new option set") var activeUrlReachable = await checkUrlStatus(window.localStorage.getItem("manager_url")) - console.log("new option verified") + console.log(window.localStorage.getItem("manager_url"),"new option verified") if(!activeUrlReachable){ $("#warning-alert").fadeTo(2000, 1000).slideUp(200, function() { @@ -76,6 +76,8 @@ async function initiateDeploy(method, selected, name){ }); return } + var manager_url = asyncLocalStorage.getItem("manager_url"); + $("#managerUrlInput").val(manager_url); if(method === "direct"){ $("#gen_pg_button").val("Generate & Deploy Physical Graph") $("#dlg_mgr_deploy").prop("checked", true) From f1ffbfdd9fda704b0efca3839ba98cec27a12904 Mon Sep 17 00:00:00 2001 From: Moritz Wicenec Date: Tue, 12 Jul 2022 16:31:25 +0800 Subject: [PATCH 3/7] potential fix 2 and some cleanup --- daliuge-translator/dlg/dropmake/web/main.js | 44 ++++++--------------- 1 file changed, 12 insertions(+), 32 deletions(-) diff --git a/daliuge-translator/dlg/dropmake/web/main.js b/daliuge-translator/dlg/dropmake/web/main.js index 47aa275c4..1db518cfb 100644 --- a/daliuge-translator/dlg/dropmake/web/main.js +++ b/daliuge-translator/dlg/dropmake/web/main.js @@ -61,14 +61,12 @@ function openSettingsModal(){ } async function initiateDeploy(method, selected, name){ + var manager_url = window.localStorage.getItem("manager_url") if (selected === false){ - console.log("new option deploying") - await changeSelectedDeployMethod(name) + changeSelectedDeployMethod(name) } - console.log("new option set") - var activeUrlReachable = await checkUrlStatus(window.localStorage.getItem("manager_url")) - console.log(window.localStorage.getItem("manager_url"),"new option verified") + var activeUrlReachable = await checkUrlStatus(manager_url) if(!activeUrlReachable){ $("#warning-alert").fadeTo(2000, 1000).slideUp(200, function() { @@ -76,7 +74,6 @@ async function initiateDeploy(method, selected, name){ }); return } - var manager_url = asyncLocalStorage.getItem("manager_url"); $("#managerUrlInput").val(manager_url); if(method === "direct"){ $("#gen_pg_button").val("Generate & Deploy Physical Graph") @@ -91,35 +88,18 @@ async function initiateDeploy(method, selected, name){ } } -async function changeSelectedDeployMethod(name) { - return new Promise((resolve, reject) => { - var deployMethodsArray = JSON.parse(localStorage.getItem("deployMethods")) - deployMethodsArray.forEach(element => { - element.active = "false" - if(element.name === name){ - element.active = "true" - } - }) - asyncLocalStorage.setItem('deployMethods', JSON.stringify(deployMethodsArray)).then(function(){ - updateDeployOptionsDropdown() - console.log("changing deploymethod") - resolve(true) - }) - +function changeSelectedDeployMethod(name) { + var deployMethodsArray = JSON.parse(localStorage.getItem("deployMethods")) + deployMethodsArray.forEach(element => { + element.active = "false" + if(element.name === name){ + element.active = "true" + } }) + window.localStorage.setItem('deployMethods', JSON.stringify(deployMethodsArray)) + updateDeployOptionsDropdown() } -const asyncLocalStorage = { - setItem: async function (key, value) { - await null; - return localStorage.setItem(key, value); - }, - getItem: async function (key) { - await null; - return localStorage.getItem(key); - } -}; - function updateDeployOptionsDropdown() { //remove old options $(".deployMethodMenuItem").remove() From de0743f83c25b3f9dee9528890e83aeeb5cefa45 Mon Sep 17 00:00:00 2001 From: Moritz Wicenec Date: Fri, 15 Jul 2022 11:11:02 +0800 Subject: [PATCH 4/7] fixed deployment --- daliuge-translator/dlg/dropmake/web/main.js | 37 +++++++++++++-------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/daliuge-translator/dlg/dropmake/web/main.js b/daliuge-translator/dlg/dropmake/web/main.js index 1db518cfb..a5e91182a 100644 --- a/daliuge-translator/dlg/dropmake/web/main.js +++ b/daliuge-translator/dlg/dropmake/web/main.js @@ -60,13 +60,19 @@ function openSettingsModal(){ $('#settingsModal').modal("show") } -async function initiateDeploy(method, selected, name){ - var manager_url = window.localStorage.getItem("manager_url") +async function initiateDeploy(method, selected, clickedName){ + var clickedUrl + JSON.parse(window.localStorage.getItem("deployMethods")).forEach(element => { + if(element.name === clickedName){ + clickedUrl = element.url + } + }) + if (selected === false){ - changeSelectedDeployMethod(name) + await changeSelectedDeployMethod(clickedName, clickedUrl) } - var activeUrlReachable = await checkUrlStatus(manager_url) + var activeUrlReachable = await checkUrlStatus(clickedUrl) if(!activeUrlReachable){ $("#warning-alert").fadeTo(2000, 1000).slideUp(200, function() { @@ -74,7 +80,6 @@ async function initiateDeploy(method, selected, name){ }); return } - $("#managerUrlInput").val(manager_url); if(method === "direct"){ $("#gen_pg_button").val("Generate & Deploy Physical Graph") $("#dlg_mgr_deploy").prop("checked", true) @@ -88,16 +93,20 @@ async function initiateDeploy(method, selected, name){ } } -function changeSelectedDeployMethod(name) { - var deployMethodsArray = JSON.parse(localStorage.getItem("deployMethods")) - deployMethodsArray.forEach(element => { - element.active = "false" - if(element.name === name){ - element.active = "true" - } +async function changeSelectedDeployMethod(name,manager_url) { + return new Promise((resolve, reject) => { + var deployMethodsArray = JSON.parse(localStorage.getItem("deployMethods")) + $("#managerUrlInput").val(manager_url); + deployMethodsArray.forEach(element => { + element.active = "false" + if(element.name === name){ + element.active = "true" + } + }) + window.localStorage.setItem('deployMethods', JSON.stringify(deployMethodsArray)) + updateDeployOptionsDropdown() + resolve(true) }) - window.localStorage.setItem('deployMethods', JSON.stringify(deployMethodsArray)) - updateDeployOptionsDropdown() } function updateDeployOptionsDropdown() { From a7abe4c21ab4d02691ebe7c188b550b1e5ce16de Mon Sep 17 00:00:00 2001 From: Andreas Wicenec Date: Tue, 19 Jul 2022 15:19:39 +0800 Subject: [PATCH 5/7] Fixed bug in documentation string of NgasDROP --- tools/xml2palette/xml2palette.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tools/xml2palette/xml2palette.py b/tools/xml2palette/xml2palette.py index c71df0c49..5b783207a 100755 --- a/tools/xml2palette/xml2palette.py +++ b/tools/xml2palette/xml2palette.py @@ -95,8 +95,15 @@ def check_environment_variables(): value = os.environ.get(variable) if value is None: - logging.error("No " + variable + " environment variable.") - return False + if variable == 'PROJECT_NAME': + os.environ['PROJECT_NAME'] = os.path.basename(os.path.abspath('.')) + elif variable == 'PROJECT_VERSION': + os.environ['PROJECT_VERSION'] = '0.1' + elif variable == 'GIT_REPO': + os.environ['GIT_REPO'] = os.environ['PROJECT_NAME'] + else: + logging.error("No " + variable + " environment variable.") + return False return True @@ -1063,6 +1070,10 @@ def params_to_nodes(params): level=logging.INFO, ) + # read environment variables + if not check_environment_variables(): + sys.exit(1) + logging.info("PROJECT_NAME:" + os.environ.get("PROJECT_NAME")) logging.info("PROJECT_VERSION:" + os.environ.get("PROJECT_VERSION")) logging.info("GIT_REPO:" + os.environ.get("GIT_REPO")) @@ -1077,10 +1088,6 @@ def params_to_nodes(params): # create a temp directory for the output of doxygen output_directory = tempfile.TemporaryDirectory() - # read environment variables - if not check_environment_variables(): - sys.exit(1) - # add extra doxygen setting for input and output locations DOXYGEN_SETTINGS.append(("PROJECT_NAME", os.environ.get("PROJECT_NAME"))) DOXYGEN_SETTINGS.append(("INPUT", inputdir)) From 3793895c76f5dab5b02acae14ec54de90874c833 Mon Sep 17 00:00:00 2001 From: Andreas Wicenec Date: Wed, 20 Jul 2022 10:03:48 +0800 Subject: [PATCH 6/7] Update kwargs treatment with named_ports --- daliuge-engine/dlg/apps/bash_shell_app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/daliuge-engine/dlg/apps/bash_shell_app.py b/daliuge-engine/dlg/apps/bash_shell_app.py index 7d6e3d3f7..a53a8c592 100644 --- a/daliuge-engine/dlg/apps/bash_shell_app.py +++ b/daliuge-engine/dlg/apps/bash_shell_app.py @@ -222,7 +222,7 @@ def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE): keyargs = {arg:appArgs[arg]["value"] for arg in appArgs if not appArgs[arg]["positional"]} logger.debug("pargs: %s; keyargs: %s, appArgs: %s",pargs, keyargs, appArgs) if "inputs" in self.parameters and isinstance(self.parameters['inputs'][0], dict): - keyargs = droputils.identify_named_ports( + pkeyargs = droputils.identify_named_ports( inputs_dict, self.parameters["inputs"], pargs, @@ -230,6 +230,7 @@ def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE): appArgs, check_len=len(inputs), mode="inputs") + keyargs.update(pkeyargs) else: for i in range(min(len(inputs), len(pargs))): keyargs.update({pargs[i]: list(inputs.values())[i]}) From 44f6c1fd43ab773dca0c091a613c895b5f87d1d0 Mon Sep 17 00:00:00 2001 From: Andreas Wicenec Date: Wed, 20 Jul 2022 10:22:38 +0800 Subject: [PATCH 7/7] Fixed replacement for output port(s) --- daliuge-engine/dlg/apps/bash_shell_app.py | 13 +++++++++++++ daliuge-engine/dlg/droputils.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/daliuge-engine/dlg/apps/bash_shell_app.py b/daliuge-engine/dlg/apps/bash_shell_app.py index a53a8c592..577610d91 100644 --- a/daliuge-engine/dlg/apps/bash_shell_app.py +++ b/daliuge-engine/dlg/apps/bash_shell_app.py @@ -234,6 +234,19 @@ def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE): else: for i in range(min(len(inputs), len(pargs))): keyargs.update({pargs[i]: list(inputs.values())[i]}) + if "outputs" in self.parameters and isinstance(self.parameters['outputs'][0], dict): + pkeyargs = droputils.identify_named_ports( + outputs_dict, + self.parameters["outputs"], + pargs, + pargsDict, + appArgs, + check_len=len(outputs), + mode="outputs") + keyargs.update(pkeyargs) + else: + for i in range(min(len(outputs), len(pargs))): + keyargs.update({pargs[i]: list(outputs.values())[i]}) keyargs = droputils.serialize_kwargs(keyargs, prefix=self._argumentPrefix, separator=self._paramValueSeparator) diff --git a/daliuge-engine/dlg/droputils.py b/daliuge-engine/dlg/droputils.py index 1781dd0a7..b07e51003 100644 --- a/daliuge-engine/dlg/droputils.py +++ b/daliuge-engine/dlg/droputils.py @@ -583,7 +583,7 @@ def identify_named_ports(ports, port_dict, posargs, pargsDict, appArgs, check_le for i in range(check_len): # key for final dict is value in named ports dict key = list(port_dict[i].values())[0] - # value for final dict is value in inputs dict + # value for final dict is value in ports dict value = ports[list(port_dict[i].keys())[0]] if not value: value = '' # make sure we are passing NULL drop events if key in posargs: