Merge branch 'master' into named-ports

daliuge-common/setup.py

@@ -32,7 +32,7 @@
 # by setuptools/pkg_resources or "final" versions.
 MAJOR = 2
 MINOR = 3
-PATCH = 0
+PATCH = 1
 VERSION = (MAJOR, MINOR, PATCH)
 VERSION_FILE = "dlg/common/version.py"
 RELEASE = True

daliuge-engine/build_engine.sh

@@ -39,16 +39,15 @@ case "$1" in
         exit 0;;
     "slim")
         C_TAG="master"
-        [[ ! -z $2 ]] && C_TAG=$2
-        echo "Building daliuge-engine slim version ${VCS_TAG} using daliuge-common:${C_TAG}"
-        docker build --build-arg VCS_TAG=${VCS_TAG} --no-cache -t icrar/daliuge-engine:${DEV_TAG} -f docker/Dockerfile.dev .
+        echo "Building daliuge-engine slim version ${VCS_TAG} using daliuge-common:${VCS_TAG}"
+        docker build --build-arg VCS_TAG=${VCS_TAG} --no-cache -t icrar/daliuge-engine.big:${VCS_TAG} -f docker/Dockerfile .
         echo "Build finished! Slimming the image now"
         echo ">>>>> docker-slim output <<<<<<<<<"
         docker run -it --rm -v /var/run/docker.sock:/var/run/docker.sock dslim/docker-slim build --include-shell \
             --include-path /etc --include-path /usr/local/lib --include-path /usr/local/bin --include-path /usr/lib/python3.8 \
             --include-path /usr/lib/python3 --include-path /dlg --include-path /daliuge --publish-exposed-ports=true \
-            --http-probe-exec start_local_managers.sh --http-probe=true --tag=icrar/daliuge-engine.slim:${DEV_TAG}\
-            icrar/daliuge-engine:${DEV_TAG} \
+            --http-probe-exec start_local_managers.sh --http-probe=true --tag=icrar/daliuge-engine:${VCS_TAG}\
+            icrar/daliuge-engine.big:${VCS_TAG} \
 	    ;;
     *)
         echo "Usage: build_engine.sh <dep|dev|devall|slim>"

daliuge-engine/dlg/apps/bash_shell_app.py

@@ -226,18 +226,31 @@ def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE):
         # pargNames = [arg for arg in pargsDict]
         keyargs = {arg:appArgs[arg]["value"] for arg in appArgs if not appArgs[arg]["positional"]}
         if "inputs" in self.parameters and isinstance(self.parameters['inputs'][0], dict):
-            portargs = droputils.identify_named_ports(
+            pkeyargs = droputils.identify_named_ports(
                             inputs_dict,
                             self.parameters["inputs"],
                             pargNames,
                             pargsDict,
                             appArgs,
                             check_len=len(inputs),
                             mode="inputs")
-            keyargs.update(portargs)
+            keyargs.update(pkeyargs)
         else:
-            for i in range(min(len(inputs), len(pargNames))):
-                keyargs.update({pargNames[i]: list(inputs.values())[i]})
+            for i in range(min(len(inputs), len(pargs))):
+                keyargs.update({pargs[i]: list(inputs.values())[i]})
+        if "outputs" in self.parameters and isinstance(self.parameters['outputs'][0], dict):
+            pkeyargs = droputils.identify_named_ports(
+                            outputs_dict,
+                            self.parameters["outputs"],
+                            pargs,
+                            pargsDict,
+                            appArgs,
+                            check_len=len(outputs),
+                            mode="outputs")
+            keyargs.update(pkeyargs)
+        else:
+            for i in range(min(len(outputs), len(pargs))):
+                keyargs.update({pargs[i]: list(outputs.values())[i]})
         keyargs = droputils.serialize_kwargs(keyargs, 
             prefix=self._argumentPrefix,
             separator=self._paramValueSeparator)

daliuge-engine/dlg/droputils.py

@@ -588,7 +588,7 @@ def identify_named_ports(ports, port_dict, posargs, pargsDict, appArgs, check_le
     for i in range(check_len):
         # key for final dict is value in named ports dict
         key = list(port_dict[i].values())[0]
-        # value for final dict is value in inputs dict
+        # value for final dict is value in ports dict
         value = ports[list(port_dict[i].keys())[0]]
         if not value: value = '' # make sure we are passing NULL drop events
         if key in posargs:

daliuge-engine/dlg/manager/web/static/js/dm.js

@@ -623,7 +623,7 @@ function startGraphStatusUpdates(serverUrl, sessionId, selectedNode, delay,
 
 			var allCompleted = statuses.reduce(function(prevVal, curVal, idx, arr) {
 				var cur_status = get_status_name(curVal);
-				return prevVal && (cur_status == 'completed' || cur_status == 'finished' || cur_status == 'error' || cur_status == 'cancelled' || cur_status == 'skipped');
+				return prevVal && (cur_status == 'completed' || cur_status == 'finished' || cur_status == 'error' || cur_status == 'cancelled' || cur_status == 'skipped' || cur_status == 'deleted' || cur_status == 'expired');
 			}, true);
 			if (!allCompleted) {
 				updateStatesDelayTimer = d3.timer(updateStates, delay);

daliuge-engine/setup.py

@@ -37,7 +37,7 @@
 # by setuptools/pkg_resources or "final" versions.
 MAJOR = 2
 MINOR = 3
-PATCH = 0
+PATCH = 1
 RELEASE = True
 VERSION = "%d.%d.%d" % (MAJOR, MINOR, PATCH)
 VERSION_FILE = "dlg/runtime/version.py"

daliuge-translator/dlg/dropmake/web/main.js

@@ -60,11 +60,19 @@ function openSettingsModal(){
     $('#settingsModal').modal("show")
 }
 
-async function initiateDeploy(method, selected, name){
+async function initiateDeploy(method, selected, clickedName){
+    var clickedUrl
+    JSON.parse(window.localStorage.getItem("deployMethods")).forEach(element => {
+        if(element.name === clickedName){
+            clickedUrl = element.url
+        }
+    })
+
     if (selected === false){
-        changeSelectedDeployMethod(name)
+        await changeSelectedDeployMethod(clickedName, clickedUrl)
     }
-    var activeUrlReachable = await checkUrlStatus(window.localStorage.getItem("manager_url"))
+
+    var activeUrlReachable = await checkUrlStatus(clickedUrl)
 
     if(!activeUrlReachable){
         $("#warning-alert").fadeTo(2000, 1000).slideUp(200, function() {
@@ -85,16 +93,20 @@ async function initiateDeploy(method, selected, name){
     }
 }
 
-function changeSelectedDeployMethod(name) {
-    var deployMethodsArray = JSON.parse(localStorage.getItem("deployMethods"))
-    deployMethodsArray.forEach(element => {
-        element.active = "false"
-        if(element.name === name){
-            element.active = "true"
-        }
+async function changeSelectedDeployMethod(name,manager_url) {
+    return new Promise((resolve, reject) => {
+        var deployMethodsArray = JSON.parse(localStorage.getItem("deployMethods"))
+        $("#managerUrlInput").val(manager_url);
+        deployMethodsArray.forEach(element => {
+            element.active = "false"
+            if(element.name === name){
+                element.active = "true"
+            }
+        })
+        window.localStorage.setItem('deployMethods', JSON.stringify(deployMethodsArray))
+        updateDeployOptionsDropdown()
+        resolve(true)
     })
-    localStorage.setItem('deployMethods', JSON.stringify(deployMethodsArray))
-    updateDeployOptionsDropdown()
 }
 
 function updateDeployOptionsDropdown() {

daliuge-translator/setup.py

@@ -34,7 +34,7 @@
 # by setuptools/pkg_resources or "final" versions.
 MAJOR = 2
 MINOR = 3
-PATCH = 0
+PATCH = 1
 RELEASE = True
 VERSION = "%d.%d.%d" % (MAJOR, MINOR, PATCH)
 VERSION_FILE = "dlg/translator/version.py"

tools/xml2palette/xml2palette.py

@@ -95,8 +95,15 @@ def check_environment_variables():
         value = os.environ.get(variable)
 
         if value is None:
-            logging.error("No " + variable + " environment variable.")
-            return False
+            if variable == 'PROJECT_NAME':
+                os.environ['PROJECT_NAME'] = os.path.basename(os.path.abspath('.'))
+            elif variable == 'PROJECT_VERSION':
+                os.environ['PROJECT_VERSION'] = '0.1'
+            elif variable == 'GIT_REPO':
+                os.environ['GIT_REPO'] = os.environ['PROJECT_NAME']
+            else:
+                logging.error("No " + variable + " environment variable.")
+                return False
 
     return True
 
@@ -1063,6 +1070,10 @@ def params_to_nodes(params):
         level=logging.INFO,
     )
 
+    # read environment variables
+    if not check_environment_variables():
+        sys.exit(1)
+
     logging.info("PROJECT_NAME:" + os.environ.get("PROJECT_NAME"))
     logging.info("PROJECT_VERSION:" + os.environ.get("PROJECT_VERSION"))
     logging.info("GIT_REPO:" + os.environ.get("GIT_REPO"))
@@ -1077,10 +1088,6 @@ def params_to_nodes(params):
     # create a temp directory for the output of doxygen
     output_directory = tempfile.TemporaryDirectory()
 
-    # read environment variables
-    if not check_environment_variables():
-        sys.exit(1)
-
     # add extra doxygen setting for input and output locations
     DOXYGEN_SETTINGS.append(("PROJECT_NAME", os.environ.get("PROJECT_NAME")))
     DOXYGEN_SETTINGS.append(("INPUT", inputdir))