From de64c6b61ada6fd134f8cb745afb2a08c7af7889 Mon Sep 17 00:00:00 2001 From: Andreas Wicenec Date: Sun, 21 May 2023 22:19:37 +0800 Subject: [PATCH 1/8] changed init to bash --- daliuge-engine/build_engine.sh | 2 +- daliuge-engine/etc/init-scripts/dlg-dim | 2 +- daliuge-engine/etc/init-scripts/dlg-nm | 2 +- daliuge-translator/build_translator.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/daliuge-engine/build_engine.sh b/daliuge-engine/build_engine.sh index ef6a8048d..c11d8e6d9 100755 --- a/daliuge-engine/build_engine.sh +++ b/daliuge-engine/build_engine.sh @@ -46,7 +46,7 @@ case "$1" in docker run -it --rm -v /var/run/docker.sock:/var/run/docker.sock dslim/docker-slim build --include-shell \ --include-path /etc --include-path /usr/local/lib --include-path /usr/local/bin --include-path /usr/lib/python3.8 \ --include-path /usr/lib/python3 --include-path /dlg --include-path /daliuge --publish-exposed-ports=true \ - --http-probe-exec start_local_managers.sh --http-probe=true --tag=icrar/daliuge-engine:${VCS_TAG}\ + --http-probe-exec start_local_managers.sh --http-probe=false --tag=icrar/daliuge-engine:${VCS_TAG}\ icrar/daliuge-engine.big:${VCS_TAG} \ ;; *) diff --git a/daliuge-engine/etc/init-scripts/dlg-dim b/daliuge-engine/etc/init-scripts/dlg-dim index 2edaea350..1f8d1a36a 100755 --- a/daliuge-engine/etc/init-scripts/dlg-dim +++ b/daliuge-engine/etc/init-scripts/dlg-dim @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash ### BEGIN INIT INFO # Provides: dlg-dim # Required-Start: $all diff --git a/daliuge-engine/etc/init-scripts/dlg-nm b/daliuge-engine/etc/init-scripts/dlg-nm index 205da59c8..3de0f2c9e 100755 --- a/daliuge-engine/etc/init-scripts/dlg-nm +++ b/daliuge-engine/etc/init-scripts/dlg-nm @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash ### BEGIN INIT INFO # Provides: dlg-nm # Required-Start: $all diff --git a/daliuge-translator/build_translator.sh b/daliuge-translator/build_translator.sh index d3255f02e..bce978c8a 100755 --- a/daliuge-translator/build_translator.sh +++ b/daliuge-translator/build_translator.sh @@ -48,7 +48,7 @@ case "$1" in echo ">>>>> docker-slim output <<<<<<<<<" docker run -it --rm -v /var/run/docker.sock:/var/run/docker.sock dslim/docker-slim build --include-shell \ --include-path /usr/local/lib --include-path /usr/local/bin --include-path /daliuge --include-path /dlg \ - --http-probe=true --tag=icrar/daliuge-translator:${VCS_TAG} icrar/daliuge-translator.big:${VCS_TAG} + --http-probe=false --tag=icrar/daliuge-translator:${VCS_TAG} icrar/daliuge-translator.big:${VCS_TAG} ;; *) echo "Usage: build_translator.sh " From c13299f68956c2fb48d6d65326f841d5561e9202 Mon Sep 17 00:00:00 2001 From: Andreas Wicenec Date: Sun, 21 May 2023 22:44:25 +0800 Subject: [PATCH 2/8] Revert "changed init to bash" This reverts commit de64c6b61ada6fd134f8cb745afb2a08c7af7889. --- daliuge-engine/build_engine.sh | 2 +- daliuge-engine/etc/init-scripts/dlg-dim | 2 +- daliuge-engine/etc/init-scripts/dlg-nm | 2 +- daliuge-translator/build_translator.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/daliuge-engine/build_engine.sh b/daliuge-engine/build_engine.sh index c11d8e6d9..ef6a8048d 100755 --- a/daliuge-engine/build_engine.sh +++ b/daliuge-engine/build_engine.sh @@ -46,7 +46,7 @@ case "$1" in docker run -it --rm -v /var/run/docker.sock:/var/run/docker.sock dslim/docker-slim build --include-shell \ --include-path /etc --include-path /usr/local/lib --include-path /usr/local/bin --include-path /usr/lib/python3.8 \ --include-path /usr/lib/python3 --include-path /dlg --include-path /daliuge --publish-exposed-ports=true \ - --http-probe-exec start_local_managers.sh --http-probe=false --tag=icrar/daliuge-engine:${VCS_TAG}\ + --http-probe-exec start_local_managers.sh --http-probe=true --tag=icrar/daliuge-engine:${VCS_TAG}\ icrar/daliuge-engine.big:${VCS_TAG} \ ;; *) diff --git a/daliuge-engine/etc/init-scripts/dlg-dim b/daliuge-engine/etc/init-scripts/dlg-dim index 1f8d1a36a..2edaea350 100755 --- a/daliuge-engine/etc/init-scripts/dlg-dim +++ b/daliuge-engine/etc/init-scripts/dlg-dim @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh ### BEGIN INIT INFO # Provides: dlg-dim # Required-Start: $all diff --git a/daliuge-engine/etc/init-scripts/dlg-nm b/daliuge-engine/etc/init-scripts/dlg-nm index 3de0f2c9e..205da59c8 100755 --- a/daliuge-engine/etc/init-scripts/dlg-nm +++ b/daliuge-engine/etc/init-scripts/dlg-nm @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh ### BEGIN INIT INFO # Provides: dlg-nm # Required-Start: $all diff --git a/daliuge-translator/build_translator.sh b/daliuge-translator/build_translator.sh index bce978c8a..d3255f02e 100755 --- a/daliuge-translator/build_translator.sh +++ b/daliuge-translator/build_translator.sh @@ -48,7 +48,7 @@ case "$1" in echo ">>>>> docker-slim output <<<<<<<<<" docker run -it --rm -v /var/run/docker.sock:/var/run/docker.sock dslim/docker-slim build --include-shell \ --include-path /usr/local/lib --include-path /usr/local/bin --include-path /daliuge --include-path /dlg \ - --http-probe=false --tag=icrar/daliuge-translator:${VCS_TAG} icrar/daliuge-translator.big:${VCS_TAG} + --http-probe=true --tag=icrar/daliuge-translator:${VCS_TAG} icrar/daliuge-translator.big:${VCS_TAG} ;; *) echo "Usage: build_translator.sh " From 33e1d30ebe479b2093a716688484ea46a935124b Mon Sep 17 00:00:00 2001 From: Andreas Wicenec Date: Sun, 21 May 2023 22:47:03 +0800 Subject: [PATCH 3/8] Revert to de64c6b6 --- daliuge-engine/build_engine.sh | 2 +- daliuge-engine/etc/init-scripts/dlg-dim | 2 +- daliuge-engine/etc/init-scripts/dlg-nm | 2 +- daliuge-translator/build_translator.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/daliuge-engine/build_engine.sh b/daliuge-engine/build_engine.sh index ef6a8048d..c11d8e6d9 100755 --- a/daliuge-engine/build_engine.sh +++ b/daliuge-engine/build_engine.sh @@ -46,7 +46,7 @@ case "$1" in docker run -it --rm -v /var/run/docker.sock:/var/run/docker.sock dslim/docker-slim build --include-shell \ --include-path /etc --include-path /usr/local/lib --include-path /usr/local/bin --include-path /usr/lib/python3.8 \ --include-path /usr/lib/python3 --include-path /dlg --include-path /daliuge --publish-exposed-ports=true \ - --http-probe-exec start_local_managers.sh --http-probe=true --tag=icrar/daliuge-engine:${VCS_TAG}\ + --http-probe-exec start_local_managers.sh --http-probe=false --tag=icrar/daliuge-engine:${VCS_TAG}\ icrar/daliuge-engine.big:${VCS_TAG} \ ;; *) diff --git a/daliuge-engine/etc/init-scripts/dlg-dim b/daliuge-engine/etc/init-scripts/dlg-dim index 2edaea350..1f8d1a36a 100755 --- a/daliuge-engine/etc/init-scripts/dlg-dim +++ b/daliuge-engine/etc/init-scripts/dlg-dim @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash ### BEGIN INIT INFO # Provides: dlg-dim # Required-Start: $all diff --git a/daliuge-engine/etc/init-scripts/dlg-nm b/daliuge-engine/etc/init-scripts/dlg-nm index 205da59c8..3de0f2c9e 100755 --- a/daliuge-engine/etc/init-scripts/dlg-nm +++ b/daliuge-engine/etc/init-scripts/dlg-nm @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash ### BEGIN INIT INFO # Provides: dlg-nm # Required-Start: $all diff --git a/daliuge-translator/build_translator.sh b/daliuge-translator/build_translator.sh index d3255f02e..bce978c8a 100755 --- a/daliuge-translator/build_translator.sh +++ b/daliuge-translator/build_translator.sh @@ -48,7 +48,7 @@ case "$1" in echo ">>>>> docker-slim output <<<<<<<<<" docker run -it --rm -v /var/run/docker.sock:/var/run/docker.sock dslim/docker-slim build --include-shell \ --include-path /usr/local/lib --include-path /usr/local/bin --include-path /daliuge --include-path /dlg \ - --http-probe=true --tag=icrar/daliuge-translator:${VCS_TAG} icrar/daliuge-translator.big:${VCS_TAG} + --http-probe=false --tag=icrar/daliuge-translator:${VCS_TAG} icrar/daliuge-translator.big:${VCS_TAG} ;; *) echo "Usage: build_translator.sh " From a21171bb13cfc7582d5f711e1a095662e4244187 Mon Sep 17 00:00:00 2001 From: Andreas Wicenec Date: Tue, 23 May 2023 23:16:57 +0800 Subject: [PATCH 4/8] First fix to enable browser submit into engine --- daliuge-engine/build_engine.sh | 4 +- .../dlg/manager/composite_manager.py | 3 +- daliuge-engine/dlg/manager/rest.py | 29 ++++--- daliuge-engine/setup.py | 7 +- daliuge-translator/dlg/dropmake/lg.py | 4 +- daliuge-translator/dlg/dropmake/pgt.py | 13 +++- daliuge-translator/dlg/dropmake/web/main.js | 78 ++++++++----------- .../dlg/dropmake/web/translator_rest.py | 62 ++++++++------- 8 files changed, 108 insertions(+), 92 deletions(-) diff --git a/daliuge-engine/build_engine.sh b/daliuge-engine/build_engine.sh index c11d8e6d9..d96904bd2 100755 --- a/daliuge-engine/build_engine.sh +++ b/daliuge-engine/build_engine.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # script builds the daliuge-engine docker container either with a tag referring to the current # branch name or with a release tag depending whether this is a development or deployment # version. @@ -46,7 +46,7 @@ case "$1" in docker run -it --rm -v /var/run/docker.sock:/var/run/docker.sock dslim/docker-slim build --include-shell \ --include-path /etc --include-path /usr/local/lib --include-path /usr/local/bin --include-path /usr/lib/python3.8 \ --include-path /usr/lib/python3 --include-path /dlg --include-path /daliuge --publish-exposed-ports=true \ - --http-probe-exec start_local_managers.sh --http-probe=false --tag=icrar/daliuge-engine:${VCS_TAG}\ + --http-probe=true --tag=icrar/daliuge-engine:${VCS_TAG}\ icrar/daliuge-engine.big:${VCS_TAG} \ ;; *) diff --git a/daliuge-engine/dlg/manager/composite_manager.py b/daliuge-engine/dlg/manager/composite_manager.py index 38db803af..9c99e87ee 100644 --- a/daliuge-engine/dlg/manager/composite_manager.py +++ b/daliuge-engine/dlg/manager/composite_manager.py @@ -490,7 +490,8 @@ def deploySession(self, sessionId, completedDrops=[]): "UIDs for completed drops not found: %r", not_found ) logger.info( - "Moving Drops to COMPLETED right away: %r", completedDrops + "Moving graph root Drops to COMPLETED right away: %r", + completedDrops, ) completed_by_host = group_by_node(completedDrops, self._graph) self.replicate( diff --git a/daliuge-engine/dlg/manager/rest.py b/daliuge-engine/dlg/manager/rest.py index 43608345a..08d69cb37 100644 --- a/daliuge-engine/dlg/manager/rest.py +++ b/daliuge-engine/dlg/manager/rest.py @@ -84,16 +84,22 @@ def fwrapper(*args, **kwargs): r"http://((localhost)|(127.0.0.1)):80[0-9][0-9]", origin ): origin = "http://localhost:8084" - bottle.response.headers["Access-Control-Allow-Origin"] = origin - bottle.response.headers[ - "Access-Control-Allow-Credentials" - ] = "true" + bottle.response.headers["Access-Control-Allow-Origin"] = "*" + # bottle.response.headers[ + # "Access-Control-Allow-Credentials" + # ] = True bottle.response.headers[ "Access-Control-Allow-Methods" ] = "GET, POST, PUT, OPTIONS" bottle.response.headers[ "Access-Control-Allow-Headers" ] = "Origin, Accept, Content-Type, Content-Encoding, X-Requested-With, X-CSRF-Token" + jres = ( + json.dumps(res) if res else json.dumps({"Status": "Success"}) + ) + logger.debug( + "Bottle sending back result: %s", jres[: min(len(jres), 80)] + ) return json.dumps(res) except Exception as e: logger.exception("Error while fulfilling request") @@ -220,7 +226,9 @@ def initializeSpecifics(self, app): @daliuge_aware def submit_methods(self): - return {"methods": [DeploymentMethods.BROWSER]} + return { + "methods": [DeploymentMethods.BROWSER, DeploymentMethods.SERVER] + } def _stop_manager(self): self.dm.shutdown() @@ -313,8 +321,8 @@ def deploySession(self, sessionId): completedDrops = [] if "completed" in bottle.request.forms: completedDrops = bottle.request.forms["completed"].split(",") - self.dm.deploySession(sessionId, completedDrops=completedDrops) - return {} + return self.dm.deploySession(sessionId, completedDrops=completedDrops) + # return {"Status": "Success"} @daliuge_aware def cancelSession(self, sessionId): @@ -336,7 +344,7 @@ def getGraphStatus(self, sessionId): @daliuge_aware def addGraphParts(self, sessionId): # WARNING: TODO: Somehow, the content_type can be overwritten to 'text/plain' - logger.debug(bottle.request.content_type) + logger.debug("Graph content type: %s", bottle.request.content_type) if ( "application/json" not in bottle.request.content_type and "text/plain" not in bottle.request.content_type @@ -346,6 +354,7 @@ def addGraphParts(self, sessionId): # We also accept gzipped content hdrs = bottle.request.headers + logger.debug("Graph hdr: %s", {k: v for k, v in hdrs.items()}) if hdrs.get("Content-Encoding", None) == "gzip": json_content = utils.ZlibUncompressedStream(bottle.request.body) else: @@ -353,8 +362,8 @@ def addGraphParts(self, sessionId): graph_parts = bottle.json_loads(json_content.read()) - self.dm.addGraphSpec(sessionId, graph_parts) - return {"graph_parts": graph_parts} + return self.dm.addGraphSpec(sessionId, graph_parts) + # return {"graph_parts": graph_parts} # =========================================================================== # non-REST methods diff --git a/daliuge-engine/setup.py b/daliuge-engine/setup.py index 79380b033..d4b494906 100644 --- a/daliuge-engine/setup.py +++ b/daliuge-engine/setup.py @@ -110,7 +110,9 @@ def run(self): lp = sysconfig.get_path("stdlib") with open(PTH_FILE, "w") as f: f.write("{0}/dist-packages".format(lp)) - install.copy_file(self, PTH_FILE, os.path.join(self.install_lib, PTH_FILE)) + install.copy_file( + self, PTH_FILE, os.path.join(self.install_lib, PTH_FILE) + ) # Core requirements of DALiuGE @@ -118,6 +120,7 @@ def run(self): install_requires = [ "wheel", # need to get wheel first... "bottle", + "urllib3<1.27,>=1.25.4", "boto3", "configobj", "crc32c", @@ -159,7 +162,7 @@ def run(self): setup( name="daliuge-engine", version=get_version_info()[0], - description=u"Data Activated \uF9CA (flow) Graph Engine - Execution Engine", + description="Data Activated \uF9CA (flow) Graph Engine - Execution Engine", long_description=""" The element of the DALiuGE system executing the workflows. This replaces the former 'runtime' package (up to version 1.0). For more information diff --git a/daliuge-translator/dlg/dropmake/lg.py b/daliuge-translator/dlg/dropmake/lg.py index 8d4e621dc..c17296ebf 100644 --- a/daliuge-translator/dlg/dropmake/lg.py +++ b/daliuge-translator/dlg/dropmake/lg.py @@ -478,7 +478,7 @@ def _link_drops(self, slgn, tlgn, src_drop, tgt_drop, llink): elif s_type in ["Application", "Control"]: sname = slgn._getPortName("outputPorts") tname = tlgn._getPortName("inputPorts") - logger.debug("Found port names: IN: %s, OUT: %s", sname, tname) + # logger.debug("Found port names: IN: %s, OUT: %s", sname, tname) sdrop.addOutput(tdrop, name=sname) tdrop.addProducer(sdrop, name=tname) if Categories.BASH_SHELL_APP == s_type: @@ -499,7 +499,7 @@ def _link_drops(self, slgn, tlgn, src_drop, tgt_drop, llink): # could be multiple ports, need to identify portId = llink["toPort"] if "toPort" in llink else None tname = tlgn._getPortName("inputPorts", portId=portId) - logger.debug("Found port names: IN: %s, OUT: %s", sname, tname) + # logger.debug("Found port names: IN: %s, OUT: %s", sname, tname) # logger.debug( # ">>> link from %s to %s (%s) (%s)", # sname, diff --git a/daliuge-translator/dlg/dropmake/pgt.py b/daliuge-translator/dlg/dropmake/pgt.py index feb2c1efd..20fab0fe0 100644 --- a/daliuge-translator/dlg/dropmake/pgt.py +++ b/daliuge-translator/dlg/dropmake/pgt.py @@ -240,10 +240,17 @@ def to_pg_spec( where the daliuge system is started up afer submission (e.g. SLURM) """ logger.debug( - "tpl_nodes_len: %s, node_list: %s", tpl_nodes_len, node_list + "# worker nodes: %s, node_list(incl. DIM): %s", + tpl_nodes_len, + node_list, ) - if tpl_nodes_len > 0: # generate pg_spec template + if ( + len(node_list) == 0 and tpl_nodes_len > 0 + ): # generate pg_spec template node_list = range(tpl_nodes_len) # create a fake list for now + tpl_fl = True + else: + tpl_fl = False if 0 == self._num_parts_done: raise GPGTException("The graph has not been partitioned yet") @@ -314,7 +321,7 @@ def to_pg_spec( # values = dict(zip(values,range(len(values)))) # dict with new values # lm = {k:values[v] for (k, v) in lm.items()} # replace old values with new - if tpl_nodes_len: + if tpl_fl: nm_list = [ "#%s" % x for x in range(nm_len) ] # so that nm_list[i] == '#i' diff --git a/daliuge-translator/dlg/dropmake/web/main.js b/daliuge-translator/dlg/dropmake/web/main.js index 4e727490d..8d6dee037 100644 --- a/daliuge-translator/dlg/dropmake/web/main.js +++ b/daliuge-translator/dlg/dropmake/web/main.js @@ -226,7 +226,7 @@ async function checkUrlSubmissionMethods(url) { resolve(response) }, error: function (jqXHR, textStatus, errorThrown) { - resolve({"methods": []}) + resolve({ "methods": [] }) }, timeout: 2000 }); @@ -332,12 +332,12 @@ function saveSettings() { let deployMethod; if (!errorFillingOut) { deployMethod = - { - name: $(this).find(".deployMethodName").val(), - url: $(this).find(".deployMethodUrl").val(), - deployMethod: $(this).find(".deployMethodMethod option:selected").val(), - active: $(this).find(".deployMethodActive").val() - } + { + name: $(this).find(".deployMethodName").val(), + url: $(this).find(".deployMethodUrl").val(), + deployMethod: $(this).find(".deployMethodMethod option:selected").val(), + active: $(this).find(".deployMethodActive").val() + } console.debug($(this).find(".deployMethodMethod option:selected").val()) deployMethodsArray.push(deployMethod) } @@ -386,7 +386,7 @@ function buildDeployMethodEntry(method, selected) { function fillOutSettings() { let deployMethodsArray; -//get setting values from local storage + //get setting values from local storage const manager_url = window.localStorage.getItem("manager_url"); $("#settingsModalErrorMessage").html('') @@ -509,7 +509,7 @@ function makePNG() { asArray[i] = data.charCodeAt(i); } - const blob = new Blob([asArray.buffer], {type: "image/png"}); + const blob = new Blob([asArray.buffer], { type: "image/png" }); saveAs(blob, pgtName + "_Template.png"); }); } @@ -577,19 +577,20 @@ async function directRestDeploy() { // sessionId must be unique or the request will fail const lgName = pgtName.substring(0, pgtName.lastIndexOf("_pgt.graph")); - const sessionId = lgName + "-" + Date.now(); + const dateId = new Date(); + const sessionId = lgName + "-" + dateId.toISOString().replace(/\:/gi, "-"); console.debug("sessionId:", sessionId); const nodes_url = manager_url + "/api/nodes"; const nodes = await fetch(nodes_url, { method: 'GET', - mode: request_mode + // mode: request_mode }) .then(handleFetchErrors) .then(response => response.json()) .catch(function (error) { - showMessageModal(`Error ${error}\nGetting nodes unsuccessful`); + showMessageModal(`Error ${error}! Getting nodes unsuccessful`); }) const pgt_url = "/gen_pg?tpl_nodes_len=" + nodes.length.toString() + "&pgt_id=" + pgtName; @@ -613,9 +614,10 @@ async function directRestDeploy() { // request pg_spec from translator const pg_spec_url = "/gen_pg_spec"; + console.debug("pg_spec_request", pg_spec_request_data); const pg_spec_response = await fetch(pg_spec_url, { method: 'POST', - mode: request_mode, + mode: 'cors', headers: { 'Content-Type': 'application/json', }, @@ -626,13 +628,14 @@ async function directRestDeploy() { .catch(function (error) { showMessageModal('Error', error + "\nGetting pg_spec unsuccessful: Unable to continue!"); }); - const session_data = {"sessionId": sessionId}; + const session_data = { "sessionId": sessionId }; + console.debug("SessionId:", sessionId); const create_session_url = manager_url + "/api/sessions"; const create_session = await fetch(create_session_url, { - credentials: 'include', + // credentials: 'include', cache: 'no-cache', method: 'POST', - mode: request_mode, + // mode: request_mode, referrerPolicy: 'no-referrer', headers: { 'Content-Type': 'application/json', @@ -646,48 +649,31 @@ async function directRestDeploy() { }); console.debug("create session response", create_session); // gzip the pg_spec - const buf = fflate.strToU8(JSON.stringify(pg_spec_response.pg_spec)); + const buf = fflate.strToU8(JSON.stringify(JSON.parse(pg_spec_response).pg_spec)); const compressed_pg_spec = fflate.zlibSync(buf); - console.debug("compressed_pg_spec", compressed_pg_spec); + // console.debug("compressed_pg_spec", compressed_pg_spec); + // console.debug("pg_spec", compressed_pg_spec); // append graph to session on engine const append_graph_url = manager_url + "/api/sessions/" + sessionId + "/graph/append"; const append_graph = await fetch(append_graph_url, { - credentials: 'include', + // credentials: 'include', method: 'POST', - mode: request_mode, - headers: { - 'Content-Type': 'application/json', - 'Content-Encoding': 'gzip' - }, + mode: "no-cors", referrerPolicy: 'origin', - //body: JSON.stringify(pg_spec_response.pg_spec) - body: new Blob([compressed_pg_spec], {type: 'application/json'}) - // body: new Blob([buf]) + // body: JSON.stringify(pg_spec_response.pg_spec) + // body: new Blob(compressed_pg_spec, { type: 'application/json' }) + body: new Blob([buf], { type: 'application/json' }) }) - .then(handleFetchErrors) - .then(response => response.json()) - .catch(function (error) { - showMessageModal('Error', error + "\nUnable to continue!"); - }); - console.debug("append graph response", append_graph); - // deploy graph - // NOTE: URLSearchParams here turns the object into a x-www-form-urlencoded form const deploy_graph_url = manager_url + "/api/sessions/" + sessionId + "/deploy"; const deploy_graph = await fetch(deploy_graph_url, { - credentials: 'include', + // credentials: 'include', method: 'POST', - mode: request_mode, + mode: "no-cors", body: new URLSearchParams({ - 'completed': pg_spec_response.root_uids, + 'completed': JSON.parse(pg_spec_response).root_uids, }) }) - .then(handleFetchErrors) - .then(response => response.json()) - .catch(function (error) { - showMessageModal('Error', error + "\nUnable to continue!"); - }); - //showMessageModal("Chart deployed" , "Check the dashboard of your k8s cluster for status updates."); const mgr_url = manager_url + "/session?sessionId=" + sessionId; window.open(mgr_url, '_blank').focus(); } @@ -706,7 +692,7 @@ async function restDeploy() { let manager_url = new URL(murl); console.info("In REST Deploy") - const request_mode = "cors"; + const request_mode = "no-cors"; manager_url = manager_url.toString(); if (manager_url.endsWith('/')) { manager_url = manager_url.substring(0, manager_url.length - 1); @@ -740,7 +726,7 @@ async function restDeploy() { var body = [pgtName, pgt]; // we send the name in the body with the pgt await fetch(create_slurm_url, { method: 'POST', - credentials: 'include', + // credentials: 'include', cache: 'no-cache', mode: request_mode, referrerPolicy: 'no-referrer', diff --git a/daliuge-translator/dlg/dropmake/web/translator_rest.py b/daliuge-translator/dlg/dropmake/web/translator_rest.py index 134cb9ec0..b5e3f9cbd 100644 --- a/daliuge-translator/dlg/dropmake/web/translator_rest.py +++ b/daliuge-translator/dlg/dropmake/web/translator_rest.py @@ -594,24 +594,7 @@ def gen_pg( RESTful interface to convert a PGT(P) into PG by mapping PGT(P) onto a given set of available resources """ - # if the 'deploy' checkbox is not checked, - # then the form submission will NOT contain a 'dlg_mgr_deploy' field - deploy = dlg_mgr_deploy is not None mprefix = "" - pgtp = pg_mgr.get_pgt(pgt_id) - if pgtp is None: - raise HTTPException( - status_code=404, - detail="PGT(P) with id {0} not found in the Physical Graph Manager".format( - pgt_id - ), - ) - - pgtpj = pgtp._gojs_json_obj - reprodata = pgtp.reprodata - num_partitions = len( - list(filter(lambda n: "isGroup" in n, pgtpj["nodeDataArray"])) - ) mport = 443 if dlg_mgr_url is not None: mparse = urlparse(dlg_mgr_url) @@ -637,6 +620,23 @@ def gen_pg( logger.debug("Manager host: %s", mhost) logger.debug("Manager port: %s", mport) logger.debug("Manager prefix: %s", mprefix) + # if the 'deploy' checkbox is not checked, + # then the form submission will NOT contain a 'dlg_mgr_deploy' field + deploy = dlg_mgr_deploy is not None + pgtp = pg_mgr.get_pgt(pgt_id) + if pgtp is None: + raise HTTPException( + status_code=404, + detail="PGT(P) with id {0} not found in the Physical Graph Manager".format( + pgt_id + ), + ) + + pgtpj = pgtp._gojs_json_obj + reprodata = pgtp.reprodata + num_partitions = len( + list(filter(lambda n: "isGroup" in n, pgtpj["nodeDataArray"])) + ) if mhost is None: if tpl_nodes_len > 0: @@ -696,7 +696,7 @@ def gen_pg( ) -@app.get("/gen_pg_spec", tags=["Original"]) +@app.post("/gen_pg_spec", tags=["Original"]) def gen_pg_spec( pgt_id: str = Body( description="The pgt_id used to internally reference this graph" @@ -708,6 +708,10 @@ def gen_pg_spec( manager_host: str = Body( description="The address of the manager host where the graph will be deployed to." ), + tpl_nodes_len: int = Body( + default=1, + description="The number of nodes requested by the graph", + ), ): """ Interface to convert a PGT(P) into pg_spec @@ -716,7 +720,7 @@ def gen_pg_spec( if manager_host == "localhost": manager_host = "localhost" logger.debug("pgt_id: %s", str(pgt_id)) - logger.debug("node_list: %s", str(node_list)) + # logger.debug("node_list: %s", str(node_list)) except Exception as ex: logger.error("%s", traceback.format_exc()) raise HTTPException( @@ -739,12 +743,18 @@ def gen_pg_spec( ) try: - pg_spec = pgtp.to_pg_spec([manager_host] + node_list, ret_str=False) + pg_spec = pgtp.to_pg_spec( + [manager_host] + node_list, + tpl_nodes_len=tpl_nodes_len, + ret_str=False, + ) root_uids = common.get_roots(pg_spec) - response = StreamingResponse( - json.dumps({"pg_spec": pg_spec, "root_uids": list(root_uids)}) + logger.debug("Root UIDs: %s", list(root_uids)) + response = JSONResponse( + json.dumps( + {"pg_spec": pg_spec, "root_uids": list(root_uids)}, + ), ) - response.content_type = "application/json" return response except Exception as ex: logger.error("%s", traceback.format_exc()) @@ -1116,6 +1126,7 @@ def get_submission_method( ) if DeploymentMethods.BROWSER in host_available_methods: available_methods.append(DeploymentMethods.SERVER) + logger.debug("Methods available: %s", available_methods) return {"methods": available_methods} @@ -1248,10 +1259,9 @@ def handler(*_args): signal.signal(signal.SIGTERM, handler) signal.signal(signal.SIGINT, handler) + logging.debug("Starting uvicorn verbose %s", options.verbose) uvicorn.run( - app=app, - host=options.host, - port=options.port, + app=app, host=options.host, port=options.port, debug=options.verbose ) From 68d3c25bbe1cfd963a3efb5fbfce149ace03063f Mon Sep 17 00:00:00 2001 From: awicenec Date: Wed, 24 May 2023 08:29:41 +0800 Subject: [PATCH 5/8] Adjusted test include SERVER method --- daliuge-engine/test/manager/test_rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daliuge-engine/test/manager/test_rest.py b/daliuge-engine/test/manager/test_rest.py index 8975465c2..7fb29e59e 100644 --- a/daliuge-engine/test/manager/test_rest.py +++ b/daliuge-engine/test/manager/test_rest.py @@ -293,4 +293,4 @@ def test_reprostatus_get(self): def test_submit_method(self): c = NodeManagerClient(hostname) response = c.get_submission_method() - self.assertEqual({"methods": [DeploymentMethods.BROWSER]}, response) + self.assertEqual({"methods": [DeploymentMethods.BROWSER, DeploymentMethods.SERVER]}, response) From d1a7940ca6fc13c04550785b03cb31f1732377ad Mon Sep 17 00:00:00 2001 From: Andreas Wicenec Date: Wed, 24 May 2023 10:35:52 +0800 Subject: [PATCH 6/8] Fixed dropclass replacement for test runs --- daliuge-translator/dlg/dropmake/pg_generator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/daliuge-translator/dlg/dropmake/pg_generator.py b/daliuge-translator/dlg/dropmake/pg_generator.py index c7e15a7fb..394fd5a0a 100644 --- a/daliuge-translator/dlg/dropmake/pg_generator.py +++ b/daliuge-translator/dlg/dropmake/pg_generator.py @@ -80,7 +80,10 @@ def unroll(lg, oid_prefix=None, zerorun=False, app=None): if app: logger.info("Replacing apps with %s", app) for dropspec in drop_list: - if "dropclass" in dropspec: + if ( + "dropclass" in dropspec + and dropspec["categoryType"] == "Application" + ): dropspec["dropclass"] = app dropspec["sleep_time"] = ( dropspec["execution_time"] From ddac0252b24c7757ad52563de6e773c270d6c8ef Mon Sep 17 00:00:00 2001 From: Andreas Wicenec Date: Mon, 29 May 2023 16:04:25 +0100 Subject: [PATCH 7/8] A few more fixes for docker deployments --- daliuge-engine/dlg/manager/rest.py | 8 ++++---- daliuge-translator/dlg/dropmake/web/main.js | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/daliuge-engine/dlg/manager/rest.py b/daliuge-engine/dlg/manager/rest.py index 08d69cb37..6e35d4ae7 100644 --- a/daliuge-engine/dlg/manager/rest.py +++ b/daliuge-engine/dlg/manager/rest.py @@ -84,10 +84,10 @@ def fwrapper(*args, **kwargs): r"http://((localhost)|(127.0.0.1)):80[0-9][0-9]", origin ): origin = "http://localhost:8084" - bottle.response.headers["Access-Control-Allow-Origin"] = "*" - # bottle.response.headers[ - # "Access-Control-Allow-Credentials" - # ] = True + bottle.response.headers["Access-Control-Allow-Origin"] = origin + bottle.response.headers[ + "Access-Control-Allow-Credentials" + ] = "true" bottle.response.headers[ "Access-Control-Allow-Methods" ] = "GET, POST, PUT, OPTIONS" diff --git a/daliuge-translator/dlg/dropmake/web/main.js b/daliuge-translator/dlg/dropmake/web/main.js index 8d6dee037..0cc5fc44a 100644 --- a/daliuge-translator/dlg/dropmake/web/main.js +++ b/daliuge-translator/dlg/dropmake/web/main.js @@ -220,6 +220,7 @@ async function checkUrlStatus(url) { async function checkUrlSubmissionMethods(url) { return new Promise((resolve) => { $.ajax({ + credentials: 'include', url: url, type: 'GET', success: function (response) { @@ -632,7 +633,7 @@ async function directRestDeploy() { console.debug("SessionId:", sessionId); const create_session_url = manager_url + "/api/sessions"; const create_session = await fetch(create_session_url, { - // credentials: 'include', + credentials: 'include', cache: 'no-cache', method: 'POST', // mode: request_mode, @@ -657,7 +658,7 @@ async function directRestDeploy() { // append graph to session on engine const append_graph_url = manager_url + "/api/sessions/" + sessionId + "/graph/append"; const append_graph = await fetch(append_graph_url, { - // credentials: 'include', + credentials: 'include', method: 'POST', mode: "no-cors", referrerPolicy: 'origin', @@ -678,6 +679,10 @@ async function directRestDeploy() { window.open(mgr_url, '_blank').focus(); } +function jsonEscape(str) { + return str.replace(/\n/g, "\\\\n").replace(/\r/g, "\\\\r").replace(/\t/g, "\\\\t"); +} + async function restDeploy() { // fetch manager host and port from local storage murl = window.localStorage.getItem("manager_url"); @@ -720,13 +725,16 @@ async function restDeploy() { .catch(function (error) { showMessageModal('Error', error + "\nGetting PGT unsuccessful: Unable to continue!"); }); - pgt = JSON.parse(pgt); + if (typeof pgt == "String") { + pgt = JSON.parse(jsonEscape(toString(pgt))); + } // This is for a deferred start of daliuge, e.g. on SLURM console.debug("sending request to ", create_slurm_url); var body = [pgtName, pgt]; // we send the name in the body with the pgt + console.debug("Submission PGT:", JSON.stringify(body)); await fetch(create_slurm_url, { method: 'POST', - // credentials: 'include', + credentials: 'include', cache: 'no-cache', mode: request_mode, referrerPolicy: 'no-referrer', From 51d2e5fc94c09be4575532da6f6bf6fbc58ada4c Mon Sep 17 00:00:00 2001 From: Andreas Wicenec Date: Wed, 31 May 2023 22:52:43 +0800 Subject: [PATCH 8/8] Fixed test for new dropclass --- daliuge-translator/test/dropmake/test_lgweb.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/daliuge-translator/test/dropmake/test_lgweb.py b/daliuge-translator/test/dropmake/test_lgweb.py index 55bc86887..7d5dcb643 100644 --- a/daliuge-translator/test/dropmake/test_lgweb.py +++ b/daliuge-translator/test/dropmake/test_lgweb.py @@ -468,7 +468,10 @@ def test_lg_unroll(self, n, graph): form_data = {"lg_content": json_data, "default_app": "test.app"} pgt = self._test_post_request(c, test_url, form_data, False) for dropspec in pgt: - if "dropclass" in dropspec: + if ( + "dropclass" in dropspec + and dropspec["category"] == "Application" + ): self.assertEqual(dropspec["dropclass"], "test.app") def test_pgt_partition(self):