Merge branch 'master' into validate-graphs

OpenAPI/tests/test.graph

@@ -33,8 +33,6 @@
             "height": 72,
             "collapsed": true,
             "flipPorts": false,
-            "streaming": false,
-            "persist": false,
             "subject": null,
             "expanded": false,
             "readonly": true,
@@ -235,8 +233,6 @@
             "height": 72,
             "collapsed": true,
             "flipPorts": false,
-            "streaming": false,
-            "persist": false,
             "subject": null,
             "expanded": false,
             "readonly": true,
@@ -304,6 +300,19 @@
                     "readonly": false,
                     "type": "Unknown",
                     "precious": false
+                },
+                {
+                    "defaultValue": true,
+                    "description": "Specifies whether this data component contains data that should not be deleted after execution",
+                    "keyAttribute": false,
+                    "name": "persist",
+                    "options": [],
+                    "positional": false,
+                    "precious": false,
+                    "readonly": false,
+                    "text": "Persist",
+                    "type": "Boolean",
+                    "value": false
                 }
             ],
             "applicationArgs": [],
@@ -337,8 +346,6 @@
             "height": 72,
             "collapsed": true,
             "flipPorts": false,
-            "streaming": false,
-            "persist": false,
             "subject": null,
             "expanded": false,
             "readonly": true,
@@ -386,6 +393,19 @@
                     "readonly": false,
                     "type": "Unknown",
                     "precious": false
+                },
+                {
+                    "defaultValue": true,
+                    "description": "Specifies whether this data component contains data that should not be deleted after execution",
+                    "keyAttribute": false,
+                    "name": "persist",
+                    "options": [],
+                    "positional": false,
+                    "precious": false,
+                    "readonly": false,
+                    "text": "Persist",
+                    "type": "Boolean",
+                    "value": false
                 }
             ],
             "applicationArgs": [],

daliuge-common/dlg/clients.py

@@ -241,6 +241,9 @@ def shutdown_node_manager(self):
     def get_log_file(self, sessionId):
         return self._request(f"/sessions/{sessionId}/logs", "GET")
 
+    def get_submission_method(self):
+        return self._get_json("/submission_method")
+
 
 class CompositeManagerClient(BaseDROPManagerClient):
     def nodes(self):
@@ -252,6 +255,9 @@ def add_node(self, node):
     def remove_node(self, node):
         self._DELETE(f"/node/{node}")
 
+    def get_submission_method(self):
+        return self._get_json("/submission_method")
+
 
 class DataIslandManagerClient(CompositeManagerClient):
     """

daliuge-common/dlg/common/deployment_methods.py

@@ -0,0 +1,30 @@
+#
+#    ICRAR - International Centre for Radio Astronomy Research
+#    (c) UWA - The University of Western Australia, 2015
+#    Copyright by UWA (in the framework of the ICRAR)
+#    All rights reserved
+#
+#    This library is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU Lesser General Public
+#    License as published by the Free Software Foundation; either
+#    version 2.1 of the License, or (at your option) any later version.
+#
+#    This library is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#    Lesser General Public License for more details.
+#
+#    You should have received a copy of the GNU Lesser General Public
+#    License along with this library; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+#    MA 02111-1307  USA
+#
+
+from enum import Enum
+
+
+class DeploymentMethods(str, Enum):
+    SERVER = "SERVER",
+    BROWSER = "BROWSER",
+    HELM = "HELM",
+    OOD = "OOD"

daliuge-common/dlg/common/k8s_utils.py

@@ -0,0 +1,40 @@
+#
+#    ICRAR - International Centre for Radio Astronomy Research
+#    (c) UWA - The University of Western Australia, 2016
+#    Copyright by UWA (in the framework of the ICRAR)
+#    All rights reserved
+#
+#    This library is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU Lesser General Public
+#    License as published by the Free Software Foundation; either
+#    version 2.1 of the License, or (at your option) any later version.
+#
+#    This library is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#    Lesser General Public License for more details.
+#
+#    You should have received a copy of the GNU Lesser General Public
+#    License along with this library; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+#    MA 02111-1307  USA
+#
+
+
+import re
+import subprocess
+
+
+def check_k8s_env():
+    """
+    Makes sure kubectl can be called and is accessible.
+    """
+    try:
+        output = subprocess.run(
+            ["kubectl version"], capture_output=True, shell=True
+        ).stdout
+        output = output.decode(encoding="utf-8").replace("\n", "")
+        pattern = re.compile(r"^Client Version:.*Server Version:.*")
+        return re.match(pattern, output)
+    except subprocess.SubprocessError:
+        return False

daliuge-common/dlg/common/network.py

@@ -95,6 +95,10 @@ def check_port(host, port, timeout=0, checking_open=True, return_socket=False):
             )
             return not checking_open
 
+        except socket.gaierror:
+            logger.debug("Endpoint service %s:%d not known", host, port)
+            return not checking_open
+
         except socket.error as e:
 
             # If the connection becomes suddenly closed from the server-side.

daliuge-common/dlg/common/reproducibility/reproducibility_fields.py

@@ -74,14 +74,12 @@ def lgt_block_fields(rmode: ReproducibilityFlags):
         "outputPorts": FieldOps.COUNT,
         "inputLocalPorts": FieldOps.COUNT,
         "outputLocalPorts": FieldOps.COUNT,  # MKN Nodes
-        "streaming": FieldOps.STORE,
     }
     if rmode == ReproducibilityFlags.REPRODUCE:
         del data["inputPorts"]
         del data["outputPorts"]
         del data["inputLocalPorts"]
         del data["outputLocalPorts"]
-        del data["streaming"]
     return data
 
 

daliuge-engine/dlg/data/drops/dataDummy_drop.py

@@ -42,6 +42,8 @@
 # @param dataclass Application Class//String/ComponentParameter/readonly//False/False/Data class
 # @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
 # @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
 # @par EAGLE_END

daliuge-engine/dlg/data/drops/environmentvar_drop.py

@@ -66,6 +66,8 @@ def _filter_parameters(parameters: dict):
 # @par EAGLE_START
 # @param category EnvironmentVariables
 # @param tag daliuge
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
 # @par EAGLE_END
 class EnvironmentVarDROP(AbstractDROP, KeyValueDROP):

daliuge-engine/dlg/data/drops/file.py

@@ -40,9 +40,12 @@
 # @param tag daliuge
 # @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
-# @param check_filepath_exists Check file path exists/True/Boolean/ComponentParameter/readwrite//False/False/Perform a check to make sure the file path exists before proceeding with the application
+# @param delete_parent_directory Delete parent directory/False/Boolean/ComponentParameter/readwrite//False/False/Also delete the parent directory of this file when deleting the file itself
+# @param check_filepath_exists Check file path exists/False/Boolean/ComponentParameter/readwrite//False/False/Perform a check to make sure the file path exists before proceeding with the application
 # @param filepath File Path//String/ComponentParameter/readwrite//False/False/Path to the file for this node
 # @param dirname Directory name//String/ComponentParameter/readwrite//False/False/Path to the file for this node
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/True/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
 # @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
 # @par EAGLE_END

daliuge-engine/dlg/data/drops/memory.py

@@ -39,6 +39,8 @@
 # @param tag daliuge
 # @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
 # @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
 # @par EAGLE_END
@@ -101,6 +103,8 @@ def generate_reproduce_data(self):
 # @param tag daliuge
 # @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
 # @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
 # @par EAGLE_END

daliuge-engine/dlg/data/drops/ngas.py

@@ -39,6 +39,8 @@
 # @param ngasConnectTimeout Connection timeout/2/Integer/ComponentParameter/readwrite//False/False/Timeout for connecting to the NGAS server
 # @param ngasMime NGAS mime-type/"text/ascii"/String/ComponentParameter/readwrite//False/False/Mime-type to be used for archiving
 # @param ngasTimeout NGAS timeout/2/Integer/ComponentParameter/readwrite//False/False/Timeout for receiving responses for NGAS
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
 # @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
 # @par EAGLE_END

daliuge-engine/dlg/data/drops/parset_drop.py

@@ -38,6 +38,8 @@
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
 # @param mode Parset mode/"YANDA"/String/ComponentParameter/readonly//False/False/To what standard DALiuGE should filter and serialize the parameters.
 # @param config_data ConfigData/""/String/ComponentParameter/readwrite//False/False/Additional configuration information to be mixed in with the initial data
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param Config ConfigFile//Object.File/OutputPort/readwrite//False/False/The output configuration file
 # @par EAGLE_END
 class ParameterSetDROP(DataDROP):

daliuge-engine/dlg/data/drops/s3_drop.py

@@ -53,6 +53,8 @@
 # @param object_name Object Name//String/ComponentParameter/readwrite//False/False/The S3 object key
 # @param profile_name Profile Name//String/ComponentParameter/readwrite//False/False/The S3 profile name
 # @param endpoint_url Endpoint URL//String/ComponentParameter/readwrite//False/False/The URL exposing the S3 REST API
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
 # @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
 # @par EAGLE_END

daliuge-engine/dlg/deploy/deployment_utils.py

@@ -113,20 +113,6 @@ def list_as_string(s):
     return _parse_list_tokens(iter(_list_tokenizer(s)))
 
 
-def check_k8s_env():
-    """
-    Makes sure kubectl can be called and is accessible.
-    """
-    try:
-        output = subprocess.run(
-            ["kubectl version"], capture_output=True, shell=True
-        ).stdout
-        pattern = re.compile(r"^Client Version:.*\nServer Version:.*")
-        return re.match(pattern, output.decode(encoding="utf-8"))
-    except subprocess.SubprocessError:
-        return False
-
-
 def find_numislands(physical_graph_template_file):
     """
     Given the physical graph data extract the graph name and the total number of

daliuge-engine/dlg/deploy/helm_client.py

@@ -42,10 +42,10 @@
     find_service_ips,
     find_pod_ips,
     wait_for_pods,
-    check_k8s_env,
 )
 from dlg.dropmake import pg_generator
 from dlg.restutils import RestClient
+from dlg.common.k8s_utils import check_k8s_env
 
 logger = logging.getLogger(__name__)
 

daliuge-engine/dlg/drop.py

@@ -1199,6 +1199,8 @@ def path(self) -> str:
 # @param dataclass Data Class/my.awesome.data.Component/String/ComponentParameter/readonly//False/False/The python class that implements this data component
 # @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
 # @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
 # @par EAGLE_END

daliuge-engine/dlg/manager/rest.py

@@ -30,6 +30,7 @@
 import json
 import logging
 import os
+import re
 import tarfile
 import threading
 
@@ -54,6 +55,7 @@
 from ..restserver import RestServer
 from ..restutils import RestClient, RestClientException
 from .session import generateLogFileName
+from dlg.common.deployment_methods import DeploymentMethods
 
 logger = logging.getLogger(__name__)
 
@@ -75,9 +77,14 @@ def fwrapper(*args, **kwargs):
             if res is not None:
                 bottle.response.content_type = "application/json"
                 # set CORS headers
+                origin = bottle.request.headers.raw("Origin")
+                if origin is None:
+                    origin = "http://localhost:8084"
+                elif not re.match(r"http://((localhost)|(127.0.0.1)):80[0-9][0-9]", origin):
+                    origin = "http://localhost:8084"
                 bottle.response.headers[
                     "Access-Control-Allow-Origin"
-                ] = "http://localhost:8084"
+                ] = origin
                 bottle.response.headers["Access-Control-Allow-Credentials"] = "true"
                 bottle.response.headers[
                     "Access-Control-Allow-Methods"
@@ -145,6 +152,7 @@ def __init__(self, dm, maxreqsize=10):
 
         # Mappings
         app = self.app
+        app.get("/api/submission_method", callback=self.submit_methods)
         app.post("/api/stop", callback=self.stop_manager)
         app.post("/api/sessions", callback=self.createSession)
         app.get("/api/sessions", callback=self.getSessions)
@@ -187,6 +195,10 @@ def initializeSpecifics(self, app):
         The default implementation does nothing.
         """
 
+    @daliuge_aware
+    def submit_methods(self):
+        return {"methods": [DeploymentMethods.BROWSER]}
+
     def _stop_manager(self):
         self.dm.shutdown()
         self.stop()

daliuge-engine/test/deploy/test_helm_deploy.py

@@ -29,7 +29,7 @@
 import unittest
 
 import yaml
-from dlg.deploy.deployment_utils import check_k8s_env
+from dlg.common.k8s_utils import check_k8s_env
 
 if check_k8s_env():
     from dlg.common.version import version as dlg_version

daliuge-engine/test/manager/test_rest.py

@@ -25,6 +25,7 @@
 
 from dlg import exceptions
 from dlg.common import Categories
+from dlg.common.deployment_methods import DeploymentMethods
 from dlg.exceptions import InvalidGraphException
 
 from dlg.manager import constants
@@ -268,3 +269,8 @@ def test_reprostatus_get(self):
         response = c.session_repro_status(sid)
         self.assertTrue(response)
         c.destroySession(sid)
+
+    def test_submit_method(self):
+        c = NodeManagerClient(hostname)
+        response = c.get_submission_method()
+        self.assertEqual({"methods": [DeploymentMethods.BROWSER]}, response)

daliuge-engine/test/reproducibility/testSingle.graph

@@ -74,7 +74,6 @@
             ],
             "selected": true,
             "showPorts": false,
-            "streaming": false,
             "subject": null,
             "text": "Bash Shell App",
             "width": 200,