Merge e82c095 into e77aaa2

ICRAR · Oct 28, 2022 · 6ff7665 · 6ff7665
2 parents e77aaa2 + e82c095
commit 6ff7665
Show file tree

Hide file tree

Showing 55 changed files with 762 additions and 606 deletions.
diff --git a/OpenAPI/tests/test.graph b/OpenAPI/tests/test.graph
@@ -33,8 +33,6 @@
             "height": 72,
             "collapsed": true,
             "flipPorts": false,
-            "streaming": false,
-            "persist": false,
             "subject": null,
             "expanded": false,
             "readonly": true,
@@ -235,8 +233,6 @@
             "height": 72,
             "collapsed": true,
             "flipPorts": false,
-            "streaming": false,
-            "persist": false,
             "subject": null,
             "expanded": false,
             "readonly": true,
@@ -304,6 +300,19 @@
                     "readonly": false,
                     "type": "Unknown",
                     "precious": false
+                },
+                {
+                    "defaultValue": true,
+                    "description": "Specifies whether this data component contains data that should not be deleted after execution",
+                    "keyAttribute": false,
+                    "name": "persist",
+                    "options": [],
+                    "positional": false,
+                    "precious": false,
+                    "readonly": false,
+                    "text": "Persist",
+                    "type": "Boolean",
+                    "value": false
                 }
             ],
             "applicationArgs": [],
@@ -337,8 +346,6 @@
             "height": 72,
             "collapsed": true,
             "flipPorts": false,
-            "streaming": false,
-            "persist": false,
             "subject": null,
             "expanded": false,
             "readonly": true,
@@ -386,6 +393,19 @@
                     "readonly": false,
                     "type": "Unknown",
                     "precious": false
+                },
+                {
+                    "defaultValue": true,
+                    "description": "Specifies whether this data component contains data that should not be deleted after execution",
+                    "keyAttribute": false,
+                    "name": "persist",
+                    "options": [],
+                    "positional": false,
+                    "precious": false,
+                    "readonly": false,
+                    "text": "Persist",
+                    "type": "Boolean",
+                    "value": false
                 }
             ],
             "applicationArgs": [],

diff --git a/daliuge-common/dlg/common/reproducibility/reproducibility_fields.py b/daliuge-common/dlg/common/reproducibility/reproducibility_fields.py
@@ -74,14 +74,12 @@ def lgt_block_fields(rmode: ReproducibilityFlags):
         "outputPorts": FieldOps.COUNT,
         "inputLocalPorts": FieldOps.COUNT,
         "outputLocalPorts": FieldOps.COUNT,  # MKN Nodes
-        "streaming": FieldOps.STORE,
     }
     if rmode == ReproducibilityFlags.REPRODUCE:
         del data["inputPorts"]
         del data["outputPorts"]
         del data["inputLocalPorts"]
         del data["outputLocalPorts"]
-        del data["streaming"]
     return data
 
 

diff --git a/daliuge-engine/dlg/data/drops/dataDummy_drop.py b/daliuge-engine/dlg/data/drops/dataDummy_drop.py
@@ -42,6 +42,8 @@
 # @param dataclass Application Class//String/ComponentParameter/readonly//False/False/Data class
 # @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
 # @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
 # @par EAGLE_END

diff --git a/daliuge-engine/dlg/data/drops/environmentvar_drop.py b/daliuge-engine/dlg/data/drops/environmentvar_drop.py
@@ -66,6 +66,8 @@ def _filter_parameters(parameters: dict):
 # @par EAGLE_START
 # @param category EnvironmentVariables
 # @param tag daliuge
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
 # @par EAGLE_END
 class EnvironmentVarDROP(AbstractDROP, KeyValueDROP):

diff --git a/daliuge-engine/dlg/data/drops/file.py b/daliuge-engine/dlg/data/drops/file.py
@@ -40,9 +40,12 @@
 # @param tag daliuge
 # @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
-# @param check_filepath_exists Check file path exists/True/Boolean/ComponentParameter/readwrite//False/False/Perform a check to make sure the file path exists before proceeding with the application
+# @param delete_parent_directory Delete parent directory/False/Boolean/ComponentParameter/readwrite//False/False/Also delete the parent directory of this file when deleting the file itself
+# @param check_filepath_exists Check file path exists/False/Boolean/ComponentParameter/readwrite//False/False/Perform a check to make sure the file path exists before proceeding with the application
 # @param filepath File Path//String/ComponentParameter/readwrite//False/False/Path to the file for this node
 # @param dirname Directory name//String/ComponentParameter/readwrite//False/False/Path to the file for this node
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/True/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
 # @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
 # @par EAGLE_END

diff --git a/daliuge-engine/dlg/data/drops/memory.py b/daliuge-engine/dlg/data/drops/memory.py
@@ -39,6 +39,8 @@
 # @param tag daliuge
 # @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
 # @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
 # @par EAGLE_END
@@ -101,6 +103,8 @@ def generate_reproduce_data(self):
 # @param tag daliuge
 # @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
 # @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
 # @par EAGLE_END

diff --git a/daliuge-engine/dlg/data/drops/ngas.py b/daliuge-engine/dlg/data/drops/ngas.py
@@ -39,6 +39,8 @@
 # @param ngasConnectTimeout Connection timeout/2/Integer/ComponentParameter/readwrite//False/False/Timeout for connecting to the NGAS server
 # @param ngasMime NGAS mime-type/"text/ascii"/String/ComponentParameter/readwrite//False/False/Mime-type to be used for archiving
 # @param ngasTimeout NGAS timeout/2/Integer/ComponentParameter/readwrite//False/False/Timeout for receiving responses for NGAS
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
 # @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
 # @par EAGLE_END

diff --git a/daliuge-engine/dlg/data/drops/parset_drop.py b/daliuge-engine/dlg/data/drops/parset_drop.py
@@ -38,6 +38,8 @@
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
 # @param mode Parset mode/"YANDA"/String/ComponentParameter/readonly//False/False/To what standard DALiuGE should filter and serialize the parameters.
 # @param config_data ConfigData/""/String/ComponentParameter/readwrite//False/False/Additional configuration information to be mixed in with the initial data
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param Config ConfigFile//Object.File/OutputPort/readwrite//False/False/The output configuration file
 # @par EAGLE_END
 class ParameterSetDROP(DataDROP):

diff --git a/daliuge-engine/dlg/data/drops/s3_drop.py b/daliuge-engine/dlg/data/drops/s3_drop.py
@@ -53,6 +53,8 @@
 # @param object_name Object Name//String/ComponentParameter/readwrite//False/False/The S3 object key
 # @param profile_name Profile Name//String/ComponentParameter/readwrite//False/False/The S3 profile name
 # @param endpoint_url Endpoint URL//String/ComponentParameter/readwrite//False/False/The URL exposing the S3 REST API
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
 # @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
 # @par EAGLE_END

diff --git a/daliuge-engine/dlg/drop.py b/daliuge-engine/dlg/drop.py
@@ -1199,6 +1199,8 @@ def path(self) -> str:
 # @param dataclass Data Class/my.awesome.data.Component/String/ComponentParameter/readonly//False/False/The python class that implements this data component
 # @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
+# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data 
+# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
 # @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
 # @par EAGLE_END

diff --git a/daliuge-engine/setup.py b/daliuge-engine/setup.py
@@ -131,7 +131,7 @@ def run(self):
     "overrides",
     "paramiko",
     "psutil",
-    "pyarrow",
+    "pyarrow==9.0",
     "python-daemon",
     "pyzmq ~= 22.3.0",
     "scp",

diff --git a/daliuge-engine/test/reproducibility/testSingle.graph b/daliuge-engine/test/reproducibility/testSingle.graph
@@ -74,7 +74,6 @@
             ],
             "selected": true,
             "showPorts": false,
-            "streaming": false,
             "subject": null,
             "text": "Bash Shell App",
             "width": 200,

diff --git a/daliuge-engine/test/reproducibility/topoGraphs/computationSandwich.graph b/daliuge-engine/test/reproducibility/topoGraphs/computationSandwich.graph
@@ -104,7 +104,6 @@
             "readonly": false,
             "selected": false,
             "showPorts": false,
-            "streaming": false,
             "subject": null,
             "text": "Bash Shell App",
             "width": 200,
@@ -165,6 +164,19 @@
                     "text": "Directory name",
                     "type": "String",
                     "value": ""
+                },
+                {
+                    "defaultValue": true,
+                    "description": "Specifies whether this data component contains data that should not be deleted after execution",
+                    "keyAttribute": false,
+                    "name": "persist",
+                    "options": [],
+                    "positional": false,
+                    "precious": false,
+                    "readonly": false,
+                    "text": "Persist",
+                    "type": "Boolean",
+                    "value": false
                 }
             ],
             "flipPorts": false,
@@ -201,7 +213,6 @@
             "readonly": false,
             "selected": false,
             "showPorts": false,
-            "streaming": false,
             "subject": null,
             "text": "File",
             "width": 200,
@@ -283,7 +294,6 @@
             "readonly": false,
             "selected": false,
             "showPorts": false,
-            "streaming": false,
             "subject": null,
             "text": "Bash Shell App",
             "width": 200,

diff --git a/daliuge-engine/test/reproducibility/topoGraphs/dataFan.graph b/daliuge-engine/test/reproducibility/topoGraphs/dataFan.graph
@@ -118,7 +118,6 @@
             "readonly": false,
             "selected": false,
             "showPorts": false,
-            "streaming": false,
             "subject": null,
             "text": "Bash Shell App",
             "width": 200,
@@ -179,6 +178,19 @@
                     "text": "Directory name",
                     "type": "String",
                     "value": ""
+                },
+                {
+                    "defaultValue": true,
+                    "description": "Specifies whether this data component contains data that should not be deleted after execution",
+                    "keyAttribute": false,
+                    "name": "persist",
+                    "options": [],
+                    "positional": false,
+                    "precious": false,
+                    "readonly": false,
+                    "text": "Persist",
+                    "type": "Boolean",
+                    "value": false
                 }
             ],
             "flipPorts": false,
@@ -208,7 +220,6 @@
             "readonly": false,
             "selected": false,
             "showPorts": false,
-            "streaming": false,
             "subject": null,
             "text": "File",
             "width": 200,
@@ -269,6 +280,19 @@
                     "text": "Directory name",
                     "type": "String",
                     "value": ""
+                },
+                {
+                    "defaultValue": true,
+                    "description": "Specifies whether this data component contains data that should not be deleted after execution",
+                    "keyAttribute": false,
+                    "name": "persist",
+                    "options": [],
+                    "positional": false,
+                    "precious": false,
+                    "readonly": false,
+                    "text": "Persist",
+                    "type": "Boolean",
+                    "value": false
                 }
             ],
             "flipPorts": false,
@@ -298,7 +322,6 @@
             "readonly": false,
             "selected": false,
             "showPorts": false,
-            "streaming": false,
             "subject": null,
             "text": "File",
             "width": 200,
@@ -359,6 +382,19 @@
                     "text": "Directory name",
                     "type": "String",
                     "value": ""
+                },
+                {
+                    "defaultValue": true,
+                    "description": "Specifies whether this data component contains data that should not be deleted after execution",
+                    "keyAttribute": false,
+                    "name": "persist",
+                    "options": [],
+                    "positional": false,
+                    "precious": false,
+                    "readonly": false,
+                    "text": "Persist",
+                    "type": "Boolean",
+                    "value": false
                 }
             ],
             "flipPorts": false,
@@ -388,7 +424,6 @@
             "readonly": false,
             "selected": false,
             "showPorts": false,
-            "streaming": false,
             "subject": null,
             "text": "File",
             "width": 200,