Merge dece7d8 into fcce6f3

.github/workflows/create-palettes.yml

@@ -24,19 +24,23 @@ jobs:
         run: |
           sudo apt-get update && sudo apt-get install -y doxygen xsltproc
           pip install BlockDAG
+          pip install dlg_paletteGen
 
       - name: Configure git
         run: |
           git config --global user.name $GITHUB_USERNAME
           git config --global user.email $GITHUB_USERNAME@gmail.com
+
+      - name: Environment variables
+        run: |
           OUTPUT_FILENAME=$PROJECT_NAME-${GITHUB_REF_NAME/\//_}
           echo "PROJECT_VERSION=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
           echo "OUTPUT_FILENAME=$OUTPUT_FILENAME" >> $GITHUB_ENV
 
       - name: Create palettes
         run: |
-          python3 tools/xml2palette/xml2palette.py -t daliuge -r ./ $OUTPUT_FILENAME.palette
-          python3 tools/xml2palette/xml2palette.py -t template -r ./ $OUTPUT_FILENAME-template.palette
+          dlg_paletteGen -r -t daliuge ./ $OUTPUT_FILENAME.palette
+          dlg_paletteGen -r -t template ./ $OUTPUT_FILENAME-template.palette
 
       - name: Commit palettes to EAGLE
         env:

daliuge-engine/dlg/apps/constructs.py

@@ -7,7 +7,7 @@
 # @par EAGLE_START
 # @param category Scatter
 # @param tag template
-# @param num_of_copies Scatter dimension/4/Integer/ComponentParameter/readwrite//False/False/Specifies the number of replications of the content of the scatter construct
+# @param num_of_copies Scatter dimension/4/Integer/ConstructParameter/readwrite//False/False/Specifies the number of replications of the content of the scatter construct
 # @param dropclass dropclass/dlg.apps.constructs.ScatterDrop/String/ComponentParameter/readwrite//False/False/Drop class
 # @par EAGLE_END
 class ScatterDrop(BarrierAppDROP):
@@ -24,7 +24,7 @@ class ScatterDrop(BarrierAppDROP):
 # @par EAGLE_START
 # @param category Gather
 # @param tag template
-# @param num_of_inputs No. of inputs/2/Integer/ApplicationArgument/readwrite//False/False/Number of inputs
+# @param num_of_inputs No. of inputs/2/Integer/ConstructParameter/readwrite//False/False/Number of inputs
 # @param gather_axis Index of gather axis/0/Integer/ApplicationArgument/readwrite//False/False/Index of gather axis
 # @param dropclass dropclass/dlg.apps.constructs.GatherDrop/String/ComponentParameter/readwrite//False/False/Drop class
 # @par EAGLE_END
@@ -42,7 +42,7 @@ class GatherDrop(BarrierAppDROP):
 # @par EAGLE_START
 # @param category Loop
 # @param tag template
-# @param num_of_iter No. of iterations/2/Integer/ApplicationArguments/readwrite//False/False/Number of iterations
+# @param num_of_iter No. of iterations/2/Integer/ConstructParameter/readwrite//False/False/Number of iterations
 # @param dropclass dropclass/dlg.apps.constructs.LoopDrop/String/ComponentParameter/readwrite//False/False/Drop class
 # @par EAGLE_END
 class LoopDrop(BarrierAppDROP):
@@ -59,7 +59,7 @@ class LoopDrop(BarrierAppDROP):
 # @par EAGLE_START
 # @param category MKN
 # @param tag template
-# @param k K/1/Integer/ApplicationArgument/readwrite//False/False/Internal multiplicity
+# @param k K/1/Integer/ConstructParameter/readwrite//False/False/Internal multiplicity
 # @param dropclass dropclass/dlg.apps.constructs.MKNDrop/String/ComponentParameter/readwrite//False/False/Drop class
 # @par EAGLE_END
 class MKNDrop(BarrierAppDROP):
@@ -76,7 +76,7 @@ class MKNDrop(BarrierAppDROP):
 # @par EAGLE_START
 # @param category GroupBy
 # @param tag template
-# @param num_of_inputs No. of inputs/2/Integer/ApplicationArgument/readwrite//False/False/Number of inputs
+# @param num_of_inputs No. of inputs/2/Integer/ConstructParameter/readwrite//False/False/Number of inputs
 # @param gather_axis Index of gather axis/0/Integer/ApplicationArgument/readwrite//False/False/Index of gather axis
 # @param dropclass dropclass/dlg.apps.constructs.GroupByDrop/String/ComponentParameter/readwrite//False/False/Drop class
 # @par EAGLE_END

daliuge-engine/dlg/apps/dynlib.py

@@ -482,6 +482,7 @@ def get_from_subprocess(proc, q):
 # @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group?
 # @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed
 # @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up
+# @param dropclass dropclass/dlg.apps.dynlib.DynlibProcApp/String/ComponentParameter/readwrite//False/False/Drop class
 # @par EAGLE_END
 class DynlibProcApp(BarrierAppDROP):
     """Loads a dynamic library in a different process and runs it"""

daliuge-engine/dlg/apps/scp.py

@@ -53,8 +53,7 @@
 # @param remoteUser Remote User//String/ApplicationArgument/readwrite//False/False/Remote user address
 # @param pkeyPath Private Key Path//String/ApplicationArgument/readwrite//False/False/Private key path
 # @param timeout Timeout/60/Float/ApplicationArgument/readwrite//False/False/Connection timeout in seconds
-# @param file File//Object.PathBasedDrop/InputPort/readwrite//False/False/Input file path
-# @param file File//Object.PathBasedDrop/OutputPort/readwrite//False/False/Output file path
+# @param file File//Object.PathBasedDrop/InputOutput/readwrite//False/False/File path
 # @par EAGLE_END
 class ScpApp(BarrierAppDROP):
     """

daliuge-engine/dlg/apps/simple.py

@@ -148,8 +148,7 @@ def run(self):
 # @param num_cpus num_cpus/1/Integer/ComponentParameter/readonly//False/False/Number of cores used
 # @param group_start group_start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group?
 # @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up
-# @param dummy_in dummy//Object/InputPort/readwrite//False/False/Dummy input port
-# @param dummy_out dummy//Object/OutputPort/readwrite//False/False/Dummy output port
+# @param dummy dummy//Object/InputOutput/readwrite//False/False/Dummy port
 # @param input_parser Input Parser/pickle/Select/ApplicationArgument/readwrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
 # @param output_parser Output Parser/pickle/Select/ApplicationArgument/readwrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
 # @par EAGLE_END
@@ -322,8 +321,7 @@ def _getArray(self):
 # @param execution_time execution_time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time
 # @param num_cpus num_cpus/1/Integer/ComponentParameter/readonly//False/False/Number of cores used
 # @param group_start group_start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group?
-# @param array_in Array//Object.Array/InputPort/readwrite//False/False/Port for the input array(s)
-# @param array_out Array//Object.Array/OutputPort/readwrite//False/False/Port carrying the averaged array
+# @param array Array//Object.Array/InputOutput/readwrite//False/False/Port for the array(s)
 # @par EAGLE_END
 class AverageArraysApp(BarrierAppDROP):
     """

daliuge-engine/dlg/data/drops/data_base.py

@@ -64,8 +64,7 @@
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
 # @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data
 # @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
-# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
-# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
+# @param dummy dummy//Object/InputOutput/readwrite//False/False/Dummy port
 # @par EAGLE_END
 class DataDROP(AbstractDROP):
     """
@@ -411,8 +410,7 @@ def path(self) -> str:
 # @param tag daliuge
 # @param dropclass dropclass/dlg.data.drops.data_base.NullDROP/String/ComponentParameter/readwrite//False/False/Drop class
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
-# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
-# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
+# @param dummy dummy//Object/InputOutput/readwrite//False/False/Dummy port
 # @par EAGLE_END
 class NullDROP(DataDROP):
     """

daliuge-engine/dlg/data/drops/file.py

@@ -46,8 +46,7 @@
 # @param persist Persist/True/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
 # @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
-# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
-# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
+# @param dummy dummy//Object/InputOutput/readwrite//False/False/Dummy port
 # @par EAGLE_END
 class FileDROP(DataDROP, PathBasedDrop):
     """

daliuge-engine/dlg/data/drops/memory.py

@@ -42,8 +42,7 @@
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
 # @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data
 # @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
-# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
-# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
+# @param dummy dummy//Object/InputOutput/readwrite//False/False/Dummy port
 # @par EAGLE_END
 class InMemoryDROP(DataDROP):
     """
@@ -107,8 +106,7 @@ def generate_reproduce_data(self):
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
 # @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data
 # @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
-# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
-# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
+# @param dummy dummy//Object/InputOutput/readwrite//False/False/Dummy port
 # @par EAGLE_END
 class SharedMemoryDROP(DataDROP):
     """

daliuge-engine/dlg/data/drops/ngas.py

@@ -42,8 +42,7 @@
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
 # @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data
 # @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
-# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
-# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
+# @param dummy dummy//Object/InputOutput/readwrite//False/False/Dummy port
 # @par EAGLE_END
 class NgasDROP(DataDROP):
     """

daliuge-engine/dlg/data/drops/plasma.py

@@ -42,9 +42,7 @@
 # @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
 # @param use_staging Use Staging/False/Boolean/ComponentParameter/readwrite//False/False/Enables writing to a dynamically resizeable staging buffer
-# @param dropclass dropclass/dlg.data.drops.plasma.PlasmaDROP/String/ComponentParameter/readwrite//False/False/Drop class
-# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
-# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
+# @param dummy dummy//Object/InputOutput/readwrite//False/False/Dummy port
 # @par EAGLE_END
 class PlasmaDROP(DataDROP):
     """
@@ -95,8 +93,7 @@ def dataURL(self) -> str:
 # @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node
 # @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group?
 # @param flight_path Flight Path//String/ComponentParameter/readwrite//False/False/IP and flight port of the drop owner
-# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
-# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
+# @param dummy dummy//Object/InputOutput/readwrite//False/False/Dummy port
 # @par EAGLE_END
 class PlasmaFlightDROP(DataDROP):
     """

daliuge-engine/dlg/data/drops/rdbms.py

@@ -43,8 +43,7 @@
 # @param condition Whats used after WHERE//String/ComponentParameter/readwrite//False/False/Condition for SELECT. For this the WHERE statement must be written using the "{X}" or "{}" placeholders
 # @param selectVals values for WHERE/{}/Json/ComponentParameter/readwrite//False/False/Values for the WHERE statement
 # @param dropclass dropclass/dlg.data.drops.rdbms.RDBMSDrop/String/ComponentParameter/readwrite//False/False/Drop class
-# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
-# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
+# @param dummy dummy//Object/InputOutput/readwrite//False/False/Dummy port
 # @par EAGLE_END
 class RDBMSDrop(DataDROP):
     """

daliuge-engine/dlg/data/drops/s3_drop.py

@@ -61,11 +61,10 @@
 # @param Key Key//String/ComponentParameter/readwrite//False/False/The S3 object key
 # @param profile_name Profile Name//String/ComponentParameter/readwrite//False/False/The S3 profile name
 # @param endpoint_url Endpoint URL//String/ComponentParameter/readwrite//False/False/The URL exposing the S3 REST API
-# @param dropclass dropclass/dlg.data.drops.s3_drop.S3DROP/String/ComponentParameter/readwrite//False/False/The URL exposing the S3 REST API
+# @param dropclass dropclass/dlg.data.drops.s3_drop.S3DROP/String/ComponentParameter/readwrite//False/False/Drop class
 # @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data
 # @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
-# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port
-# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
+# @param dummy dummy//Object/InputOutput/readwrite//False/False/Dummy port
 # @par EAGLE_END
 class S3DROP(DataDROP):
     """