Merge 481f985 into fcce6f3

.github/workflows/create-palettes.yml

@@ -24,19 +24,23 @@ jobs:
         run: |
           sudo apt-get update && sudo apt-get install -y doxygen xsltproc
           pip install BlockDAG
+          pip install dlg_paletteGen
 
       - name: Configure git
         run: |
           git config --global user.name $GITHUB_USERNAME
           git config --global user.email $GITHUB_USERNAME@gmail.com
+
+      - name: Environment variables
+        run: |
           OUTPUT_FILENAME=$PROJECT_NAME-${GITHUB_REF_NAME/\//_}
           echo "PROJECT_VERSION=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
           echo "OUTPUT_FILENAME=$OUTPUT_FILENAME" >> $GITHUB_ENV
 
       - name: Create palettes
         run: |
-          python3 tools/xml2palette/xml2palette.py -t daliuge -r ./ $OUTPUT_FILENAME.palette
-          python3 tools/xml2palette/xml2palette.py -t template -r ./ $OUTPUT_FILENAME-template.palette
+          dlg_paletteGen -r -t daliuge ./ $OUTPUT_FILENAME.palette
+          dlg_paletteGen -r -t template ./ $OUTPUT_FILENAME-template.palette
 
       - name: Commit palettes to EAGLE
         env:

daliuge-engine/dlg/apps/archiving.py

@@ -83,20 +83,20 @@ def store(self, inputDrop):
 # @par EAGLE_START
 # @param category PythonApp
 # @param tag daliuge
-# @param dropclass Application Class/dlg.apps.archiving.NgasArchivingApp/String/ComponentParameter/readonly//False/False/Application class
-# @param input_parser Input Parser/pickle/Select/ApplicationArgument/readwrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
-# @param output_parser Output Parser/pickle/Select/ApplicationArgument/readwrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
-# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time
-# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used
-# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group?
-# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed
-# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up
-# @param ngasSrv NGAS Server URL/localhost/String/ApplicationArgument/readwrite//False/False/URL of the NGAS Server
-# @param ngasPort NGAS Server Port/7777/Integer/ApplicationArgument/readwrite//False/False/TCP/IP Port on the NGAS Server
-# @param ngasMime NGAS Mime Type/"application/octet-stream"/String/ApplicationArgument/readwrite//False/False/Mime-type of the NGAS payload
-# @param ngasTimeout NGAS Server Timeout/2/Integer/ApplicationArgument/readonly//False/False/Archiving request timeout
-# @param ngasConnectTimeout NGAS Server Connect Timeout/2/Integer/ApplicationArgument/readonly//False/False/NGAS Server connection timeout
-# @param fileObject File Object//Object.File/InputPort/readwrite//False/False/Input File Object
+# @param dropclass dlg.apps.archiving.NgasArchivingApp/String/ComponentParameter/NoPort/ReadOnly//False/False/Application class
+# @param input_parser pickle/Select/ApplicationArgument/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
+# @param output_parser pickle/Select/ApplicationArgument/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
+# @param execution_time 5/Float/ComponentParameter/NoPort/ReadOnly//False/False/Estimated execution time
+# @param num_cpus 1/Integer/ComponentParameter/NoPort/ReadOnly//False/False/Number of cores used
+# @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group?
+# @param input_error_threshold 0/Integer/ComponentParameter/NoPort/ReadWrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed
+# @param n_tries 1/Integer/ComponentParameter/NoPort/ReadWrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up
+# @param ngasSrv localhost/String/ApplicationArgument/NoPort/ReadWrite//False/False/URL of the NGAS Server
+# @param ngasPort 7777/Integer/ApplicationArgument/NoPort/ReadWrite//False/False/"TCP/IP Port on the NGAS Server"
+# @param ngasMime "application/octet-stream"/String/ApplicationArgument/NoPort/ReadWrite//False/False/Mime-type of the NGAS payload
+# @param ngasTimeout 2/Integer/ApplicationArgument/NoPort/ReadOnly//False/False/Archiving request timeout
+# @param ngasConnectTimeout 2/Integer/ApplicationArgument/NoPort/ReadOnly//False/False/NGAS Server connection timeout
+# @param fileObject /Object.File/ApplicationArgument/InputPort/ReadWrite//False/False/Input File Object
 # @par EAGLE_END
 class NgasArchivingApp(ExternalStoreApp):
     """

daliuge-engine/dlg/apps/bash_shell_app.py

@@ -381,20 +381,20 @@ def execute(self, data):
 # @par EAGLE_START
 # @param category BashShellApp
 # @param tag template
-# @param command Command//String/ComponentParameter/readwrite//False/False/The command to be executed
-# @param input_redirection Input Redirection//String/ComponentParameter/readwrite//False/False/The command line argument that specifies the input into this application
-# @param output_redirection Output Redirection//String/ComponentParameter/readwrite//False/False/The command line argument that specifies the output from this application
-# @param command_line_arguments Command Line Arguments//String/ComponentParameter/readwrite//False/False/Additional command line arguments to be added to the command line to be executed
-# @param paramValueSeparator Param value separator/ /String/ComponentParameter/readwrite//False/False/Separator character(s) between parameters on the command line
-# @param argumentPrefix Argument prefix/"--"/String/ComponentParameter/readwrite//False/False/Prefix to each keyed argument on the command line
-# @param dropclass dropclass/dlg.apps.bash_shell_app.BashShellApp/String/ComponentParameter/readwrite//False/False/Drop class
-# @param input_parser Input Parser/pickle/Select/ApplicationArgument/readwrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
-# @param output_parser Output Parser/pickle/Select/ApplicationArgument/readwrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
-# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time
-# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used
-# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group?
-# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed
-# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up
+# @param command /String/ComponentParameter/NoPort/ReadWrite//False/False/The command to be executed
+# @param input_redirection /String/ComponentParameter/NoPort/ReadWrite//False/False/The command line argument that specifies the input into this application
+# @param output_redirection /String/ComponentParameter/NoPort/ReadWrite//False/False/The command line argument that specifies the output from this application
+# @param command_line_arguments /String/ComponentParameter/NoPort/ReadWrite//False/False/Additional command line arguments to be added to the command line to be executed
+# @param paramValueSeparator " "/String/ComponentParameter/NoPort/ReadWrite//False/False/Separator character(s) between parameters on the command line
+# @param argumentPrefix "--"/String/ComponentParameter/NoPort/ReadWrite//False/False/Prefix to each keyed argument on the command line
+# @param dropclass dlg.apps.bash_shell_app.BashShellApp/String/ComponentParameter/NoPort/ReadWrite//False/False/Drop class
+# @param input_parser pickle/Select/ApplicationArgument/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
+# @param output_parser pickle/Select/ApplicationArgument/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
+# @param execution_time 5/Float/ComponentParameter/NoPort/ReadOnly//False/False/Estimated execution time
+# @param num_cpus 1/Integer/ComponentParameter/NoPort/ReadOnly//False/False/Number of cores used
+# @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group?
+# @param input_error_threshold 0/Integer/ComponentParameter/NoPort/ReadWrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed
+# @param n_tries 1/Integer/ComponentParameter/NoPort/ReadWrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up
 # @par EAGLE_END
 class BashShellApp(BashShellBase, BarrierAppDROP):
     """

daliuge-engine/dlg/apps/branch.py

@@ -9,16 +9,17 @@
 # @par EAGLE_START
 # @param category Branch
 # @param tag template
-# @param dropclass Application Class/dlg.apps.simple.SimpleBranch/String/ComponentParameter/readonly//False/False/Application class
-# @param input_parser Input Parser/pickle/Select/ApplicationArgument/readwrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
-# @param output_parser Output Parser/pickle/Select/ApplicationArgument/readwrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
-# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time
-# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used
-# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group?
-# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed
-# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up
-# @param dummy0 dummy0//Object/OutputPort/readwrite//False/False/Dummy output port
-# @param dummy1 dummy1//Object/OutputPort/readwrite//False/False/Dummy output port
+# @param dropclass dlg.apps.simple.SimpleBranch/String/ComponentParameter/NoPort/ReadOnly//False/False/Application class
+# @param input_parser pickle/Select/ApplicationArgument/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
+# @param output_parser pickle/Select/ApplicationArgument/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
+# @param execution_time 5/Float/ComponentParameter/NoPort/ReadOnly//False/False/Estimated execution time
+# @param num_cpus 1/Integer/ComponentParameter/NoPort/ReadOnly//False/False/Number of cores used
+# @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group?
+# @param input_error_threshold 0/Integer/ComponentParameter/NoPort/ReadWrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed
+# @param n_tries 1/Integer/ComponentParameter/NoPort/ReadWrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up
+# @param dummy_input /Object/ApplicationArgument/InputPort/ReadWrite//False/False/Dummy input port
+# @param dummy0 /Object/ApplicationArgument/OutputPort/ReadWrite//False/False/Dummy output port
+# @param dummy1 /Object/ApplicationArgument/OutputPort/ReadWrite//False/False/Dummy output port
 # @par EAGLE_END
 class BranchAppDrop(BarrierAppDROP):
     """

daliuge-engine/dlg/apps/constructs.py

@@ -7,8 +7,8 @@
 # @par EAGLE_START
 # @param category Scatter
 # @param tag template
-# @param num_of_copies Scatter dimension/4/Integer/ComponentParameter/readwrite//False/False/Specifies the number of replications of the content of the scatter construct
-# @param dropclass dropclass/dlg.apps.constructs.ScatterDrop/String/ComponentParameter/readwrite//False/False/Drop class
+# @param num_of_copies 4/Integer/ConstructParameter/NoPort/ReadWrite//False/False/Specifies the number of replications of the content of the scatter construct
+# @param dropclass dlg.apps.constructs.ScatterDrop/String/ComponentParameter/NoPort/ReadWrite//False/False/Drop class
 # @par EAGLE_END
 class ScatterDrop(BarrierAppDROP):
     """
@@ -24,9 +24,9 @@ class ScatterDrop(BarrierAppDROP):
 # @par EAGLE_START
 # @param category Gather
 # @param tag template
-# @param num_of_inputs No. of inputs/2/Integer/ApplicationArgument/readwrite//False/False/Number of inputs
-# @param gather_axis Index of gather axis/0/Integer/ApplicationArgument/readwrite//False/False/Index of gather axis
-# @param dropclass dropclass/dlg.apps.constructs.GatherDrop/String/ComponentParameter/readwrite//False/False/Drop class
+# @param num_of_inputs 2/Integer/ConstructParameter/NoPort/ReadWrite//False/False/Number of inputs
+# @param gather_axis 0/Integer/ApplicationArgument/NoPort/ReadWrite//False/False/Index of gather axis
+# @param dropclass dlg.apps.constructs.GatherDrop/String/ComponentParameter/NoPort/ReadWrite//False/False/Drop class
 # @par EAGLE_END
 class GatherDrop(BarrierAppDROP):
     """
@@ -42,8 +42,8 @@ class GatherDrop(BarrierAppDROP):
 # @par EAGLE_START
 # @param category Loop
 # @param tag template
-# @param num_of_iter No. of iterations/2/Integer/ApplicationArguments/readwrite//False/False/Number of iterations
-# @param dropclass dropclass/dlg.apps.constructs.LoopDrop/String/ComponentParameter/readwrite//False/False/Drop class
+# @param num_of_iter 2/Integer/ConstructParameter/NoPort/ReadWrite//False/False/Number of iterations
+# @param dropclass dlg.apps.constructs.LoopDrop/String/ComponentParameter/NoPort/ReadWrite//False/False/Drop class
 # @par EAGLE_END
 class LoopDrop(BarrierAppDROP):
     """
@@ -59,8 +59,8 @@ class LoopDrop(BarrierAppDROP):
 # @par EAGLE_START
 # @param category MKN
 # @param tag template
-# @param k K/1/Integer/ApplicationArgument/readwrite//False/False/Internal multiplicity
-# @param dropclass dropclass/dlg.apps.constructs.MKNDrop/String/ComponentParameter/readwrite//False/False/Drop class
+# @param k 1/Integer/ConstructParameter/NoPort/ReadWrite//False/False/Internal multiplicity
+# @param dropclass dlg.apps.constructs.MKNDrop/String/ComponentParameter/NoPort/ReadWrite//False/False/Drop class
 # @par EAGLE_END
 class MKNDrop(BarrierAppDROP):
     """
@@ -76,9 +76,9 @@ class MKNDrop(BarrierAppDROP):
 # @par EAGLE_START
 # @param category GroupBy
 # @param tag template
-# @param num_of_inputs No. of inputs/2/Integer/ApplicationArgument/readwrite//False/False/Number of inputs
-# @param gather_axis Index of gather axis/0/Integer/ApplicationArgument/readwrite//False/False/Index of gather axis
-# @param dropclass dropclass/dlg.apps.constructs.GroupByDrop/String/ComponentParameter/readwrite//False/False/Drop class
+# @param num_of_inputs 2/Integer/ConstructParameter/NoPort/ReadWrite//False/False/Number of inputs
+# @param gather_axis 0/Integer/ApplicationArgument/NoPort/ReadWrite//False/False/Index of gather axis
+# @param dropclass dlg.apps.constructs.GroupByDrop/String/ComponentParameter/NoPort/ReadWrite//False/False/Drop class
 # @par EAGLE_END
 class GroupByDrop(BarrierAppDROP):
     """
@@ -93,7 +93,7 @@ class GroupByDrop(BarrierAppDROP):
 # @details A SubGraph template drop
 # @par EAGLE_START
 # @param category SubGraph
-# @param dropclass dropclass/dlg.apps.constructs.SubGraphDrop/String/ComponentParameter/readwrite//False/False/Drop class
+# @param dropclass dlg.apps.constructs.SubGraphDrop/String/ComponentParameter/NoPort/ReadWrite//False/False/Drop class
 # @param tag template
 # @par EAGLE_END
 class SubGraphDrop(BarrierAppDROP):
@@ -109,7 +109,7 @@ class SubGraphDrop(BarrierAppDROP):
 # @details A comment template drop
 # @par EAGLE_START
 # @param category Comment
-# @param dropclass dropclass/dlg.apps.constructs.CommentDrop/String/ComponentParameter/readwrite//False/False/Drop class
+# @param dropclass dlg.apps.constructs.CommentDrop/String/ComponentParameter/NoPort/ReadWrite//False/False/Drop class
 # @param tag template
 # @par EAGLE_END
 class CommentDrop(BarrierAppDROP):
@@ -125,7 +125,7 @@ class CommentDrop(BarrierAppDROP):
 # @details A description template drop
 # @par EAGLE_START
 # @param category Description
-# @param dropclass dropclass/dlg.apps.constructs.DescriptionDrop/String/ComponentParameter/readwrite//False/False/Drop class
+# @param dropclass dlg.apps.constructs.DescriptionDrop/String/ComponentParameter/NoPort/ReadWrite//False/False/Drop class
 # @param tag template
 # @par EAGLE_END
 class DescriptionDrop(BarrierAppDROP):
@@ -141,7 +141,7 @@ class DescriptionDrop(BarrierAppDROP):
 # @details An Exclusive Force Node
 # @par EAGLE_START
 # @param category ExclusiveForceNode
-# @param dropclass dropclass/dlg.apps.constructs.ExclusiveForceDrop/String/ComponentParameter/readwrite//False/False/Drop class
+# @param dropclass dlg.apps.constructs.ExclusiveForceDrop/String/ComponentParameter/NoPort/ReadWrite//False/False/Drop class
 # @param tag template
 # @par EAGLE_END
 class ExclusiveForceDrop(BarrierAppDROP):

daliuge-engine/dlg/apps/crc.py

@@ -86,15 +86,15 @@ def run(self):
 # @par EAGLE_START
 # @param category PythonApp
 # @param tag daliuge
-# @param dropclass Application Class/dlg.apps.crc.CRCStreamApp/String/ComponentParameter/readonly//False/False/Application class
-# @param input_parser Input Parser/pickle/Select/ApplicationArgument/readwrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
-# @param output_parser Output Parser/pickle/Select/ApplicationArgument/readwrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
-# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time
-# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used
-# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group?
-# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed
-# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up
-# @param data Data//String/OutputPort/readwrite//False/False/Input data stream
+# @param dropclass dlg.apps.crc.CRCStreamApp/String/ComponentParameter/NoPort/ReadOnly//False/False/Application class
+# @param input_parser pickle/Select/ApplicationArgument/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Input port parsing technique
+# @param output_parser pickle/Select/ApplicationArgument/NoPort/ReadWrite/raw,pickle,eval,npy,path,dataurl/False/False/Output port parsing technique
+# @param execution_time 5/Float/ComponentParameter/NoPort/ReadOnly//False/False/Estimated execution time
+# @param num_cpus 1/Integer/ComponentParameter/NoPort/ReadOnly//False/False/Number of cores used
+# @param group_start False/Boolean/ComponentParameter/NoPort/ReadWrite//False/False/Is this node the start of a group?
+# @param input_error_threshold 0/Integer/ComponentParameter/NoPort/ReadWrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed
+# @param n_tries 1/Integer/ComponentParameter/NoPort/ReadWrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up
+# @param data /String/ApplicationArgument/OutputPort/ReadWrite//False/False/Input data stream
 # @par EAGLE_END
 class CRCStreamApp(AppDROP):
     """