Merge branch 'master' into yan-893

# Conflicts:
#	daliuge-translator/dlg/dropmake/web/main.js
#	daliuge-translator/dlg/dropmake/web/pg_viewer.html

.travis.yml

@@ -36,7 +36,9 @@ matrix:
     # NOTE: The OpenAPI code still needs to be removed
     # - python: "3.8"
     #   env: TEST_OPENAPI=1
-    - python: "3.7"
+    # NOTE: commenting 3.7 for now.
+    # travis early build stage fails non-reproducible
+    # - python: "3.7"
     - name: doxygen
       before_install:
       install:

daliuge-engine/dlg/apps/archiving.py

@@ -82,27 +82,27 @@ def store(self, inputDrop):
 # @par EAGLE_START
 # @param category PythonApp
 # @param tag daliuge
-# @param[in] cparam/appclass Application class/dlg.apps.archiving.NgasArchivingApp/String/readonly/False/
+# @param[in] cparam/appclass Application class/dlg.apps.archiving.NgasArchivingApp/String/readonly/False//False/
 #     \~English Application class
-# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False/
+# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/
 #     \~English Estimated execution time
-# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False/
+# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/
 #     \~English Number of cores used
-# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False/
+# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/
 #     \~English Is this node the start of a group?
-# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False/
+# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/
 #     \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed
-# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False/
+# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/
 #     \~English Specifies the number of times the 'run' method will be executed before finally giving up
-# @param[in] aparam/ngasSrv NGAS Server URL/localhost/String/readwrite/False/
+# @param[in] aparam/ngasSrv NGAS Server URL/localhost/String/readwrite/False//False/
 #     \~English URL of the NGAS Server
-# @param[in] aparam/ngasPort NGAS Server Port/7777/Integer/readwrite/False/
+# @param[in] aparam/ngasPort NGAS Server Port/7777/Integer/readwrite/False//False/
 #     \~English TCP/IP Port on the NGAS Server
-# @param[in] aparam/ngasMime NGAS Mime Type/"application/octet-stream"/String/readwrite/False/
+# @param[in] aparam/ngasMime NGAS Mime Type/"application/octet-stream"/String/readwrite/False//False/
 #     \~English Mime-type of the NGAS payload
-# @param[in] aparam/ngasTimeout NGAS Server Timeout/2/Integer/readonly/False/
+# @param[in] aparam/ngasTimeout NGAS Server Timeout/2/Integer/readonly/False//False/
 #     \~English Archiving request timeout
-# @param[in] aparam/ngasConnectTimeout NGAS Server Connect Timeout/2/Integer/readonly/False/
+# @param[in] aparam/ngasConnectTimeout NGAS Server Connect Timeout/2/Integer/readonly/False//False/
 #     \~English NGAS Server connection timeout
 # @param[in] port/fileObject File Object/File/
 #     \~English Input File Object

daliuge-engine/dlg/apps/bash_shell_app.py

@@ -265,7 +265,7 @@ def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE):
             logger.error(mesage_stdouts(message, pstdout, pstderr))
             raise Exception(message)
 
-    def dataURL(self):
+    def dataURL(self) -> str:
         return type(self).__name__
 
     def cancel(self):
@@ -331,27 +331,27 @@ def execute(self, data):
 # @par EAGLE_START
 # @param category BashShellApp
 # @param tag template
-# @param[in] cparam/command Command//String/readwrite/False/
+# @param[in] cparam/command Command//String/readwrite/False//False/
 #     \~English The command to be executed
-# @param[in] cparam/input_redirection Input Redirection//String/readwrite/False/
+# @param[in] cparam/input_redirection Input Redirection//String/readwrite/False//False/
 #     \~English The command line argument that specifies the input into this application
-# @param[in] cparam/output_redirection Output Redirection//String/readwrite/False/
+# @param[in] cparam/output_redirection Output Redirection//String/readwrite/False//False/
 #     \~English The command line argument that specifies the output from this application
-# @param[in] cparam/command_line_arguments Command Line Arguments//String/readwrite/False/
+# @param[in] cparam/command_line_arguments Command Line Arguments//String/readwrite/False//False/
 #     \~English Additional command line arguments to be added to the command line to be executed
-# @param[in] cparam/paramValueSeparator Param value separator/ /String/readwrite/False/
+# @param[in] cparam/paramValueSeparator Param value separator/ /String/readwrite/False//False/
 #     \~English Separator character(s) between parameters on the command line
-# @param[in] cparam/argumentPrefix Argument prefix/"--"/String/readwrite/False/
+# @param[in] cparam/argumentPrefix Argument prefix/"--"/String/readwrite/False//False/
 #     \~English Prefix to each keyed argument on the command line
-# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False/
+# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/
 #     \~English Estimated execution time
-# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False/
+# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/
 #     \~English Number of cores used
-# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False/
+# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/
 #     \~English Is this node the start of a group?
-# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False/
+# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/
 #     \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed
-# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False/
+# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/
 #     \~English Specifies the number of times the 'run' method will be executed before finally giving up
 # @par EAGLE_END
 class BashShellApp(BashShellBase, BarrierAppDROP):

daliuge-engine/dlg/apps/crc.py

@@ -79,17 +79,17 @@ def run(self):
 # @par EAGLE_START
 # @param category PythonApp
 # @param tag daliuge
-# @param[in] cparam/appclass Application Class/dlg.apps.crc.CRCStreamApp/String/readonly/False/
+# @param[in] cparam/appclass Application Class/dlg.apps.crc.CRCStreamApp/String/readonly/False//False/
 #     \~English Application class
-# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False/
+# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/
 #     \~English Estimated execution time
-# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False/
+# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/
 #     \~English Number of cores used
-# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False/
+# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/
 #     \~English Is this node the start of a group?
-# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False/
+# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/
 #     \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed
-# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False/
+# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/
 #     \~English Specifies the number of times the 'run' method will be executed before finally giving up
 # @param[out] port/data Data/String/
 # @par EAGLE_END

daliuge-engine/dlg/apps/dockerapp.py

@@ -74,43 +74,43 @@ def waitForIp(self, timeout=None):
 # @par EAGLE_START
 # @param category Docker
 # @param tag template
-# @param[in] cparam/image Image//String/readwrite/False/
+# @param[in] cparam/image Image//String/readwrite/False//False/
 #     \~English The name of the docker image to be used for this application
-# @param[in] cparam/tag Tag/1.0/String/readwrite/False/
+# @param[in] cparam/tag Tag/1.0/String/readwrite/False//False/
 #     \~English The tag of the docker image to be used for this application
-# @param[in] cparam/digest Digest//String/readwrite/False/
+# @param[in] cparam/digest Digest//String/readwrite/False//False/
 #     \~English The hexadecimal hash (long version) of the docker image to be used for this application
-# @param[in] cparam/command Command//String/readwrite/False/
+# @param[in] cparam/command Command//String/readwrite/False//False/
 #     \~English The command line to run within the docker instance. The specified command will be executed in a bash shell. That means that images will need a bash shell.
-# @param[in] cparam/input_redirection Input Redirection//String/readwrite/False/
+# @param[in] cparam/input_redirection Input Redirection//String/readwrite/False//False/
 #     \~English The command line argument that specifies the input into this application
-# @param[in] cparam/output_redirection Output Redirection//String/readwrite/False/
+# @param[in] cparam/output_redirection Output Redirection//String/readwrite/False//False/
 #     \~English The command line argument that specifies the output from this application
-# @param[in] cparam/command_line_arguments Command Line Arguments//String/readwrite/False/
+# @param[in] cparam/command_line_arguments Command Line Arguments//String/readwrite/False//False/
 #     \~English Additional command line arguments to be added to the command line to be executed
-# @param[in] cparam/paramValueSeparator Param value separator/ /String/readwrite/False/
+# @param[in] cparam/paramValueSeparator Param value separator/ /String/readwrite/False//False/
 #     \~English Separator character(s) between parameters on the command line
-# @param[in] cparam/argumentPrefix Argument prefix/"--"/String/readwrite/False/
+# @param[in] cparam/argumentPrefix Argument prefix/"--"/String/readwrite/False//False/
 #     \~English Prefix to each keyed argument on the command line
-# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False/
+# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/
 #     \~English Estimated execution time
-# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False/
+# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/
 #     \~English Number of cores used
-# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False/
+# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/
 #     \~English Is this node the start of a group?
-# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False/
+# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/
 #     \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed
-# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False/
+# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/
 #     \~English Specifies the number of times the 'run' method will be executed before finally giving up
-# @param[in] cparam/user User//String/readwrite/False/
+# @param[in] cparam/user User//String/readwrite/False//False/
 #     \~English Username of the user who will run the application within the docker image
-# @param[in] cparam/ensureUserAndSwitch Ensure User And Switch/False/Boolean/readwrite/False/
+# @param[in] cparam/ensureUserAndSwitch Ensure User And Switch/False/Boolean/readwrite/False//False/
 #     \~English Make sure the user specified in the User parameter exists and then run the docker container as that user
-# @param[in] cparam/removeContainer Remove Container/True/Boolean/readwrite/False/
+# @param[in] cparam/removeContainer Remove Container/True/Boolean/readwrite/False//False/
 #     \~English Instruct Docker engine to delete the container after execution is complete
-# @param[in] cparam/additionalBindings Additional Bindings//String/readwrite/False/
+# @param[in] cparam/additionalBindings Additional Bindings//String/readwrite/False//False/
 #     \~English Directories which will be visible inside the container during run-time. Format is srcdir_on_host:trgtdir_on_container. Multiple entries can be separated by commas.
-# @param[in] cparam/portMappings Port Mappings//String/readwrite/False/
+# @param[in] cparam/portMappings Port Mappings//String/readwrite/False//False/
 #     \~English Port mappings on the host machine
 # @par EAGLE_END
 class DockerApp(BarrierAppDROP):
@@ -309,8 +309,10 @@ def initialize(self, **kwargs):
                     f"{utils.getDlgDir()}/workspace/settings/passwd:/etc/passwd",
                     f"{utils.getDlgDir()}/workspace/settings/group:/etc/group"
         ]
-        bindings += self._getArg(kwargs, "additionalBindings", [])
-        bindings = bindings.split(",") if isinstance(bindings, str) else bindings
+        additionalBindings = self._getArg(kwargs, "additionalBindings", [])
+        additionalBindings = additionalBindings.split(",") if isinstance(additionalBindings, str) \
+            else additionalBindings
+        bindings += additionalBindings
         for binding in bindings:
             if len(binding) == 0:
                 continue
@@ -345,7 +347,7 @@ def initialize(self, **kwargs):
             logger.debug("Image '%s' found, no need to pull it", self._image)
 
         # Check if the image specifies a working directory
-        # If it doesn't use the one provided by the user. 
+        # If it doesn't use the one provided by the user.
         # If none is provided use the session directory
         inspection = c.api.inspect_image(self._image)
         logger.debug("Docker Image inspection: %r", inspection)
@@ -498,7 +500,7 @@ def run(self):
                         addEnv[key] = value
                     env.update(addEnv)
                 elif isinstance(addEnv, list): # if it is a list populate from host environment
-                    for e in addEnv: 
+                    for e in addEnv:
                         env.update(os.environ[e])
             else:
                 logger.warning("Ignoring provided environment variables: Format wrong! Check documentation")

daliuge-engine/dlg/apps/dynlib.py

@@ -354,17 +354,17 @@ def addStreamingInput(self, streamingInputDrop, back=True):
 # @par EAGLE_START
 # @param category DynlibApp
 # @param tag template
-# @param[in] cparam/libpath Library Path//String/readwrite/False/
+# @param[in] cparam/libpath Library Path//String/readwrite/False//False/
 #     \~English The location of the shared object/DLL that implements this application
-# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False/
+# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/
 #     \~English Estimated execution time
-# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False/
+# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/
 #     \~English Number of cores used
-# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False/
+# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/
 #     \~English Is this node the start of a group?
-# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False/
+# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/
 #     \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed
-# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False/
+# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/
 #     \~English Specifies the number of times the 'run' method will be executed before finally giving up
 # @par EAGLE_END
 class DynlibApp(DynlibAppBase, BarrierAppDROP):

daliuge-engine/dlg/apps/mpi.py

@@ -38,29 +38,29 @@
 # @par EAGLE_START
 # @param category Mpi
 # @param tag template
-# @param[in] cparam/num_of_procs Num procs/1/Integer/readwrite/False/
+# @param[in] cparam/num_of_procs Num procs/1/Integer/readwrite/False//False/
 #     \~English Number of processes used for this application
-# @param[in] cparam/command Command//String/readwrite/False/
+# @param[in] cparam/command Command//String/readwrite/False//False/
 #     \~English The command to be executed
-# @param[in] cparam/input_redirection Input Redirection//String/readwrite/False/
+# @param[in] cparam/input_redirection Input Redirection//String/readwrite/False//False/
 #     \~English The command line argument that specifies the input into this application
-# @param[in] cparam/output_redirection Output Redirection//String/readwrite/False/
+# @param[in] cparam/output_redirection Output Redirection//String/readwrite/False//False/
 #     \~English The command line argument that specifies the output from this application
-# @param[in] cparam/command_line_arguments Command Line Arguments//String/readwrite/False/
+# @param[in] cparam/command_line_arguments Command Line Arguments//String/readwrite/False//False/
 #     \~English Additional command line arguments to be added to the command line to be executed
-# @param[in] cparam/paramValueSeparator Param value separator/ /String/readwrite/False/
+# @param[in] cparam/paramValueSeparator Param value separator/ /String/readwrite/False//False/
 #     \~English Separator character(s) between parameters on the command line
-# @param[in] cparam/argumentPrefix Argument prefix/"--"/String/readwrite/False/
+# @param[in] cparam/argumentPrefix Argument prefix/"--"/String/readwrite/False//False/
 #     \~English Prefix to each keyed argument on the command line
-# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False/
+# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/
 #     \~English Estimated execution time
-# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False/
+# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/
 #     \~English Number of cores used
-# @param[in] cparam/group_end Group end/False/Boolean/readwrite/False/
+# @param[in] cparam/group_end Group end/False/Boolean/readwrite/False//False/
 #     \~English Is this node the end of a group?
-# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False/
+# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/
 #     \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed
-# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False/
+# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/
 #     \~English Specifies the number of times the 'run' method will be executed before finally giving up
 # @par EAGLE_END
 class MPIApp(BarrierAppDROP):