diff --git a/daliuge-common/docker/Dockerfile.dev b/daliuge-common/docker/Dockerfile.dev index dd7c0a51b..d309e5ab5 100644 --- a/daliuge-common/docker/Dockerfile.dev +++ b/daliuge-common/docker/Dockerfile.dev @@ -7,18 +7,13 @@ FROM ubuntu:20.04 ARG BUILD_ID LABEL stage=builder LABEL build=$BUILD_ID -RUN apt-get update && apt-get install -y gcc python3 python3.8-venv && apt-get clean +RUN apt-get update && apt-get install -y gcc python3 python3.8-venv python3-pip python3-distutils libmetis-dev curl && apt-get clean COPY / /daliuge -RUN cd && python3 -m venv dlg && cd /daliuge && \ - . ${HOME}/dlg/bin/activate && \ - pip install numpy && \ - pip install . && \ - apt-get remove -y gcc && \ - apt-get autoremove -y +RUN cd / && python3 -m venv dlg && cd /daliuge && \ + . /dlg/bin/activate && \ + pip install wheel numpy && \ + pip install . - -FROM ubuntu:20.04 -RUN apt-get update && apt-get install -y bash -COPY --from=0 /root/dlg /root/dlg +# we don't clean this up, will be done in the derived images \ No newline at end of file diff --git a/daliuge-engine/dlg/apps/bash_shell_app.py b/daliuge-engine/dlg/apps/bash_shell_app.py index b28bc3da4..82998be6d 100644 --- a/daliuge-engine/dlg/apps/bash_shell_app.py +++ b/daliuge-engine/dlg/apps/bash_shell_app.py @@ -157,17 +157,26 @@ class BashShellBase(object): Common class for BashShell apps. It simply requires a command to be specified. """ - + #TODO: use the shlex module for most of the construction of the + # command line to get a proper and safe shell syntax command = dlg_string_param("Bash command", None) def initialize(self, **kwargs): super(BashShellBase, self).initialize(**kwargs) self.proc = None + self._inputRedirect = self._getArg(kwargs, "input_redirection", "") + self._outputRedirect = self._getArg(kwargs, "output_redirection", "") + self._cmdLineArgs = self._getArg(kwargs, "command_line_arguments", "") + self._applicationParams = self._getArg(kwargs, "applicationParams", {}) + self._argumentPrefix = self._getArg(kwargs, "argumentPrefix", "--") + if not self.command: - raise InvalidDropException( - self, "No command specified, cannot create BashShellApp" - ) + self.command = self._getArg(kwargs, "command", None) + if not self.command: + raise InvalidDropException( + self, "No command specified, cannot create BashShellApp" + ) def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE): """ @@ -186,7 +195,16 @@ def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE): session_id = ( self._dlg_session.sessionId if self._dlg_session is not None else "" ) - cmd = self.command + argumentString = droputils.serialize_applicationParams(self._applicationParams, \ + self._argumentPrefix) + # complete command including all additional parameters and optional redirects + cmd = f"{self.command} {self._cmdLineArgs} {argumentString}" + if self._outputRedirect: + cmd = f"{cmd} > {self._outputRedirect}" + if self._inputRedirect: + cmd = f"cat {self._inputRedirect} > {cmd}" + cmd = cmd.strip() + app_uid = self.uid # self.run_bash(self._command, self.uid, session_id, *args, **kwargs) @@ -210,7 +228,7 @@ def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE): # Wrap everything inside bash cmd = ("/bin/bash", "-c", cmd) - logger.debug("Command after user creation and wrapping is: %s", cmd) + logger.debug("Command after wrapping is: %s", cmd) start = time.time() diff --git a/daliuge-engine/dlg/apps/dockerapp.py b/daliuge-engine/dlg/apps/dockerapp.py index bec25b7bd..7c1ff5726 100644 --- a/daliuge-engine/dlg/apps/dockerapp.py +++ b/daliuge-engine/dlg/apps/dockerapp.py @@ -204,17 +204,28 @@ def initialize(self, **kwargs): ) self._command = self._getArg(kwargs, "command", None) + if not self._command: logger.warning( "No command specified. Assume that a default command is executed in the container" ) + # The above also means that we can't pass applicationParams # raise InvalidDropException( # self, "No command specified, cannot create DockerApp") + else: + self._applicationParams = self._getArg(kwargs, "applicationParams", {}) + + # construct the actual command line from all application parameters + argumentPrefix = self._getArg(kwargs, "argumentPrefix", "--") + argumentString = droputils.serialize_applicationParams(self._applicationParams, \ + argumentPrefix) + self._command = f"{self._command} {argumentString}" # The user used to run the process in the docker container # By default docker containers run as root, but we don't want to run # a process using a different user because otherwise anything that that # process writes to the filesystem + # TODO: User switching should be changed to be transparent self._user = self._getArg(kwargs, "user", None) # In some cases we want to make sure the command in the container runs diff --git a/daliuge-engine/dlg/droputils.py b/daliuge-engine/dlg/droputils.py index 8cfbf91c6..4a3ab701b 100644 --- a/daliuge-engine/dlg/droputils.py +++ b/daliuge-engine/dlg/droputils.py @@ -446,6 +446,30 @@ def replace_dataurl_placeholders(cmd, inputs, outputs): return cmd +def serialize_applicationParams(applicationParams, prefix='--'): + """ + Unpacks the applicationParams dictionary and returns a string + that can be used as command line parameters. + """ + if not isinstance(applicationParams, dict): + logger.info("applicationParams are not passed as a dict. Ignored!") + # construct the actual command line from all application parameters + args = [] + + for (name, value) in applicationParams.items(): + if value in [None, False, ""]: + continue + elif value is True: + value = '' + # short and long version of keywords + if prefix == "--" and len(name) == 1: + arg = [f'-{name} {value}'] + else: + arg = [f'{prefix}{name} {value}'] + args += arg.strip() # remove unneccesary blanks + + return f"{' '.join(args)}" + # Easing the transition from single- to multi-package get_leaves = common.get_leaves diff --git a/daliuge-engine/docker/Dockerfile b/daliuge-engine/docker/Dockerfile index 5b619bfc0..87f42cd55 100644 --- a/daliuge-engine/docker/Dockerfile +++ b/daliuge-engine/docker/Dockerfile @@ -22,6 +22,6 @@ EXPOSE 8002 # enable the virtualenv path from daliuge-common ENV VIRTUAL_ENV=/home/ray/dlg ENV PATH="$VIRTUAL_ENV/bin:$PATH" -ENV DLG_ROOT="/tmp/dlg/var/dlg_home" +ENV DLG_ROOT="/var/dlg_home" CMD ["dlg", "daemon", "-vv"] \ No newline at end of file diff --git a/daliuge-engine/docker/Dockerfile.dev b/daliuge-engine/docker/Dockerfile.dev index 4ad67b797..5f44af744 100644 --- a/daliuge-engine/docker/Dockerfile.dev +++ b/daliuge-engine/docker/Dockerfile.dev @@ -5,11 +5,8 @@ FROM icrar/daliuge-common:${VCS_TAG:-latest} # RUN sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata \ # gcc g++ gdb casacore-dev clang-tidy-10 clang-tidy libboost1.71-all-dev libgsl-dev -RUN apt-get update &&\ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gcc python3-pip curl - COPY / /daliuge -RUN . /root/dlg/bin/activate && pip install wheel && cd /daliuge && \ +RUN . /dlg/bin/activate && pip install wheel && cd /daliuge && \ pip install . EXPOSE 9000 @@ -20,8 +17,8 @@ EXPOSE 8001 EXPOSE 8002 # enable the virtualenv path from daliuge-common -ENV VIRTUAL_ENV=/root/dlg +ENV VIRTUAL_ENV=/dlg ENV PATH="$VIRTUAL_ENV/bin:$PATH" -ENV DLG_ROOT="/tmp/dlg/var/dlg_home" +ENV DLG_ROOT="/tmp/dlg" CMD ["dlg", "daemon", "-vv"] \ No newline at end of file diff --git a/daliuge-engine/docker/group.template b/daliuge-engine/docker/group.template new file mode 100644 index 000000000..135ac16a3 --- /dev/null +++ b/daliuge-engine/docker/group.template @@ -0,0 +1,42 @@ +root:x:0: +daemon:x:1: +bin:x:2: +sys:x:3: +adm:x:4: +tty:x:5: +disk:x:6: +lp:x:7: +mail:x:8: +news:x:9: +uucp:x:10: +man:x:12: +proxy:x:13: +kmem:x:15: +dialout:x:20: +fax:x:21: +voice:x:22: +cdrom:x:24: +floppy:x:25: +tape:x:26: +sudo:x:27: +audio:x:29: +dip:x:30: +www-data:x:33: +backup:x:34: +operator:x:37: +list:x:38: +irc:x:39: +src:x:40: +gnats:x:41: +shadow:x:42: +utmp:x:43: +video:x:44: +sasl:x:45: +plugdev:x:46: +staff:x:50: +games:x:60: +users:x:100: +nogroup:x:65534: +crontab:x:101: +messagebus:x:102: +ssh:x:103: diff --git a/daliuge-engine/docker/passwd.template b/daliuge-engine/docker/passwd.template new file mode 100644 index 000000000..c9043069a --- /dev/null +++ b/daliuge-engine/docker/passwd.template @@ -0,0 +1,20 @@ +root:x:0:0:root:/root:/bin/bash +daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin +bin:x:2:2:bin:/bin:/usr/sbin/nologin +sys:x:3:3:sys:/dev:/usr/sbin/nologin +sync:x:4:65534:sync:/bin:/bin/sync +games:x:5:60:games:/usr/games:/usr/sbin/nologin +man:x:6:12:man:/var/cache/man:/usr/sbin/nologin +lp:x:7:7:lp:/var/spool/lpd:/usr/sbin/nologin +mail:x:8:8:mail:/var/mail:/usr/sbin/nologin +news:x:9:9:news:/var/spool/news:/usr/sbin/nologin +uucp:x:10:10:uucp:/var/spool/uucp:/usr/sbin/nologin +proxy:x:13:13:proxy:/bin:/usr/sbin/nologin +www-data:x:33:33:www-data:/var/www:/usr/sbin/nologin +backup:x:34:34:backup:/var/backups:/usr/sbin/nologin +list:x:38:38:Mailing List Manager:/var/list:/usr/sbin/nologin +irc:x:39:39:ircd:/var/run/ircd:/usr/sbin/nologin +gnats:x:41:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/usr/sbin/nologin +nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin +_apt:x:100:65534::/nonexistent:/usr/sbin/nologin +messagebus:x:101:102::/nonexistent:/usr/sbin/nologin diff --git a/daliuge-engine/docker/prepare_user.py b/daliuge-engine/docker/prepare_user.py new file mode 100644 index 000000000..72cb8e674 --- /dev/null +++ b/daliuge-engine/docker/prepare_user.py @@ -0,0 +1,48 @@ +# +# ICRAR - International Centre for Radio Astronomy Research +# (c) UWA - The University of Western Australia, 2014 +# Copyright by UWA (in the framework of the ICRAR) +# All rights reserved +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307 USA +# +""" +Script to generate passwd and group files for docker containers to mount. +This will make sure that the apps in the containers are running as the +current user and thus the generated files have the correct owner. + +Inspired by Stimela code +""" +import pwd, grp, os + +workdir = f"{os.environ['DLG_ROOT']}/workspace/settings" +try: + os.mkdir(workdir) +except FileExistsError: + pass +except: + raise +template_dir = os.path.join(os.path.dirname(__file__), ".") +# get current user info +pw = pwd.getpwuid(os.getuid()) +gr = grp.getgrgid(pw.pw_gid) +with open(os.path.join(workdir, "passwd"), "wt") as file: + file.write(open(os.path.join(template_dir, "passwd.template"), "rt").read()) + file.write(f"{pw.pw_name}:x:{pw.pw_uid}:{pw.pw_gid}:{pw.pw_gecos}:/:/bin/bash") +with open(os.path.join(workdir, "group"), "wt") as file: + file.write(open(os.path.join(template_dir, "group.template"), "rt").read()) + file.write(f"{gr.gr_name}:x:{gr.gr_gid}:") + diff --git a/daliuge-engine/run_engine.sh b/daliuge-engine/run_engine.sh index 082a3d1bf..8841971cc 100755 --- a/daliuge-engine/run_engine.sh +++ b/daliuge-engine/run_engine.sh @@ -6,7 +6,20 @@ DOCKER_OPTS="\ -p 5555:5555 -p 6666:6666 \ -p 8000:8000 -p 8001:8001 \ -p 8002:8002 -p 9000:9000 \ +--user $(id -u):$(id -g) \ " +common_prep () +{ + mkdir -p ${DLG_ROOT}/workspace + mkdir -p ${DLG_ROOT}/testdata + mkdir -p ${DLG_ROOT}/code + # get current user and group id and prepare passwd and group files + python docker/prepare_user.py + DOCKER_OPTS=${DOCKER_OPTS}" -v ${DLG_ROOT}/workspace/settings/passwd:/etc/passwd" + DOCKER_OPTS=${DOCKER_OPTS}" -v ${DLG_ROOT}/workspace/settings/group:/etc/group" + DOCKER_OPTS=${DOCKER_OPTS}" -v ${PWD}/dlg/manager:/dlg/lib/python3.8/site-packages/dlg/manager" + DOCKER_OPTS=${DOCKER_OPTS}" -v ${DLG_ROOT}:${DLG_ROOT} --env DLG_ROOT=${DLG_ROOT}" +} case "$1" in "dep") @@ -17,24 +30,19 @@ case "$1" in echo "Please either create and grant access to $USER or build and run the development version." else VCS_TAG=`git describe --tags --abbrev=0|sed s/v//` - DOCKER_OPTS=${DOCKER_OPTS}"-v ${DLG_ROOT}:${DLG_ROOT} --env DLG_ROOT=${DLG_ROOT} " + common_prep() echo "Running Engine deployment version in background..." echo "docker run -td "${DOCKER_OPTS}" icrar/daliuge-engine:${VCS_TAG}" docker run -td ${DOCKER_OPTS} icrar/daliuge-engine:${VCS_TAG} exit 0 fi;; "dev") - DLG_ROOT="/tmp/dlg" + export DLG_ROOT="/tmp/dlg" export VCS_TAG=`git rev-parse --abbrev-ref HEAD | tr '[:upper:]' '[:lower:]'` + common_prep echo "Running Engine development version in background..." - mkdir -p ${DLG_ROOT}/workspace - mkdir -p ${DLG_ROOT}/testdata - mkdir -p ${DLG_ROOT}/code - DOCKER_OPTS=${DOCKER_OPTS}"-v ${PWD}/dlg/manager:/root/dlg/lib/python3.8/site-packages/dlg/manager" - DOCKER_OPTS=${DOCKER_OPTS}" -v ${DLG_ROOT}:${DLG_ROOT} --env DLG_ROOT=${DLG_ROOT}" echo "docker run -td ${DOCKER_OPTS} icrar/daliuge-engine:${VCS_TAG}" docker run -td ${DOCKER_OPTS} icrar/daliuge-engine:${VCS_TAG} -# docker run -td ${DOCKER_OPTS} icrar/dlg-engine:casa sleep 3 ./start_local_managers.sh exit 0;; @@ -42,11 +50,7 @@ case "$1" in DLG_ROOT="/tmp/dlg" export VCS_TAG=`git rev-parse --abbrev-ref HEAD | tr '[:upper:]' '[:lower:]'` echo "Running Engine development version in background..." - mkdir -p ${DLG_ROOT}/workspace - mkdir -p ${DLG_ROOT}/testdata - mkdir -p ${DLG_ROOT}/code - DOCKER_OPTS=${DOCKER_OPTS}"-v ${PWD}/dlg/manager:/root/dlg/lib/python3.8/site-packages/dlg/manager" - DOCKER_OPTS=${DOCKER_OPTS}" -v ${DLG_ROOT}:${DLG_ROOT} --env DLG_ROOT=${DLG_ROOT}" + common_prep CONTAINER_NM="icrar/daliuge-engine:${VCS_TAG}-casa" echo "docker run -td ${DOCKER_OPTS} ${CONTAINER_NM}" docker run -td ${DOCKER_OPTS} ${CONTAINER_NM} diff --git a/daliuge-translator/dlg/dropmake/pg_generator.py b/daliuge-translator/dlg/dropmake/pg_generator.py index 18f6c1ec1..05cad9102 100644 --- a/daliuge-translator/dlg/dropmake/pg_generator.py +++ b/daliuge-translator/dlg/dropmake/pg_generator.py @@ -542,12 +542,19 @@ def make_oid(self, iid="0"): return "{0}_{1}_{2}".format(self._ssid, self.id, iid), rank def _update_key_value_attributes(self, kwargs): + # NOTE: We should really just pass all of these on un-altered and finally drop + # support for the Arg%02d arguments. # get the arguments from new fields dictionary in a backwards compatible way if "fields" in self.jd: for je in self.jd["fields"]: # The field to be used is not the text, but the name field self.jd[je["name"]] = je["value"] kwargs[je["name"]] = je["value"] + kwargs["applicationParams"] = {} # make sure the dict always exists downstream + if "applicationParams" in self.jd: # and fill it if provided + for je in self.jd["applicationParams"]: + self.jd[je["name"]] = je["value"] + kwargs["applicationParams"][je["name"]] = je["value"] for i in range(10): k = "Arg%02d" % (i + 1) if k not in self.jd: @@ -624,13 +631,43 @@ def _create_test_drop_spec(self, oid, rank, kwargs) -> dropdict: kwargs["filepath"] = fp self._update_key_value_attributes(kwargs) drop_spec.update(kwargs) - elif drop_type in [Categories.COMPONENT, Categories.PYTHON_APP, Categories.BRANCH]: + elif drop_type in [Categories.COMPONENT, Categories.PYTHON_APP, Categories.BRANCH, Categories.DOCKER]: # default generic component becomes "sleep and copy" - if "appclass" not in self.jd or len(self.jd["appclass"]) == 0: - app_class = "dlg.apps.simple.SleepApp" + if drop_type not in [Categories.DOCKER]: + if "appclass" not in self.jd or len(self.jd["appclass"]) == 0: + app_class = "dlg.apps.simple.SleepApp" + else: + app_class = self.jd["appclass"] else: - app_class = self.jd["appclass"] - + # deal with the Docker specific component params + app_class = "dlg.apps.dockerapp.DockerApp" + typ = DropType.APP + image = str(self.jd.get("image")) + if image == "": + raise GraphException("Missing image for Docker component '%s'" % self.text) + + command = str(self.jd.get("command")) + # There ARE containers which don't need/want a command + # if command == "": + # raise GraphException("Missing command for Construct '%s'" % self.text) + + kwargs["image"] = image + kwargs["command"] = command + # TODO: User inside docker should follow user of engine. + kwargs["user"] = str(self.jd.get("user", "")) + kwargs["ensureUserAndSwitch"] = self.str_to_bool( + str(self.jd.get("ensureUserAndSwitch", "0")) + ) + kwargs["removeContainer"] = self.str_to_bool( + str(self.jd.get("removeContainer", "1")) + ) + kwargs["additionalBindings"] = str(self.jd.get("additionalBindings", "")) + if kwargs["additionalBindings"]: + kwargs["additionalBindings"] += "," + # always mount DLG_ROOT directory. ENV variable is only known in engine + kwargs["additionalBindings"] += "${DLG_ROOT}:${DLG_ROOT}" + kwargs["portMappings"] = str(self.jd.get("portMappings", "")) + kwargs["shmSize"] = str(self.jd.get("shmSize","")) if "execution_time" in self.jd: execTime = int(self.jd["execution_time"]) if execTime < 0: @@ -656,7 +693,7 @@ def _create_test_drop_spec(self, oid, rank, kwargs) -> dropdict: kwargs["num_cpus"] = int(self.jd.get("num_cpus", 1)) if "mkn" in self.jd: kwargs["mkn"] = self.jd["mkn"] - self._update_key_value_attributes(kwargs) + self._update_key_value_attributes(kwargs) # pass on all other kw-value pairs drop_spec.update(kwargs) elif drop_type in [Categories.DYNLIB_APP, Categories.DYNLIB_PROC_APP]: @@ -696,61 +733,44 @@ def _create_test_drop_spec(self, oid, rank, kwargs) -> dropdict: ) # add more arguments cmds = [] - for i in range(10): - k = "Arg%02d" % (i + 1,) - if k not in self.jd: - k = "arg%02d" % (i + 1,) - if k not in self.jd: - continue - v = self.jd[k] - if v is not None and len(str(v)) > 0: - cmds.append(str(v)) - # add more arguments - this is the new method of adding arguments in EAGLE - # the method above (Arg**) is retained for compatibility, but eventually should be removed - for k in [ - "command", - "input_redirection", - "output_redirection", - "command_line_arguments", - ]: - if k in self.jd: - cmds.append(self.jd[k]) - # kwargs['command'] = ' '.join(cmds) - kwargs["command"] = BashCommand(cmds) + if "command" in self.jd: + cmds = [self.jd["command"]] + self._update_key_value_attributes(kwargs) # get all the other params + kwargs["command"] = BashCommand(cmds) # NOTE: Not really required anymore? kwargs["num_cpus"] = int(self.jd.get("num_cpus", 1)) drop_spec.update(kwargs) - elif drop_type == Categories.DOCKER: - # Docker application. - app_class = "dlg.apps.dockerapp.DockerApp" - typ = DropType.APP - drop_spec = dropdict( - {"oid": oid, "type": typ, "app": app_class, "rank": rank} - ) - - image = str(self.jd.get("image")) - if image == "": - raise GraphException("Missing image for Construct '%s'" % self.text) - - command = str(self.jd.get("command")) - # There ARE containers which don't need/want a command - # if command == "": - # raise GraphException("Missing command for Construct '%s'" % self.text) - - kwargs["tw"] = int(self.jd.get("execution_time", "5")) - kwargs["image"] = image - kwargs["command"] = command - kwargs["user"] = str(self.jd.get("user", "")) - kwargs["ensureUserAndSwitch"] = self.str_to_bool( - str(self.jd.get("ensureUserAndSwitch", "0")) - ) - kwargs["removeContainer"] = self.str_to_bool( - str(self.jd.get("removeContainer", "1")) - ) - kwargs["additionalBindings"] = str(self.jd.get("additionalBindings", "")) - kwargs["portMappings"] = str(self.jd.get("portMappings", "")) - kwargs["shmSize"] = str(self.jd.get("shmSize","")) - drop_spec.update(kwargs) + # elif drop_type == Categories.DOCKER: + # # Docker application. + # app_class = "dlg.apps.dockerapp.DockerApp" + # typ = DropType.APP + # drop_spec = dropdict( + # {"oid": oid, "type": typ, "app": app_class, "rank": rank} + # ) + + # image = str(self.jd.get("image")) + # if image == "": + # raise GraphException("Missing image for Construct '%s'" % self.text) + + # command = str(self.jd.get("command")) + # # There ARE containers which don't need/want a command + # # if command == "": + # # raise GraphException("Missing command for Construct '%s'" % self.text) + + # kwargs["tw"] = int(self.jd.get("execution_time", "5")) + # kwargs["image"] = image + # kwargs["command"] = command + # kwargs["user"] = str(self.jd.get("user", "")) + # kwargs["ensureUserAndSwitch"] = self.str_to_bool( + # str(self.jd.get("ensureUserAndSwitch", "0")) + # ) + # kwargs["removeContainer"] = self.str_to_bool( + # str(self.jd.get("removeContainer", "1")) + # ) + # kwargs["additionalBindings"] = str(self.jd.get("additionalBindings", "")) + # kwargs["portMappings"] = str(self.jd.get("portMappings", "")) + # kwargs["shmSize"] = str(self.jd.get("shmSize","")) + # drop_spec.update(kwargs) elif drop_type == Categories.GROUP_BY: drop_spec = dropdict( diff --git a/daliuge-translator/docker/Dockerfile.dev b/daliuge-translator/docker/Dockerfile.dev index d5bc0be59..e081213c0 100644 --- a/daliuge-translator/docker/Dockerfile.dev +++ b/daliuge-translator/docker/Dockerfile.dev @@ -7,13 +7,14 @@ ARG BUILD_ID FROM icrar/daliuge-common:${VCS_TAG} LABEL stage=builder LABEL build=$BUILD_ID -RUN apt-get update && \ - apt-get clean && \ - apt install -y gcc python3-venv python3-distutils +# all dependencies are already installed in daliuge-common +# RUN apt-get update && \ +# apt-get clean && \ +# apt install -y gcc python3-venv python3-distutils COPY / /daliuge -RUN . /root/dlg/bin/activate && \ +RUN . /dlg/bin/activate && \ cd /daliuge && \ pip3 install wheel && \ pip3 install . @@ -21,11 +22,11 @@ RUN . /root/dlg/bin/activate && \ # Second stage build taking what's required from first stage FROM icrar/daliuge-common:${VCS_TAG} COPY --from=0 /daliuge/. /daliuge/. -COPY --from=0 /root/dlg /root/dlg -RUN apt-get update && apt-get install -y libmetis-dev python3 +COPY --from=0 /dlg /dlg +# RUN apt-get update && apt-get install -y libmetis-dev python3 # enable the virtualenv path from daliuge-common -ENV VIRTUAL_ENV=/root/dlg +ENV VIRTUAL_ENV=/dlg ENV PATH="$VIRTUAL_ENV/bin:$PATH" EXPOSE 8084 diff --git a/daliuge-translator/run_translator.sh b/daliuge-translator/run_translator.sh index 0eca0ea05..c79557e9b 100755 --- a/daliuge-translator/run_translator.sh +++ b/daliuge-translator/run_translator.sh @@ -7,12 +7,12 @@ case "$1" in "dev") export VCS_TAG=`git rev-parse --abbrev-ref HEAD| tr '[:upper:]' '[:lower:]'` echo "Running Translator development version in foreground..." - docker run --volume $PWD/dlg/dropmake:/root/dlg/lib/python3.8/site-packages/dlg/dropmake --name daliuge-translator --rm -t -p 8084:8084 icrar/daliuge-translator:${VCS_TAG} + docker run --volume $PWD/dlg/dropmake:/dlg/lib/python3.8/site-packages/dlg/dropmake --name daliuge-translator --rm -t -p 8084:8084 icrar/daliuge-translator:${VCS_TAG} exit 0;; "casa") export VCS_TAG=`git rev-parse --abbrev-ref HEAD| tr '[:upper:]' '[:lower:]'`-casa echo "Running Translator development version in foreground..." - docker run --volume $PWD/dlg/dropmake:/root/dlg/lib/python3.8/site-packages/dlg/dropmake --name daliuge-translator --rm -t -p 8084:8084 icrar/daliuge-translator:${VCS_TAG} + docker run --volume $PWD/dlg/dropmake:/dlg/lib/python3.8/site-packages/dlg/dropmake --name daliuge-translator --rm -t -p 8084:8084 icrar/daliuge-translator:${VCS_TAG} exit 0;; *) echo "Usage run_translator.sh " diff --git a/daliuge-translator/test/dropmake/logical_graphs/test-20190830-110556.graph b/daliuge-translator/test/dropmake/logical_graphs/test-20190830-110556.graph index e6f6d80be..be1e85b34 100644 --- a/daliuge-translator/test/dropmake/logical_graphs/test-20190830-110556.graph +++ b/daliuge-translator/test/dropmake/logical_graphs/test-20190830-110556.graph @@ -1,415 +1,320 @@ { + "linkDataArray": [ + { + "from": -2, + "fromPort": "47c421b8-5cdc-4ff7-ab7e-b75140f2d951", + "loop_aware": "0", + "to": -4, + "toPort": "c05689f4-6c5a-47dc-b3b1-fcbdfa09e4df" + } + ], "modelData": { - "fileType": "", - "repoService": "GitHub", + "eagleCommitHash": "dce847d911db5b8d2245775bb6d5c719eb4aa061", + "eagleVersion": "v4.1.0", + "filePath": "test-20190830-110556.graph", + "fileType": "Graph", + "gitUrl": "", + "lastModifiedDatetime": 0, + "lastModifiedEmail": "", + "lastModifiedName": "", + "readonly": true, + "repo": "", "repoBranch": "", - "repo": "james-strauss-uwa/eagle-test", - "filePath": "test/test-20190830-110556.graph", - "sha": "", - "git_url": "" + "repoService": "Unknown", + "schemaVersion": "OJS", + "sha": "" }, "nodeDataArray": [ { - "category": "Memory", - "categoryType": "Data", - "isData": true, - "isGroup": false, - "canHaveInputs": true, - "canHaveOutputs": true, - "color": "#394BB2", - "drawOrderHint": 0, - "key": -1, - "text": "Enter label", - "description": "", - "x": 200, - "y": 100, - "width": 200, - "height": 200, - "collapsed": false, - "showPorts": false, - "streaming": false, - "subject": null, - "selected": false, - "expanded": false, - "inputApplicationName": "", - "outputApplicationName": "", - "exitApplicationName": "", - "inputApplicationType": "None", - "outputApplicationType": "None", - "exitApplicationType": "None", - "inputPorts": [], - "outputPorts": [ - { - "Id": "ab5ada14-04d7-4b03-816d-c43428d4e2e4", - "IdText": "event" - } - ], - "inputLocalPorts": [], - "outputLocalPorts": [], - "inputAppFields": [], - "outputAppFields": [], - "fields": [ - { - "text": "Data volume", - "name": "data_volume", - "value": "5", - "description": "" - }, - { - "text": "Group end", - "name": "group_end", - "value": "0", - "description": "" - } - ] - }, - { + "applicationParams": [], "category": "BashShellApp", - "categoryType": "Application", - "isData": false, - "isGroup": false, - "canHaveInputs": true, - "canHaveOutputs": true, - "color": "#1C2833", - "drawOrderHint": 0, - "key": -2, - "text": "Enter label", + "collapsed": true, + "color": "#0059a5", "description": "", - "x": 400, - "y": 200, - "width": 200, - "height": 200, - "collapsed": false, - "showPorts": false, - "streaming": false, - "subject": null, - "selected": false, + "drawOrderHint": 0, "expanded": false, - "inputApplicationName": "", - "outputApplicationName": "", - "exitApplicationName": "", - "inputApplicationType": "None", - "outputApplicationType": "None", - "exitApplicationType": "None", - "inputPorts": [ - { - "Id": "c12aa833-43a9-4c1e-abaa-c77396010a31", - "IdText": "event" - } - ], - "outputPorts": [ - { - "Id": "47c421b8-5cdc-4ff7-ab7e-b75140f2d951", - "IdText": "event" - } - ], - "inputLocalPorts": [], - "outputLocalPorts": [], - "inputAppFields": [], - "outputAppFields": [], "fields": [ { - "text": "Execution time", + "defaultValue": "", + "description": "", "name": "execution_time", - "value": "5", - "description": "" + "precious": false, + "readonly": false, + "text": "Execution time", + "type": "Unknown", + "value": "5" }, { - "text": "Num CPUs", + "defaultValue": "", + "description": "", "name": "num_cpus", - "value": "1", - "description": "" + "precious": false, + "readonly": false, + "text": "Num CPUs", + "type": "Unknown", + "value": "1" }, { - "text": "Group start", + "defaultValue": "", + "description": "", "name": "group_start", - "value": "0", - "description": "" + "precious": false, + "readonly": false, + "text": "Group start", + "type": "Unknown", + "value": "0" }, { - "text": "Arg01", + "defaultValue": "", + "description": "", "name": "Arg01", - "value": "", - "description": "" + "precious": false, + "readonly": false, + "text": "Arg01", + "type": "Unknown", + "value": "" }, { - "text": "Arg02", + "defaultValue": "", + "description": "", "name": "Arg02", - "value": "", - "description": "" + "precious": false, + "readonly": false, + "text": "Arg02", + "type": "Unknown", + "value": "" }, { - "text": "Arg03", + "defaultValue": "", + "description": "", "name": "Arg03", - "value": "", - "description": "" + "precious": false, + "readonly": false, + "text": "Arg03", + "type": "Unknown", + "value": "" }, { - "text": "Arg04", + "defaultValue": "", + "description": "", "name": "Arg04", - "value": "", - "description": "" + "precious": false, + "readonly": false, + "text": "Arg04", + "type": "Unknown", + "value": "" }, { - "text": "Arg05", + "defaultValue": "", + "description": "", "name": "Arg05", - "value": "", - "description": "" + "precious": false, + "readonly": false, + "text": "Arg05", + "type": "Unknown", + "value": "" }, { - "text": "Arg06", + "defaultValue": "", + "description": "", "name": "Arg06", - "value": "", - "description": "" + "precious": false, + "readonly": false, + "text": "Arg06", + "type": "Unknown", + "value": "" }, { - "text": "Arg07", + "defaultValue": "", + "description": "", "name": "Arg07", - "value": "", - "description": "" + "precious": false, + "readonly": false, + "text": "Arg07", + "type": "Unknown", + "value": "" }, { - "text": "Arg08", + "defaultValue": "", + "description": "", "name": "Arg08", - "value": "", - "description": "" + "precious": false, + "readonly": false, + "text": "Arg08", + "type": "Unknown", + "value": "" }, { - "text": "Arg09", + "defaultValue": "", + "description": "", "name": "Arg09", - "value": "", - "description": "" + "precious": false, + "readonly": false, + "text": "Arg09", + "type": "Unknown", + "value": "" }, { - "text": "Arg10", + "defaultValue": "", + "description": "", "name": "Arg10", - "value": "", - "description": "" + "precious": false, + "readonly": false, + "text": "Arg10", + "type": "Unknown", + "value": "" + }, + { + "defaultValue": "echo Hello", + "description": "just a dummy command", + "name": "command", + "precious": false, + "readonly": false, + "text": "command", + "type": "String", + "value": "echo Hello" } - ] - }, - { - "category": "Component", - "categoryType": "Application", - "isData": false, - "isGroup": false, - "canHaveInputs": true, - "canHaveOutputs": true, - "color": "#3498DB", - "drawOrderHint": 0, - "key": -3, - "text": "Enter label", - "description": "", - "x": 600, - "y": 300, - "width": 200, - "height": 200, - "collapsed": false, - "showPorts": false, - "streaming": false, - "subject": null, - "selected": false, - "expanded": false, + ], + "flipPorts": false, + "git_url": "", + "height": 72, + "inputAppFields": [], + "inputApplicationKey": null, "inputApplicationName": "", - "outputApplicationName": "", - "exitApplicationName": "", "inputApplicationType": "None", - "outputApplicationType": "None", - "exitApplicationType": "None", + "inputLocalPorts": [], "inputPorts": [ { - "Id": "0178b7ce-79ed-406d-9e4b-ee2a53c168ec", - "IdText": "event" + "Id": "c12aa833-43a9-4c1e-abaa-c77396010a31", + "IdText": "event", + "description": "", + "event": false, + "text": "", + "type": "" } ], + "isGroup": false, + "key": -2, + "outputAppFields": [], + "outputApplicationKey": null, + "outputApplicationName": "", + "outputApplicationType": "None", + "outputLocalPorts": [], "outputPorts": [ { - "Id": "f487361b-f633-43ba-978d-c65d7a52c34d", - "IdText": "event" + "Id": "47c421b8-5cdc-4ff7-ab7e-b75140f2d951", + "IdText": "event", + "description": "", + "event": false, + "text": "", + "type": "" } ], - "inputLocalPorts": [], - "outputLocalPorts": [], - "inputAppFields": [], - "outputAppFields": [], - "fields": [ - { - "text": "Execution time", - "name": "execution_time", - "value": "5", - "description": "" - }, - { - "text": "Num CPUs", - "name": "num_cpus", - "value": "1", - "description": "" - }, - { - "text": "Group start", - "name": "group_start", - "value": "0", - "description": "" - }, - { - "text": "Appclass", - "name": "appclass", - "value": "test.graphsRepository", - "description": "" - }, - { - "text": "Arg01", - "name": "Arg01", - "value": "", - "description": "" - }, - { - "text": "Arg02", - "name": "Arg02", - "value": "", - "description": "" - }, - { - "text": "Arg03", - "name": "Arg03", - "value": "", - "description": "" - }, - { - "text": "Arg04", - "name": "Arg04", - "value": "", - "description": "" - }, - { - "text": "Arg05", - "name": "Arg05", - "value": "", - "description": "" - }, - { - "text": "Arg06", - "name": "Arg06", - "value": "", - "description": "" - }, - { - "text": "Arg07", - "name": "Arg07", - "value": "", - "description": "" - }, - { - "text": "Arg08", - "name": "Arg08", - "value": "", - "description": "" - }, - { - "text": "Arg09", - "name": "Arg09", - "value": "", - "description": "" - }, - { - "text": "Arg10", - "name": "Arg10", - "value": "", - "description": "" - } - ] + "precious": false, + "readonly": true, + "sha": "", + "streaming": false, + "subject": null, + "text": "Enter label", + "width": 200, + "x": 400, + "y": 200 }, { + "applicationParams": [], "category": "File", - "categoryType": "Data", - "isData": true, - "isGroup": false, - "canHaveInputs": true, - "canHaveOutputs": true, - "color": "#394BB2", - "drawOrderHint": 0, - "key": -4, - "text": "Enter label", + "collapsed": true, + "color": "#2c2c2c", "description": "", - "x": 800, - "y": 400, - "width": 200, - "height": 200, - "collapsed": false, - "showPorts": false, - "streaming": false, - "subject": null, - "selected": false, + "drawOrderHint": 0, "expanded": false, - "inputApplicationName": "", - "outputApplicationName": "", - "exitApplicationName": "", - "inputApplicationType": "None", - "outputApplicationType": "None", - "exitApplicationType": "None", - "inputPorts": [ - { - "Id": "c05689f4-6c5a-47dc-b3b1-fcbdfa09e4df", - "IdText": "event" - } - ], - "outputPorts": [], - "inputLocalPorts": [], - "outputLocalPorts": [], - "inputAppFields": [], - "outputAppFields": [], "fields": [ { - "text": "Data volume", + "defaultValue": "", + "description": "", "name": "data_volume", - "value": "5", - "description": "" + "precious": false, + "readonly": false, + "text": "Data volume", + "type": "Unknown", + "value": "5" }, { - "text": "Group end", + "defaultValue": "", + "description": "", "name": "group_end", - "value": "0", - "description": "" + "precious": false, + "readonly": false, + "text": "Group end", + "type": "Unknown", + "value": "0" }, { - "text": "Check file path exists", + "defaultValue": "", + "description": "", "name": "check_filepath_exists", - "value": "1", - "description": "" + "precious": false, + "readonly": false, + "text": "Check file path exists", + "type": "Unknown", + "value": "1" }, { - "text": "File path", + "defaultValue": "", + "description": "", "name": "filepath", - "value": "", - "description": "" + "precious": false, + "readonly": false, + "text": "File path", + "type": "Unknown", + "value": "" }, { - "text": "Directory name", + "defaultValue": "", + "description": "", "name": "dirname", - "value": "", - "description": "" + "precious": false, + "readonly": false, + "text": "Directory name", + "type": "Unknown", + "value": "" } - ] - } - ], - "linkDataArray": [ - { - "from": -1, - "fromPort": "ab5ada14-04d7-4b03-816d-c43428d4e2e4", - "to": -2, - "toPort": "c12aa833-43a9-4c1e-abaa-c77396010a31" - }, - { - "from": -2, - "fromPort": "47c421b8-5cdc-4ff7-ab7e-b75140f2d951", - "to": -3, - "toPort": "0178b7ce-79ed-406d-9e4b-ee2a53c168ec" - }, - { - "from": -3, - "fromPort": "f487361b-f633-43ba-978d-c65d7a52c34d", - "to": -4, - "toPort": "c05689f4-6c5a-47dc-b3b1-fcbdfa09e4df" + ], + "flipPorts": false, + "git_url": "", + "height": 72, + "inputAppFields": [], + "inputApplicationKey": null, + "inputApplicationName": "", + "inputApplicationType": "None", + "inputLocalPorts": [], + "inputPorts": [ + { + "Id": "c05689f4-6c5a-47dc-b3b1-fcbdfa09e4df", + "IdText": "event", + "description": "", + "event": false, + "text": "", + "type": "" + } + ], + "isGroup": false, + "key": -4, + "outputAppFields": [], + "outputApplicationKey": null, + "outputApplicationName": "", + "outputApplicationType": "None", + "outputLocalPorts": [], + "outputPorts": [], + "precious": false, + "readonly": true, + "sha": "", + "streaming": false, + "subject": null, + "text": "Enter label", + "width": 200, + "x": 800, + "y": 400 } ] } \ No newline at end of file diff --git a/daliuge-translator/test/dropmake/logical_graphs/testLoop.graph b/daliuge-translator/test/dropmake/logical_graphs/testLoop.graph index 2ed50cbd1..ab83e978c 100644 --- a/daliuge-translator/test/dropmake/logical_graphs/testLoop.graph +++ b/daliuge-translator/test/dropmake/logical_graphs/testLoop.graph @@ -1,414 +1,426 @@ { + "linkDataArray": [ + { + "from": -4, + "fromPort": "1765f421-782c-4972-ab76-ac0c23bd95e3", + "loop_aware": "1", + "to": -3, + "toPort": "102d6fc6-eea7-434b-9196-65a42a79cf13" + }, + { + "from": -3, + "fromPort": "b8739916-45c0-41e1-b38d-afede242bcfd", + "loop_aware": "1", + "to": -2, + "toPort": "b133f803-42f7-4c70-83b7-803bf8ec3035" + } + ], "modelData": { + "eagleCommitHash": "dce847d911db5b8d2245775bb6d5c719eb4aa061", + "eagleVersion": "v4.1.0", + "filePath": "testLoop.graph", "fileType": "Graph", - "repoService": "GitHub", - "repoBranch": "yan-812-2", - "repo": "ICRAR/daliuge", - "filePath": "daliuge-translator/test/dropmake/logical_graphs/testLoop.graph", - "eagleVersion": "Unknown", - "eagleCommitHash": "Unknown", - "schemaVersion": "OJS", - "readonly": true, - "sha": "", "gitUrl": "", - "lastModifiedName": "", + "lastModifiedDatetime": 1644027730, "lastModifiedEmail": "", - "lastModifiedDatetime": "" + "lastModifiedName": "", + "readonly": true, + "repo": "", + "repoBranch": "", + "repoService": "Unknown", + "schemaVersion": "OJS", + "sha": "" }, "nodeDataArray": [ { + "applicationParams": [], "category": "Loop", - "isData": false, - "isGroup": true, - "canHaveInputs": false, - "canHaveOutputs": false, + "collapsed": false, "color": "rgb(221, 173, 0)", - "drawOrderHint": 0, - "key": -1, - "text": "Loop", "description": "Placeholder 'loop' description", - "x": 606.4859101595122, - "y": 337.06816127722925, - "width": 385.98717625064404, - "height": 378.12856316963075, - "collapsed": false, - "flipPorts": false, - "streaming": false, - "precious": false, - "subject": null, + "drawOrderHint": 0, "expanded": true, - "readonly": true, - "git_url": "", - "sha": "", - "inputPorts": [ + "fields": [ { - "Id": "123e02e1-0e35-47a3-8dca-2525f71a6766", - "IdText": "event", - "text": "", - "event": false, - "type": "", - "description": "" + "defaultValue": "", + "description": "", + "name": "num_of_iter", + "precious": false, + "readonly": false, + "text": "Number loops", + "type": "Unknown", + "value": "5" } ], - "outputPorts": [ + "flipPorts": false, + "git_url": "", + "height": 378.12856316963075, + "inputAppFields": [], + "inputApplicationKey": -5, + "inputApplicationName": "event", + "inputApplicationType": "UnknownApplication", + "inputLocalPorts": [ { "Id": "1eeee75a-c2d6-49ea-b94b-5c208b8f3f0c", "IdText": "event", - "text": "", + "description": "", "event": false, - "type": "", - "description": "" - } - ], - "inputLocalPorts": [ + "text": "", + "type": "" + }, { "Id": "a21aba04-cf68-4dab-b57f-dd0dba9e4c91", "IdText": "event", - "text": "", + "description": "", "event": false, - "type": "", - "description": "" - } - ], - "outputLocalPorts": [ - { - "Id": "05863afc-5933-4eef-9f64-5e10a4b86ac6", - "IdText": "event", "text": "", - "event": false, - "type": "", - "description": "" + "type": "" } ], - "fields": [ + "inputPorts": [ { - "text": "Number loops", - "name": "num_of_iter", - "value": "5", - "defaultValue": "", + "Id": "123e02e1-0e35-47a3-8dca-2525f71a6766", + "IdText": "event", "description": "", - "readonly": false, - "type": "Unknown", - "precious": false + "event": false, + "text": "", + "type": "" } ], - "applicationParams": [], - "inputAppFields": [], + "isGroup": true, + "key": -1, "outputAppFields": [], - "exitAppFields": [], - "inputApplicationName": "event", - "inputApplicationType": "UnknownApplication", - "inputApplicationKey": -5, + "outputApplicationKey": null, "outputApplicationName": "", "outputApplicationType": "None", - "outputApplicationKey": null, - "exitApplicationName": "event", - "exitApplicationType": "UnknownApplication", - "exitApplicationKey": -6 + "outputLocalPorts": [], + "outputPorts": [], + "precious": false, + "readonly": true, + "sha": "", + "streaming": false, + "subject": null, + "text": "Loop", + "width": 385.98717625064404, + "x": 606.4859101595122, + "y": 337.06816127722925 }, { + "applicationParams": [], "category": "BashShellApp", - "isData": false, - "isGroup": false, - "canHaveInputs": true, - "canHaveOutputs": true, + "collapsed": true, "color": "#0059a5", - "drawOrderHint": 0, - "key": -2, - "text": "SleepExternal", "description": "An application component run within the Bash Shell", - "x": 736.1530259962292, - "y": 855.7366246240957, - "width": 200, - "height": 72, - "collapsed": true, - "flipPorts": false, - "streaming": false, - "precious": false, - "subject": null, + "drawOrderHint": 0, "expanded": false, - "readonly": true, - "git_url": "", - "sha": "", - "inputPorts": [ - { - "Id": "b133f803-42f7-4c70-83b7-803bf8ec3035", - "IdText": "event", - "text": "", - "event": false, - "type": "", - "description": "" - } - ], - "outputPorts": [ - { - "Id": "1765f421-782c-4972-ab76-ac0c23bd95e3", - "IdText": "event", - "text": "", - "event": false, - "type": "", - "description": "" - } - ], - "inputLocalPorts": [], - "outputLocalPorts": [], "fields": [ { - "text": "Execution time", - "name": "execution_time", - "value": "5", "defaultValue": "", "description": "", + "name": "execution_time", + "precious": false, "readonly": false, + "text": "Execution time", "type": "Unknown", - "precious": false + "value": "5" }, { - "text": "Num CPUs", - "name": "num_cpus", - "value": "1", "defaultValue": "", "description": "", + "name": "num_cpus", + "precious": false, "readonly": false, + "text": "Num CPUs", "type": "Unknown", - "precious": false + "value": "1" }, { - "text": "Group start", - "name": "group_start", - "value": "0", "defaultValue": "", "description": "", + "name": "group_start", + "precious": false, "readonly": false, + "text": "Group start", "type": "Unknown", - "precious": false + "value": "0" }, { - "text": "Arg01", - "name": "Arg01", - "value": "sleep 5", "defaultValue": "", "description": "The command line to be executed", + "name": "Arg01", + "precious": false, "readonly": false, + "text": "Arg01", "type": "Unknown", - "precious": false + "value": "5" + }, + { + "defaultValue": "sleep", + "description": "", + "name": "command", + "precious": false, + "readonly": false, + "text": "command", + "type": "String", + "value": "sleep" } ], - "applicationParams": [], + "flipPorts": false, + "git_url": "", + "height": 72, "inputAppFields": [], - "outputAppFields": [], - "exitAppFields": [], + "inputApplicationKey": null, "inputApplicationName": "", "inputApplicationType": "None", - "inputApplicationKey": null, - "outputApplicationName": "", - "outputApplicationType": "None", - "outputApplicationKey": null, - "exitApplicationName": "", - "exitApplicationType": "None", - "exitApplicationKey": null - }, - { - "category": "Memory", - "isData": true, - "isGroup": false, - "canHaveInputs": true, - "canHaveOutputs": true, - "color": "#2c2c2c", - "drawOrderHint": 0, - "key": -3, - "text": "Memory", - "description": "", - "x": 700.7892671316696, - "y": 593.7828552569915, - "width": 200, - "height": 72, - "collapsed": true, - "flipPorts": false, - "streaming": false, - "precious": false, - "subject": null, - "expanded": false, - "readonly": true, - "git_url": "", - "sha": "", - "group": -1, + "inputLocalPorts": [], "inputPorts": [ { - "Id": "ae981288-d839-4890-bb11-f54f336cfad0", + "Id": "b133f803-42f7-4c70-83b7-803bf8ec3035", "IdText": "event", - "text": "", + "description": "", "event": false, - "type": "", - "description": "" + "text": "", + "type": "" } ], + "isGroup": false, + "key": -2, + "outputAppFields": [], + "outputApplicationKey": null, + "outputApplicationName": "", + "outputApplicationType": "None", + "outputLocalPorts": [], "outputPorts": [ { - "Id": "b7c36b5e-dbe7-4d9d-ad2f-abec81071de5", + "Id": "1765f421-782c-4972-ab76-ac0c23bd95e3", "IdText": "event", - "text": "", + "description": "", "event": false, - "type": "", - "description": "" + "text": "", + "type": "" } ], - "inputLocalPorts": [], - "outputLocalPorts": [], + "precious": false, + "readonly": true, + "sha": "", + "streaming": false, + "subject": null, + "text": "SleepExternal", + "width": 200, + "x": 736.1530259962292, + "y": 855.7366246240957 + }, + { + "applicationParams": [], + "category": "BashShellApp", + "collapsed": false, + "color": "#0059a5", + "description": "An application component run within the Bash Shell", + "drawOrderHint": 0, + "expanded": false, "fields": [ { - "text": "Data volume", - "name": "data_volume", - "value": "5", "defaultValue": "", "description": "", + "name": "execution_time", + "precious": false, "readonly": false, + "text": "Execution time", "type": "Unknown", - "precious": false + "value": "5" }, { - "text": "Group end", - "name": "group_end", - "value": "1", "defaultValue": "", "description": "", + "name": "num_cpus", + "precious": false, "readonly": false, + "text": "Num CPUs", "type": "Unknown", - "precious": false + "value": "1" + }, + { + "defaultValue": "false", + "description": "", + "name": "group_start", + "precious": false, + "readonly": false, + "text": "Group start", + "type": "Boolean", + "value": true + }, + { + "defaultValue": "", + "description": "The command line to be executed", + "name": "Arg01", + "precious": false, + "readonly": false, + "text": "Arg01", + "type": "Unknown", + "value": "3" + }, + { + "defaultValue": "sleep", + "description": "", + "name": "command", + "precious": false, + "readonly": false, + "text": "command", + "type": "String", + "value": "sleep" } ], - "applicationParams": [], - "inputAppFields": [], - "outputAppFields": [], - "exitAppFields": [], - "inputApplicationName": "", - "inputApplicationType": "None", - "inputApplicationKey": null, - "outputApplicationName": "", - "outputApplicationType": "None", - "outputApplicationKey": null, - "exitApplicationName": "", - "exitApplicationType": "None", - "exitApplicationKey": null - }, - { - "category": "BashShellApp", - "isData": false, - "isGroup": false, - "canHaveInputs": true, - "canHaveOutputs": true, - "color": "#0059a5", - "drawOrderHint": 0, - "key": -4, - "text": "SleepInternal", - "description": "An application component run within the Bash Shell", - "x": 692.9306540506565, - "y": 454.94735749242614, - "width": 200, - "height": 72, - "collapsed": false, "flipPorts": false, - "streaming": false, - "precious": false, - "subject": null, - "expanded": false, - "readonly": true, "git_url": "", - "sha": "", "group": -1, + "height": 72, + "inputAppFields": [], + "inputApplicationKey": null, + "inputApplicationName": "", + "inputApplicationType": "None", + "inputLocalPorts": [], "inputPorts": [ { "Id": "b133f803-42f7-4c70-83b7-803bf8ec3035", "IdText": "event", - "text": "", + "description": "", "event": false, - "type": "", - "description": "" + "text": "", + "type": "" } ], + "isGroup": false, + "key": -4, + "outputAppFields": [], + "outputApplicationKey": null, + "outputApplicationName": "", + "outputApplicationType": "None", + "outputLocalPorts": [], "outputPorts": [ { "Id": "1765f421-782c-4972-ab76-ac0c23bd95e3", "IdText": "event", - "text": "", + "description": "", "event": false, - "type": "", - "description": "" + "text": "", + "type": "" } ], - "inputLocalPorts": [], - "outputLocalPorts": [], + "precious": false, + "readonly": true, + "sha": "", + "streaming": false, + "subject": null, + "text": "SleepInternal", + "width": 200, + "x": 692.9306540506565, + "y": 454.94735749242614 + }, + { + "applicationParams": [], + "category": "File", + "collapsed": true, + "color": "#2c2c2c", + "description": "A standard file on a filesystem mounted to the deployment machine", + "drawOrderHint": 0, + "expanded": false, "fields": [ { - "text": "Execution time", - "name": "execution_time", - "value": "5", - "defaultValue": "", - "description": "", + "defaultValue": "5", + "description": "Estimated size of the data contained in this node", + "name": "data_volume", + "precious": false, "readonly": false, - "type": "Unknown", - "precious": false + "text": "Data volume", + "type": "Float", + "value": 5 }, { - "text": "Num CPUs", - "name": "num_cpus", - "value": "1", - "defaultValue": "", - "description": "", + "defaultValue": "false", + "description": "Is this node the end of a group?", + "name": "group_end", + "precious": false, "readonly": false, - "type": "Unknown", - "precious": false + "text": "Group end", + "type": "Boolean", + "value": true + }, + { + "defaultValue": "true", + "description": "Perform a check to make sure the file path exists before proceeding with the application", + "name": "check_filepath_exists", + "precious": false, + "readonly": false, + "text": "Check file path exists", + "type": "Boolean", + "value": true }, { - "text": "Group start", - "name": "group_start", - "value": "1", "defaultValue": "", - "description": "", + "description": "Path to the file for this node", + "name": "filepath", + "precious": false, "readonly": false, - "type": "Unknown", - "precious": false + "text": "File path", + "type": "String", + "value": "" }, { - "text": "Arg01", - "name": "Arg01", - "value": "sleep 3", "defaultValue": "", - "description": "The command line to be executed", + "description": "Name of the directory containing the file for this node", + "name": "dirname", + "precious": false, "readonly": false, - "type": "Unknown", - "precious": false + "text": "Directory name", + "type": "String", + "value": "" } ], - "applicationParams": [], + "flipPorts": false, + "git_url": "", + "group": -1, + "height": 72, "inputAppFields": [], - "outputAppFields": [], - "exitAppFields": [], + "inputApplicationKey": null, "inputApplicationName": "", "inputApplicationType": "None", - "inputApplicationKey": null, + "inputLocalPorts": [], + "inputPorts": [ + { + "Id": "102d6fc6-eea7-434b-9196-65a42a79cf13", + "IdText": "event", + "description": "", + "event": false, + "text": "event", + "type": "event" + } + ], + "isGroup": false, + "key": -3, + "outputAppFields": [], + "outputApplicationKey": null, "outputApplicationName": "", "outputApplicationType": "None", - "outputApplicationKey": null, - "exitApplicationName": "", - "exitApplicationType": "None", - "exitApplicationKey": null - } - ], - "linkDataArray": [ - { - "from": -4, - "fromPort": "1765f421-782c-4972-ab76-ac0c23bd95e3", - "to": -3, - "toPort": "ae981288-d839-4890-bb11-f54f336cfad0", - "loop_aware": "0" - }, - { - "from": -3, - "fromPort": "b7c36b5e-dbe7-4d9d-ad2f-abec81071de5", - "to": -2, - "toPort": "b133f803-42f7-4c70-83b7-803bf8ec3035", - "loop_aware": "1" + "outputLocalPorts": [], + "outputPorts": [ + { + "Id": "b8739916-45c0-41e1-b38d-afede242bcfd", + "IdText": "event", + "description": "", + "event": false, + "text": "event", + "type": "event" + } + ], + "precious": false, + "readonly": true, + "sha": "", + "streaming": false, + "subject": null, + "text": "event", + "width": 200, + "x": 723.5418400234428, + "y": 594.3419910582609 } ] } \ No newline at end of file