Yan 852

daliuge-common/docker/Dockerfile.dev

@@ -7,18 +7,13 @@ FROM ubuntu:20.04
 ARG BUILD_ID
 LABEL stage=builder
 LABEL build=$BUILD_ID
-RUN apt-get update && apt-get install -y gcc python3 python3.8-venv && apt-get clean
+RUN apt-get update && apt-get install -y gcc python3 python3.8-venv python3-pip python3-distutils libmetis-dev curl && apt-get clean
 
 COPY / /daliuge
 
-RUN cd && python3 -m venv dlg && cd /daliuge && \
-    . ${HOME}/dlg/bin/activate && \
-    pip install numpy && \
-    pip install . && \
-    apt-get remove -y gcc && \
-    apt-get autoremove -y
+RUN cd / && python3 -m venv dlg && cd /daliuge && \
+    . /dlg/bin/activate && \
+    pip install wheel numpy && \
+    pip install .
 
-
-FROM ubuntu:20.04
-RUN apt-get update && apt-get install -y bash
-COPY --from=0 /root/dlg /root/dlg
+# we don't clean this up, will be done in the derived images

daliuge-engine/dlg/apps/bash_shell_app.py

@@ -157,17 +157,26 @@ class BashShellBase(object):
     Common class for BashShell apps. It simply requires a command to be
     specified.
     """
-
+    #TODO: use the shlex module for most of the construction of the
+    # command line to get a proper and safe shell syntax
     command = dlg_string_param("Bash command", None)
 
     def initialize(self, **kwargs):
         super(BashShellBase, self).initialize(**kwargs)
 
         self.proc = None
+        self._inputRedirect = self._getArg(kwargs, "input_redirection", "")
+        self._outputRedirect = self._getArg(kwargs, "output_redirection", "")
+        self._cmdLineArgs = self._getArg(kwargs, "command_line_arguments", "")
+        self._applicationParams = self._getArg(kwargs, "applicationParams", {})
+        self._argumentPrefix = self._getArg(kwargs, "argumentPrefix", "--")
+
         if not self.command:
-            raise InvalidDropException(
-                self, "No command specified, cannot create BashShellApp"
-            )
+            self.command = self._getArg(kwargs, "command", None)
+            if not self.command:
+                raise InvalidDropException(
+                    self, "No command specified, cannot create BashShellApp"
+                )
 
     def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE):
         """
@@ -186,7 +195,16 @@ def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE):
         session_id = (
             self._dlg_session.sessionId if self._dlg_session is not None else ""
         )
-        cmd = self.command
+        argumentString = droputils.serialize_applicationParams(self._applicationParams, \
+            self._argumentPrefix)
+        # complete command including all additional parameters and optional redirects
+        cmd = f"{self.command} {self._cmdLineArgs} {argumentString}"
+        if self._outputRedirect:
+            cmd = f"{cmd} > {self._outputRedirect}"
+        if self._inputRedirect:
+            cmd = f"cat {self._inputRedirect} > {cmd}"
+        cmd = cmd.strip()
+
         app_uid = self.uid
         # self.run_bash(self._command, self.uid, session_id, *args, **kwargs)
 
@@ -210,7 +228,7 @@ def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE):
 
         # Wrap everything inside bash
         cmd = ("/bin/bash", "-c", cmd)
-        logger.debug("Command after user creation and wrapping is: %s", cmd)
+        logger.debug("Command after wrapping is: %s", cmd)
 
         start = time.time()
 

daliuge-engine/dlg/apps/dockerapp.py

@@ -204,17 +204,28 @@ def initialize(self, **kwargs):
             )
 
         self._command = self._getArg(kwargs, "command", None)
+
         if not self._command:
             logger.warning(
                 "No command specified. Assume that a default command is executed in the container"
             )
+            # The above also means that we can't pass applicationParams
             # raise InvalidDropException(
             #     self, "No command specified, cannot create DockerApp")
+        else:
+            self._applicationParams = self._getArg(kwargs, "applicationParams", {})
+
+            # construct the actual command line from all application parameters
+            argumentPrefix = self._getArg(kwargs, "argumentPrefix", "--")
+            argumentString = droputils.serialize_applicationParams(self._applicationParams, \
+                argumentPrefix)
+            self._command = f"{self._command} {argumentString}"
 
         # The user used to run the process in the docker container
         # By default docker containers run as root, but we don't want to run
         # a process using a different user because otherwise anything that that
         # process writes to the filesystem
+        # TODO: User switching should be changed to be transparent
         self._user = self._getArg(kwargs, "user", None)
 
         # In some cases we want to make sure the command in the container runs

daliuge-engine/dlg/droputils.py

@@ -446,6 +446,30 @@ def replace_dataurl_placeholders(cmd, inputs, outputs):
 
     return cmd
 
+def serialize_applicationParams(applicationParams, prefix='--'):
+    """
+    Unpacks the applicationParams dictionary and returns a string
+    that can be used as command line parameters.
+    """
+    if not isinstance(applicationParams, dict):
+        logger.info("applicationParams are not passed as a dict. Ignored!")
+    # construct the actual command line from all application parameters
+    args = []
+
+    for (name, value) in applicationParams.items():
+        if value in [None, False, ""]:
+            continue
+        elif value is True:
+            value = ''
+        # short and long version of keywords
+        if prefix == "--" and len(name) == 1:
+            arg = [f'-{name} {value}']
+        else:
+            arg = [f'{prefix}{name} {value}']
+        args += arg.strip() # remove unneccesary blanks
+
+    return f"{' '.join(args)}"
+
 
 # Easing the transition from single- to multi-package
 get_leaves = common.get_leaves

daliuge-engine/docker/Dockerfile

@@ -22,6 +22,6 @@ EXPOSE 8002
 # enable the virtualenv path from daliuge-common
 ENV VIRTUAL_ENV=/home/ray/dlg
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-ENV DLG_ROOT="/tmp/dlg/var/dlg_home"
+ENV DLG_ROOT="/var/dlg_home"
 
 CMD ["dlg", "daemon", "-vv"]

daliuge-engine/docker/Dockerfile.dev

@@ -5,11 +5,8 @@ FROM icrar/daliuge-common:${VCS_TAG:-latest}
 # RUN sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata \
 #     gcc g++ gdb casacore-dev clang-tidy-10 clang-tidy libboost1.71-all-dev libgsl-dev
 
-RUN apt-get update &&\
-    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gcc python3-pip curl
-
 COPY / /daliuge
-RUN . /root/dlg/bin/activate && pip install wheel && cd /daliuge && \
+RUN . /dlg/bin/activate && pip install wheel && cd /daliuge && \
     pip install . 
 
 EXPOSE 9000
@@ -20,8 +17,8 @@ EXPOSE 8001
 EXPOSE 8002
 
 # enable the virtualenv path from daliuge-common
-ENV VIRTUAL_ENV=/root/dlg
+ENV VIRTUAL_ENV=/dlg
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-ENV DLG_ROOT="/tmp/dlg/var/dlg_home"
+ENV DLG_ROOT="/tmp/dlg"
 
 CMD ["dlg", "daemon", "-vv"]

daliuge-engine/docker/group.template

@@ -0,0 +1,42 @@
+root:x:0:
+daemon:x:1:
+bin:x:2:
+sys:x:3:
+adm:x:4:
+tty:x:5:
+disk:x:6:
+lp:x:7:
+mail:x:8:
+news:x:9:
+uucp:x:10:
+man:x:12:
+proxy:x:13:
+kmem:x:15:
+dialout:x:20:
+fax:x:21:
+voice:x:22:
+cdrom:x:24:
+floppy:x:25:
+tape:x:26:
+sudo:x:27:
+audio:x:29:
+dip:x:30:
+www-data:x:33:
+backup:x:34:
+operator:x:37:
+list:x:38:
+irc:x:39:
+src:x:40:
+gnats:x:41:
+shadow:x:42:
+utmp:x:43:
+video:x:44:
+sasl:x:45:
+plugdev:x:46:
+staff:x:50:
+games:x:60:
+users:x:100:
+nogroup:x:65534:
+crontab:x:101:
+messagebus:x:102:
+ssh:x:103:

daliuge-engine/docker/passwd.template

@@ -0,0 +1,20 @@
+root:x:0:0:root:/root:/bin/bash
+daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin
+bin:x:2:2:bin:/bin:/usr/sbin/nologin
+sys:x:3:3:sys:/dev:/usr/sbin/nologin
+sync:x:4:65534:sync:/bin:/bin/sync
+games:x:5:60:games:/usr/games:/usr/sbin/nologin
+man:x:6:12:man:/var/cache/man:/usr/sbin/nologin
+lp:x:7:7:lp:/var/spool/lpd:/usr/sbin/nologin
+mail:x:8:8:mail:/var/mail:/usr/sbin/nologin
+news:x:9:9:news:/var/spool/news:/usr/sbin/nologin
+uucp:x:10:10:uucp:/var/spool/uucp:/usr/sbin/nologin
+proxy:x:13:13:proxy:/bin:/usr/sbin/nologin
+www-data:x:33:33:www-data:/var/www:/usr/sbin/nologin
+backup:x:34:34:backup:/var/backups:/usr/sbin/nologin
+list:x:38:38:Mailing List Manager:/var/list:/usr/sbin/nologin
+irc:x:39:39:ircd:/var/run/ircd:/usr/sbin/nologin
+gnats:x:41:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/usr/sbin/nologin
+nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin
+_apt:x:100:65534::/nonexistent:/usr/sbin/nologin
+messagebus:x:101:102::/nonexistent:/usr/sbin/nologin

daliuge-engine/docker/prepare_user.py

@@ -0,0 +1,48 @@
+#
+#    ICRAR - International Centre for Radio Astronomy Research
+#    (c) UWA - The University of Western Australia, 2014
+#    Copyright by UWA (in the framework of the ICRAR)
+#    All rights reserved
+#
+#    This library is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU Lesser General Public
+#    License as published by the Free Software Foundation; either
+#    version 2.1 of the License, or (at your option) any later version.
+#
+#    This library is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#    Lesser General Public License for more details.
+#
+#    You should have received a copy of the GNU Lesser General Public
+#    License along with this library; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+#    MA 02111-1307  USA
+#
+"""
+Script to generate passwd and group files for docker containers to mount.
+This will make sure that the apps in the containers are running as the 
+current user and thus the generated files have the correct owner.
+
+Inspired by Stimela code
+"""
+import pwd, grp, os
+
+workdir = f"{os.environ['DLG_ROOT']}/workspace/settings"
+try:
+    os.mkdir(workdir)
+except FileExistsError:
+    pass
+except:
+    raise
+template_dir = os.path.join(os.path.dirname(__file__), ".")
+# get current user info
+pw = pwd.getpwuid(os.getuid())
+gr = grp.getgrgid(pw.pw_gid)
+with open(os.path.join(workdir, "passwd"), "wt") as file:
+    file.write(open(os.path.join(template_dir, "passwd.template"), "rt").read())
+    file.write(f"{pw.pw_name}:x:{pw.pw_uid}:{pw.pw_gid}:{pw.pw_gecos}:/:/bin/bash")
+with open(os.path.join(workdir, "group"), "wt") as file:
+    file.write(open(os.path.join(template_dir, "group.template"), "rt").read())
+    file.write(f"{gr.gr_name}:x:{gr.gr_gid}:")
+

daliuge-engine/run_engine.sh

@@ -6,7 +6,20 @@ DOCKER_OPTS="\
 -p 5555:5555 -p 6666:6666 \
 -p 8000:8000 -p 8001:8001 \
 -p 8002:8002 -p 9000:9000 \
+--user $(id -u):$(id -g) \
 " 
+common_prep ()
+{
+    mkdir -p ${DLG_ROOT}/workspace
+    mkdir -p ${DLG_ROOT}/testdata
+    mkdir -p ${DLG_ROOT}/code
+    # get current user and group id and prepare passwd and group files
+    python docker/prepare_user.py
+    DOCKER_OPTS=${DOCKER_OPTS}" -v ${DLG_ROOT}/workspace/settings/passwd:/etc/passwd"
+    DOCKER_OPTS=${DOCKER_OPTS}" -v ${DLG_ROOT}/workspace/settings/group:/etc/group"
+    DOCKER_OPTS=${DOCKER_OPTS}" -v ${PWD}/dlg/manager:/dlg/lib/python3.8/site-packages/dlg/manager"
+    DOCKER_OPTS=${DOCKER_OPTS}" -v ${DLG_ROOT}:${DLG_ROOT} --env DLG_ROOT=${DLG_ROOT}"
+}
 
 case "$1" in
     "dep")
@@ -17,36 +30,27 @@ case "$1" in
             echo "Please either create and grant access to $USER or build and run the development version."
         else
             VCS_TAG=`git describe --tags --abbrev=0|sed s/v//`
-            DOCKER_OPTS=${DOCKER_OPTS}"-v ${DLG_ROOT}:${DLG_ROOT} --env DLG_ROOT=${DLG_ROOT} "
+            common_prep()
             echo "Running Engine deployment version in background..."
             echo "docker run -td "${DOCKER_OPTS}"  icrar/daliuge-engine:${VCS_TAG}"
             docker run -td ${DOCKER_OPTS}  icrar/daliuge-engine:${VCS_TAG}
             exit 0
         fi;;
     "dev")
-        DLG_ROOT="/tmp/dlg"
+        export DLG_ROOT="/tmp/dlg"
         export VCS_TAG=`git rev-parse --abbrev-ref HEAD | tr '[:upper:]' '[:lower:]'`
+        common_prep
         echo "Running Engine development version in background..."
-        mkdir -p ${DLG_ROOT}/workspace
-        mkdir -p ${DLG_ROOT}/testdata
-        mkdir -p ${DLG_ROOT}/code
-        DOCKER_OPTS=${DOCKER_OPTS}"-v ${PWD}/dlg/manager:/root/dlg/lib/python3.8/site-packages/dlg/manager"
-        DOCKER_OPTS=${DOCKER_OPTS}" -v ${DLG_ROOT}:${DLG_ROOT} --env DLG_ROOT=${DLG_ROOT}"
         echo "docker run -td ${DOCKER_OPTS}  icrar/daliuge-engine:${VCS_TAG}"
         docker run -td ${DOCKER_OPTS}  icrar/daliuge-engine:${VCS_TAG}
-#        docker run -td ${DOCKER_OPTS} icrar/dlg-engine:casa
         sleep 3
         ./start_local_managers.sh
         exit 0;;
     "casa")
         DLG_ROOT="/tmp/dlg"
         export VCS_TAG=`git rev-parse --abbrev-ref HEAD | tr '[:upper:]' '[:lower:]'`
         echo "Running Engine development version in background..."
-        mkdir -p ${DLG_ROOT}/workspace
-        mkdir -p ${DLG_ROOT}/testdata
-        mkdir -p ${DLG_ROOT}/code
-        DOCKER_OPTS=${DOCKER_OPTS}"-v ${PWD}/dlg/manager:/root/dlg/lib/python3.8/site-packages/dlg/manager"
-        DOCKER_OPTS=${DOCKER_OPTS}" -v ${DLG_ROOT}:${DLG_ROOT} --env DLG_ROOT=${DLG_ROOT}"
+        common_prep
         CONTAINER_NM="icrar/daliuge-engine:${VCS_TAG}-casa"
         echo "docker run -td ${DOCKER_OPTS}  ${CONTAINER_NM}"
         docker run -td ${DOCKER_OPTS}  ${CONTAINER_NM}