Merge branch 'master' into liu-150

ICRAR · Aug 25, 2021 · 9fe442f · 9fe442f
2 parents 7d63d66 + 9d094d9
commit 9fe442f
Show file tree

Hide file tree

Showing 15 changed files with 579 additions and 24 deletions.
diff --git a/daliuge-engine/build_engine.sh b/daliuge-engine/build_engine.sh
@@ -21,7 +21,7 @@ case "$1" in
         echo "$VERSION:$VCS_TAG" > dlg/manager/web/VERSION
         git rev-parse --verify HEAD >> dlg/manager/web/VERSION
         cp ../LICENSE dlg/manager/web/.
-        docker build --build-arg VCS_TAG=${C_TAG} --no-cache -t icrar/daliuge-engine:${VCS_TAG} -f docker/Dockerfile .
+        docker build --build-arg VCS_TAG=${C_TAG} --no-cache -t icrar/daliuge-engine:${VCS_TAG} -f docker/Dockerfile.dev .
         echo "Build finished!"
         exit 1;;
     "casa")

diff --git a/daliuge-engine/dlg/apps/dockerapp.py b/daliuge-engine/dlg/apps/dockerapp.py
@@ -28,6 +28,7 @@
 import os
 import threading
 import time
+from overrides import overrides
 
 from configobj import ConfigObj
 import docker
@@ -350,27 +351,22 @@ def run(self):
 
         c = DockerApp._get_client()
 
-        # Remove the container unless it's specified that we should keep it
-        # (used below)
-        def rm(container):
-            if self._removeContainer:
-                container.remove()
-
         # Create container
-        container = c.containers.create(
+        self.container = c.containers.create(
                 self._image,
                 cmd,
                 volumes=binds,
                 user=user,
                 environment=env,
-                working_dir=self.workdir
+                working_dir=self.workdir,
+                auto_remove=self._removeContainer
         )
-        self._containerId = cId = container.id
+        self._containerId = cId = self.container.id
         logger.info("Created container %s for %r", cId, self)
 
         # Start it
         start = time.time()
-        container.start()
+        self.container.start()
         logger.info("Started container %s", cId)
 
         # Figure out the container's IP and save it
@@ -383,7 +379,7 @@ def rm(container):
         # Wait until it finishes
         # In docker-py < 3 the .wait() method returns the exit code directly
         # In docker-py >= 3 the .wait() method returns a dictionary with the API response
-        x = container.wait()
+        x = self.container.wait()
         if isinstance(x, dict) and 'StatusCode' in x:
             self._exitCode = x['StatusCode']
         else:
@@ -401,12 +397,30 @@ def rm(container):
             stderr = container.logs(stream=False, stdout=False, stderr=True)
             msg = "Container %s didn't finish successfully (exit code %d)" % (cId, self._exitCode)
             logger.error(msg + ", output follows.\n==STDOUT==\n%s==STDERR==\n%s", stdout, stderr)
-            rm(container)
             raise Exception(msg)
 
-        rm(container)
         c.api.close()
 
+    @overrides
+    def setCompleted(self):
+        super(BarrierAppDROP, self).setCompleted()
+        self.container.stop()
+
+    @overrides
+    def setError(self):
+        super(BarrierAppDROP, self).setError()
+        self.container.stop()
+
+    @overrides
+    def cancel(self):
+        super(BarrierAppDROP, self).cancel()
+        self.container.stop()
+
+    @overrides
+    def skip(self):
+        super(BarrierAppDROP, self).skip()
+        self.container.stop()
+
     @classmethod
     def _get_client(cls):
         return docker.from_env(version='auto', **cls._kwargs_from_env())

diff --git a/daliuge-engine/dlg/manager/web/LICENSE b/daliuge-engine/dlg/manager/web/LICENSE
diff --git a/daliuge-engine/dlg/manager/web/VERSION b/daliuge-engine/dlg/manager/web/VERSION
@@ -0,0 +1,2 @@
+1.0.0:liu-153
+87d81814ab01da6b2aeb41927f4478280df4d30c
diff --git a/daliuge-engine/docker/Dockerfile b/daliuge-engine/docker/Dockerfile
@@ -5,10 +5,10 @@ FROM icrar/daliuge-common:${VCS_TAG:-latest}
 # RUN sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata \
 #     gcc g++ gdb casacore-dev clang-tidy-10 clang-tidy libboost1.71-all-dev libgsl-dev
 
-RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends gcc python3-pip curl python3-casacore
+RUN sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends gcc python3-pip curl python3-casacore
 
 COPY / /daliuge
-RUN . /root/dlg/bin/activate && cd /daliuge && \
+RUN . /home/ray/dlg/bin/activate && cd /daliuge && \
     pip install . 
 
 EXPOSE 9000
@@ -19,7 +19,7 @@ EXPOSE 8001
 EXPOSE 8002
 
 # enable the virtualenv path from daliuge-common
-ENV VIRTUAL_ENV=/root/dlg
+ENV VIRTUAL_ENV=/home/ray/dlg
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 ENV DLG_ROOT="/tmp/.dlg/var/dlg_home"
 

diff --git a/daliuge-engine/docker/Dockerfile.dev b/daliuge-engine/docker/Dockerfile.dev
@@ -0,0 +1,26 @@
+ARG VCS_TAG
+# We need the base image we build with the other Dockerfile
+FROM icrar/daliuge-common:${VCS_TAG:-latest}
+
+# RUN sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata \
+#     gcc g++ gdb casacore-dev clang-tidy-10 clang-tidy libboost1.71-all-dev libgsl-dev
+
+RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends gcc python3-pip curl python3-casacore
+
+COPY / /daliuge
+RUN . /root/dlg/bin/activate && cd /daliuge && \
+    pip install . 
+
+EXPOSE 9000
+EXPOSE 5555
+EXPOSE 6666
+EXPOSE 8000
+EXPOSE 8001
+EXPOSE 8002
+
+# enable the virtualenv path from daliuge-common
+ENV VIRTUAL_ENV=/root/dlg
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV DLG_ROOT="/tmp/.dlg/var/dlg_home"
+
+CMD ["dlg", "daemon", "-vv"]
diff --git a/daliuge-engine/setup.py b/daliuge-engine/setup.py
@@ -124,6 +124,7 @@ def run(self):
     "lockfile",
     # 0.10.6 builds correctly with old (<=3.10) Linux kernels
     "netifaces>=0.10.6",
+    "overrides",
     "paramiko",
     "psutil",
     "python-daemon",

diff --git a/daliuge-engine/test/pytest.ini b/daliuge-engine/test/pytest.ini
@@ -0,0 +1,6 @@
+[pytest]
+minversion = 6.0
+addopts = -ra -q
+testpaths =
+    daliuge-engine/test
+    daliuge-translator/test
diff --git a/daliuge-translator/build_translator.sh b/daliuge-translator/build_translator.sh
@@ -24,7 +24,7 @@ case "$1" in
         cp ../LICENSE dlg/dropmake/web/.
         # The complete casa and arrow installation is only required for the Plasma streaming
         # and should not go much further.
-        docker build --build-arg VCS_TAG=${C_TAG} --no-cache -t icrar/daliuge-translator:${VCS_TAG} -f docker/Dockerfile .
+        docker build --build-arg VCS_TAG=${C_TAG} --no-cache -t icrar/daliuge-translator:${VCS_TAG} -f docker/Dockerfile.dev .
         echo "Build finished!"
         exit 1;;
     "casa")
@@ -35,7 +35,7 @@ case "$1" in
         cp ../LICENSE dlg/dropmake/web/.
         # The complete casa and arrow installation is only required for the Plasma streaming
         # and should not go much further.
-        docker build --build-arg VCS_TAG=${VCS_TAG}-casa --no-cache -t icrar/daliuge-translator:${VCS_TAG}-casa -f docker/Dockerfile .
+        docker build --build-arg VCS_TAG=${VCS_TAG}-casa --no-cache -t icrar/daliuge-translator:${VCS_TAG}-casa -f docker/Dockerfile.dev .
         echo "Build finished!"
         exit 1;;
     *)

diff --git a/daliuge-translator/dlg/dropmake/pg_generator.py b/daliuge-translator/dlg/dropmake/pg_generator.py
@@ -816,7 +816,7 @@ def make_single_drop(self, iid="0", **kwargs):
 
     @staticmethod
     def str_to_bool(value, default_value=False):
-        res = True if value in ["1", "true", "yes"] else default_value
+        res = True if value in ["1", "true", "True", "yes"] else default_value
         return res
 
     @staticmethod

diff --git a/daliuge-translator/dlg/dropmake/web/VERSION b/daliuge-translator/dlg/dropmake/web/VERSION
@@ -1,2 +1,2 @@
 1.0.0:liu-150
-df28e36db30e12662802d1c1a81e88f1f67240a7
+7d63d6614a0adf606acd07a5c8283b9eacaa5f0d
diff --git a/daliuge-translator/dlg/dropmake/web/lg_web.py b/daliuge-translator/dlg/dropmake/web/lg_web.py
@@ -568,6 +568,7 @@ def save(lg_name, logical_graph):
     return new_path
 
 
+@get("/")
 @get("/")
 def root():
     tpl = file_as_string("pg_viewer.html")

diff --git a/daliuge-translator/docker/Dockerfile b/daliuge-translator/docker/Dockerfile
@@ -6,7 +6,7 @@ FROM icrar/daliuge-common:${VCS_TAG:-latest}
 ARG BUILD_ID
 LABEL stage=builder
 LABEL build=$BUILD_ID
-RUN apt-get update && apt-get install -y python3-pip gcc && apt-get clean
+RUN sudo apt-get update && sudo apt-get install -y python3-pip gcc && sudo apt-get clean
 
 COPY / /daliuge
 
@@ -17,10 +17,10 @@ RUN . ~/dlg/bin/activate && \
 # Second stage build taking what's required from first stage
 FROM icrar/daliuge-common:${VCS_TAG:-latest}
 COPY --from=0 /daliuge/. /daliuge/.
-COPY --from=0 /root/dlg /root/dlg
+COPY --from=0 /home/ray/dlg /home/ray/dlg
 
 # enable the virtualenv path from daliuge-common
-ENV VIRTUAL_ENV=/root/dlg
+ENV VIRTUAL_ENV=/home/ray/dlg
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
 EXPOSE 8084

diff --git a/daliuge-translator/stop_translator.sh b/daliuge-translator/stop_translator.sh
@@ -0,0 +1,2 @@
+# This utlity script stops a locally running docker-engine container
+docker stop daliuge-translator 
diff --git a/docs/conf.py b/docs/conf.py
@@ -59,6 +59,7 @@ def __getattr__(cls, _):
  "netifaces",
  "networkx",
  "numpy",
+ "overrides",
  "paramiko",
  "paramiko.client",
  "paramiko.rsakey",