Merge pull request #100 from ICRAR/devall-docker

Extra docker build targets

daliuge-common/build_common.sh

@@ -12,13 +12,16 @@ case "$1" in
         exit 0 ;;
     "dev")
         export VCS_TAG=`git rev-parse --abbrev-ref HEAD | tr '[:upper:]' '[:lower:]'`
-        # export VCS_TAG="casa"
         echo "Building daliuge-common development version using tag ${VCS_TAG}"
-        # The complete casa and arrow installation is only required for the Plasma streaming
-        # and should not go much further.
         docker build --build-arg VCS_TAG=${VCS_TAG} --no-cache -t icrar/daliuge-common:${VCS_TAG} -f docker/Dockerfile.dev .
         echo "Build finished!"
         exit 0;;
+    "devcuda")
+        export VCS_TAG=`git rev-parse --abbrev-ref HEAD | tr '[:upper:]' '[:lower:]'`
+        echo "Building daliuge-common development version using tag ${VCS_TAG}"
+        docker build --build-arg VCS_TAG=${VCS_TAG} --no-cache -t icrar/daliuge-common:${VCS_TAG} -f docker/Dockerfile.cuda .
+        echo "Build finished!"
+        exit 0;;
     "casa")
         export VCS_TAG=`git rev-parse --abbrev-ref HEAD | tr '[:upper:]' '[:lower:]'`
         # export VCS_TAG="casa"
@@ -29,6 +32,6 @@ case "$1" in
         echo "Build finished!"
         exit 0;;
     *)
-        echo "Usage: build_common.sh <dep|dev|casa>"
+        echo "Usage: build_common.sh <dep|dev|devcuda|casa>"
         exit 0;;
 esac

daliuge-common/docker/Dockerfile.cuda

@@ -0,0 +1,38 @@
+# we are doing a two-stage build to keep the size of
+# the final image low.
+
+# First stage build and cleanup
+#FROM python:3.8-slim
+FROM ubuntu:20.04
+ARG BUILD_ID
+LABEL stage=builder
+LABEL build=$BUILD_ID
+RUN apt-get update && apt-get install -y gcc python3 python3.8-venv && apt-get clean
+
+COPY / /daliuge
+
+RUN cd && python3 -m venv dlg && cd /daliuge && \
+    . ${HOME}/dlg/bin/activate && \
+    pip install numpy && \
+    pip install . && \
+    apt-get remove -y gcc && \
+    apt-get autoremove -y
+
+
+FROM ubuntu:20.04
+RUN apt-get update && apt-get install -y bash
+COPY --from=0 /root/dlg /root/dlg
+
+RUN apt install -y wget gnupg2 software-properties-common
+RUN mkdir -p /code && cd /code &&\
+    wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin &&\
+    mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 &&\
+    apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub &&\
+    add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" &&\
+    apt update
+
+RUN DEBIAN_FRONTEND=noninteractive apt -y --no-install-recommends install \
+    cuda-minimal-build-11-2 cuda-libraries-11-2 cuda-libraries-dev-11-2 &&\
+    ln -s /usr/local/cuda-11.2 /usr/local/cuda &&\
+    ln -s /usr/local/cuda/targets/x86_64-linux/lib /usr/local/cuda/lib &&\
+    ln -s /usr/local/cuda/targets/x86_64-linux/include /usr/local/cuda/include

daliuge-engine/build_engine.sh

@@ -24,13 +24,25 @@ case "$1" in
         docker build --build-arg VCS_TAG=${C_TAG} --no-cache -t icrar/daliuge-engine:${VCS_TAG} -f docker/Dockerfile.dev .
         echo "Build finished!"
         exit 0;;
+    "devall")
+        C_TAG="master"
+        [[ ! -z $2 ]] && C_TAG=$2
+        export VERSION=`git describe --tags --abbrev=0|sed s/v//`
+        export VCS_TAG=`git rev-parse --abbrev-ref HEAD | tr '[:upper:]' '[:lower:]'`
+        echo "Building daliuge-engine development version using daliuge-common:${C_TAG}"
+        echo "$VERSION:$VCS_TAG" > dlg/manager/web/VERSION
+        git rev-parse --verify HEAD >> dlg/manager/web/VERSION
+        cp ../LICENSE dlg/manager/web/.
+        docker build --build-arg VCS_TAG=${C_TAG} --no-cache -t icrar/daliuge-engine:${VCS_TAG} -f docker/Dockerfile.devall .
+        echo "Build finished!"
+        exit 0;;
     "casa")
         export VCS_TAG=`git rev-parse --abbrev-ref HEAD | tr '[:upper:]' '[:lower:]'`
         echo "Building daliuge-engine development version"
         docker build --build-arg VCS_TAG=${VCS_TAG}-casa --no-cache -t icrar/daliuge-engine:${VCS_TAG}-casa -f docker/Dockerfile.casa .
         echo "Build finished!"
         exit 0;;
     *)
-        echo "Usage: build_engine.sh <dep|dev>"
+        echo "Usage: build_engine.sh <dep|dev|devall|casa>"
         exit 0;;
 esac

daliuge-engine/dlg/apps/simple.py

@@ -609,7 +609,7 @@ def run(self):
         if len(self.inputs) != len(self.scatter_axes):
             raise DaliugeException(\
                 f"expected {len(self.inputs)} axes,\
-                 got {len(self.scatter_axes)}")
+                 got {len(self.scatter_axes)}, {self.scatter_axes}")
 
         # split it as many times as we have outputs
         self.num_of_copies = self.num_of_copies

daliuge-engine/docker/Dockerfile.devall

@@ -0,0 +1,32 @@
+ARG VCS_TAG
+# We need the base image we build with the other Dockerfile
+FROM icrar/daliuge-common:${VCS_TAG:-latest}
+
+# RUN sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata \
+#     gcc g++ gdb casacore-dev clang-tidy-10 clang-tidy libboost1.71-all-dev libgsl-dev
+
+RUN apt-get update &&\
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gcc python3-pip curl
+
+COPY / /daliuge
+RUN . /root/dlg/bin/activate && pip install wheel && cd /daliuge && \
+    pip install . 
+
+EXPOSE 9000
+EXPOSE 5555
+EXPOSE 6666
+EXPOSE 8000
+EXPOSE 8001
+EXPOSE 8002
+
+# enable the virtualenv path from daliuge-common
+ENV VIRTUAL_ENV=/root/dlg
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV DLG_ROOT="/tmp/dlg/var/dlg_home"
+
+RUN apt install -y git python3-dev
+RUN pip install 'git+https://github.com/ICRAR/dlg-nifty-components.git'
+RUN pip install --extra-index-url=https://artefact.skao.int/repository/pypi-internal/simple 'ska-sdp-cbf-emulator[plasma]>=1.6.4'
+RUN pip install 'git+https://github.com/ICRAR/dlg-casacore-components.git'
+
+CMD ["dlg", "daemon", "-vv"]