Required changes to get the ray installation working again

Dockerfile

@@ -1,10 +1,13 @@
 # Build a docker image 
 # Merging the ray engine (https://docs.ray.io/en/master/installation.html)
 # and DALiuGE
-FROM rayproject/ray:latest-cpu
-RUN sudo apt update && sudo apt install -y gcc && test -e daliuge || cd && git clone --branch ray_test https://github.com/ICRAR/daliuge.git
+FROM rayproject/ray:74cbf0-py38
+
+#FROM kernsuite/base:7
+RUN sudo apt update && sudo apt install -y gcc && test -e daliuge || cd && git clone --branch YAN-708 https://github.com/ICRAR/daliuge.git
 RUN cd /home/ray/daliuge/daliuge-common && pip install . \
     && cd ../daliuge-engine && pip install . \
+    && pip install ray \
     && rm -rf /home/ray/anaconda3/lib/python3.7/site-packages/azure \
     && sudo apt-get remove cmake gcc -y \
     && sudo apt-get clean 

Dockerfile.ray

@@ -1,5 +1,6 @@
 # We need the base image we build with the other Dockerfile
-FROM rayproject/ray:latest-cpu
+# latest ray image does not work because compilation of netifaces fails
+FROM rayproject/ray:74cbf0-py38
 
 RUN git clone https://github.com/ICRAR/daliuge && \
     apt update && \

build_dlg_ray.sh

@@ -2,4 +2,4 @@
 
 # Helper script for building the DALiuGE Ray docker image.
 # For more details please refer to the Dockerfile
-docker build --no-cache -t icrar/dlg_ray:1.3 .
+docker build --no-cache -t icrar/dlg_ray:1.5 .

daliuge-engine/dlg/graph_loader.py

@@ -219,7 +219,8 @@ def createGraphFromDropSpecList(dropSpecList, session=None):
     for n,dropSpec in enumerate(dropSpecList):
 
         check_dropspec(n, dropSpec)
-        dropType = dropSpec.pop("type")
+#        dropType = dropSpec.pop("type")
+        dropType = dropSpec["type"]
 
         cf = __CREATION_FUNCTIONS[dropType]
         drop = cf(dropSpec, session=session)

daliuge-engine/run_engine.sh

@@ -1,6 +1,8 @@
 docker run --shm-size=1g --ipc="shareable" --rm -td --name daliuge-engine -v /var/dlg_home:/var/dlg_home -v /var/run/docker.sock:/var/run/docker.sock -p 5555:5555 -p 6666:6666 -p 8000:8000 -p 8001:8001 -p 8002:8002 -p 9000:9000  icrar/daliuge-engine:ray
 sleep 2
-./start_local_managers.sh
+# ./start_local_managers.sh
 # start the plasma store. NOTE: the container has been started with shareable memory
 # which means that the plasma store is accessible across containers on the same host.
-docker  exec -ti daliuge-engine bash -c 'plasma_store -m 600000000 -s /var/dlg_home/tmp/plasma 1> /var/dlg_home/logs/plasma 2> /var/dlg_home/logs/plasma &'
+
+# TODO: The following command just leaves a defunct process, but not the plasma store running (problem of quoting??)
+# docker  exec -t daliuge-engine bash -c 'plasma_store -m 600000000 -s /var/dlg_home/tmp/plasma 1> /var/dlg_home/logs/plasma 2> /var/dlg_home/logs/plasma &'

daliuge-ray.yaml

@@ -27,7 +27,7 @@ autoscaling_mode: default
 # and opens all the necessary ports to support the Ray cluster.
 # Empty string means disabled.
 docker:
-    image: "icrar/dlg_ray:1.4"
+    image: "icrar/dlg_ray:1.5"
     # image: "rayproject/ray:latest-cpu" # You can change this to latest-cpu if you don't need GPU support and want a faster startup
     container_name: "ray_container"
     # If true, pulls latest version of image. Otherwise, `docker run` will only pull the image
@@ -39,7 +39,7 @@ docker:
     # head_image: "rayproject/ray:latest-gpu"
     # Allow Ray to automatically detect GPUs
 
-    worker_image: "icrar/dlg_ray:1.4"
+    worker_image: "icrar/dlg_ray:1.5"
     worker_run_options: [-v /var/run/docker.sock:/var/run/docker.sock  -p 8265:8265]
 
 # The autoscaler will scale up the cluster to this target fraction of resource