Testing CI : Fix the intermittent failing build with SGE (#515)

* Keeping some debug message, trying to not restart SGE slaves at startup

* Use run-master and run-slave from git repo (instead of docker image), remove some useless debug commands

* Fix no SGE worker logs in cleanup CI

ci/sge.sh

@@ -12,6 +12,7 @@ function jobqueue_before_install {
 
     docker ps -a
     docker images
+    docker exec sge_master qconf -sq dask.q
 }
 
 function jobqueue_install {
@@ -23,7 +24,8 @@ function jobqueue_script {
 }
 
 function jobqueue_after_script {
-    docker exec sge_master bash -c 'cat /tmp/sge*'
-    docker exec slave_one bash -c 'cat /tmp/exec*'
-    docker exec slave_two bash -c 'cat /tmp/exec*'
+    echo "Daemon logs"
+    docker exec sge_master bash -c 'cat /tmp/sge*' || echo "No sge_master logs"
+    docker exec slave_one bash -c 'cat /tmp/exec*' || echo "No slave_one logs"
+    docker exec slave_two bash -c 'cat /tmp/exec*' || echo "No slave_two logs"
 }

ci/sge/docker-compose.yml

@@ -14,7 +14,7 @@ services:
     #network_mode: host
     volumes:
       - ../..:/dask-jobqueue
-    command: bash /run-master.sh
+    command: bash /dask-jobqueue/ci/sge/run-master.sh
 
   slave-one:
     image: daskdev/dask-jobqueue:sge-slave
@@ -28,7 +28,7 @@ services:
     #network_mode: host
     volumes:
       - ../..:/dask-jobqueue
-    command: bash /run-slave.sh
+    command: bash /dask-jobqueue/ci/sge/run-slave.sh
     links:
       - "master:sge_master"
     depends_on:
@@ -46,7 +46,7 @@ services:
     #network_mode: host
     volumes:
       - ../..:/dask-jobqueue
-    command: bash /run-slave.sh
+    command: bash /dask-jobqueue/ci/sge/run-slave.sh
     links:
       - "master:sge_master"
     depends_on:

ci/sge/run-master.sh

@@ -5,6 +5,9 @@
 sudo service gridengine-master restart
 
 while ! ping -c1 slave_one &>/dev/null; do :; done
+#Sometimes conf is inaccessible at first
+while ! qconf -sconf &>/dev/null; do sleep 0.1; done
+cat /var/lib/gridengine//default/common/act_qmaster
 
 qconf -Msconf /scheduler.txt
 qconf -Ahgrp /hosts.txt

ci/sge/run-slave.sh

@@ -1,9 +1,8 @@
 #!/bin/bash
 
 # start sge
-sudo service gridengine-exec restart
-
-sleep 4
+#wait a bit for master configuration
+sleep 3
 
 sudo service gridengine-exec restart