Add docker-compose htcondor setup (#441)

Co-authored-by: Loïc Estève <loic.esteve@ymail.com>

ci/htcondor.sh

@@ -2,41 +2,32 @@
 
 function jobqueue_before_install {
     docker version
+    docker-compose version
 
-    docker run -d --name jobqueue-htcondor-mini htcondor/mini:el7 # might fail if called as script
+    # start htcondor cluster
+    cd ./ci/htcondor
+    docker-compose pull
+    ./start-htcondor.sh
+    cd -
 
     docker ps -a
     docker images
 }
-[[ "${BASH_SOURCE[0]}" != "${0}" ]] || jobqueue_before_install # excute if called as script
 
 function jobqueue_install {
-    docker exec --user root jobqueue-htcondor-mini  /bin/bash -c "
-    	rm -rf /dask-jobqueue
-   "
-    docker cp . jobqueue-htcondor-mini:/dask-jobqueue 
-
-    docker exec --user root jobqueue-htcondor-mini /bin/bash -c "
-    	python3 -c 'import psutil' 2>/dev/null || yum -y install python3-psutil; # psutil has no wheel , install gcc even slower
-	cd /dask-jobqueue; 
-	pip3 install -e .;
-	pip3 install pytest;
-	rm -f /var/log/condor/*
-	chown -R submituser:submituser /dask-jobqueue 
-    "
+    cd ./ci/htcondor
+    docker-compose exec -T submit /bin/bash -c "cd /dask-jobqueue; pip3 install -e .;chown -R submituser ."
+    cd -
 }
-[[ "${BASH_SOURCE[0]}" != "${0}" ]] || jobqueue_install # excute if called as script
 
 function jobqueue_script {
-    docker exec --user=submituser jobqueue-htcondor-mini /bin/bash -c "
-    	cd /dask-jobqueue; 
-	pytest dask_jobqueue --verbose -s -E htcondor"
+    cd ./ci/htcondor
+    docker-compose exec -T --user submituser submit /bin/bash -c "cd; pytest /dask-jobqueue/dask_jobqueue --verbose -E htcondor -s"
+    cd -
 }
-[[ "${BASH_SOURCE[0]}" != "${0}" ]] || jobqueue_script # excute if called as script
 
 function jobqueue_after_script {
-    docker exec --user root jobqueue-htcondor-mini /bin/bash -c "
-    	grep -R \"\" /var/log/condor/	
-   "
+    cd ./ci/htcondor
+    docker-compose exec -T cm /bin/bash -c " grep -R \"\" /var/log/condor/	"
+    cd -
 }
-[[ "${BASH_SOURCE[0]}" != "${0}" ]] || jobqueue_after_script # excute if called as script

ci/htcondor/Dockerfile

@@ -0,0 +1,10 @@
+FROM  htcondor/submit:el7 as submit
+
+RUN yum install -y gcc git 
+RUN yum install -y python3-devel python3-pip 
+RUN pip3 install dask distributed pytest
+
+FROM  htcondor/execute:el7 as execute 
+RUN yum install -y python3
+COPY --from=submit /usr/local/lib/python3.6 /usr/local/lib/python3.6
+COPY --from=submit /usr/local/lib64/python3.6 /usr/local/lib64/python3.6

ci/htcondor/docker-compose.yml

@@ -0,0 +1,57 @@
+version: "3.4"
+
+services:
+  cm:
+    image: htcondor/cm:el7
+    hostname: cm.htcondor
+    environment:
+      - USE_POOL_PASSWORD=yes
+    volumes:
+      - secrets:/root/secrets
+    command: bash -c 'condor_store_cred -p password -f /root/secrets/pool_password ; exec bash -x /start.sh'
+
+  submit:
+    image: daskdev/dask-jobqueue:htcondor-submit
+    build:
+      context: .
+      target: submit
+    hostname: submit.htcondor
+    environment:
+      - CONDOR_HOST=cm
+      - USE_POOL_PASSWORD=yes
+    depends_on:
+      - cm
+    volumes:
+      - secrets:/root/secrets
+      - ../..:/dask-jobqueue
+
+  execute1:
+    image: daskdev/dask-jobqueue:htcondor-execute
+    build:
+      context: .
+      target: execute
+    hostname: execute1.htcondor
+    environment:
+      - CONDOR_HOST=cm
+      - USE_POOL_PASSWORD=yes
+    depends_on:
+      - cm
+    volumes:
+      - secrets:/root/secrets
+
+  execute2:
+    image: daskdev/dask-jobqueue:htcondor-execute
+    build:
+      context: .
+      target: execute
+    hostname: execute2.htcondor
+    environment:
+      - CONDOR_HOST=cm
+      - USE_POOL_PASSWORD=yes
+    depends_on:
+      - cm
+    volumes:
+      - secrets:/root/secrets
+
+volumes:
+  secrets:

ci/htcondor/start-htcondor.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+docker-compose up -d
+
+while [ `docker-compose exec -T submit condor_status -af activity|grep Idle|wc -l` -ne 2 ]
+  do
+    echo "Waiting for cluster to become ready";
+    sleep 2
+  done
+echo "HTCondor properly configured"