Merge pull request #7 from duedil-ltd/fix/terminate-docker

Terminate the ephemeral docker daemon with a SIGTERM

portainer/app/executor.py

@@ -3,17 +3,18 @@
 
 import docker
 import functools
+import io
 import json
 import logging
 import mesos.interface
-import pesos.executor
 import os
-import threading
-import time
+import pesos.executor
 import re
+import signal
 import subprocess
+import threading
+import time
 import traceback
-import io
 
 from pesos.vendor.mesos import mesos_pb2
 
@@ -70,7 +71,6 @@ def launch_docker_daemon():
 
             # TODO(tarnfeld): This should be made a little more flexible
             proc = subprocess.Popen(["/usr/local/bin/wrapdocker"])
-            # os.environ["MESOS_DIRECTORY"]
 
             self.docker = docker.Client()
             while True:
@@ -94,10 +94,10 @@ def launch_docker_daemon():
             self.docker_daemon_up = True
 
     def disconnected(self, driver):
-        log.info("Disconnected from master! Ahh!")
+        logger.info("Disconnected from master! Ahh!")
 
     def reregistered(self, driver, slaveInfo):
-        log.info("Re-registered from the master! Ahh!")
+        logger.info("Re-registered from the master! Ahh!")
 
     def launchTask(self, driver, taskInfo):
 
@@ -114,6 +114,18 @@ def launchTask(self, driver, taskInfo):
         thread.setDaemon(True)
         thread.start()
 
+    def shutdown(self, driver):
+        logger.info("Shutting down the executor")
+        if os.path.exists("/var/run/docker.pid"):
+            try:
+                docker_pid = int(open("/var/run/docker.pid", "r").read())
+                os.kill(docker_pid, signal.SIGTERM)
+            except Exception, e:
+                logger.error("Caught exception killing docker daemon")
+                logger.error(e)
+        else:
+            logger.warning("Unable to locate docker pidfile")
+
     def _wrap_docker_stream(self, stream):
         """Wrapper to parse the different types of messages from the
         Docker Remote API and spit them out in a friendly format."""
@@ -160,17 +172,22 @@ def _build_image(self, driver, taskInfo, buildTask):
 
         logger.info("Waiting for docker daemon to be available")
 
-        # Wait for the docker daemon to be ready
-        while not self.docker_daemon_up:
+        # Wait for the docker daemon to be ready (up to 30 seconds)
+        timeout = 30
+        while timeout > 1 and not self.docker_daemon_up:
+            timeout -= 1
             time.sleep(1)
 
-        # Now that docker is up, let's go and do stuff
-        driver.sendStatusUpdate(mesos_pb2.TaskStatus(
-            task_id=taskInfo.task_id,
-            state=self.TASK_RUNNING
-        ))
-
         try:
+            if not self.docker_daemon_up:
+                raise Exception("Timed out waiting for docker daemon")
+
+            # Now that docker is up, let's go and do stuff
+            driver.sendStatusUpdate(mesos_pb2.TaskStatus(
+                task_id=taskInfo.task_id,
+                state=self.TASK_RUNNING
+            ))
+
             if not buildTask:
                 raise Exception("Failed to decode the BuildTask protobuf data")