Improve resilience of docker startup

concourse/concourse#215

assets/common.sh

@@ -1,5 +1,6 @@
 LOG_FILE=${LOG_FILE:-/tmp/docker.log}
 SKIP_PRIVILEGED=${SKIP_PRIVILEGED:-false}
+STARTUP_TIMEOUT=${STARTUP_TIMEOUT:-120}
 
 sanitize_cgroups() {
   mkdir -p /sys/fs/cgroup
@@ -73,18 +74,28 @@ start_docker() {
     server_args="${server_args} --registry-mirror $4"
   fi
 
-  dockerd --data-root /scratch/docker ${server_args} >$LOG_FILE 2>&1 &
-  echo $! > /tmp/docker.pid
+  try_start() {
+    dockerd --data-root /scratch/docker ${server_args} >$LOG_FILE 2>&1 &
+    echo $! > /tmp/docker.pid
 
-  trap stop_docker EXIT
+    trap stop_docker EXIT
+    sleep 1
 
-  sleep 1
-
-  if ! docker info >/dev/null 2>&1; then
     echo waiting for docker to come up...
     until docker info >/dev/null 2>&1; do
       sleep 1
+      if ! kill -0 "$(cat /tmp/docker.pid)" 2>/dev/null; then
+        return 1
+      fi
     done
+  }
+
+  export server_args LOG_FILE
+  declare -fxr stop_docker try_start
+
+  if ! timeout -t ${STARTUP_TIMEOUT} bash -ce 'while true; do try_start && break; done'; then
+    echo Docker failed to start within ${STARTUP_TIMEOUT} seconds.
+    return 1
   fi
 }
 

tests/fixtures/bin/docker

@@ -4,3 +4,7 @@ echo "DOCKER:" "$@"
 for var in "$@"; do
     echo "DOCKER ARG:" "$var"
 done
+
+if [ "$1" == "info" ] && [ -f /tmp/docker_failing ]; then
+    exit 1
+fi

tests/fixtures/bin/dockerd

@@ -2,5 +2,11 @@
 
 echo "DOCKERD:" "$@"
 
-# pretend to actually be running something, as dockerd would
-exec sleep 9999
+if [ -f /tmp/docker_failing ]; then
+  rm /tmp/docker_failing
+elif [ "$FAIL_ONCE" == "true" ]; then
+  touch /tmp/docker_failing
+  exec sleep 1
+fi
+
+exec sleep 999

tests/out_test.go

@@ -64,6 +64,38 @@ var _ = Describe("Out", func() {
 		return "DOCKERD: " + cmd
 	}
 
+	It("retries starting dockerd if it fails", func() {
+		session := putWithEnv(map[string]interface{}{
+			"source": map[string]interface{}{
+				"repository": "test",
+			},
+			"params": map[string]interface{}{
+				"build": "/docker-image-resource/tests/fixtures/build",
+			},
+		}, map[string]string{
+			"STARTUP_TIMEOUT": "5",
+			"FAIL_ONCE": "true",
+		})
+
+		Expect(session.Err).To(gbytes.Say("(?s:DOCKERD.*DOCKERD.*)"))
+	})
+
+	It("times out retrying dockerd", func() {
+		session := putWithEnv(map[string]interface{}{
+			"source": map[string]interface{}{
+				"repository": "test",
+			},
+			"params": map[string]interface{}{
+				"build": "/docker-image-resource/tests/fixtures/build",
+			},
+		}, map[string]string{
+			"STARTUP_TIMEOUT": "1",
+			"FAIL_ONCE": "true",
+		})
+
+		Expect(session.Err).To(gbytes.Say(".*Docker failed to start.*"))
+	})
+
 	It("starts dockerd with --data-root under /scratch", func() {
 		session := put(map[string]interface{}{
 			"source": map[string]interface{}{