fix(services): prevent units from stopping as "failed/failed"

Deis' systemd units always return "failed/failed" after being stopped by fleetctl. Somehow our ExecStop command to remove the service container conflicts with our ExecStart command and causes it to return an error code. This is fixed by removing the ExecStop commands and adding a `--rm` flag to the docker command in ExecStart. Additionally, our on_exit() handler in each /app/bin/boot script was not returning 0. TESTING: Run `make run`, then `make stop` on a Deis cluster. You should see all units stop and return to "inactive/dead" status, whereas before they would show "failed/failed."
deis · Jun 26, 2014 · 6b452e5 · 6b452e5
1 parent 813057e
commit 6b452e5
Show file tree

Hide file tree

Showing 14 changed files with 16 additions and 16 deletions.
diff --git a/builder/bin/boot b/builder/bin/boot
@@ -65,6 +65,7 @@ SSHD_PID=$!
 function on_exit() {
 	kill -TERM $DOCKER_PID $SSHD_PID
 	wait $DOCKER_PID $SSHD_PID 2>/dev/null
+	exit 0
 }
 trap on_exit INT TERM EXIT
 

diff --git a/builder/systemd/deis-builder.service b/builder/systemd/deis-builder.service
@@ -7,9 +7,8 @@ TimeoutStartSec=20m
 ExecStartPre=/bin/sh -c "docker history deis/builder >/dev/null || docker pull deis/builder:latest"
 ExecStartPre=/bin/sh -c "docker inspect deis-builder >/dev/null && docker rm -f deis-builder || true"
 ExecStartPre=/bin/sh -c "docker inspect deis-builder-data >/dev/null 2>&1 || docker run --name deis-builder-data -v /var/lib/docker deis/base true"
-ExecStart=/usr/bin/docker run --name deis-builder -p 2223:22 -e PUBLISH=22 -e HOST=${COREOS_PRIVATE_IPV4} -e PORT=2223 --volumes-from deis-builder-data --privileged deis/builder
+ExecStart=/usr/bin/docker run --name deis-builder --rm -p 2223:22 -e PUBLISH=22 -e HOST=${COREOS_PRIVATE_IPV4} -e PORT=2223 --volumes-from deis-builder-data --privileged deis/builder
 ExecStartPost=/bin/sh -c "echo 'Waiting for builder on 2223/tcp...' && until cat </dev/null>/dev/tcp/$COREOS_PRIVATE_IPV4/2223; do sleep 1; done"
-ExecStop=/usr/bin/docker rm -f deis-builder
 
 [Install]
 WantedBy=multi-user.target
diff --git a/cache/bin/boot b/cache/bin/boot
@@ -44,6 +44,7 @@ SERVICE_PID=$!
 function on_exit() {
 	kill -TERM $SERVICE_PID
 	wait $SERVICE_PID 2>/dev/null
+	exit 0
 }
 trap on_exit INT TERM
 

diff --git a/cache/systemd/deis-cache.service b/cache/systemd/deis-cache.service
@@ -6,8 +6,7 @@ EnvironmentFile=/etc/environment
 TimeoutStartSec=20m
 ExecStartPre=/bin/sh -c "docker history deis/cache >/dev/null || docker pull deis/cache:latest"
 ExecStartPre=/bin/sh -c "docker inspect deis-cache >/dev/null && docker rm -f deis-cache || true"
-ExecStart=/usr/bin/docker run --name deis-cache -p 6379:6379 -e PUBLISH=6379 -e HOST=${COREOS_PRIVATE_IPV4} deis/cache
-ExecStop=/usr/bin/docker rm -f deis-cache
+ExecStart=/usr/bin/docker run --name deis-cache --rm -p 6379:6379 -e PUBLISH=6379 -e HOST=${COREOS_PRIVATE_IPV4} deis/cache
 
 [Install]
 WantedBy=multi-user.target
diff --git a/controller/bin/boot b/controller/bin/boot
@@ -66,6 +66,7 @@ GUNICORN_PID=$!
 function on_exit() {
 	kill -TERM $CELERY_PID $GUNICORN_PID
 	wait $CELERY_PID $GUNICORN_PID 2>/dev/null
+	exit 0
 }
 trap on_exit INT TERM
 

diff --git a/controller/systemd/deis-controller.service b/controller/systemd/deis-controller.service
@@ -8,8 +8,7 @@ EnvironmentFile=/etc/environment
 TimeoutStartSec=20m
 ExecStartPre=/bin/sh -c "docker history deis/controller >/dev/null || docker pull deis/controller:latest"
 ExecStartPre=/bin/sh -c "docker inspect deis-controller >/dev/null && docker rm -f deis-controller || true"
-ExecStart=/usr/bin/docker run --name deis-controller -p 8000:8000 -e PUBLISH=8000 -e HOST=${COREOS_PRIVATE_IPV4} --volumes-from=deis-logger deis/controller
-ExecStop=/usr/bin/docker rm -f deis-controller
+ExecStart=/usr/bin/docker run --name deis-controller --rm -p 8000:8000 -e PUBLISH=8000 -e HOST=${COREOS_PRIVATE_IPV4} --volumes-from=deis-logger deis/controller
 
 [Install]
 WantedBy=multi-user.target

diff --git a/database/bin/boot b/database/bin/boot
@@ -59,8 +59,9 @@ SERVICE_PID=$!
 
 # smart shutdown on SIGINT and SIGTERM
 function on_exit() {
-	kill -TERM $SERVICE_PID
-	wait $SERVICE_PID 2>/dev/null
+    kill -TERM $SERVICE_PID
+    wait $SERVICE_PID 2>/dev/null
+    exit 0
 }
 trap on_exit INT TERM
 

diff --git a/database/systemd/deis-database.service b/database/systemd/deis-database.service
@@ -7,8 +7,7 @@ TimeoutStartSec=20m
 ExecStartPre=/bin/sh -c "docker history deis/database >/dev/null || docker pull deis/database:latest"
 ExecStartPre=/bin/sh -c "docker inspect deis-database >/dev/null && docker rm -f deis-database || true"
 ExecStartPre=/bin/sh -c "docker inspect deis-database-data >/dev/null 2>&1 || docker run --name deis-database-data -v /var/lib/postgresql deis/base true"
-ExecStart=/usr/bin/docker run --name deis-database -p 5432:5432 -e PUBLISH=5432 -e HOST=${COREOS_PRIVATE_IPV4} --volumes-from deis-database-data deis/database
-ExecStop=/usr/bin/docker rm -f deis-database
+ExecStart=/usr/bin/docker run --name deis-database --rm -p 5432:5432 -e PUBLISH=5432 -e HOST=${COREOS_PRIVATE_IPV4} --volumes-from deis-database-data deis/database
 
 [Install]
 WantedBy=multi-user.target
diff --git a/logger/bin/boot b/logger/bin/boot
@@ -26,6 +26,7 @@ SERVICE_PID=$!
 function on_exit() {
     kill -TERM $SERVICE_PID
     wait $SERVICE_PID 2>/dev/null
+    exit 0
 }
 trap on_exit INT TERM
 

diff --git a/logger/systemd/deis-logger.service b/logger/systemd/deis-logger.service
@@ -7,8 +7,7 @@ TimeoutStartSec=20m
 ExecStartPre=/bin/sh -c "docker history deis/logger >/dev/null || docker pull deis/logger:latest"
 ExecStartPre=/bin/sh -c "docker inspect deis-logger >/dev/null && docker rm -f deis-logger || true"
 ExecStartPre=/bin/sh -c "docker inspect deis-logger-data >/dev/null 2>&1 || docker run --name deis-logger-data -v /var/log/deis deis/base true"
-ExecStart=/usr/bin/docker run --name deis-logger -p 514:514/udp -e PUBLISH=514 -e HOST=${COREOS_PRIVATE_IPV4} --volumes-from deis-logger-data deis/logger
-ExecStop=/usr/bin/docker rm -f deis-logger
+ExecStart=/usr/bin/docker run --name deis-logger --rm -p 514:514/udp -e PUBLISH=514 -e HOST=${COREOS_PRIVATE_IPV4} --volumes-from deis-logger-data deis/logger
 
 [Install]
 WantedBy=multi-user.target
diff --git a/registry/bin/boot b/registry/bin/boot
@@ -56,6 +56,7 @@ SERVICE_PID=$!
 function on_exit() {
 	kill -TERM $SERVICE_PID
 	wait $SERVICE_PID 2>/dev/null
+	exit 0
 }
 trap on_exit INT TERM
 

diff --git a/registry/systemd/deis-registry.service b/registry/systemd/deis-registry.service
@@ -7,9 +7,8 @@ TimeoutStartSec=20m
 ExecStartPre=/bin/sh -c "docker history deis/registry >/dev/null || docker pull deis/registry:latest"
 ExecStartPre=/bin/sh -c "docker inspect deis-registry >/dev/null && docker rm -f deis-registry || true"
 ExecStartPre=/bin/sh -c "docker inspect deis-registry-data >/dev/null 2>&1 || docker run --name deis-registry-data -v /data deis/base /bin/true"
-ExecStart=/usr/bin/docker run --name deis-registry -p 5000:5000 -e PUBLISH=5000 -e HOST=${COREOS_PRIVATE_IPV4} --volumes-from deis-registry-data deis/registry
+ExecStart=/usr/bin/docker run --name deis-registry --rm -p 5000:5000 -e PUBLISH=5000 -e HOST=${COREOS_PRIVATE_IPV4} --volumes-from deis-registry-data deis/registry
 ExecStartPost=/bin/sh -c "echo 'Waiting for listener on 5000/tcp...' && until cat </dev/null>/dev/tcp/$COREOS_PRIVATE_IPV4/5000; do sleep 1; done && docker pull deis/slugrunner:latest && docker tag deis/slugrunner $COREOS_PRIVATE_IPV4:5000/deis/slugrunner && docker push $COREOS_PRIVATE_IPV4:5000/deis/slugrunner"
-ExecStop=/usr/bin/docker rm -f deis-registry
 
 [Install]
 WantedBy=multi-user.target
diff --git a/router/bin/boot b/router/bin/boot
@@ -62,6 +62,7 @@ SERVICE_PID=$!
 function on_exit() {
     kill -TERM $SERVICE_PID
     wait $SERVICE_PID 2>/dev/null
+    exit 0
 }
 trap on_exit INT TERM
 

diff --git a/router/systemd/deis-router.service b/router/systemd/deis-router.service
@@ -6,8 +6,7 @@ EnvironmentFile=/etc/environment
 TimeoutStartSec=20m
 ExecStartPre=/bin/sh -c "docker history deis/router >/dev/null || docker pull deis/router:latest"
 ExecStartPre=/bin/sh -c "docker inspect deis-router >/dev/null && docker rm -f deis-router || true"
-ExecStart=/usr/bin/docker run --name deis-router -p 80:80 -p 2222:2222 -e PUBLISH=80 -e HOST=${COREOS_PRIVATE_IPV4} deis/router
-ExecStop=/usr/bin/docker rm -f deis-router
+ExecStart=/usr/bin/docker run --name deis-router --rm -p 80:80 -p 2222:2222 -e PUBLISH=80 -e HOST=${COREOS_PRIVATE_IPV4} deis/router
 
 [Install]
 WantedBy=multi-user.target