Kill process instead of container, on container restart.

When Docker restarts a container and something goes wrong attaching the network, kill the process inside the container instead of killing the container, so that Docker will restart it again
weaveworks · May 6, 2016 · 8d71467 · 8d71467
1 parent 01e3d1d
commit 8d71467
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 4 deletions.
diff --git a/proxy/proxy.go b/proxy/proxy.go
@@ -103,7 +103,7 @@ func (proxy *Proxy) attachWithRetry(id string) {
 
 	j := &attachJob{id: id, tryInterval: initialInterval}
 	j.timer = time.AfterFunc(time.Duration(0), func() {
-		if err := proxy.attach(id, false); err != nil {
+		if err := proxy.attach(id, false, false); err != nil {
 			// The delay at the nth retry is a random value in the range
 			// [i-i/2,i+i/2], where i = initialInterval * 1.5^(n-1).
 			j.timer.Reset(j.tryInterval)
@@ -383,7 +383,10 @@ func (proxy *Proxy) listen(protoAndAddr string) (net.Listener, string, error) {
 
 // weavedocker.ContainerObserver interface
 func (proxy *Proxy) ContainerStarted(ident string) {
-	proxy.notifyWaiters(ident, proxy.attach(ident, true))
+	// In case of attach failure, if we have a request waiting on the start, kill the container,
+	// otherwise assume it is a Docker-initated restart and kill the process inside.
+	err := proxy.attach(ident, true, proxy.waitChan(ident) == nil)
+	proxy.notifyWaiters(ident, err)
 }
 
 func containerShouldAttach(container *docker.Container) bool {
@@ -463,7 +466,7 @@ func (proxy *Proxy) ContainerDestroyed(ident string) {}
 
 // Check if this container needs to be attached, if so then attach it,
 // and return nil on success or not needed.
-func (proxy *Proxy) attach(containerID string, orDie bool) error {
+func (proxy *Proxy) attach(containerID string, orDie, killProcess bool) error {
 	container, err := proxy.client.InspectContainer(containerID)
 	if err != nil {
 		if _, ok := err.(*docker.NoSuchContainer); !ok {
@@ -500,7 +503,11 @@ func (proxy *Proxy) attach(containerID string, orDie bool) error {
 		args = append(args, "--no-multicast-route")
 	}
 	if orDie {
-		args = append(args, "--or-die")
+		if killProcess {
+			args = append(args, "--or-kill")
+		} else {
+			args = append(args, "--or-die")
+		}
 	}
 	args = append(args, container.ID)
 	return callWeaveAttach(container, args)

diff --git a/weave b/weave
@@ -832,13 +832,24 @@ kill_container() {
     docker kill $1 >/dev/null 2>&1 || true
 }
 
+kill_container_process() {
+    kill -9 $(docker inspect --format='{{.State.Pid}}' $1) >/dev/null 2>&1 || true
+}
+
 with_container_netns_or_die() {
     if ! with_container_netns "$@" >/dev/null ; then
         kill_container $1
         exit 1
     fi
 }
 
+with_container_netns_or_kill() {
+    if ! with_container_netns "$@" >/dev/null ; then
+        kill_container_process $1
+        exit 1
+    fi
+}
+
 # Execute arguments as a command within the $CONTAINER_NETNS network namespace
 netnsenter() {
     nsenter --net=$CONTAINER_NETNS "$@"
@@ -1460,6 +1471,13 @@ ipam_cidrs_or_die() {
     fi
 }
 
+ipam_cidrs_or_kill() {
+    if ! ipam_cidrs "$@" ; then
+        kill_container_process $2
+        exit 1
+    fi
+}
+
 show_addrs() {
     addrs=
     for cidr in "$@" ; do
@@ -2136,6 +2154,9 @@ EOF
                 --or-die)
                     ATTACH_TYPE="_or_die"
                     ;;
+                --or-kill)
+                    ATTACH_TYPE="_or_kill"
+                    ;;
                 --rewrite-hosts)
                     REWRITE_HOSTS=1
                     ;;