E2e liveness 4253

test/e2e/pods.go

@@ -30,10 +30,58 @@ import (
 	. "github.com/onsi/gomega"
 )
 
+func runLivenessTest(c *client.Client, podDescr *api.Pod) {
+	defer GinkgoRecover()
+	ns := "e2e-test-" + string(util.NewUUID())
+
+	By(fmt.Sprintf("Creating pod %s in namespace %s", podDescr.Name, ns))
+	_, err := c.Pods(ns).Create(podDescr)
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("creating pod %s", podDescr.Name))
+
+	// At the end of the test, clean up by removing the pod.
+	defer func() {
+		By("deleting the pod")
+		c.Pods(ns).Delete(podDescr.Name)
+	}()
+
+	// Wait until the pod is not pending. (Here we need to check for something other than
+	// 'Pending' other than checking for 'Running', since when failures occur, we go to
+	// 'Terminated' which can cause indefinite blocking.)
+	By("waiting for the pod to be something other than pending")
+	err = waitForPodNotPending(c, ns, podDescr.Name, 60*time.Second)
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("starting pod %s in namespace %s", podDescr.Name, ns))
+	By(fmt.Sprintf("Started pod %s in namespace %s", podDescr.Name, ns))
+
+	// Check the pod's current state and verify that restartCount is present.
+	By("checking the pod's current state and verifying that restartCount is present")
+	pod, err := c.Pods(ns).Get(podDescr.Name)
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("getting pod %s in namespace %s", podDescr.Name, ns))
+	initialRestartCount := pod.Status.Info["liveness"].RestartCount
+	By(fmt.Sprintf("Initial restart count of pod %s is %d", podDescr.Name, initialRestartCount))
+
+	// Wait for at most 48 * 5 = 240s = 4 minutes until restartCount is incremented
+	pass := false
+	for i := 0; i < 48; i++ {
+		// Wait until restartCount is incremented.
+		time.Sleep(5 * time.Second)
+		pod, err = c.Pods(ns).Get(podDescr.Name)
+		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("getting pod %s", podDescr.Name))
+		restartCount := pod.Status.Info["liveness"].RestartCount
+		By(fmt.Sprintf("Restart count of pod %s in namespace %s is now %d", podDescr.Name, ns, restartCount))
+		if restartCount > initialRestartCount {
+			By(fmt.Sprintf("Restart count of pod %s in namespace %s increased from %d to %d during the test", podDescr.Name, ns, initialRestartCount, restartCount))
+			pass = true
+			break
+		}
+	}
+
+	if !pass {
+		Fail(fmt.Sprintf("Did not see the restart count of pod %s in namespace %s increase from %d during the test", podDescr.Name, ns, initialRestartCount))
+	}
+}
+
 var _ = Describe("Pods", func() {
-	var (
-		c *client.Client
-	)
+	var c *client.Client
 
 	BeforeEach(func() {
 		var err error
@@ -314,4 +362,57 @@ var _ = Describe("Pods", func() {
 
 		// We could try a wget the service from the client pod.  But services.sh e2e test covers that pretty well.
 	})
+
+	It("should be restarted with a docker exec \"cat /tmp/health\" liveness probe", func() {
+		runLivenessTest(c, &api.Pod{
+			ObjectMeta: api.ObjectMeta{
+				Name:   "liveness-exec",
+				Labels: map[string]string{"test": "liveness"},
+			},
+			Spec: api.PodSpec{
+				Containers: []api.Container{
+					{
+						Name:    "liveness",
+						Image:   "busybox",
+						Command: []string{"/bin/sh", "-c", "echo ok >/tmp/health; sleep 10; echo fail >/tmp/health; sleep 600"},
+						LivenessProbe: &api.Probe{
+							Handler: api.Handler{
+								Exec: &api.ExecAction{
+									Command: []string{"cat", "/tmp/health"},
+								},
+							},
+							InitialDelaySeconds: 15,
+						},
+					},
+				},
+			},
+		})
+	})
+
+	It("should be restarted with a /healthz http liveness probe", func() {
+		runLivenessTest(c, &api.Pod{
+			ObjectMeta: api.ObjectMeta{
+				Name:   "liveness-http",
+				Labels: map[string]string{"test": "liveness"},
+			},
+			Spec: api.PodSpec{
+				Containers: []api.Container{
+					{
+						Name:    "liveness",
+						Image:   "kubernetes/liveness",
+						Command: []string{"/server"},
+						LivenessProbe: &api.Probe{
+							Handler: api.Handler{
+								HTTPGet: &api.HTTPGetAction{
+									Path: "/healthz",
+									Port: util.NewIntOrStringFromInt(8080),
+								},
+							},
+							InitialDelaySeconds: 15,
+						},
+					},
+				},
+			},
+		})
+	})
 })