From 8b354556640d15ab327b37dac655a5afe529a406 Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Sat, 7 Nov 2015 16:48:11 -0800 Subject: [PATCH 1/2] Dump kubectl logs of Ingress controller at on e2e failure --- test/e2e/ingress.go | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 0d378b95f26e..331d2b5da906 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -62,6 +62,9 @@ var ( expectedLBCreationTime = 7 * time.Minute expectedLBHealthCheckTime = 7 * time.Minute + // Name of the loadbalancer controller within the cluster addon + lbContainerName = "l7-lb-controller" + // On average it takes ~6 minutes for a single backend to come online. // We *don't* expect this poll to consistently take 15 minutes for every // Ingress as GCE is creating/checking backends in parallel, but at the @@ -201,6 +204,35 @@ func checkLeakedResources() error { return nil } +// run kubectl log on the L7 controller pod. +func kubectlLogLBController(c *client.Client) { + selector := labels.SelectorFromSet(labels.Set(map[string]string{"name": "glbc"})) + podList, err := c.Pods(api.NamespaceAll).List(selector, fields.Everything()) + if err != nil { + Logf("Cannot log L7 controller output, error listing pods %v", err) + return + } + if len(podList.Items) == 0 { + Logf("Loadbalancer controller pod not found") + return + } + for _, p := range podList.Items { + Logf("\nLast 100 log lines of %v\n", p.Name) + Logf(runKubectl("logs", p.Name, "--namespace=kube-system", "-c", lbContainerName, "--tail=100")) + } +} + +// dumpDebugAndFail dumps verbose debug output before failing. +func dumpDebugAndFail(err string, ns string, c *client.Client) { + kubectlLogLBController(c) + Logf("\nOutput of kubectl describe ing:\n") + + // TODO: runKubectl will hard fail if kubectl fails, swap it out for + // something more befitting for a debug dumper. + Logf(runKubectl("describe", "ing", fmt.Sprintf("--namespace=%v", ns))) + Failf(err) +} + var _ = Describe("GCE L7 LoadBalancer Controller", func() { // These variables are initialized after framework's beforeEach. var ns string @@ -268,6 +300,9 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() { // Wait for the loadbalancer IP. start := time.Now() address, err := waitForIngressAddress(client, ing.Namespace, ing.Name, lbPollTimeout) + if err != nil { + dumpDebugAndFail(fmt.Sprintf("Ingress failed to acquire an IP address within %v", lbPollTimeout), ns, client) + } Expect(err).NotTo(HaveOccurred()) By(fmt.Sprintf("Found address %v for ingress %v, took %v to come online", address, ing.Name, time.Since(start))) @@ -295,8 +330,8 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() { return true, nil }) if pollErr != nil { - Failf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v", - lbPollTimeout, route, rules.Host, lastBody, pollErr) + dumpDebugAndFail(fmt.Sprintf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v", + lbPollTimeout, route, rules.Host, lastBody, pollErr), ns, client) } rt := time.Since(GETStart) By(fmt.Sprintf("Route %v host %v took %v to respond", route, rules.Host, rt)) @@ -310,12 +345,12 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() { sort.Sort(timeSlice(creationTimes)) perc50 := creationTimes[len(creationTimes)/2] if perc50 > expectedLBCreationTime { - Failf("Average creation time is too high: %+v", creationTimes) + dumpDebugAndFail(fmt.Sprintf("Average creation time is too high: %+v", creationTimes), ns, client) } sort.Sort(timeSlice(responseTimes)) perc50 = responseTimes[len(responseTimes)/2] if perc50 > expectedLBHealthCheckTime { - Failf("Average startup time is too high: %+v", responseTimes) + dumpDebugAndFail(fmt.Sprintf("Average startup time is too high: %+v", responseTimes), ns, client) } }) }) From eb4106fe29dfe23655431aa1be47dc18bae2f29a Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Sun, 8 Nov 2015 10:58:21 -0800 Subject: [PATCH 2/2] Refactor debug logging methods. --- test/e2e/examples.go | 86 ++++++++++++++++----------------- test/e2e/ingress.go | 47 ++++++++++-------- test/e2e/kubectl.go | 104 ++++++++++++++++++++-------------------- test/e2e/portforward.go | 2 +- test/e2e/util.go | 40 ++++++++++------ 5 files changed, 149 insertions(+), 130 deletions(-) diff --git a/test/e2e/examples.go b/test/e2e/examples.go index 71a299ddb54e..70208ae1e0ea 100644 --- a/test/e2e/examples.go +++ b/test/e2e/examples.go @@ -74,7 +74,7 @@ var _ = Describe("Examples e2e", func() { expectedOnSentinel := "+monitor master" By("starting redis bootstrap") - runKubectl("create", "-f", bootstrapYaml, nsFlag) + runKubectlOrDie("create", "-f", bootstrapYaml, nsFlag) err := waitForPodRunningInNamespace(c, bootstrapPodName, ns) Expect(err).NotTo(HaveOccurred()) @@ -84,13 +84,13 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) By("setting up services and controllers") - runKubectl("create", "-f", sentinelServiceYaml, nsFlag) - runKubectl("create", "-f", sentinelControllerYaml, nsFlag) - runKubectl("create", "-f", controllerYaml, nsFlag) + runKubectlOrDie("create", "-f", sentinelServiceYaml, nsFlag) + runKubectlOrDie("create", "-f", sentinelControllerYaml, nsFlag) + runKubectlOrDie("create", "-f", controllerYaml, nsFlag) By("scaling up the deployment") - runKubectl("scale", "rc", redisRC, "--replicas=3", nsFlag) - runKubectl("scale", "rc", sentinelRC, "--replicas=3", nsFlag) + runKubectlOrDie("scale", "rc", redisRC, "--replicas=3", nsFlag) + runKubectlOrDie("scale", "rc", sentinelRC, "--replicas=3", nsFlag) By("checking up the services") checkAllLogs := func() { @@ -110,7 +110,7 @@ var _ = Describe("Examples e2e", func() { checkAllLogs() By("turning down bootstrap") - runKubectl("delete", "-f", bootstrapYaml, nsFlag) + runKubectlOrDie("delete", "-f", bootstrapYaml, nsFlag) err = waitForRCPodToDisappear(c, ns, redisRC, bootstrapPodName) Expect(err).NotTo(HaveOccurred()) By("waiting for the new master election") @@ -131,8 +131,8 @@ var _ = Describe("Examples e2e", func() { nsFlag := fmt.Sprintf("--namespace=%v", ns) By("starting rabbitmq") - runKubectl("create", "-f", rabbitmqServiceYaml, nsFlag) - runKubectl("create", "-f", rabbitmqControllerYaml, nsFlag) + runKubectlOrDie("create", "-f", rabbitmqServiceYaml, nsFlag) + runKubectlOrDie("create", "-f", rabbitmqControllerYaml, nsFlag) forEachPod(c, ns, "component", "rabbitmq", func(pod api.Pod) { _, err := lookForStringInLog(ns, pod.Name, "rabbitmq", "Server startup complete", serverStartTimeout) Expect(err).NotTo(HaveOccurred()) @@ -141,15 +141,15 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) By("starting celery") - runKubectl("create", "-f", celeryControllerYaml, nsFlag) + runKubectlOrDie("create", "-f", celeryControllerYaml, nsFlag) forEachPod(c, ns, "component", "celery", func(pod api.Pod) { _, err := lookForStringInFile(ns, pod.Name, "celery", "/data/celery.log", " ready.", serverStartTimeout) Expect(err).NotTo(HaveOccurred()) }) By("starting flower") - runKubectl("create", "-f", flowerServiceYaml, nsFlag) - runKubectl("create", "-f", flowerControllerYaml, nsFlag) + runKubectlOrDie("create", "-f", flowerServiceYaml, nsFlag) + runKubectlOrDie("create", "-f", flowerControllerYaml, nsFlag) forEachPod(c, ns, "component", "flower", func(pod api.Pod) { // Do nothing. just wait for it to be up and running. }) @@ -173,9 +173,9 @@ var _ = Describe("Examples e2e", func() { nsFlag := fmt.Sprintf("--namespace=%v", ns) By("starting master") - runKubectl("create", "-f", serviceJson, nsFlag) - runKubectl("create", "-f", masterJson, nsFlag) - runKubectl("create", "-f", driverJson, nsFlag) + runKubectlOrDie("create", "-f", serviceJson, nsFlag) + runKubectlOrDie("create", "-f", masterJson, nsFlag) + runKubectlOrDie("create", "-f", driverJson, nsFlag) err := waitForPodRunningInNamespace(c, "spark-master", ns) Expect(err).NotTo(HaveOccurred()) _, err = lookForStringInLog(ns, "spark-master", "spark-master", "Starting Spark master at", serverStartTimeout) @@ -188,7 +188,7 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) By("starting workers") - runKubectl("create", "-f", workerControllerJson, nsFlag) + runKubectlOrDie("create", "-f", workerControllerJson, nsFlag) ScaleRC(c, ns, "spark-worker-controller", 2, true) forEachPod(c, ns, "name", "spark-worker", func(pod api.Pod) { _, err := lookForStringInLog(ns, pod.Name, "spark-worker", "Successfully registered with master", serverStartTimeout) @@ -208,8 +208,8 @@ var _ = Describe("Examples e2e", func() { nsFlag := fmt.Sprintf("--namespace=%v", ns) By("starting service and pod") - runKubectl("create", "-f", serviceYaml, nsFlag) - runKubectl("create", "-f", podYaml, nsFlag) + runKubectlOrDie("create", "-f", serviceYaml, nsFlag) + runKubectlOrDie("create", "-f", podYaml, nsFlag) err := waitForPodRunningInNamespace(c, "cassandra", ns) Expect(err).NotTo(HaveOccurred()) @@ -220,7 +220,7 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) By("create and scale rc") - runKubectl("create", "-f", controllerYaml, nsFlag) + runKubectlOrDie("create", "-f", controllerYaml, nsFlag) err = ScaleRC(c, ns, "cassandra", 2, true) Expect(err).NotTo(HaveOccurred()) forEachPod(c, ns, "name", "cassandra", func(pod api.Pod) { @@ -230,7 +230,7 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) }) - output := runKubectl("exec", "cassandra", nsFlag, "--", "nodetool", "status") + output := runKubectlOrDie("exec", "cassandra", nsFlag, "--", "nodetool", "status") forEachPod(c, ns, "name", "cassandra", func(pod api.Pod) { if !strings.Contains(output, pod.Status.PodIP) { Failf("Pod ip %s not found in nodetool status", pod.Status.PodIP) @@ -253,8 +253,8 @@ var _ = Describe("Examples e2e", func() { zookeeperPod := "zookeeper" By("starting Zookeeper") - runKubectl("create", "-f", zookeeperPodJson, nsFlag) - runKubectl("create", "-f", zookeeperServiceJson, nsFlag) + runKubectlOrDie("create", "-f", zookeeperPodJson, nsFlag) + runKubectlOrDie("create", "-f", zookeeperServiceJson, nsFlag) err := waitForPodRunningInNamespace(c, zookeeperPod, ns) Expect(err).NotTo(HaveOccurred()) @@ -265,8 +265,8 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) By("starting Nimbus") - runKubectl("create", "-f", nimbusPodJson, nsFlag) - runKubectl("create", "-f", nimbusServiceJson, nsFlag) + runKubectlOrDie("create", "-f", nimbusPodJson, nsFlag) + runKubectlOrDie("create", "-f", nimbusServiceJson, nsFlag) err = waitForPodRunningInNamespace(c, "nimbus", ns) Expect(err).NotTo(HaveOccurred()) @@ -274,7 +274,7 @@ var _ = Describe("Examples e2e", func() { Expect(err).NotTo(HaveOccurred()) By("starting workers") - runKubectl("create", "-f", workerControllerJson, nsFlag) + runKubectlOrDie("create", "-f", workerControllerJson, nsFlag) forEachPod(c, ns, "name", "storm-worker", func(pod api.Pod) { //do nothing, just wait for the pod to be running }) @@ -288,7 +288,7 @@ var _ = Describe("Examples e2e", func() { By("checking if Nimbus responds to requests") lookForString("No topologies running.", time.Minute, func() string { - return runKubectl("exec", "nimbus", nsFlag, "--", "bin/storm", "list") + return runKubectlOrDie("exec", "nimbus", nsFlag, "--", "bin/storm", "list") }) }) }) @@ -302,8 +302,8 @@ var _ = Describe("Examples e2e", func() { httpYaml := mkpath("http-liveness.yaml") nsFlag := fmt.Sprintf("--namespace=%v", ns) - runKubectl("create", "-f", execYaml, nsFlag) - runKubectl("create", "-f", httpYaml, nsFlag) + runKubectlOrDie("create", "-f", execYaml, nsFlag) + runKubectlOrDie("create", "-f", httpYaml, nsFlag) checkRestart := func(podName string, timeout time.Duration) { err := waitForPodRunningInNamespace(c, podName, ns) Expect(err).NotTo(HaveOccurred()) @@ -335,8 +335,8 @@ var _ = Describe("Examples e2e", func() { nsFlag := fmt.Sprintf("--namespace=%v", ns) By("creating secret and pod") - runKubectl("create", "-f", secretYaml, nsFlag) - runKubectl("create", "-f", podYaml, nsFlag) + runKubectlOrDie("create", "-f", secretYaml, nsFlag) + runKubectlOrDie("create", "-f", podYaml, nsFlag) By("checking if secret was read correctly") _, err := lookForStringInLog(ns, "secret-test-pod", "test-container", "value-1", serverStartTimeout) @@ -354,7 +354,7 @@ var _ = Describe("Examples e2e", func() { podName := "dapi-test-pod" By("creating the pod") - runKubectl("create", "-f", podYaml, nsFlag) + runKubectlOrDie("create", "-f", podYaml, nsFlag) By("checking if name and namespace were passed correctly") _, err := lookForStringInLog(ns, podName, "test-container", fmt.Sprintf("POD_NAMESPACE=%v", ns), serverStartTimeout) @@ -376,8 +376,8 @@ var _ = Describe("Examples e2e", func() { nsFlag := fmt.Sprintf("--namespace=%v", ns) By("starting rethinkdb") - runKubectl("create", "-f", driverServiceYaml, nsFlag) - runKubectl("create", "-f", rethinkDbControllerYaml, nsFlag) + runKubectlOrDie("create", "-f", driverServiceYaml, nsFlag) + runKubectlOrDie("create", "-f", rethinkDbControllerYaml, nsFlag) checkDbInstances := func() { forEachPod(c, ns, "db", "rethinkdb", func(pod api.Pod) { _, err := lookForStringInLog(ns, pod.Name, "rethinkdb", "Server ready", serverStartTimeout) @@ -393,8 +393,8 @@ var _ = Describe("Examples e2e", func() { checkDbInstances() By("starting admin") - runKubectl("create", "-f", adminServiceYaml, nsFlag) - runKubectl("create", "-f", adminPodYaml, nsFlag) + runKubectlOrDie("create", "-f", adminServiceYaml, nsFlag) + runKubectlOrDie("create", "-f", adminPodYaml, nsFlag) err = waitForPodRunningInNamespace(c, "rethinkdb-admin", ns) Expect(err).NotTo(HaveOccurred()) checkDbInstances() @@ -416,8 +416,8 @@ var _ = Describe("Examples e2e", func() { nsFlag := fmt.Sprintf("--namespace=%v", ns) By("starting hazelcast") - runKubectl("create", "-f", serviceYaml, nsFlag) - runKubectl("create", "-f", controllerYaml, nsFlag) + runKubectlOrDie("create", "-f", serviceYaml, nsFlag) + runKubectlOrDie("create", "-f", controllerYaml, nsFlag) forEachPod(c, ns, "name", "hazelcast", func(pod api.Pod) { _, err := lookForStringInLog(ns, pod.Name, "hazelcast", "Members [1]", serverStartTimeout) Expect(err).NotTo(HaveOccurred()) @@ -476,11 +476,11 @@ var _ = Describe("Examples e2e", func() { } for _, ns := range namespaces { - runKubectl("create", "-f", backendRcYaml, getNsCmdFlag(ns)) + runKubectlOrDie("create", "-f", backendRcYaml, getNsCmdFlag(ns)) } for _, ns := range namespaces { - runKubectl("create", "-f", backendSvcYaml, getNsCmdFlag(ns)) + runKubectlOrDie("create", "-f", backendSvcYaml, getNsCmdFlag(ns)) } // wait for objects @@ -526,7 +526,7 @@ var _ = Describe("Examples e2e", func() { // create a pod in each namespace for _, ns := range namespaces { - newKubectlCommand("create", "-f", "-", getNsCmdFlag(ns)).withStdinData(updatedPodYaml).exec() + newKubectlCommand("create", "-f", "-", getNsCmdFlag(ns)).withStdinData(updatedPodYaml).execOrDie() } // wait until the pods have been scheduler, i.e. are not Pending anymore. Remember @@ -605,13 +605,13 @@ func forEachPod(c *client.Client, ns, selectorKey, selectorValue string, fn func func lookForStringInLog(ns, podName, container, expectedString string, timeout time.Duration) (result string, err error) { return lookForString(expectedString, timeout, func() string { - return runKubectl("log", podName, container, fmt.Sprintf("--namespace=%v", ns)) + return runKubectlOrDie("log", podName, container, fmt.Sprintf("--namespace=%v", ns)) }) } func lookForStringInFile(ns, podName, container, file, expectedString string, timeout time.Duration) (result string, err error) { return lookForString(expectedString, timeout, func() string { - return runKubectl("exec", podName, "-c", container, fmt.Sprintf("--namespace=%v", ns), "--", "cat", file) + return runKubectlOrDie("exec", podName, "-c", container, fmt.Sprintf("--namespace=%v", ns), "--", "cat", file) }) } @@ -620,7 +620,7 @@ func lookForStringInPodExec(ns, podName string, command []string, expectedString // use the first container args := []string{"exec", podName, fmt.Sprintf("--namespace=%v", ns), "--"} args = append(args, command...) - return runKubectl(args...) + return runKubectlOrDie(args...) }) } diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 331d2b5da906..6acf33833c16 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -65,6 +65,10 @@ var ( // Name of the loadbalancer controller within the cluster addon lbContainerName = "l7-lb-controller" + // If set, the test tries to perform an HTTP GET on each url endpoint of + // the Ingress. Only set to false to short-circuit test runs in debugging. + verifyHTTPGET = true + // On average it takes ~6 minutes for a single backend to come online. // We *don't* expect this poll to consistently take 15 minutes for every // Ingress as GCE is creating/checking backends in parallel, but at the @@ -173,7 +177,9 @@ func createApp(c *client.Client, ns string, i int) { // gcloudUnmarshal unmarshals json output of gcloud into given out interface. func gcloudUnmarshal(resource, regex string, out interface{}) { output, err := exec.Command("gcloud", "compute", resource, "list", - fmt.Sprintf("--regex=%v", regex), "-q", "--format=json").CombinedOutput() + fmt.Sprintf("--regex=%v", regex), + fmt.Sprintf("--project=%v", testContext.CloudConfig.ProjectID), + "-q", "--format=json").CombinedOutput() if err != nil { Failf("Error unmarshalling gcloud output: %v", err) } @@ -204,7 +210,7 @@ func checkLeakedResources() error { return nil } -// run kubectl log on the L7 controller pod. +// kubectlLogLBController logs kubectl debug output for the L7 controller pod. func kubectlLogLBController(c *client.Client) { selector := labels.SelectorFromSet(labels.Set(map[string]string{"name": "glbc"})) podList, err := c.Pods(api.NamespaceAll).List(selector, fields.Everything()) @@ -218,21 +224,11 @@ func kubectlLogLBController(c *client.Client) { } for _, p := range podList.Items { Logf("\nLast 100 log lines of %v\n", p.Name) - Logf(runKubectl("logs", p.Name, "--namespace=kube-system", "-c", lbContainerName, "--tail=100")) + l, _ := runKubectl("logs", p.Name, fmt.Sprintf("--namespace=%v", api.NamespaceSystem), "-c", lbContainerName, "--tail=100") + Logf(l) } } -// dumpDebugAndFail dumps verbose debug output before failing. -func dumpDebugAndFail(err string, ns string, c *client.Client) { - kubectlLogLBController(c) - Logf("\nOutput of kubectl describe ing:\n") - - // TODO: runKubectl will hard fail if kubectl fails, swap it out for - // something more befitting for a debug dumper. - Logf(runKubectl("describe", "ing", fmt.Sprintf("--namespace=%v", ns))) - Failf(err) -} - var _ = Describe("GCE L7 LoadBalancer Controller", func() { // These variables are initialized after framework's beforeEach. var ns string @@ -254,6 +250,12 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() { }) AfterEach(func() { + if CurrentGinkgoTestDescription().Failed { + kubectlLogLBController(client) + Logf("\nOutput of kubectl describe ing:\n") + desc, _ := runKubectl("describe", "ing", fmt.Sprintf("--namespace=%v", ns)) + Logf(desc) + } framework.afterEach() err := wait.Poll(lbPollInterval, lbPollTimeout, func() (bool, error) { if err := checkLeakedResources(); err != nil { @@ -301,13 +303,17 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() { start := time.Now() address, err := waitForIngressAddress(client, ing.Namespace, ing.Name, lbPollTimeout) if err != nil { - dumpDebugAndFail(fmt.Sprintf("Ingress failed to acquire an IP address within %v", lbPollTimeout), ns, client) + Failf("Ingress failed to acquire an IP address within %v", lbPollTimeout) } Expect(err).NotTo(HaveOccurred()) By(fmt.Sprintf("Found address %v for ingress %v, took %v to come online", address, ing.Name, time.Since(start))) creationTimes = append(creationTimes, time.Since(start)) + if !verifyHTTPGET { + continue + } + // Check that all rules respond to a simple GET. for _, rules := range ing.Spec.Rules { // As of Kubernetes 1.1 we only support HTTP Ingress. @@ -330,8 +336,8 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() { return true, nil }) if pollErr != nil { - dumpDebugAndFail(fmt.Sprintf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v", - lbPollTimeout, route, rules.Host, lastBody, pollErr), ns, client) + Failf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v", + lbPollTimeout, route, rules.Host, lastBody, pollErr) } rt := time.Since(GETStart) By(fmt.Sprintf("Route %v host %v took %v to respond", route, rules.Host, rt)) @@ -345,12 +351,15 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() { sort.Sort(timeSlice(creationTimes)) perc50 := creationTimes[len(creationTimes)/2] if perc50 > expectedLBCreationTime { - dumpDebugAndFail(fmt.Sprintf("Average creation time is too high: %+v", creationTimes), ns, client) + Failf("Average creation time is too high: %+v", creationTimes) + } + if !verifyHTTPGET { + return } sort.Sort(timeSlice(responseTimes)) perc50 = responseTimes[len(responseTimes)/2] if perc50 > expectedLBHealthCheckTime { - dumpDebugAndFail(fmt.Sprintf("Average startup time is too high: %+v", responseTimes), ns, client) + Failf("Average startup time is too high: %+v", responseTimes) } }) }) diff --git a/test/e2e/kubectl.go b/test/e2e/kubectl.go index 58e1ecf808e7..3f4ddc79a3c7 100644 --- a/test/e2e/kubectl.go +++ b/test/e2e/kubectl.go @@ -95,7 +95,7 @@ var _ = Describe("Kubectl client", func() { defer cleanup(nautilusPath, ns, updateDemoSelector) By("creating a replication controller") - runKubectl("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns)) validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg", ns), ns) }) @@ -103,22 +103,22 @@ var _ = Describe("Kubectl client", func() { defer cleanup(nautilusPath, ns, updateDemoSelector) By("creating a replication controller") - runKubectl("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns)) validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg", ns), ns) By("scaling down the replication controller") - runKubectl("scale", "rc", "update-demo-nautilus", "--replicas=1", "--timeout=5m", fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("scale", "rc", "update-demo-nautilus", "--replicas=1", "--timeout=5m", fmt.Sprintf("--namespace=%v", ns)) validateController(c, nautilusImage, 1, "update-demo", updateDemoSelector, getUDData("nautilus.jpg", ns), ns) By("scaling up the replication controller") - runKubectl("scale", "rc", "update-demo-nautilus", "--replicas=2", "--timeout=5m", fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("scale", "rc", "update-demo-nautilus", "--replicas=2", "--timeout=5m", fmt.Sprintf("--namespace=%v", ns)) validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg", ns), ns) }) It("should do a rolling update of a replication controller [Conformance]", func() { By("creating the initial replication controller") - runKubectl("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns)) validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg", ns), ns) By("rolling-update to new replication controller") - runKubectl("rolling-update", "update-demo-nautilus", "--update-period=1s", "-f", kittenPath, fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("rolling-update", "update-demo-nautilus", "--update-period=1s", "-f", kittenPath, fmt.Sprintf("--namespace=%v", ns)) validateController(c, kittenImage, 2, "update-demo", updateDemoSelector, getUDData("kitten.jpg", ns), ns) // Everything will hopefully be cleaned up when the namespace is deleted. }) @@ -135,7 +135,7 @@ var _ = Describe("Kubectl client", func() { defer cleanup(guestbookPath, ns, frontendSelector, redisMasterSelector, redisSlaveSelector) By("creating all guestbook components") - runKubectl("create", "-f", guestbookPath, fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("create", "-f", guestbookPath, fmt.Sprintf("--namespace=%v", ns)) By("validating guestbook app") validateGuestbookApp(c, ns) @@ -148,7 +148,7 @@ var _ = Describe("Kubectl client", func() { BeforeEach(func() { podPath = filepath.Join(testContext.RepoRoot, "docs/user-guide/pod.yaml") By("creating the pod") - runKubectl("create", "-f", podPath, fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("create", "-f", podPath, fmt.Sprintf("--namespace=%v", ns)) checkPodsRunningReady(c, ns, []string{simplePodName}, podStartTimeout) }) AfterEach(func() { @@ -157,7 +157,7 @@ var _ = Describe("Kubectl client", func() { It("should support exec", func() { By("executing a command in the container") - execOutput := runKubectl("exec", fmt.Sprintf("--namespace=%v", ns), simplePodName, "echo", "running", "in", "container") + execOutput := runKubectlOrDie("exec", fmt.Sprintf("--namespace=%v", ns), simplePodName, "echo", "running", "in", "container") if e, a := "running in container", execOutput; e != a { Failf("Unexpected kubectl exec output. Wanted %q, got %q", e, a) } @@ -165,7 +165,7 @@ var _ = Describe("Kubectl client", func() { By("executing a command in the container with noninteractive stdin") execOutput = newKubectlCommand("exec", fmt.Sprintf("--namespace=%v", ns), "-i", simplePodName, "cat"). withStdinData("abcd1234"). - exec() + execOrDie() if e, a := "abcd1234", execOutput; e != a { Failf("Unexpected kubectl exec output. Wanted %q, got %q", e, a) } @@ -181,7 +181,7 @@ var _ = Describe("Kubectl client", func() { By("executing a command in the container with pseudo-interactive stdin") execOutput = newKubectlCommand("exec", fmt.Sprintf("--namespace=%v", ns), "-i", simplePodName, "bash"). withStdinReader(r). - exec() + execOrDie() if e, a := "hi", execOutput; e != a { Failf("Unexpected kubectl exec output. Wanted %q, got %q", e, a) } @@ -241,7 +241,7 @@ var _ = Describe("Kubectl client", func() { } // start exec-proxy-tester container netexecPodPath := filepath.Join(testContext.RepoRoot, "test/images/netexec/pod.yaml") - runKubectl("create", "-f", netexecPodPath, fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("create", "-f", netexecPodPath, fmt.Sprintf("--namespace=%v", ns)) checkPodsRunningReady(c, ns, []string{netexecContainer}, podStartTimeout) // Clean up defer cleanup(netexecPodPath, ns, netexecPodSelector) @@ -311,7 +311,7 @@ var _ = Describe("Kubectl client", func() { By("Running kubectl in netexec via an HTTP proxy using " + proxyVar) // start the proxy container goproxyPodPath := filepath.Join(testContext.RepoRoot, "test/images/goproxy/pod.yaml") - runKubectl("create", "-f", goproxyPodPath, fmt.Sprintf("--namespace=%v", ns)) + runKubectlOrDie("create", "-f", goproxyPodPath, fmt.Sprintf("--namespace=%v", ns)) checkPodsRunningReady(c, ns, []string{goproxyContainer}, podStartTimeout) // get the proxy address @@ -348,7 +348,7 @@ var _ = Describe("Kubectl client", func() { // Verify the proxy server logs saw the connection expectedProxyLog := fmt.Sprintf("Accepting CONNECT to %s", strings.TrimRight(strings.TrimLeft(testContext.Host, "https://"), "/api")) - proxyLog := runKubectl("log", "goproxy", fmt.Sprintf("--namespace=%v", ns)) + proxyLog := runKubectlOrDie("log", "goproxy", fmt.Sprintf("--namespace=%v", ns)) if !strings.Contains(proxyLog, expectedProxyLog) { Failf("Missing expected log result on proxy server for %s. Expected: %q, got %q", proxyVar, expectedProxyLog, proxyLog) @@ -364,7 +364,7 @@ var _ = Describe("Kubectl client", func() { By("executing a command with run and attach with stdin") runOutput := newKubectlCommand(nsFlag, "run", "run-test", "--image=busybox", "--restart=Never", "--attach=true", "--stdin", "--", "sh", "-c", "cat && echo 'stdin closed'"). withStdinData("abcd1234"). - exec() + execOrDie() Expect(runOutput).To(ContainSubstring("abcd1234")) Expect(runOutput).To(ContainSubstring("stdin closed")) Expect(c.Pods(ns).Delete("run-test", api.NewDeleteOptions(0))).To(BeNil()) @@ -372,7 +372,7 @@ var _ = Describe("Kubectl client", func() { By("executing a command with run and attach without stdin") runOutput = newKubectlCommand(fmt.Sprintf("--namespace=%v", ns), "run", "run-test-2", "--image=busybox", "--restart=Never", "--attach=true", "--leave-stdin-open=true", "--", "sh", "-c", "cat && echo 'stdin closed'"). withStdinData("abcd1234"). - exec() + execOrDie() Expect(runOutput).ToNot(ContainSubstring("abcd1234")) Expect(runOutput).To(ContainSubstring("stdin closed")) Expect(c.Pods(ns).Delete("run-test-2", api.NewDeleteOptions(0))).To(BeNil()) @@ -380,7 +380,7 @@ var _ = Describe("Kubectl client", func() { By("executing a command with run and attach with stdin with open stdin should remain running") runOutput = newKubectlCommand(nsFlag, "run", "run-test-3", "--image=busybox", "--restart=Never", "--attach=true", "--leave-stdin-open=true", "--stdin", "--", "sh", "-c", "cat && echo 'stdin closed'"). withStdinData("abcd1234\n"). - exec() + execOrDie() Expect(runOutput).ToNot(ContainSubstring("stdin closed")) if !checkPodsRunningReady(c, ns, []string{"run-test-3"}, time.Minute) { Failf("Pod %q should still be running", "run-test-3") @@ -392,7 +392,7 @@ var _ = Describe("Kubectl client", func() { if !checkPodsRunningReady(c, ns, []string{"run-test-3"}, 1*time.Second) { Failf("Pod %q should still be running", "run-test-3") } - logOutput := runKubectl(nsFlag, "logs", "run-test-3") + logOutput := runKubectlOrDie(nsFlag, "logs", "run-test-3") Expect(logOutput).ToNot(ContainSubstring("stdin closed")) return strings.Contains(logOutput, "abcd1234"), nil }) @@ -425,7 +425,7 @@ var _ = Describe("Kubectl client", func() { Describe("Kubectl api-versions", func() { It("should check if v1 is in available api versions [Conformance]", func() { By("validating api verions") - output := runKubectl("api-versions") + output := runKubectlOrDie("api-versions") if !strings.Contains(output, "v1") { Failf("No v1 in kubectl api-versions") } @@ -440,12 +440,12 @@ var _ = Describe("Kubectl client", func() { controllerJson := mkpath("redis-master-controller.json") nsFlag := fmt.Sprintf("--namespace=%v", ns) By("creating Redis RC") - runKubectl("create", "-f", controllerJson, nsFlag) + runKubectlOrDie("create", "-f", controllerJson, nsFlag) By("applying a modified configuration") stdin := modifyReplicationControllerConfiguration(controllerJson) newKubectlCommand("apply", "-f", "-", nsFlag). withStdinReader(stdin). - exec() + execOrDie() By("checking the result") forEachReplicationController(c, ns, "app", "redis", validateReplicationControllerConfiguration) }) @@ -454,7 +454,7 @@ var _ = Describe("Kubectl client", func() { Describe("Kubectl cluster-info", func() { It("should check if Kubernetes master services is included in cluster-info [Conformance]", func() { By("validating cluster-info") - output := runKubectl("cluster-info") + output := runKubectlOrDie("cluster-info") // Can't check exact strings due to terminal control commands (colors) requiredItems := []string{"Kubernetes master", "is running at"} if providerIs("gce", "gke") { @@ -477,12 +477,12 @@ var _ = Describe("Kubectl client", func() { serviceJson := mkpath("redis-master-service.json") nsFlag := fmt.Sprintf("--namespace=%v", ns) - runKubectl("create", "-f", controllerJson, nsFlag) - runKubectl("create", "-f", serviceJson, nsFlag) + runKubectlOrDie("create", "-f", controllerJson, nsFlag) + runKubectlOrDie("create", "-f", serviceJson, nsFlag) // Pod forEachPod(c, ns, "app", "redis", func(pod api.Pod) { - output := runKubectl("describe", "pod", pod.Name, nsFlag) + output := runKubectlOrDie("describe", "pod", pod.Name, nsFlag) requiredStrings := [][]string{ {"Name:", "redis-master-"}, {"Namespace:", ns}, @@ -498,7 +498,7 @@ var _ = Describe("Kubectl client", func() { }) // Rc - output := runKubectl("describe", "rc", "redis-master", nsFlag) + output := runKubectlOrDie("describe", "rc", "redis-master", nsFlag) requiredStrings := [][]string{ {"Name:", "redis-master"}, {"Namespace:", ns}, @@ -511,7 +511,7 @@ var _ = Describe("Kubectl client", func() { checkOutput(output, requiredStrings) // Service - output = runKubectl("describe", "service", "redis-master", nsFlag) + output = runKubectlOrDie("describe", "service", "redis-master", nsFlag) requiredStrings = [][]string{ {"Name:", "redis-master"}, {"Namespace:", ns}, @@ -528,7 +528,7 @@ var _ = Describe("Kubectl client", func() { nodes, err := c.Nodes().List(labels.Everything(), fields.Everything()) Expect(err).NotTo(HaveOccurred()) node := nodes.Items[0] - output = runKubectl("describe", "node", node.Name) + output = runKubectlOrDie("describe", "node", node.Name) requiredStrings = [][]string{ {"Name:", node.Name}, {"Labels:"}, @@ -547,7 +547,7 @@ var _ = Describe("Kubectl client", func() { checkOutput(output, requiredStrings) // Namespace - output = runKubectl("describe", "namespace", ns) + output = runKubectlOrDie("describe", "namespace", ns) requiredStrings = [][]string{ {"Name:", ns}, {"Labels:"}, @@ -569,7 +569,7 @@ var _ = Describe("Kubectl client", func() { redisPort := 6379 By("creating Redis RC") - runKubectl("create", "-f", controllerJson, nsFlag) + runKubectlOrDie("create", "-f", controllerJson, nsFlag) forEachPod(c, ns, "app", "redis", func(pod api.Pod) { lookForStringInLog(ns, pod.Name, "redis-master", "The server is now ready to accept connections", podStartTimeout) }) @@ -617,12 +617,12 @@ var _ = Describe("Kubectl client", func() { } By("exposing RC") - runKubectl("expose", "rc", "redis-master", "--name=rm2", "--port=1234", fmt.Sprintf("--target-port=%d", redisPort), nsFlag) + runKubectlOrDie("expose", "rc", "redis-master", "--name=rm2", "--port=1234", fmt.Sprintf("--target-port=%d", redisPort), nsFlag) waitForService(c, ns, "rm2", true, poll, serviceStartTimeout) validateService("rm2", 1234, serviceStartTimeout) By("exposing service") - runKubectl("expose", "service", "rm2", "--name=rm3", "--port=2345", fmt.Sprintf("--target-port=%d", redisPort), nsFlag) + runKubectlOrDie("expose", "service", "rm2", "--name=rm3", "--port=2345", fmt.Sprintf("--target-port=%d", redisPort), nsFlag) waitForService(c, ns, "rm3", true, poll, serviceStartTimeout) validateService("rm3", 2345, serviceStartTimeout) }) @@ -635,7 +635,7 @@ var _ = Describe("Kubectl client", func() { podPath = filepath.Join(testContext.RepoRoot, "docs/user-guide/pod.yaml") By("creating the pod") nsFlag = fmt.Sprintf("--namespace=%v", ns) - runKubectl("create", "-f", podPath, nsFlag) + runKubectlOrDie("create", "-f", podPath, nsFlag) checkPodsRunningReady(c, ns, []string{simplePodName}, podStartTimeout) }) AfterEach(func() { @@ -647,17 +647,17 @@ var _ = Describe("Kubectl client", func() { labelValue := "testing-label-value" By("adding the label " + labelName + " with value " + labelValue + " to a pod") - runKubectl("label", "pods", simplePodName, labelName+"="+labelValue, nsFlag) + runKubectlOrDie("label", "pods", simplePodName, labelName+"="+labelValue, nsFlag) By("verifying the pod has the label " + labelName + " with the value " + labelValue) - output := runKubectl("get", "pod", simplePodName, "-L", labelName, nsFlag) + output := runKubectlOrDie("get", "pod", simplePodName, "-L", labelName, nsFlag) if !strings.Contains(output, labelValue) { Failf("Failed updating label " + labelName + " to the pod " + simplePodName) } By("removing the label " + labelName + " of a pod") - runKubectl("label", "pods", simplePodName, labelName+"-", nsFlag) + runKubectlOrDie("label", "pods", simplePodName, labelName+"-", nsFlag) By("verifying the pod doesn't have the label " + labelName) - output = runKubectl("get", "pod", simplePodName, "-L", labelName, nsFlag) + output = runKubectlOrDie("get", "pod", simplePodName, "-L", labelName, nsFlag) if strings.Contains(output, labelValue) { Failf("Failed removing label " + labelName + " of the pod " + simplePodName) } @@ -675,7 +675,7 @@ var _ = Describe("Kubectl client", func() { rcPath = mkpath("redis-master-controller.json") By("creating an rc") nsFlag = fmt.Sprintf("--namespace=%v", ns) - runKubectl("create", "-f", rcPath, nsFlag) + runKubectlOrDie("create", "-f", rcPath, nsFlag) }) AfterEach(func() { cleanup(rcPath, ns, simplePodSelector) @@ -688,17 +688,17 @@ var _ = Describe("Kubectl client", func() { Expect(err).NotTo(HaveOccurred()) By("limiting log lines") - out := runKubectl("log", pod.Name, containerName, nsFlag, "--tail=1") + out := runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--tail=1") Expect(len(out)).NotTo(BeZero()) Expect(len(strings.Split(out, "\n"))).To(Equal(1)) By("limiting log bytes") - out = runKubectl("log", pod.Name, containerName, nsFlag, "--limit-bytes=1") + out = runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--limit-bytes=1") Expect(len(strings.Split(out, "\n"))).To(Equal(1)) Expect(len(out)).To(Equal(1)) By("exposing timestamps") - out = runKubectl("log", pod.Name, containerName, nsFlag, "--tail=1", "--timestamps") + out = runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--tail=1", "--timestamps") lines := strings.Split(out, "\n") Expect(len(lines)).To(Equal(1)) words := strings.Split(lines[0], " ") @@ -711,9 +711,9 @@ var _ = Describe("Kubectl client", func() { By("restricting to a time range") time.Sleep(1500 * time.Millisecond) // ensure that startup logs on the node are seen as older than 1s - out = runKubectl("log", pod.Name, containerName, nsFlag, "--since=1s") + out = runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--since=1s") recent := len(strings.Split(out, "\n")) - out = runKubectl("log", pod.Name, containerName, nsFlag, "--since=24h") + out = runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--since=24h") older := len(strings.Split(out, "\n")) Expect(recent).To(BeNumerically("<", older)) }) @@ -728,10 +728,10 @@ var _ = Describe("Kubectl client", func() { controllerJson := mkpath("redis-master-controller.json") nsFlag := fmt.Sprintf("--namespace=%v", ns) By("creating Redis RC") - runKubectl("create", "-f", controllerJson, nsFlag) + runKubectlOrDie("create", "-f", controllerJson, nsFlag) By("patching all pods") forEachPod(c, ns, "app", "redis", func(pod api.Pod) { - runKubectl("patch", "pod", pod.Name, nsFlag, "-p", "{\"metadata\":{\"annotations\":{\"x\":\"y\"}}}") + runKubectlOrDie("patch", "pod", pod.Name, nsFlag, "-p", "{\"metadata\":{\"annotations\":{\"x\":\"y\"}}}") }) By("checking annotations") @@ -751,7 +751,7 @@ var _ = Describe("Kubectl client", func() { Describe("Kubectl version", func() { It("should check is all data is printed [Conformance]", func() { - version := runKubectl("version") + version := runKubectlOrDie("version") requiredItems := []string{"Client Version:", "Server Version:", "Major:", "Minor:", "GitCommit:"} for _, item := range requiredItems { if !strings.Contains(version, item) { @@ -771,14 +771,14 @@ var _ = Describe("Kubectl client", func() { }) AfterEach(func() { - runKubectl("stop", "rc", rcName, nsFlag) + runKubectlOrDie("stop", "rc", rcName, nsFlag) }) It("should create an rc from an image [Conformance]", func() { image := "nginx" By("running the image " + image) - runKubectl("run", rcName, "--image="+image, nsFlag) + runKubectlOrDie("run", rcName, "--image="+image, nsFlag) By("verifying the rc " + rcName + " was created") rc, err := c.ReplicationControllers(ns).Get(rcName) if err != nil { @@ -797,7 +797,7 @@ var _ = Describe("Kubectl client", func() { } pods := podlist.Items if pods == nil || len(pods) != 1 || len(pods[0].Spec.Containers) != 1 || pods[0].Spec.Containers[0].Image != image { - runKubectl("get", "pods", "-L", "run", nsFlag) + runKubectlOrDie("get", "pods", "-L", "run", nsFlag) Failf("Failed creating 1 pod with expected image %s. Number of pods = %v", image, len(pods)) } }) @@ -814,14 +814,14 @@ var _ = Describe("Kubectl client", func() { }) AfterEach(func() { - runKubectl("stop", "pods", podName, nsFlag) + runKubectlOrDie("stop", "pods", podName, nsFlag) }) It("should create a pod from an image when restart is OnFailure [Conformance]", func() { image := "nginx" By("running the image " + image) - runKubectl("run", podName, "--restart=OnFailure", "--image="+image, nsFlag) + runKubectlOrDie("run", podName, "--restart=OnFailure", "--image="+image, nsFlag) By("verifying the pod " + podName + " was created") pod, err := c.Pods(ns).Get(podName) if err != nil { @@ -840,7 +840,7 @@ var _ = Describe("Kubectl client", func() { image := "nginx" By("running the image " + image) - runKubectl("run", podName, "--restart=Never", "--image="+image, nsFlag) + runKubectlOrDie("run", podName, "--restart=Never", "--image="+image, nsFlag) By("verifying the pod " + podName + " was created") pod, err := c.Pods(ns).Get(podName) if err != nil { diff --git a/test/e2e/portforward.go b/test/e2e/portforward.go index 3bc392083301..c07a3ae9458d 100644 --- a/test/e2e/portforward.go +++ b/test/e2e/portforward.go @@ -116,7 +116,7 @@ func runKubectlWithTimeout(timeout time.Duration, args ...string) string { logOutput := make(chan string) go func() { defer GinkgoRecover() - logOutput <- runKubectl(args...) + logOutput <- runKubectlOrDie(args...) }() select { case <-time.After(timeout): diff --git a/test/e2e/util.go b/test/e2e/util.go index 201df2626211..f81719d08c86 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -926,14 +926,14 @@ func cleanup(filePath string, ns string, selectors ...string) { if ns != "" { nsArg = fmt.Sprintf("--namespace=%s", ns) } - runKubectl("stop", "--grace-period=0", "-f", filePath, nsArg) + runKubectlOrDie("stop", "--grace-period=0", "-f", filePath, nsArg) for _, selector := range selectors { - resources := runKubectl("get", "rc,svc", "-l", selector, "--no-headers", nsArg) + resources := runKubectlOrDie("get", "rc,svc", "-l", selector, "--no-headers", nsArg) if resources != "" { Failf("Resources left running after stop:\n%s", resources) } - pods := runKubectl("get", "pods", "-l", selector, nsArg, "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}{{ \"\\n\" }}{{ end }}{{ end }}") + pods := runKubectlOrDie("get", "pods", "-l", selector, nsArg, "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}{{ \"\\n\" }}{{ end }}{{ end }}") if pods != "" { Failf("Pods left unterminated after stop:\n%s", pods) } @@ -967,7 +967,7 @@ func validateController(c *client.Client, containerImage string, replicas int, c By(fmt.Sprintf("waiting for all containers in %s pods to come up.", testname)) //testname should be selector waitLoop: for start := time.Now(); time.Since(start) < podStartTimeout; time.Sleep(5 * time.Second) { - getPodsOutput := runKubectl("get", "pods", "-o", "template", getPodsTemplate, "--api-version=v1", "-l", testname, fmt.Sprintf("--namespace=%v", ns)) + getPodsOutput := runKubectlOrDie("get", "pods", "-o", "template", getPodsTemplate, "--api-version=v1", "-l", testname, fmt.Sprintf("--namespace=%v", ns)) pods := strings.Fields(getPodsOutput) if numPods := len(pods); numPods != replicas { By(fmt.Sprintf("Replicas for %s: expected=%d actual=%d", testname, replicas, numPods)) @@ -975,13 +975,13 @@ waitLoop: } var runningPods []string for _, podID := range pods { - running := runKubectl("get", "pods", podID, "-o", "template", getContainerStateTemplate, "--api-version=v1", fmt.Sprintf("--namespace=%v", ns)) + running := runKubectlOrDie("get", "pods", podID, "-o", "template", getContainerStateTemplate, "--api-version=v1", fmt.Sprintf("--namespace=%v", ns)) if running != "true" { Logf("%s is created but not running", podID) continue waitLoop } - currentImage := runKubectl("get", "pods", podID, "-o", "template", getImageTemplate, "--api-version=v1", fmt.Sprintf("--namespace=%v", ns)) + currentImage := runKubectlOrDie("get", "pods", podID, "-o", "template", getImageTemplate, "--api-version=v1", fmt.Sprintf("--namespace=%v", ns)) if currentImage != containerImage { Logf("%s is created but running wrong image; expected: %s, actual: %s", podID, containerImage, currentImage) continue waitLoop @@ -1062,29 +1062,39 @@ func (b kubectlBuilder) withStdinReader(reader io.Reader) *kubectlBuilder { return &b } -func (b kubectlBuilder) exec() string { +func (b kubectlBuilder) execOrDie() string { + str, err := b.exec() + Expect(err).NotTo(HaveOccurred()) + return str +} + +func (b kubectlBuilder) exec() (string, error) { var stdout, stderr bytes.Buffer cmd := b.cmd cmd.Stdout, cmd.Stderr = &stdout, &stderr Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args[1:], " ")) // skip arg[0] as it is printed separately if err := cmd.Run(); err != nil { - Failf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr) - return "" + return "", fmt.Errorf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr) } Logf(stdout.String()) // TODO: trimspace should be unnecessary after switching to use kubectl binary directly - return strings.TrimSpace(stdout.String()) + return strings.TrimSpace(stdout.String()), nil +} + +// runKubectlOrDie is a convenience wrapper over kubectlBuilder +func runKubectlOrDie(args ...string) string { + return newKubectlCommand(args...).execOrDie() } // runKubectl is a convenience wrapper over kubectlBuilder -func runKubectl(args ...string) string { +func runKubectl(args ...string) (string, error) { return newKubectlCommand(args...).exec() } -// runKubectlInput is a convenience wrapper over kubectlBuilder that takes input to stdin -func runKubectlInput(data string, args ...string) string { - return newKubectlCommand(args...).withStdinData(data).exec() +// runKubectlOrDieInput is a convenience wrapper over kubectlBuilder that takes input to stdin +func runKubectlOrDieInput(data string, args ...string) string { + return newKubectlCommand(args...).withStdinData(data).execOrDie() } func startCmdAndStreamOutput(cmd *exec.Cmd) (stdout, stderr io.ReadCloser, err error) { @@ -1973,7 +1983,7 @@ func NewHostExecPodSpec(ns, name string) *api.Pod { // RunHostCmd runs the given cmd in the context of the given pod using `kubectl exec` // inside of a shell. func RunHostCmd(ns, name, cmd string) string { - return runKubectl("exec", fmt.Sprintf("--namespace=%v", ns), name, "--", "/bin/sh", "-c", cmd) + return runKubectlOrDie("exec", fmt.Sprintf("--namespace=%v", ns), name, "--", "/bin/sh", "-c", cmd) } // LaunchHostExecPod launches a hostexec pod in the given namespace and waits