Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Dump kubectl logs of Ingress controller at on e2e failure #16979

Merged
merged 2 commits into from Nov 17, 2015
Merged

Dump kubectl logs of Ingress controller at on e2e failure #16979

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8b35455
Dump kubectl logs of Ingress controller at on e2e failure
bprashanth Nov 8, 2015
eb4106f
Refactor debug logging methods.
bprashanth Nov 8, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
86 changes: 43 additions & 43 deletions test/e2e/examples.go
View file
Open in desktop
Expand Up @@ -74,7 +74,7 @@ var _ = Describe("Examples e2e", func() {
expectedOnSentinel := "+monitor master"

By("starting redis bootstrap")
runKubectl("create", "-f", bootstrapYaml, nsFlag)
runKubectlOrDie("create", "-f", bootstrapYaml, nsFlag)
err := waitForPodRunningInNamespace(c, bootstrapPodName, ns)
Expect(err).NotTo(HaveOccurred())

Expand All @@ -84,13 +84,13 @@ var _ = Describe("Examples e2e", func() {
Expect(err).NotTo(HaveOccurred())

By("setting up services and controllers")
runKubectl("create", "-f", sentinelServiceYaml, nsFlag)
runKubectl("create", "-f", sentinelControllerYaml, nsFlag)
runKubectl("create", "-f", controllerYaml, nsFlag)
runKubectlOrDie("create", "-f", sentinelServiceYaml, nsFlag)
runKubectlOrDie("create", "-f", sentinelControllerYaml, nsFlag)
runKubectlOrDie("create", "-f", controllerYaml, nsFlag)

By("scaling up the deployment")
runKubectl("scale", "rc", redisRC, "--replicas=3", nsFlag)
runKubectl("scale", "rc", sentinelRC, "--replicas=3", nsFlag)
runKubectlOrDie("scale", "rc", redisRC, "--replicas=3", nsFlag)
runKubectlOrDie("scale", "rc", sentinelRC, "--replicas=3", nsFlag)

By("checking up the services")
checkAllLogs := func() {
Expand All @@ -110,7 +110,7 @@ var _ = Describe("Examples e2e", func() {
checkAllLogs()

By("turning down bootstrap")
runKubectl("delete", "-f", bootstrapYaml, nsFlag)
runKubectlOrDie("delete", "-f", bootstrapYaml, nsFlag)
err = waitForRCPodToDisappear(c, ns, redisRC, bootstrapPodName)
Expect(err).NotTo(HaveOccurred())
By("waiting for the new master election")
Expand All @@ -131,8 +131,8 @@ var _ = Describe("Examples e2e", func() {
nsFlag := fmt.Sprintf("--namespace=%v", ns)

By("starting rabbitmq")
runKubectl("create", "-f", rabbitmqServiceYaml, nsFlag)
runKubectl("create", "-f", rabbitmqControllerYaml, nsFlag)
runKubectlOrDie("create", "-f", rabbitmqServiceYaml, nsFlag)
runKubectlOrDie("create", "-f", rabbitmqControllerYaml, nsFlag)
forEachPod(c, ns, "component", "rabbitmq", func(pod api.Pod) {
_, err := lookForStringInLog(ns, pod.Name, "rabbitmq", "Server startup complete", serverStartTimeout)
Expect(err).NotTo(HaveOccurred())
Expand All @@ -141,15 +141,15 @@ var _ = Describe("Examples e2e", func() {
Expect(err).NotTo(HaveOccurred())

By("starting celery")
runKubectl("create", "-f", celeryControllerYaml, nsFlag)
runKubectlOrDie("create", "-f", celeryControllerYaml, nsFlag)
forEachPod(c, ns, "component", "celery", func(pod api.Pod) {
_, err := lookForStringInFile(ns, pod.Name, "celery", "/data/celery.log", " ready.", serverStartTimeout)
Expect(err).NotTo(HaveOccurred())
})

By("starting flower")
runKubectl("create", "-f", flowerServiceYaml, nsFlag)
runKubectl("create", "-f", flowerControllerYaml, nsFlag)
runKubectlOrDie("create", "-f", flowerServiceYaml, nsFlag)
runKubectlOrDie("create", "-f", flowerControllerYaml, nsFlag)
forEachPod(c, ns, "component", "flower", func(pod api.Pod) {
// Do nothing. just wait for it to be up and running.
})
Expand All @@ -173,9 +173,9 @@ var _ = Describe("Examples e2e", func() {
nsFlag := fmt.Sprintf("--namespace=%v", ns)

By("starting master")
runKubectl("create", "-f", serviceJson, nsFlag)
runKubectl("create", "-f", masterJson, nsFlag)
runKubectl("create", "-f", driverJson, nsFlag)
runKubectlOrDie("create", "-f", serviceJson, nsFlag)
runKubectlOrDie("create", "-f", masterJson, nsFlag)
runKubectlOrDie("create", "-f", driverJson, nsFlag)
err := waitForPodRunningInNamespace(c, "spark-master", ns)
Expect(err).NotTo(HaveOccurred())
_, err = lookForStringInLog(ns, "spark-master", "spark-master", "Starting Spark master at", serverStartTimeout)
Expand All @@ -188,7 +188,7 @@ var _ = Describe("Examples e2e", func() {
Expect(err).NotTo(HaveOccurred())

By("starting workers")
runKubectl("create", "-f", workerControllerJson, nsFlag)
runKubectlOrDie("create", "-f", workerControllerJson, nsFlag)
ScaleRC(c, ns, "spark-worker-controller", 2, true)
forEachPod(c, ns, "name", "spark-worker", func(pod api.Pod) {
_, err := lookForStringInLog(ns, pod.Name, "spark-worker", "Successfully registered with master", serverStartTimeout)
Expand All @@ -208,8 +208,8 @@ var _ = Describe("Examples e2e", func() {
nsFlag := fmt.Sprintf("--namespace=%v", ns)

By("starting service and pod")
runKubectl("create", "-f", serviceYaml, nsFlag)
runKubectl("create", "-f", podYaml, nsFlag)
runKubectlOrDie("create", "-f", serviceYaml, nsFlag)
runKubectlOrDie("create", "-f", podYaml, nsFlag)
err := waitForPodRunningInNamespace(c, "cassandra", ns)
Expect(err).NotTo(HaveOccurred())

Expand All @@ -220,7 +220,7 @@ var _ = Describe("Examples e2e", func() {
Expect(err).NotTo(HaveOccurred())

By("create and scale rc")
runKubectl("create", "-f", controllerYaml, nsFlag)
runKubectlOrDie("create", "-f", controllerYaml, nsFlag)
err = ScaleRC(c, ns, "cassandra", 2, true)
Expect(err).NotTo(HaveOccurred())
forEachPod(c, ns, "name", "cassandra", func(pod api.Pod) {
Expand All @@ -230,7 +230,7 @@ var _ = Describe("Examples e2e", func() {
Expect(err).NotTo(HaveOccurred())
})

output := runKubectl("exec", "cassandra", nsFlag, "--", "nodetool", "status")
output := runKubectlOrDie("exec", "cassandra", nsFlag, "--", "nodetool", "status")
forEachPod(c, ns, "name", "cassandra", func(pod api.Pod) {
if !strings.Contains(output, pod.Status.PodIP) {
Failf("Pod ip %s not found in nodetool status", pod.Status.PodIP)
Expand All @@ -253,8 +253,8 @@ var _ = Describe("Examples e2e", func() {
zookeeperPod := "zookeeper"

By("starting Zookeeper")
runKubectl("create", "-f", zookeeperPodJson, nsFlag)
runKubectl("create", "-f", zookeeperServiceJson, nsFlag)
runKubectlOrDie("create", "-f", zookeeperPodJson, nsFlag)
runKubectlOrDie("create", "-f", zookeeperServiceJson, nsFlag)
err := waitForPodRunningInNamespace(c, zookeeperPod, ns)
Expect(err).NotTo(HaveOccurred())

Expand All @@ -265,16 +265,16 @@ var _ = Describe("Examples e2e", func() {
Expect(err).NotTo(HaveOccurred())

By("starting Nimbus")
runKubectl("create", "-f", nimbusPodJson, nsFlag)
runKubectl("create", "-f", nimbusServiceJson, nsFlag)
runKubectlOrDie("create", "-f", nimbusPodJson, nsFlag)
runKubectlOrDie("create", "-f", nimbusServiceJson, nsFlag)
err = waitForPodRunningInNamespace(c, "nimbus", ns)
Expect(err).NotTo(HaveOccurred())

err = waitForEndpoint(c, ns, "nimbus")
Expect(err).NotTo(HaveOccurred())

By("starting workers")
runKubectl("create", "-f", workerControllerJson, nsFlag)
runKubectlOrDie("create", "-f", workerControllerJson, nsFlag)
forEachPod(c, ns, "name", "storm-worker", func(pod api.Pod) {
//do nothing, just wait for the pod to be running
})
Expand All @@ -288,7 +288,7 @@ var _ = Describe("Examples e2e", func() {

By("checking if Nimbus responds to requests")
lookForString("No topologies running.", time.Minute, func() string {
return runKubectl("exec", "nimbus", nsFlag, "--", "bin/storm", "list")
return runKubectlOrDie("exec", "nimbus", nsFlag, "--", "bin/storm", "list")
})
})
})
Expand All @@ -302,8 +302,8 @@ var _ = Describe("Examples e2e", func() {
httpYaml := mkpath("http-liveness.yaml")
nsFlag := fmt.Sprintf("--namespace=%v", ns)

runKubectl("create", "-f", execYaml, nsFlag)
runKubectl("create", "-f", httpYaml, nsFlag)
runKubectlOrDie("create", "-f", execYaml, nsFlag)
runKubectlOrDie("create", "-f", httpYaml, nsFlag)
checkRestart := func(podName string, timeout time.Duration) {
err := waitForPodRunningInNamespace(c, podName, ns)
Expect(err).NotTo(HaveOccurred())
Expand Down Expand Up @@ -335,8 +335,8 @@ var _ = Describe("Examples e2e", func() {
nsFlag := fmt.Sprintf("--namespace=%v", ns)

By("creating secret and pod")
runKubectl("create", "-f", secretYaml, nsFlag)
runKubectl("create", "-f", podYaml, nsFlag)
runKubectlOrDie("create", "-f", secretYaml, nsFlag)
runKubectlOrDie("create", "-f", podYaml, nsFlag)

By("checking if secret was read correctly")
_, err := lookForStringInLog(ns, "secret-test-pod", "test-container", "value-1", serverStartTimeout)
Expand All @@ -354,7 +354,7 @@ var _ = Describe("Examples e2e", func() {
podName := "dapi-test-pod"

By("creating the pod")
runKubectl("create", "-f", podYaml, nsFlag)
runKubectlOrDie("create", "-f", podYaml, nsFlag)

By("checking if name and namespace were passed correctly")
_, err := lookForStringInLog(ns, podName, "test-container", fmt.Sprintf("POD_NAMESPACE=%v", ns), serverStartTimeout)
Expand All @@ -376,8 +376,8 @@ var _ = Describe("Examples e2e", func() {
nsFlag := fmt.Sprintf("--namespace=%v", ns)

By("starting rethinkdb")
runKubectl("create", "-f", driverServiceYaml, nsFlag)
runKubectl("create", "-f", rethinkDbControllerYaml, nsFlag)
runKubectlOrDie("create", "-f", driverServiceYaml, nsFlag)
runKubectlOrDie("create", "-f", rethinkDbControllerYaml, nsFlag)
checkDbInstances := func() {
forEachPod(c, ns, "db", "rethinkdb", func(pod api.Pod) {
_, err := lookForStringInLog(ns, pod.Name, "rethinkdb", "Server ready", serverStartTimeout)
Expand All @@ -393,8 +393,8 @@ var _ = Describe("Examples e2e", func() {
checkDbInstances()

By("starting admin")
runKubectl("create", "-f", adminServiceYaml, nsFlag)
runKubectl("create", "-f", adminPodYaml, nsFlag)
runKubectlOrDie("create", "-f", adminServiceYaml, nsFlag)
runKubectlOrDie("create", "-f", adminPodYaml, nsFlag)
err = waitForPodRunningInNamespace(c, "rethinkdb-admin", ns)
Expect(err).NotTo(HaveOccurred())
checkDbInstances()
Expand All @@ -416,8 +416,8 @@ var _ = Describe("Examples e2e", func() {
nsFlag := fmt.Sprintf("--namespace=%v", ns)

By("starting hazelcast")
runKubectl("create", "-f", serviceYaml, nsFlag)
runKubectl("create", "-f", controllerYaml, nsFlag)
runKubectlOrDie("create", "-f", serviceYaml, nsFlag)
runKubectlOrDie("create", "-f", controllerYaml, nsFlag)
forEachPod(c, ns, "name", "hazelcast", func(pod api.Pod) {
_, err := lookForStringInLog(ns, pod.Name, "hazelcast", "Members [1]", serverStartTimeout)
Expect(err).NotTo(HaveOccurred())
Expand Down Expand Up @@ -476,11 +476,11 @@ var _ = Describe("Examples e2e", func() {
}

for _, ns := range namespaces {
runKubectl("create", "-f", backendRcYaml, getNsCmdFlag(ns))
runKubectlOrDie("create", "-f", backendRcYaml, getNsCmdFlag(ns))
}

for _, ns := range namespaces {
runKubectl("create", "-f", backendSvcYaml, getNsCmdFlag(ns))
runKubectlOrDie("create", "-f", backendSvcYaml, getNsCmdFlag(ns))
}

// wait for objects
Expand Down Expand Up @@ -526,7 +526,7 @@ var _ = Describe("Examples e2e", func() {

// create a pod in each namespace
for _, ns := range namespaces {
newKubectlCommand("create", "-f", "-", getNsCmdFlag(ns)).withStdinData(updatedPodYaml).exec()
newKubectlCommand("create", "-f", "-", getNsCmdFlag(ns)).withStdinData(updatedPodYaml).execOrDie()
}

// wait until the pods have been scheduler, i.e. are not Pending anymore. Remember
Expand Down Expand Up @@ -605,13 +605,13 @@ func forEachPod(c *client.Client, ns, selectorKey, selectorValue string, fn func

func lookForStringInLog(ns, podName, container, expectedString string, timeout time.Duration) (result string, err error) {
return lookForString(expectedString, timeout, func() string {
return runKubectl("log", podName, container, fmt.Sprintf("--namespace=%v", ns))
return runKubectlOrDie("log", podName, container, fmt.Sprintf("--namespace=%v", ns))
})
}

func lookForStringInFile(ns, podName, container, file, expectedString string, timeout time.Duration) (result string, err error) {
return lookForString(expectedString, timeout, func() string {
return runKubectl("exec", podName, "-c", container, fmt.Sprintf("--namespace=%v", ns), "--", "cat", file)
return runKubectlOrDie("exec", podName, "-c", container, fmt.Sprintf("--namespace=%v", ns), "--", "cat", file)
})
}

Expand All @@ -620,7 +620,7 @@ func lookForStringInPodExec(ns, podName string, command []string, expectedString
// use the first container
args := []string{"exec", podName, fmt.Sprintf("--namespace=%v", ns), "--"}
args = append(args, command...)
return runKubectl(args...)
return runKubectlOrDie(args...)
})
}

Expand Down
46 changes: 45 additions & 1 deletion test/e2e/ingress.go
View file
Open in desktop
Expand Up @@ -62,6 +62,13 @@ var (
expectedLBCreationTime = 7 * time.Minute
expectedLBHealthCheckTime = 7 * time.Minute

// Name of the loadbalancer controller within the cluster addon
lbContainerName = "l7-lb-controller"

// If set, the test tries to perform an HTTP GET on each url endpoint of
// the Ingress. Only set to false to short-circuit test runs in debugging.
verifyHTTPGET = true

// On average it takes ~6 minutes for a single backend to come online.
// We *don't* expect this poll to consistently take 15 minutes for every
// Ingress as GCE is creating/checking backends in parallel, but at the
Expand Down Expand Up @@ -170,7 +177,9 @@ func createApp(c *client.Client, ns string, i int) {
// gcloudUnmarshal unmarshals json output of gcloud into given out interface.
func gcloudUnmarshal(resource, regex string, out interface{}) {
output, err := exec.Command("gcloud", "compute", resource, "list",
fmt.Sprintf("--regex=%v", regex), "-q", "--format=json").CombinedOutput()
fmt.Sprintf("--regex=%v", regex),
fmt.Sprintf("--project=%v", testContext.CloudConfig.ProjectID),
"-q", "--format=json").CombinedOutput()
if err != nil {
Failf("Error unmarshalling gcloud output: %v", err)
}
Expand Down Expand Up @@ -201,6 +210,25 @@ func checkLeakedResources() error {
return nil
}

// kubectlLogLBController logs kubectl debug output for the L7 controller pod.
func kubectlLogLBController(c *client.Client) {
selector := labels.SelectorFromSet(labels.Set(map[string]string{"name": "glbc"}))
podList, err := c.Pods(api.NamespaceAll).List(selector, fields.Everything())
if err != nil {
Logf("Cannot log L7 controller output, error listing pods %v", err)
return
}
if len(podList.Items) == 0 {
Logf("Loadbalancer controller pod not found")
return
}
for _, p := range podList.Items {
Logf("\nLast 100 log lines of %v\n", p.Name)
l, _ := runKubectl("logs", p.Name, fmt.Sprintf("--namespace=%v", api.NamespaceSystem), "-c", lbContainerName, "--tail=100")
Logf(l)
}
}

var _ = Describe("GCE L7 LoadBalancer Controller", func() {
// These variables are initialized after framework's beforeEach.
var ns string
Expand All @@ -222,6 +250,12 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() {
})

AfterEach(func() {
if CurrentGinkgoTestDescription().Failed {
kubectlLogLBController(client)
Logf("\nOutput of kubectl describe ing:\n")
desc, _ := runKubectl("describe", "ing", fmt.Sprintf("--namespace=%v", ns))
Logf(desc)
}
framework.afterEach()
err := wait.Poll(lbPollInterval, lbPollTimeout, func() (bool, error) {
if err := checkLeakedResources(); err != nil {
Expand Down Expand Up @@ -268,11 +302,18 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() {
// Wait for the loadbalancer IP.
start := time.Now()
address, err := waitForIngressAddress(client, ing.Namespace, ing.Name, lbPollTimeout)
if err != nil {
Failf("Ingress failed to acquire an IP address within %v", lbPollTimeout)
}
Expect(err).NotTo(HaveOccurred())
By(fmt.Sprintf("Found address %v for ingress %v, took %v to come online",
address, ing.Name, time.Since(start)))
creationTimes = append(creationTimes, time.Since(start))

if !verifyHTTPGET {
continue
}

// Check that all rules respond to a simple GET.
for _, rules := range ing.Spec.Rules {
// As of Kubernetes 1.1 we only support HTTP Ingress.
Expand Down Expand Up @@ -312,6 +353,9 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() {
if perc50 > expectedLBCreationTime {
Failf("Average creation time is too high: %+v", creationTimes)
}
if !verifyHTTPGET {
return
}
sort.Sort(timeSlice(responseTimes))
perc50 = responseTimes[len(responseTimes)/2]
if perc50 > expectedLBHealthCheckTime {
Expand Down