From b1c1187cca7f7986b321073760d067994d1d8483 Mon Sep 17 00:00:00 2001 From: Kubernetes Prow Robot Date: Sat, 29 Feb 2020 07:54:40 -0800 Subject: [PATCH] e2e: wait for controller manager pod to be ready The MetricsGrabber may use the controller-manager pod to gather metrics, however, it doesn't wait until it is ready to serve, failing the test if this is the case. We wait until the controller-manager pod is running before trying to get metrics from it. --- test/e2e/framework/metrics/BUILD | 1 + test/e2e/framework/metrics/metrics_grabber.go | 28 ++++++++++++------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/test/e2e/framework/metrics/BUILD b/test/e2e/framework/metrics/BUILD index 3713c9f49f30..6ea745a7b373 100644 --- a/test/e2e/framework/metrics/BUILD +++ b/test/e2e/framework/metrics/BUILD @@ -32,6 +32,7 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/component-base/metrics/testutil:go_default_library", "//test/e2e/framework/log:go_default_library", + "//test/e2e/framework/pod:go_default_library", "//test/e2e/perftype:go_default_library", "//test/e2e/system:go_default_library", "//vendor/k8s.io/klog:go_default_library", diff --git a/test/e2e/framework/metrics/metrics_grabber.go b/test/e2e/framework/metrics/metrics_grabber.go index c1e46264002c..3540cc54a315 100644 --- a/test/e2e/framework/metrics/metrics_grabber.go +++ b/test/e2e/framework/metrics/metrics_grabber.go @@ -19,12 +19,14 @@ package metrics import ( "context" "fmt" + "sync" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" clientset "k8s.io/client-go/kubernetes" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/master/ports" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/kubernetes/test/e2e/system" "k8s.io/klog" @@ -41,15 +43,16 @@ type Collection struct { // Grabber provides functions which grab metrics from components type Grabber struct { - client clientset.Interface - externalClient clientset.Interface - grabFromAPIServer bool - grabFromControllerManager bool - grabFromKubelets bool - grabFromScheduler bool - grabFromClusterAutoscaler bool - masterName string - registeredMaster bool + client clientset.Interface + externalClient clientset.Interface + grabFromAPIServer bool + grabFromControllerManager bool + grabFromKubelets bool + grabFromScheduler bool + grabFromClusterAutoscaler bool + masterName string + registeredMaster bool + waitForControllerManagerReadyOnce sync.Once } // NewMetricsGrabber returns new metrics which are initialized. @@ -161,7 +164,12 @@ func (g *Grabber) GrabFromControllerManager() (ControllerManagerMetrics, error) if !g.registeredMaster { return ControllerManagerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ControllerManager's metrics gathering") } - output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName), metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort) + + podName := fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName) + g.waitForControllerManagerReadyOnce.Do(func() { + e2epod.WaitForPodNameRunningInNamespace(g.client, podName, metav1.NamespaceSystem) + }) + output, err := g.getMetricsFromPod(g.client, podName, metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort) if err != nil { return ControllerManagerMetrics{}, err }