test for excessive etcd leadership changes

test/extended/etcd/leader_changes.go

@@ -0,0 +1,86 @@
+package etcd
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/util/json"
+
+	g "github.com/onsi/ginkgo"
+	o "github.com/onsi/gomega"
+	routev1 "github.com/openshift/api/route/v1"
+	routev1client "github.com/openshift/client-go/route/clientset/versioned"
+	exutil "github.com/openshift/origin/test/extended/util"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+var _ = g.Describe("etcd", func() {
+	defer g.GinkgoRecover()
+	var (
+		oc          = exutil.NewCLI("etcd-leader-change", exutil.KubeConfigPath())
+		kubeClient  kubernetes.Interface
+		routeClient routev1client.Interface
+	)
+
+	g.BeforeEach(func() {
+		kubeClient = oc.AdminKubeClient()
+		routeClient = oc.AdminRouteClient()
+	})
+
+	g.It("leader changes are not excessive", func() {
+
+		g.By("Waiting for prometheus service to exist")
+		o.Eventually(func() error {
+			_, err := kubeClient.CoreV1().Services("openshift-monitoring").Get("prometheus-k8s", metav1.GetOptions{})
+			return err
+		}, "2m", "1s").ShouldNot(o.HaveOccurred())
+
+		g.By("Waiting for the prometheus route to exist")
+		var route *routev1.Route
+		o.Eventually(func() error {
+			var err error
+			route, err = routeClient.RouteV1().Routes("openshift-monitoring").Get("prometheus-k8s", metav1.GetOptions{})
+			return err
+		}).ShouldNot(o.HaveOccurred())
+		prometheusHost := route.Spec.Host
+
+		g.By("Getting the prometheus bearer token")
+		var prometheusToken string
+		secrets, err := kubeClient.CoreV1().Secrets("openshift-monitoring").List(metav1.ListOptions{})
+		o.Expect(err).ToNot(o.HaveOccurred())
+		for _, secret := range secrets.Items {
+			if secret.Type == corev1.SecretTypeServiceAccountToken && strings.HasPrefix(secret.Name, "prometheus-") {
+				prometheusToken = string(secret.Data[corev1.ServiceAccountTokenKey])
+				break
+			}
+		}
+		o.Expect(prometheusToken).ToNot(o.BeEmpty())
+
+		g.By("Querying promethesus")
+		queryUrl := fmt.Sprintf("https://%s/api/v1/query", prometheusHost)
+		request, err := http.NewRequest("GET", queryUrl, nil)
+		o.Expect(err).ToNot(o.HaveOccurred())
+		request.URL.RawQuery = url.Values{"query": []string{"sum(etcd_server_leader_changes_seen_total)"}}.Encode()
+		request.Header.Add("Authorization", "Bearer "+prometheusToken)
+		client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
+		response, err := client.Do(request)
+		o.Expect(err).ToNot(o.HaveOccurred())
+		defer response.Body.Close()
+		o.Expect(response.StatusCode).To(o.Equal(http.StatusOK))
+		body, err := ioutil.ReadAll(response.Body)
+		o.Expect(err).ToNot(o.HaveOccurred())
+		result := map[string]interface{}{}
+		err = json.Unmarshal(body, &result)
+		o.Expect(err).ToNot(o.HaveOccurred())
+
+		g.By("Expect total number of ectd leadership changes seen to be less than 10")
+		totalLeaderChanges := result["data"].(map[string]interface{})["result"].([]interface{})[0].(map[string]interface{})["value"].([]interface{})[1]
+		o.Expect(totalLeaderChanges).To(o.BeNumerically("<", 10))
+	})
+})