e2e: test case for health probe annotations (#1117)

* e2e test case for health probe annotations

* add the e2e prefix

* correct yaml, test health-probe-hostname in ssl-e2e-redirect

* use the right https endpoint

* correct typo

* set namespace when apply manifest

* add helper to update k8s resource object

* add missing namespace

scripts/e2e/cmd/runner/helper.go

@@ -234,6 +234,99 @@ func parseK8sYaml(fileName string) ([]runtime.Object, error) {
 	return retVal, nil
 }
 
+func updateYaml(clientset *kubernetes.Clientset, namespaceName string, fileName string) error {
+	// create objects in the yaml
+	fileObjects, err := parseK8sYaml(fileName)
+	if err != nil {
+		return err
+	}
+
+	for _, objs := range fileObjects {
+		if secret, ok := objs.(*v1.Secret); ok {
+			nm := secret.Namespace
+			if len(nm) == 0 && len(namespaceName) != 0 {
+				if _, err := clientset.CoreV1().Secrets(namespaceName).Update(secret); err != nil {
+					return err
+				}
+			} else if len(nm) != 0 {
+				if _, err := clientset.CoreV1().Secrets(nm).Update(secret); err != nil {
+					return err
+				}
+			} else {
+				return errors.New("namespace is not defined for secrets when update")
+			}
+		} else if ingress, ok := objs.(*v1beta1.Ingress); ok {
+			nm := ingress.Namespace
+			if len(nm) == 0 && len(namespaceName) != 0 {
+				if _, err := clientset.ExtensionsV1beta1().Ingresses(namespaceName).Update(ingress); err != nil {
+					return err
+				}
+			} else if len(nm) != 0 {
+				if _, err := clientset.ExtensionsV1beta1().Ingresses(nm).Update(ingress); err != nil {
+					return err
+				}
+			} else {
+				return errors.New("namespace is not defined for ingress when update")
+			}
+		} else if service, ok := objs.(*v1.Service); ok {
+			nm := service.Namespace
+			if len(nm) == 0 && len(namespaceName) != 0 {
+				if _, err := clientset.CoreV1().Services(namespaceName).Update(service); err != nil {
+					return err
+				}
+			} else if len(nm) != 0 {
+				if _, err := clientset.CoreV1().Services(nm).Update(service); err != nil {
+					return err
+				}
+			} else {
+				return errors.New("namespace is not defined for service when update")
+			}
+		} else if deployment, ok := objs.(*appsv1.Deployment); ok {
+			nm := deployment.Namespace
+			if len(nm) == 0 && len(namespaceName) != 0 {
+				if _, err := clientset.AppsV1().Deployments(namespaceName).Update(deployment); err != nil {
+					return err
+				}
+			} else if len(nm) != 0 {
+				if _, err := clientset.AppsV1().Deployments(nm).Update(deployment); err != nil {
+					return err
+				}
+			} else {
+				return errors.New("namespace is not defined for deployment when update")
+			}
+		} else if cm, ok := objs.(*v1.ConfigMap); ok {
+			nm := cm.Namespace
+			if len(nm) == 0 && len(namespaceName) != 0 {
+				if _, err := clientset.CoreV1().ConfigMaps(namespaceName).Update(cm); err != nil {
+					return err
+				}
+			} else if len(nm) != 0 {
+				if _, err := clientset.CoreV1().ConfigMaps(nm).Update(cm); err != nil {
+					return err
+				}
+			} else {
+				return errors.New("namespace is not defined for configmaps when update")
+			}
+		} else if pod, ok := objs.(*v1.Pod); ok {
+			nm := pod.Namespace
+			if len(nm) == 0 && len(namespaceName) != 0 {
+				if _, err := clientset.CoreV1().Pods(namespaceName).Update(pod); err != nil {
+					return err
+				}
+			} else if len(nm) != 0 {
+				if _, err := clientset.CoreV1().Pods(nm).Update(pod); err != nil {
+					return err
+				}
+			} else {
+				return errors.New("namespace is not defined for pods when update")
+			}
+		} else {
+			return fmt.Errorf("unable to update YAML. Unknown object type: %v", objs)
+		}
+	}
+	return nil
+}
+
 func applyYaml(clientset *kubernetes.Clientset, namespaceName string, fileName string) error {
 	// create objects in the yaml
 	fileObjects, err := parseK8sYaml(fileName)
@@ -253,7 +346,7 @@ func applyYaml(clientset *kubernetes.Clientset, namespaceName string, fileName s
 					return err
 				}
 			} else {
-				return errors.New("namespace is not defined for secrets")
+				return errors.New("namespace is not defined for secrets when create")
 			}
 		} else if ingress, ok := objs.(*v1beta1.Ingress); ok {
 			nm := ingress.Namespace
@@ -266,7 +359,7 @@ func applyYaml(clientset *kubernetes.Clientset, namespaceName string, fileName s
 					return err
 				}
 			} else {
-				return errors.New("namespace is not defined for ingress")
+				return errors.New("namespace is not defined for ingress when create")
 			}
 		} else if service, ok := objs.(*v1.Service); ok {
 			nm := service.Namespace
@@ -279,7 +372,7 @@ func applyYaml(clientset *kubernetes.Clientset, namespaceName string, fileName s
 					return err
 				}
 			} else {
-				return errors.New("namespace is not defined for service")
+				return errors.New("namespace is not defined for service when create")
 			}
 		} else if deployment, ok := objs.(*appsv1.Deployment); ok {
 			nm := deployment.Namespace
@@ -292,7 +385,7 @@ func applyYaml(clientset *kubernetes.Clientset, namespaceName string, fileName s
 					return err
 				}
 			} else {
-				return errors.New("namespace is not defined for deployment")
+				return errors.New("namespace is not defined for deployment when create")
 			}
 		} else if cm, ok := objs.(*v1.ConfigMap); ok {
 			nm := cm.Namespace
@@ -305,7 +398,7 @@ func applyYaml(clientset *kubernetes.Clientset, namespaceName string, fileName s
 					return err
 				}
 			} else {
-				return errors.New("namespace is not defined for configmaps")
+				return errors.New("namespace is not defined for configmaps when create")
 			}
 		} else if pod, ok := objs.(*v1.Pod); ok {
 			nm := pod.Namespace

scripts/e2e/cmd/runner/mfu_one_namespace_one_ingress_test.go

@@ -66,6 +66,24 @@ var _ = Describe("MFU", func() {
 			// https get to return 200 ok
 			_, err = makeGetRequest(urlHttps, "", 200, true)
 			Expect(err).To(BeNil())
+
+			//start to configure with bad hostname, 502 is expected
+			healthConfigProbeBadHostnameYamlPath := "testdata/one-namespace-one-ingress/ssl-e2e-redirect/probe-hostname-bad.yaml"
+			klog.Info("Updating ingress with bad hostname annotation")
+			err = updateYaml(clientset, namespaceName, healthConfigProbeBadHostnameYamlPath)
+			Expect(err).To(BeNil())
+			time.Sleep(30 * time.Second)
+			_, err = makeGetRequest(urlHttps, "", 502, true)
+			Expect(err).To(BeNil())
+
+			// start to configure with good hostname, 200 is expected
+			healthConfigProbeGoodHostnameYamlPath := "testdata/one-namespace-one-ingress/ssl-e2e-redirect/probe-hostname-good.yaml"
+			klog.Info("Updating ingress with good hostname annotation")
+			err = updateYaml(clientset, namespaceName, healthConfigProbeGoodHostnameYamlPath)
+			Expect(err).To(BeNil())
+			time.Sleep(30 * time.Second)
+			_, err = makeGetRequest(urlHttps, "", 200, true)
+			Expect(err).To(BeNil())
 		})
 
 		It("[three-namespaces] containers with the same probe and labels in 3 different namespaces should have unique and working health probes", func() {
@@ -99,6 +117,88 @@ var _ = Describe("MFU", func() {
 			}
 		})
 
+		It("[health-probe-config] health probe configuration with annotation should be applied first", func() {
+			namespaceName := "e2e-health-probe-config"
+			ns := &v1.Namespace{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: namespaceName,
+				},
+			}
+			klog.Info("Creating namespace: ", namespaceName)
+			_, err = clientset.CoreV1().Namespaces().Create(ns)
+			Expect(err).To(BeNil())
+
+			healthConfigProbeYamlPath := "testdata/one-namespace-one-ingress/health-probe-configurations/app.yaml"
+			klog.Info("Applying yaml: ", healthConfigProbeYamlPath)
+			err = applyYaml(clientset, namespaceName, healthConfigProbeYamlPath)
+			Expect(err).To(BeNil())
+			time.Sleep(30 * time.Second)
+
+			// get ip address for 1 ingress
+			klog.Info("Getting public IP from Ingress...")
+			publicIP, _ := getPublicIP(clientset, namespaceName)
+			Expect(publicIP).ToNot(Equal(""))
+
+			// initial deployment should be ok for the request
+			url := fmt.Sprintf("http://%s/status/200", publicIP)
+			_, err = makeGetRequest(url, "", 200, true)
+			Expect(err).To(BeNil())
+
+			// start to configure with bad path, 502 is expected
+			healthConfigProbeBadPathYamlPath := "testdata/one-namespace-one-ingress/health-probe-configurations/probe-path-bad.yaml"
+			klog.Info("Updating ingress with bad path annotation")
+			err = updateYaml(clientset, namespaceName, healthConfigProbeBadPathYamlPath)
+			Expect(err).To(BeNil())
+			time.Sleep(15 * time.Second)
+			_, err = makeGetRequest(url, "", 502, true)
+			Expect(err).To(BeNil())
+
+			// start to configure with good path, 200 is expected
+			healthConfigProbeGoodPathYamlPath := "testdata/one-namespace-one-ingress/health-probe-configurations/probe-path-good.yaml"
+			klog.Info("Updating ingress with good path annotation")
+			err = updateYaml(clientset, namespaceName, healthConfigProbeGoodPathYamlPath)
+			Expect(err).To(BeNil())
+			time.Sleep(15 * time.Second)
+			_, err = makeGetRequest(url, "", 200, true)
+			Expect(err).To(BeNil())
+
+			// start to configure with bad port, 502 is expected
+			healthConfigProbeBadPortYamlPath := "testdata/one-namespace-one-ingress/health-probe-configurations/probe-port-bad.yaml"
+			klog.Info("Updating ingress with bad port annotation")
+			err = updateYaml(clientset, namespaceName, healthConfigProbeBadPortYamlPath)
+			Expect(err).To(BeNil())
+			time.Sleep(15 * time.Second)
+			_, err = makeGetRequest(url, "", 502, true)
+			Expect(err).To(BeNil())
+
+			// start to configure with good port, 200 is expected
+			healthConfigProbeGoodPortYamlPath := "testdata/one-namespace-one-ingress/health-probe-configurations/probe-port-good.yaml"
+			klog.Info("Updating ingress with good port annotation")
+			err = updateYaml(clientset, namespaceName, healthConfigProbeGoodPortYamlPath)
+			Expect(err).To(BeNil())
+			time.Sleep(15 * time.Second)
+			_, err = makeGetRequest(url, "", 200, true)
+			Expect(err).To(BeNil())
+
+			// start to configure with bad status, 502 is expected
+			healthConfigProbeBadStatusYamlPath := "testdata/one-namespace-one-ingress/health-probe-configurations/probe-status-bad.yaml"
+			klog.Info("Updating ingress with bad status annotation")
+			err = updateYaml(clientset, namespaceName, healthConfigProbeBadStatusYamlPath)
+			Expect(err).To(BeNil())
+			time.Sleep(15 * time.Second)
+			_, err = makeGetRequest(url, "", 502, true)
+			Expect(err).To(BeNil())
+
+			// start to configure with good status, 200 is expected
+			healthConfigProbeGoodStatusYamlPath := "testdata/one-namespace-one-ingress/health-probe-configurations/probe-status-good.yaml"
+			klog.Info("Updating ingress with good status annotation")
+			err = updateYaml(clientset, namespaceName, healthConfigProbeGoodStatusYamlPath)
+			Expect(err).To(BeNil())
+			time.Sleep(15 * time.Second)
+			_, err = makeGetRequest(url, "", 200, true)
+			Expect(err).To(BeNil())
+		})
+
 		It("[container-readiness-probe] backend should be removed when health probe is not healthy", func() {
 			// http get to return 200 ok
 			for _, nm := range []string{"e2e-probe1", "e2e-probe2"} {

scripts/e2e/cmd/runner/testdata/one-namespace-one-ingress/health-probe-configurations/app.yaml

@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-deployment
+spec:
+  selector:
+    matchLabels:
+      app: test-app
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: test-app
+    spec:
+      containers:
+        - name: test-app
+          imagePullPolicy: Always
+          image: docker.io/kennethreitz/httpbin
+          ports:
+            - containerPort: 80
+          livenessProbe:
+            httpGet:
+              path: /status/200
+              port: 80
+            initialDelaySeconds: 3
+            periodSeconds: 3
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: test-service
+spec:
+  selector:
+    app: test-app
+  ports:
+  - protocol: TCP
+    port: 80
+    targetPort: 80
+
+---
+
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: test-ingress
+  annotations:
+    kubernetes.io/ingress.class: azure/application-gateway
+    appgw.ingress.kubernetes.io/health-probe-interval: "1"
+    appgw.ingress.kubernetes.io/health-probe-unhealthy-threshold: "3"
+    appgw.ingress.kubernetes.io/health-probe-timeout: "1"
+spec:
+  rules:
+    - http:
+        paths:
+        - path: /status/*
+          backend:
+            serviceName: test-service
+            servicePort: 80

scripts/e2e/cmd/runner/testdata/one-namespace-one-ingress/health-probe-configurations/probe-path-bad.yaml

@@ -0,0 +1,18 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: test-ingress
+  annotations:
+    kubernetes.io/ingress.class: azure/application-gateway
+    appgw.ingress.kubernetes.io/health-probe-path: "/statuss/200"
+    appgw.ingress.kubernetes.io/health-probe-interval: "1"
+    appgw.ingress.kubernetes.io/health-probe-unhealthy-threshold: "3"
+    appgw.ingress.kubernetes.io/health-probe-timeout: "1"
+spec:
+  rules:
+    - http:
+        paths:
+        - path: /status/*
+          backend:
+            serviceName: test-service
+            servicePort: 80

scripts/e2e/cmd/runner/testdata/one-namespace-one-ingress/health-probe-configurations/probe-path-good.yaml

@@ -0,0 +1,18 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: test-ingress
+  annotations:
+    kubernetes.io/ingress.class: azure/application-gateway
+    appgw.ingress.kubernetes.io/health-probe-path: "/status/200"
+    appgw.ingress.kubernetes.io/health-probe-interval: "1"
+    appgw.ingress.kubernetes.io/health-probe-unhealthy-threshold: "3"
+    appgw.ingress.kubernetes.io/health-probe-timeout: "1"
+spec:
+  rules:
+    - http:
+        paths:
+        - path: /status/*
+          backend:
+            serviceName: test-service
+            servicePort: 80

scripts/e2e/cmd/runner/testdata/one-namespace-one-ingress/health-probe-configurations/probe-port-bad.yaml

@@ -0,0 +1,18 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: test-ingress
+  annotations:
+    kubernetes.io/ingress.class: azure/application-gateway
+    appgw.ingress.kubernetes.io/health-probe-port: "81"
+    appgw.ingress.kubernetes.io/health-probe-interval: "1"
+    appgw.ingress.kubernetes.io/health-probe-unhealthy-threshold: "3"
+    appgw.ingress.kubernetes.io/health-probe-timeout: "1"
+spec:
+  rules:
+    - http:
+        paths:
+        - path: /status/*
+          backend:
+            serviceName: test-service
+            servicePort: 80