From dee2f347d429bac4dffd67e42a70acee57743adb Mon Sep 17 00:00:00 2001
From: qianyong <qianyong@api7.ai>
Date: Thu, 11 Nov 2021 17:19:21 +0800
Subject: [PATCH 1/2] fix: use independent dns service for UDP e2e test, avoid
 create resource in kube-system namespace

---
 test/e2e/ingress/stream.go   | 54 ++++++++++++++++++++++++++++++++++--
 test/e2e/scaffold/ingress.go |  2 --
 test/e2e/scaffold/k8s.go     | 30 ++++++++++++++++++++
 3 files changed, 81 insertions(+), 5 deletions(-)

diff --git a/test/e2e/ingress/stream.go b/test/e2e/ingress/stream.go
index a38c48d9d0..7fcf777131 100644
--- a/test/e2e/ingress/stream.go
+++ b/test/e2e/ingress/stream.go
@@ -72,6 +72,56 @@ spec:
 		resp.Body().Contains("x-my-value")
 	})
 	ginkgo.It("stream udp proxy", func() {
+		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coredns
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: coredns
+  template:
+    metadata:
+      labels:
+        app: coredns
+    spec:
+      containers:
+      - name: coredns
+        image: coredns/coredns:1.8.4
+        livenessProbe:
+          tcpSocket:
+            port: 53
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        readinessProbe:
+          tcpSocket:
+            port: 53
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        ports:    
+        - name: dns
+          containerPort: 53
+          protocol: UDP
+`))
+		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(`
+kind: Service
+apiVersion: v1
+metadata:
+  name: coredns
+spec:
+  selector:
+    app: coredns
+  type: ClusterIP
+  ports:
+  - port: 53
+    targetPort: 53
+    protocol: UDP
+`))
+
+		s.EnsureNumEndpointsReady(ginkgo.GinkgoT(), "coredns", 1)
+
 		apisixRoute := fmt.Sprintf(`
 apiVersion: apisix.apache.org/v2beta2
 kind: ApisixRoute
@@ -84,11 +134,9 @@ spec:
     match:
       ingressPort: 9200
     backend:
-      serviceName: kube-dns
+      serviceName: coredns
       servicePort: 53
 `)
-		// update namespace only for this case
-		s.UpdateNamespace("kube-system")
 		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(apisixRoute))
 		defer func() {
 			err := s.RemoveResourceByString(apisixRoute)
diff --git a/test/e2e/scaffold/ingress.go b/test/e2e/scaffold/ingress.go
index a2c732d140..8908602b4f 100644
--- a/test/e2e/scaffold/ingress.go
+++ b/test/e2e/scaffold/ingress.go
@@ -277,8 +277,6 @@ spec:
             - http://apisix-service-e2e-test:9180/apisix/admin
             - --default-apisix-cluster-admin-key
             - edd1c9f034335f136f87ad84b625c8f1
-            - --app-namespace
-            - kube-system
             - --namespace-selector
             - %s
             - --apisix-route-version
diff --git a/test/e2e/scaffold/k8s.go b/test/e2e/scaffold/k8s.go
index 8761980647..6e79a103bd 100644
--- a/test/e2e/scaffold/k8s.go
+++ b/test/e2e/scaffold/k8s.go
@@ -17,6 +17,7 @@ package scaffold
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -27,6 +28,8 @@ import (
 	"github.com/apache/apisix-ingress-controller/pkg/apisix"
 	v1 "github.com/apache/apisix-ingress-controller/pkg/types/apisix/v1"
 	"github.com/gruntwork-io/terratest/modules/k8s"
+	"github.com/gruntwork-io/terratest/modules/retry"
+	"github.com/gruntwork-io/terratest/modules/testing"
 	"github.com/onsi/ginkgo"
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
@@ -467,3 +470,30 @@ func (s *Scaffold) newAPISIXTunnels() error {
 func (s *Scaffold) Namespace() string {
 	return s.kubectlOptions.Namespace
 }
+
+func (s *Scaffold) EnsureNumEndpointsReady(t testing.TestingT, endpointsName string, desired int) {
+	e, err := k8s.GetKubernetesClientFromOptionsE(t, s.kubectlOptions)
+	assert.Nil(t, err, "get kubernetes client")
+	statusMsg := fmt.Sprintf("Wait for endpoints %s to be ready.", endpointsName)
+	message := retry.DoWithRetry(
+		t,
+		statusMsg,
+		20,
+		5*time.Second,
+		func() (string, error) {
+			endpoints, err := e.CoreV1().Endpoints(s.Namespace()).Get(context.Background(), endpointsName, metav1.GetOptions{})
+			if err != nil {
+				return "", err
+			}
+			readyNum := 0
+			for _, subset := range endpoints.Subsets {
+				readyNum += len(subset.Addresses)
+			}
+			if readyNum == desired {
+				return "Service is now available", nil
+			}
+			return "Endpoints not ready yet", nil
+		},
+	)
+	ginkgo.GinkgoT().Log(message)
+}

From a17917d4022aa5d10d3e0235d63905a43a7b1409 Mon Sep 17 00:00:00 2001
From: qianyong <qianyong@api7.ai>
Date: Thu, 11 Nov 2021 18:21:24 +0800
Subject: [PATCH 2/2] fix

---
 test/e2e/scaffold/k8s.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/scaffold/k8s.go b/test/e2e/scaffold/k8s.go
index 6e79a103bd..0d0bd92a9f 100644
--- a/test/e2e/scaffold/k8s.go
+++ b/test/e2e/scaffold/k8s.go
@@ -492,7 +492,7 @@ func (s *Scaffold) EnsureNumEndpointsReady(t testing.TestingT, endpointsName str
 			if readyNum == desired {
 				return "Service is now available", nil
 			}
-			return "Endpoints not ready yet", nil
+			return fmt.Sprintf("Endpoints not ready yet, expect %v, actual %v", desired, readyNum), nil
 		},
 	)
 	ginkgo.GinkgoT().Log(message)