fix e2e testing

kubeovn · Sep 10, 2021 · 8ad46da · 8ad46da
1 parent 5a12637
commit 8ad46da
Show file tree

Hide file tree

Showing 8 changed files with 114 additions and 108 deletions.
diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
@@ -53,14 +53,13 @@ jobs:
 
       - name: Unit test
         run: |
-          go get -u github.com/onsi/ginkgo/ginkgo
-          go get -u github.com/onsi/gomega/...
+          go install github.com/onsi/ginkgo/ginkgo@latest
           make ut
 
       - name: Build
         run: |
           go mod tidy
-          go get -u github.com/securego/gosec/cmd/gosec
+          go install github.com/securego/gosec/cmd/gosec@latest
           make release
           make tar
 
@@ -109,8 +108,7 @@ jobs:
 
       - name: Run E2E
         run: |
-          go get -u github.com/onsi/ginkgo/ginkgo
-          go get -u github.com/onsi/gomega/...
+          go install github.com/onsi/ginkgo/ginkgo@latest
           sudo kubectl cluster-info
           sudo chmod 666 /home/runner/.kube/config
           make e2e
@@ -154,8 +152,7 @@ jobs:
 
       - name: Run E2E
         run: |
-          go get -u github.com/onsi/ginkgo/ginkgo
-          go get -u github.com/onsi/gomega/...
+          go install github.com/onsi/ginkgo/ginkgo@latest
           sudo kubectl cluster-info
           sudo chmod 666 /home/runner/.kube/config
           make e2e
@@ -199,8 +196,7 @@ jobs:
 
       - name: Run E2E
         run: |
-          go get -u github.com/onsi/ginkgo/ginkgo
-          go get -u github.com/onsi/gomega/...
+          go install github.com/onsi/ginkgo/ginkgo@latest
           sudo kubectl cluster-info
           sudo chmod 666 /home/runner/.kube/config
           make e2e-vlan
@@ -244,8 +240,7 @@ jobs:
 
       - name: Run E2E
         run: |
-          go get -u github.com/onsi/ginkgo/ginkgo
-          go get -u github.com/onsi/gomega/...
+          go install github.com/onsi/ginkgo/ginkgo@latest
           sudo kubectl cluster-info
           sudo chmod 666 /home/runner/.kube/config
           make e2e-underlay-single-nic
@@ -289,8 +284,7 @@ jobs:
 
       - name: Run E2E
         run: |
-          go get -u github.com/onsi/ginkgo/ginkgo
-          go get -u github.com/onsi/gomega/...
+          go install github.com/onsi/ginkgo/ginkgo@latest
           sudo kubectl cluster-info
           sudo chmod 666 /home/runner/.kube/config
           make e2e
@@ -369,8 +363,7 @@ jobs:
 
       - name: Run E2E
         run: |
-          go get -u github.com/onsi/ginkgo/ginkgo
-          go get -u github.com/onsi/gomega/...
+          go install github.com/onsi/ginkgo/ginkgo@latest
           sudo kubectl cluster-info
           sudo chmod 666 /home/runner/.kube/config
           make e2e-ipv6
@@ -418,8 +411,7 @@ jobs:
 
       - name: Run E2E
         run: |
-          go get -u github.com/onsi/ginkgo/ginkgo
-          go get -u github.com/onsi/gomega/...
+          go install github.com/onsi/ginkgo/ginkgo@latest
           sudo kubectl cluster-info
           sudo chmod 666 /home/runner/.kube/config
           make e2e-vlan-ipv6
@@ -467,8 +459,7 @@ jobs:
 
       - name: Run E2E
         run: |
-          go get -u github.com/onsi/ginkgo/ginkgo
-          go get -u github.com/onsi/gomega/...
+          go install github.com/onsi/ginkgo/ginkgo@latest
           sudo kubectl cluster-info
           sudo chmod 666 /home/runner/.kube/config
           make e2e-underlay-single-nic
@@ -516,8 +507,7 @@ jobs:
 
       - name: Run E2E
         run: |
-          go get -u github.com/onsi/ginkgo/ginkgo
-          go get -u github.com/onsi/gomega/...
+          go install github.com/onsi/ginkgo/ginkgo@latest
           sudo kubectl cluster-info
           sudo chmod 666 /home/runner/.kube/config
           make e2e
@@ -563,8 +553,7 @@ jobs:
 
       - name: Run E2E
         run: |
-          go get -u github.com/onsi/ginkgo/ginkgo
-          go get -u github.com/onsi/gomega/...
+          go install github.com/onsi/ginkgo/ginkgo@latest
           sudo kubectl cluster-info
           sudo chmod 666 /home/runner/.kube/config
           make e2e
@@ -610,8 +599,7 @@ jobs:
 
       - name: Run E2E
         run: |
-          go get -u github.com/onsi/ginkgo/ginkgo
-          go get -u github.com/onsi/gomega/...
+          go install github.com/onsi/ginkgo/ginkgo@latest
           sudo kubectl cluster-info
           sudo chmod 666 /home/runner/.kube/config
           make e2e
@@ -657,8 +645,7 @@ jobs:
 
       - name: Run E2E
         run: |
-          go get -u github.com/onsi/ginkgo/ginkgo
-          go get -u github.com/onsi/gomega/...
+          go install github.com/onsi/ginkgo/ginkgo@latest
           sudo kubectl cluster-info
           sudo chmod 666 /home/runner/.kube/config
           make e2e

diff --git a/go.mod b/go.mod
@@ -12,6 +12,7 @@ require (
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/elazarl/goproxy v0.0.0-20190630181448-f1e96bc0f4c5 // indirect
 	github.com/emicklei/go-restful v2.11.1+incompatible
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-ini/ini v1.42.0 // indirect
 	github.com/golang/protobuf v1.5.2
 	github.com/greenpau/ovsdb v0.0.0-20181114004433-3582b85e8968
@@ -24,7 +25,7 @@ require (
 	github.com/neverlee/keymutex v0.0.0-20171121013845-f593aa834bf9
 	github.com/oilbeater/go-ping v0.0.0-20200413021620-332b7197c5b5
 	github.com/onsi/ginkgo v1.16.4
-	github.com/onsi/gomega v1.15.0
+	github.com/onsi/gomega v1.16.0
 	github.com/osrg/gobgp v0.0.0-20190401195721-805d02fdfbc5
 	github.com/parnurzeal/gorequest v0.2.15
 	github.com/projectcalico/go-json v0.0.0-20161128004156-6219dc7339ba // indirect
@@ -39,8 +40,8 @@ require (
 	github.com/stretchr/testify v1.6.1
 	github.com/vishvananda/netlink v1.1.0
 	golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect
-	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
-	golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e
+	golang.org/x/net v0.0.0-20210907225631-ff17edfbf26d // indirect
+	golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/tools v0.1.5 // indirect
 	google.golang.org/grpc v1.27.1

diff --git a/go.sum b/go.sum
@@ -149,8 +149,9 @@ github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.2/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
+github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
@@ -416,8 +417,8 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU=
-github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
+github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c=
+github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/osrg/gobgp v0.0.0-20190401195721-805d02fdfbc5 h1:ZFqHsIsRU5o6JY4HrCrkBYOEEHe3QqF1XR7f0FwAJn4=
 github.com/osrg/gobgp v0.0.0-20190401195721-805d02fdfbc5/go.mod h1:CM+XMF5H4FfrlOm3OCFsOXJkVmWB7sx1AfoS+EKA47g=
 github.com/parnurzeal/gorequest v0.2.15 h1:oPjDCsF5IkD4gUk6vIgsxYNaSgvAnIh1EJeROn3HdJU=
@@ -662,8 +663,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
-golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210907225631-ff17edfbf26d h1:kuk8nKPQ25KCDODLCDXt99tnTVeOyOM8HGvtJ0NzAvw=
+golang.org/x/net v0.0.0-20210907225631-ff17edfbf26d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -726,8 +727,9 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34 h1:GkvMjFtXUmahfDtashnc1mnrCtuBVcwse5QV2lUk/tI=
+golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

diff --git a/test/e2e-underlay-single-nic/e2e_suite_test.go b/test/e2e-underlay-single-nic/e2e_suite_test.go
@@ -11,9 +11,11 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"github.com/kubeovn/kube-ovn/test/e2e/framework"
+
+	// tests to run
 	_ "github.com/kubeovn/kube-ovn/test/e2e-underlay-single-nic/kubectl-ko"
 	_ "github.com/kubeovn/kube-ovn/test/e2e-underlay-single-nic/node"
-	"github.com/kubeovn/kube-ovn/test/e2e/framework"
 )
 
 func TestE2e(t *testing.T) {

diff --git a/test/e2e-underlay-single-nic/node/node.go b/test/e2e-underlay-single-nic/node/node.go
@@ -36,7 +36,7 @@ type nodeNetwork struct {
 	MacAddress          string
 }
 
-var _ = Describe("[Vlan Node]", func() {
+var _ = Describe("[Underlay Node]", func() {
 	f := framework.NewFramework("node", fmt.Sprintf("%s/.kube/config", os.Getenv("HOME")))
 
 	var network *nodeNetwork
@@ -53,29 +53,38 @@ var _ = Describe("[Vlan Node]", func() {
 		Expect(nodes).NotTo(BeNil())
 		Expect(len(nodes.Items)).NotTo(BeZero())
 
-		nodeIPs := make([]string, 0, len(nodes.Items))
-		nodeRoutes := make([]string, 0, len(nodes.Items))
+		nodeIPs := make([]string, 0, len(nodes.Items)*2)
+		nodeRoutes := make([]string, 0, len(nodes.Items)*4)
 		if network != nil {
 			if network.IPAddress != "" {
-				nodeIPs = append(nodeIPs, fmt.Sprintf("%s/%d", network.IPAddress, network.IPPrefixLen))
+				addr := fmt.Sprintf("%s/%d", network.IPAddress, network.IPPrefixLen)
+				nodeIPs = append(nodeIPs, addr)
+				_, ipnet, err := net.ParseCIDR(addr)
+				Expect(err).NotTo(HaveOccurred())
+				nodeRoutes = append(nodeRoutes, fmt.Sprintf("%s ", ipnet.String()))
 			}
 			if network.GlobalIPv6Address != "" {
-				nodeIPs = append(nodeIPs, fmt.Sprintf("%s/%d", network.GlobalIPv6Address, network.GlobalIPv6PrefixLen))
+				addr := fmt.Sprintf("%s/%d", network.GlobalIPv6Address, network.GlobalIPv6PrefixLen)
+				nodeIPs = append(nodeIPs, addr)
+				_, ipnet, err := net.ParseCIDR(addr)
+				Expect(err).NotTo(HaveOccurred())
+				nodeRoutes = append(nodeRoutes, fmt.Sprintf("%s ", ipnet.String()))
 			}
 			if network.Gateway != "" {
 				nodeRoutes = append(nodeRoutes, fmt.Sprintf("default via %s ", network.Gateway))
 			}
 			if network.IPv6Gateway != "" {
-				nodeRoutes = append(nodeRoutes, fmt.Sprintf("default via %s ", network.Gateway))
+				nodeRoutes = append(nodeRoutes, fmt.Sprintf("default via %s ", network.IPv6Gateway))
 			}
 		} else {
 			for _, node := range nodes.Items {
 				if node.Name == "kube-ovn-control-plane" {
-					for _, addr := range node.Status.Addresses {
-						if addr.Type == corev1.NodeInternalIP {
-							nodeIPs = append(nodeIPs, addr.Address+"/")
-							break
-						}
+					ipv4, ipv6 := util.GetNodeInternalIP(node)
+					if ipv4 != "" {
+						nodeIPs = append(nodeIPs, ipv4+"/")
+					}
+					if ipv6 != "" {
+						nodeIPs = append(nodeIPs, ipv6+"/")
 					}
 					break
 				}
@@ -104,31 +113,33 @@ var _ = Describe("[Vlan Node]", func() {
 		stdout, _, err := f.ExecToPodThroughAPI("ovs-vsctl list-ports "+vlanBr, "openvswitch", ovsPod.Name, ovsPod.Namespace, nil)
 		Expect(err).NotTo(HaveOccurred())
 
-		var portFound bool
+		var found bool
 		for _, port := range strings.Split(stdout, "\n") {
 			if port == vlanNic {
-				portFound = true
+				found = true
 				break
 			}
 		}
-		Expect(portFound).To(BeTrue())
+		Expect(found).To(BeTrue())
 
 		stdout, _, err = f.ExecToPodThroughAPI("ip addr show "+vlanBr, "openvswitch", ovsPod.Name, ovsPod.Namespace, nil)
 		Expect(err).NotTo(HaveOccurred())
+		Expect(stdout).NotTo(BeEmpty())
 
-		var isUp bool
 		ipFound := make([]bool, len(nodeIPs))
 		for i, s := range strings.Split(stdout, "\n") {
 			if i == 0 {
+				var linkUp bool
 				idx1, idx2 := strings.IndexRune(s, '<'), strings.IndexRune(s, '>')
 				if idx1 > 0 && idx2 > idx1+1 {
 					for _, state := range strings.Split(s[idx1+1:idx2], ",") {
 						if state == "UP" {
-							isUp = true
+							linkUp = true
 							break
 						}
 					}
 				}
+				Expect(linkUp).To(BeTrue())
 				continue
 			}
 			if i == 1 && network != nil && network.MacAddress != "" {
@@ -144,13 +155,13 @@ var _ = Describe("[Vlan Node]", func() {
 				}
 			}
 		}
-		Expect(isUp).To(BeTrue())
 		for _, found := range ipFound {
 			Expect(found).To(BeTrue())
 		}
 
 		stdout, _, err = f.ExecToPodThroughAPI("ip addr show "+vlanNic, "openvswitch", ovsPod.Name, ovsPod.Namespace, nil)
 		Expect(err).NotTo(HaveOccurred())
+		Expect(stdout).NotTo(BeEmpty())
 
 		var hasAddr bool
 		for _, s := range strings.Split(stdout, "\n") {
@@ -168,7 +179,7 @@ var _ = Describe("[Vlan Node]", func() {
 		}
 		Expect(hasAddr).To(BeFalse())
 
-		stdout, _, err = f.ExecToPodThroughAPI("ip route show dev "+vlanBr, "openvswitch", ovsPod.Name, ovsPod.Namespace, nil)
+		stdout, _, err = f.ExecToPodThroughAPI("ip -4 route show dev "+vlanBr, "openvswitch", ovsPod.Name, ovsPod.Namespace, nil)
 		Expect(err).NotTo(HaveOccurred())
 		routes := strings.Split(stdout, "\n")
 
@@ -189,7 +200,7 @@ var _ = Describe("[Vlan Node]", func() {
 			Expect(found).To(BeTrue())
 		}
 
-		stdout, _, err = f.ExecToPodThroughAPI("ip route show dev "+vlanNic, "openvswitch", ovsPod.Name, ovsPod.Namespace, nil)
+		stdout, _, err = f.ExecToPodThroughAPI("ip -4 route show dev "+vlanNic, "openvswitch", ovsPod.Name, ovsPod.Namespace, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(strings.TrimSpace(stdout)).To(BeEmpty())
 

diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
@@ -8,6 +8,7 @@ import (
 	"os"
 	"strconv"
 	"testing"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -211,6 +212,8 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	if err = f.WaitProviderNetworkReady(pn.Name); err != nil {
 		Fail("provider network failed: " + err.Error())
 	}
+	// FIXME: wait 3 seconds to ensure node conditions are up to date
+	time.Sleep(3 * time.Second)
 	if pn, err = f.OvnClientSet.KubeovnV1().ProviderNetworks().Get(context.Background(), pn.Name, metav1.GetOptions{}); err != nil {
 		Fail("failed to get provider network: " + err.Error())
 	}
@@ -234,9 +237,6 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	if _, err = f.OvnClientSet.KubeovnV1().Vlans().Create(context.Background(), &vlan, metav1.CreateOptions{}); err != nil {
 		Fail("failed to create vlan: " + err.Error())
 	}
-	if err = f.WaitProviderNetworkReady(pn.Name); err != nil {
-		Fail("provider network failed: " + err.Error())
-	}
 
 	// create subnet
 	subnet := kubeovn.Subnet{