From 01cee24de11e5e78618fb6fa7b09019ef40cd4dc Mon Sep 17 00:00:00 2001 From: soulbird Date: Sun, 24 Apr 2022 21:30:05 +0800 Subject: [PATCH 1/2] feat(ci): add zookeeper e2e test --- test/e2e/go.mod | 1 + test/e2e/go.sum | 2 + test/e2e/regcenter/nacos_test.go | 47 --------------- test/e2e/regcenter/regcenter_suite_test.go | 2 +- test/e2e/regcenter/regcenter_test.go | 59 +++++++++++++++++++ test/e2e/tools/common/common.go | 1 + test/e2e/tools/regcenter.go | 2 + test/e2e/tools/regcenter/nacos.go | 2 +- test/e2e/tools/regcenter/zookeeper.go | 66 ++++++++++++++++++++++ test/e2e/tools/servers.go | 8 +-- 10 files changed, 137 insertions(+), 53 deletions(-) delete mode 100644 test/e2e/regcenter/nacos_test.go create mode 100644 test/e2e/regcenter/regcenter_test.go create mode 100644 test/e2e/tools/regcenter/zookeeper.go diff --git a/test/e2e/go.mod b/test/e2e/go.mod index e18ac48..8f1e75c 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -3,6 +3,7 @@ module e2e go 1.17 require ( + github.com/go-zookeeper/zk v1.0.2 github.com/onsi/ginkgo/v2 v2.0.0 github.com/onsi/gomega v1.18.1 ) diff --git a/test/e2e/go.sum b/test/e2e/go.sum index c9b712e..f21e60c 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -6,6 +6,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-zookeeper/zk v1.0.2 h1:4mx0EYENAdX/B/rbunjlt5+4RTA/a9SMHBRuSKdGxPM= +github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= diff --git a/test/e2e/regcenter/nacos_test.go b/test/e2e/regcenter/nacos_test.go deleted file mode 100644 index bdd48d0..0000000 --- a/test/e2e/regcenter/nacos_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package regcenter_test - -import ( - "e2e/tools" - "e2e/tools/common" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Nacos", Ordered, func() { - Context("single route, one server", func() { - var s1 *tools.SimServer - var reg tools.IRegCenter - BeforeAll(func() { - Expect(tools.CreateRoutes([]*tools.Route{ - tools.NewRoute("1", "/test1", "APISIX-NACOS", "nacos"), - })).To(BeNil()) - //create sim server - s1 = tools.NewSimServer("0.0.0.0", "9990", "APISIX-NACOS") - Expect(tools.CreateSimServer([]*tools.SimServer{ - s1, - })).To(BeNil()) - reg = tools.NewIRegCenter("nacos") - }) - - It("request successful, returns 200", func() { - // request - // register server to discover center - s1.Register(reg) - time.Sleep(3 * time.Second) - status, body, err := common.Request("/test1") - Expect(err).To(BeNil()) - Expect(status).To(Equal(200)) - Expect(body).To(Equal("response: 0.0.0.0:9990")) - }) - - It("request failed, returns 503", func() { - s1.LogOut(reg) - time.Sleep(3 * time.Second) - status, _, err := common.Request("/test1") - Expect(err).To(BeNil()) - Expect(status).To(Equal(503)) - }) - }) -}) diff --git a/test/e2e/regcenter/regcenter_suite_test.go b/test/e2e/regcenter/regcenter_suite_test.go index 99e1752..ee0a9c9 100644 --- a/test/e2e/regcenter/regcenter_suite_test.go +++ b/test/e2e/regcenter/regcenter_suite_test.go @@ -11,7 +11,7 @@ import ( var _ = BeforeSuite(func() { Expect(tools.CleanRoutes()).To(BeNil()) Expect(tools.NewIRegCenter("nacos").Clean()).To(BeNil()) - + Expect(tools.NewIRegCenter("zookeeper").Clean()).To(BeNil()) }) func TestRegcenter(t *testing.T) { diff --git a/test/e2e/regcenter/regcenter_test.go b/test/e2e/regcenter/regcenter_test.go new file mode 100644 index 0000000..e64070b --- /dev/null +++ b/test/e2e/regcenter/regcenter_test.go @@ -0,0 +1,59 @@ +package regcenter_test + +import ( + "e2e/tools" + "e2e/tools/common" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Normal test", Ordered, func() { + Context("single route, one server", func() { + + type normalCase struct { + URI string + Route *tools.Route + Server *tools.SimServer + Reg tools.IRegCenter + ExpectBody string + } + + DescribeTable("general logic", Ordered, + func(tc normalCase) { + Expect(tools.CreateRoutes([]*tools.Route{tc.Route})).To(BeNil()) + //create sim server + Expect(tools.CreateSimServer([]*tools.SimServer{tc.Server})).To(BeNil()) + + // upstream server online + Expect(tc.Server.Register(tc.Reg)).To(BeNil()) + time.Sleep(3 * time.Second) + status, body, err := common.Request(tc.URI) + Expect(err).To(BeNil()) + Expect(status).To(Equal(200)) + Expect(body).To(Equal(tc.ExpectBody)) + // upstream server offline + Expect(tc.Server.LogOut(tc.Reg)).To(BeNil()) + time.Sleep(3 * time.Second) + status, _, err = common.Request(tc.URI) + Expect(err).To(BeNil()) + Expect(status).To(Equal(503)) + }, + Entry("Nacos", normalCase{ + URI: "/test1", + Route: tools.NewRoute("1", "/test1", "APISIX-NACOS", "nacos"), + Server: tools.NewSimServer("0.0.0.0", "9990", "APISIX-NACOS"), + Reg: tools.NewIRegCenter("nacos"), + ExpectBody: "response: 0.0.0.0:9990", + }), + Entry("Zookeeper", normalCase{ + URI: "/test2", + Route: tools.NewRoute("2", "/test2", "APISIX-ZK", "zookeeper"), + Server: tools.NewSimServer("0.0.0.0", "9991", "APISIX-ZK"), + Reg: tools.NewIRegCenter("zookeeper"), + ExpectBody: "response: 0.0.0.0:9991", + }), + ) + }) +}) diff --git a/test/e2e/tools/common/common.go b/test/e2e/tools/common/common.go index 942a053..0ce9fa1 100644 --- a/test/e2e/tools/common/common.go +++ b/test/e2e/tools/common/common.go @@ -7,6 +7,7 @@ import ( const ( NACOS_HOST = "http://127.0.0.1:8848" + ZK_HOST = "127.0.0.1:2181" APISIX_HOST = "http://127.0.0.1:9080" APISIX_TOKEN = "edd1c9f034335f136f87ad84b625c8f1" DOCKER_GATEWAY = "172.50.238.1" diff --git a/test/e2e/tools/regcenter.go b/test/e2e/tools/regcenter.go index d6b53e2..6d0a691 100644 --- a/test/e2e/tools/regcenter.go +++ b/test/e2e/tools/regcenter.go @@ -16,6 +16,8 @@ func NewIRegCenter(name string) IRegCenter { switch name { case "nacos": return regcenter.NewNacos() + case "zookeeper": + return regcenter.NewZookeeper() } return nil } diff --git a/test/e2e/tools/regcenter/nacos.go b/test/e2e/tools/regcenter/nacos.go index b3cd831..7f721d4 100644 --- a/test/e2e/tools/regcenter/nacos.go +++ b/test/e2e/tools/regcenter/nacos.go @@ -110,7 +110,7 @@ func (n *Nacos) deleteService(service string) error { if resp.StatusCode != 200 { return errors.New("delete service failed, serviceName=:" + service) } - fmt.Println("delete service, serviceName=:" + service) + fmt.Println("delete service, serviceName=" + service) return nil } diff --git a/test/e2e/tools/regcenter/zookeeper.go b/test/e2e/tools/regcenter/zookeeper.go new file mode 100644 index 0000000..191b354 --- /dev/null +++ b/test/e2e/tools/regcenter/zookeeper.go @@ -0,0 +1,66 @@ +package regcenter + +import ( + "e2e/tools/common" + "fmt" + "time" + + "github.com/go-zookeeper/zk" +) + +type Zookeeper struct { + conn *zk.Conn + prefix string +} + +func NewZookeeper() *Zookeeper { + conn, _, err := zk.Connect([]string{common.ZK_HOST}, time.Second*5) + if err != nil { + panic(err) + } + return &Zookeeper{ + conn: conn, + prefix: "/zookeeper", + } +} + +func (zookeeper *Zookeeper) Online(node *common.Node) error { + nodeStr := `{"host":"` + common.DOCKER_GATEWAY + `","port":` + node.Port + `}` + _, err := zookeeper.conn.Create(zookeeper.prefix+"/"+node.ServiceName, []byte(nodeStr), 0, zk.WorldACL(zk.PermAll)) + if err != nil { + return err + } + + fmt.Println("register instance to Zookeeper: ", node.String()) + return err +} + +func (zookeeper *Zookeeper) Offline(node *common.Node) error { + path := zookeeper.prefix + "/" + node.ServiceName + _, stat, err := zookeeper.conn.Exists(path) + if err != nil { + return err + } + + fmt.Println("offline instance to Zookeeper: ", node.String()) + return zookeeper.conn.Delete(path, stat.Version) + +} + +func (zookeeper *Zookeeper) Clean() error { + fmt.Println("clean all service form zookeeper...") + children, stat, err := zookeeper.conn.Children(zookeeper.prefix) + if err != nil { + return err + } + for _, p := range children { + if p == "config" || p == "quota" { + continue + } + if err := zookeeper.conn.Delete(zookeeper.prefix+"/"+p, stat.Version); err != nil { + return err + } + fmt.Println("delete service, serviceName=", p) + } + return nil +} diff --git a/test/e2e/tools/servers.go b/test/e2e/tools/servers.go index 0e9b541..c7809dd 100644 --- a/test/e2e/tools/servers.go +++ b/test/e2e/tools/servers.go @@ -36,12 +36,12 @@ func (server *SimServer) Run() { }() wg.Wait() } -func (server *SimServer) Register(reg IRegCenter) { - reg.Online(server.Node) +func (server *SimServer) Register(reg IRegCenter) error { + return reg.Online(server.Node) } -func (server *SimServer) LogOut(reg IRegCenter) { - reg.Offline(server.Node) +func (server *SimServer) LogOut(reg IRegCenter) error { + return reg.Offline(server.Node) } func (server *SimServer) Stop() { From 4b0d5ca1f7ccc71b4355caa32941ecf6ccbf92f8 Mon Sep 17 00:00:00 2001 From: soulbird Date: Sun, 24 Apr 2022 21:41:54 +0800 Subject: [PATCH 2/2] add zookeeper e2e config --- ci/apisix-seed/conf.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ci/apisix-seed/conf.yaml b/ci/apisix-seed/conf.yaml index fba90b5..9f1080f 100644 --- a/ci/apisix-seed/conf.yaml +++ b/ci/apisix-seed/conf.yaml @@ -23,3 +23,9 @@ discovery: # service discovery center connect: 2000 # default 2000ms send: 2000 # default 2000ms read: 5000 # default 5000ms + zookeeper: + hosts: + - "172.50.238.40:2181" + prefix: /zookeeper + weight: 100 # default weight for node + timeout: 10 # default 10s