e2e-test: add e2e tests and CRDs for ApisixClusterConfig v2 (#1016)

Signed-off-by: Ling Samuel <lingsamuelgrace@gmail.com>

samples/deploy/crd/v1/ApisixClusterConfig.yaml

@@ -31,6 +31,45 @@ spec:
   preserveUnknownFields: false
   versions:
     - name: v2beta3
+      served: true
+      storage: false
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                admin:
+                  type: object
+                  required:
+                    - baseURL
+                  properties:
+                    baseURL:
+                      type: string
+                      pattern: "https?://[^:]+:(\\d+)"
+                    adminKey:
+                      type: string
+                monitoring:
+                  type: object
+                  properties:
+                    prometheus:
+                      type: object
+                      properties:
+                        enable:
+                          type: boolean
+                    skywalking:
+                      type: object
+                      properties:
+                        enable:
+                          type: boolean
+                        sampleRatio:
+                          type: number
+                          minimum: 0.00001
+                          maximum: 1
+    - name: v2
       served: true
       storage: true
       subresources:

test/e2e/scaffold/cluster_config.go

@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package scaffold
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/gruntwork-io/terratest/modules/k8s"
+)
+
+var (
+	_apisixClusterConfigTemplate = `
+apiVersion: %s
+kind: ApisixClusterConfig
+metadata:
+  name: %s
+spec:
+  monitoring:
+    prometheus:
+      enable: %v
+`
+)
+
+// NewApisixClusterConfig creates an ApisixClusterConfig CRD
+func (s *Scaffold) NewApisixClusterConfig(name string, enable bool) error {
+	cc := fmt.Sprintf(_apisixClusterConfigTemplate, s.opts.APISIXClusterConfigVersion, name, enable)
+	if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, cc); err != nil {
+		return err
+	}
+	time.Sleep(5 * time.Second)
+	return nil
+}
+
+// DeleteApisixClusterConfig removes an ApisixClusterConfig CRD
+func (s *Scaffold) DeleteApisixClusterConfig(name string, enable bool) error {
+	cc := fmt.Sprintf(_apisixClusterConfigTemplate, s.opts.APISIXClusterConfigVersion, name, enable)
+	if err := k8s.KubectlDeleteFromStringE(s.t, s.kubectlOptions, cc); err != nil {
+		return err
+	}
+	time.Sleep(5 * time.Second)
+	return nil
+}

test/e2e/scaffold/scaffold.go

@@ -45,17 +45,18 @@ import (
 )
 
 type Options struct {
-	Name                     string
-	Kubeconfig               string
-	APISIXConfigPath         string
-	IngressAPISIXReplicas    int
-	HTTPBinServicePort       int
-	APISIXRouteVersion       string
-	APISIXTlsVersion         string
-	APISIXAdminAPIKey        string
-	EnableWebhooks           bool
-	APISIXPublishAddress     string
-	disableNamespaceSelector bool
+	Name                       string
+	Kubeconfig                 string
+	APISIXConfigPath           string
+	IngressAPISIXReplicas      int
+	HTTPBinServicePort         int
+	APISIXRouteVersion         string
+	APISIXTlsVersion           string
+	APISIXClusterConfigVersion string
+	APISIXAdminAPIKey          string
+	EnableWebhooks             bool
+	APISIXPublishAddress       string
+	disableNamespaceSelector   bool
 }
 
 type Scaffold struct {
@@ -109,6 +110,9 @@ func NewScaffold(o *Options) *Scaffold {
 	if o.APISIXTlsVersion == "" {
 		o.APISIXTlsVersion = config.ApisixV2beta3
 	}
+	if o.APISIXClusterConfigVersion == "" {
+		o.APISIXClusterConfigVersion = config.ApisixV2beta3
+	}
 	if o.APISIXAdminAPIKey == "" {
 		o.APISIXAdminAPIKey = "edd1c9f034335f136f87ad84b625c8f1"
 	}
@@ -128,31 +132,33 @@ func NewScaffold(o *Options) *Scaffold {
 // NewDefaultScaffold creates a scaffold with some default options.
 func NewDefaultScaffold() *Scaffold {
 	opts := &Options{
-		Name:                  "default",
-		Kubeconfig:            GetKubeconfig(),
-		APISIXConfigPath:      "testdata/apisix-gw-config.yaml",
-		IngressAPISIXReplicas: 1,
-		HTTPBinServicePort:    80,
-		APISIXRouteVersion:    kube.ApisixRouteV2beta3,
-		APISIXTlsVersion:      config.ApisixV2beta3,
-		EnableWebhooks:        false,
-		APISIXPublishAddress:  "",
+		Name:                       "default",
+		Kubeconfig:                 GetKubeconfig(),
+		APISIXConfigPath:           "testdata/apisix-gw-config.yaml",
+		IngressAPISIXReplicas:      1,
+		HTTPBinServicePort:         80,
+		APISIXRouteVersion:         kube.ApisixRouteV2beta3,
+		APISIXTlsVersion:           config.ApisixV2beta3,
+		APISIXClusterConfigVersion: config.ApisixV2beta3,
+		EnableWebhooks:             false,
+		APISIXPublishAddress:       "",
 	}
 	return NewScaffold(opts)
 }
 
 // NewDefaultV2Scaffold creates a scaffold with some default options.
 func NewDefaultV2Scaffold() *Scaffold {
 	opts := &Options{
-		Name:                  "default",
-		Kubeconfig:            GetKubeconfig(),
-		APISIXConfigPath:      "testdata/apisix-gw-config.yaml",
-		IngressAPISIXReplicas: 1,
-		HTTPBinServicePort:    80,
-		APISIXRouteVersion:    kube.ApisixRouteV2,
-		APISIXTlsVersion:      config.ApisixV2,
-		EnableWebhooks:        false,
-		APISIXPublishAddress:  "",
+		Name:                       "default",
+		Kubeconfig:                 GetKubeconfig(),
+		APISIXConfigPath:           "testdata/apisix-gw-config.yaml",
+		IngressAPISIXReplicas:      1,
+		HTTPBinServicePort:         80,
+		APISIXRouteVersion:         kube.ApisixRouteV2,
+		APISIXTlsVersion:           config.ApisixV2,
+		APISIXClusterConfigVersion: config.ApisixV2,
+		EnableWebhooks:             false,
+		APISIXPublishAddress:       "",
 	}
 	return NewScaffold(opts)
 }

test/e2e/suite-features/global_rule.go

@@ -27,39 +27,21 @@ import (
 )
 
 var _ = ginkgo.Describe("suite-features: ApisixClusterConfig", func() {
-	opts := &scaffold.Options{
-		Name:                  "default",
-		Kubeconfig:            scaffold.GetKubeconfig(),
-		APISIXConfigPath:      "testdata/apisix-gw-config.yaml",
-		IngressAPISIXReplicas: 1,
-		HTTPBinServicePort:    80,
-		APISIXRouteVersion:    "apisix.apache.org/v2beta3",
-	}
-	s := scaffold.NewScaffold(opts)
-	ginkgo.It("enable prometheus", func() {
-		adminSvc, adminPort := s.ApisixAdminServiceAndPort()
-		acc := `
-apiVersion: apisix.apache.org/v2beta3
-kind: ApisixClusterConfig
-metadata:
-  name: default
-spec:
-  monitoring:
-    prometheus:
-      enable: true
-`
-		err := s.CreateResourceFromString(acc)
-		assert.Nil(ginkgo.GinkgoT(), err, "creating ApisixClusterConfig")
+	suites := func(scaffoldFunc func() *scaffold.Scaffold) {
+		s := scaffoldFunc()
+
+		ginkgo.It("enable prometheus", func() {
+			adminSvc, adminPort := s.ApisixAdminServiceAndPort()
+			assert.Nil(ginkgo.GinkgoT(), s.NewApisixClusterConfig("default", true), "creating ApisixClusterConfig")
 
-		defer func() {
-			err := s.RemoveResourceByString(acc)
-			assert.Nil(ginkgo.GinkgoT(), err)
-		}()
+			defer func() {
+				assert.Nil(ginkgo.GinkgoT(), s.DeleteApisixClusterConfig("default", true))
+			}()
 
-		// Wait until the ApisixClusterConfig create event was delivered.
-		time.Sleep(3 * time.Second)
+			// Wait until the ApisixClusterConfig create event was delivered.
+			time.Sleep(3 * time.Second)
 
-		ar := fmt.Sprintf(`
+			ar := fmt.Sprintf(`
 apiVersion: apisix.apache.org/v2beta3
 kind: ApisixRoute
 metadata:
@@ -78,23 +60,27 @@ spec:
       enable: true
 `, adminSvc, adminPort)
 
-		err = s.CreateResourceFromString(ar)
-		assert.Nil(ginkgo.GinkgoT(), err, "creating ApisixRouteConfig")
+			err := s.CreateResourceFromString(ar)
+			assert.Nil(ginkgo.GinkgoT(), err, "creating ApisixRouteConfig")
 
-		time.Sleep(3 * time.Second)
+			time.Sleep(3 * time.Second)
 
-		grs, err := s.ListApisixGlobalRules()
-		assert.Nil(ginkgo.GinkgoT(), err, "listing global_rules")
-		assert.Len(ginkgo.GinkgoT(), grs, 1)
-		assert.Equal(ginkgo.GinkgoT(), grs[0].ID, id.GenID("default"))
-		assert.Len(ginkgo.GinkgoT(), grs[0].Plugins, 1)
-		_, ok := grs[0].Plugins["prometheus"]
-		assert.Equal(ginkgo.GinkgoT(), ok, true)
+			grs, err := s.ListApisixGlobalRules()
+			assert.Nil(ginkgo.GinkgoT(), err, "listing global_rules")
+			assert.Len(ginkgo.GinkgoT(), grs, 1)
+			assert.Equal(ginkgo.GinkgoT(), grs[0].ID, id.GenID("default"))
+			assert.Len(ginkgo.GinkgoT(), grs[0].Plugins, 1)
+			_, ok := grs[0].Plugins["prometheus"]
+			assert.Equal(ginkgo.GinkgoT(), ok, true)
+
+			resp := s.NewAPISIXClient().GET("/apisix/prometheus/metrics").Expect()
+			resp.Status(http.StatusOK)
+			resp.Body().Contains("# HELP apisix_etcd_modify_indexes Etcd modify index for APISIX keys")
+			resp.Body().Contains("# HELP apisix_etcd_reachable Config server etcd reachable from APISIX, 0 is unreachable")
+			resp.Body().Contains("# HELP apisix_node_info Info of APISIX node")
+		})
+	}
 
-		resp := s.NewAPISIXClient().GET("/apisix/prometheus/metrics").Expect()
-		resp.Status(http.StatusOK)
-		resp.Body().Contains("# HELP apisix_etcd_modify_indexes Etcd modify index for APISIX keys")
-		resp.Body().Contains("# HELP apisix_etcd_reachable Config server etcd reachable from APISIX, 0 is unreachable")
-		resp.Body().Contains("# HELP apisix_node_info Info of APISIX node")
-	})
+	suites(scaffold.NewDefaultScaffold)
+	suites(scaffold.NewDefaultV2Scaffold)
 })