From 9f2cd7f856f1879ae2586f2a84f4f39d2654996d Mon Sep 17 00:00:00 2001 From: Nic Date: Mon, 22 Nov 2021 10:18:27 +0800 Subject: [PATCH] feat: support environment variable in config file (#745) --- pkg/config/config.go | 24 +++++++- pkg/config/config_test.go | 107 ++++++++++++++++++++++++++++++++++ test/e2e/config/config.go | 84 ++++++++++++++++++++++++++ test/e2e/config/manifests.go | 39 +++++++++++++ test/e2e/e2e.go | 1 + test/e2e/go.mod | 1 + test/e2e/scaffold/ingress.go | 2 +- test/e2e/scaffold/k8s.go | 8 +++ test/e2e/scaffold/scaffold.go | 2 +- 9 files changed, 264 insertions(+), 4 deletions(-) create mode 100644 test/e2e/config/config.go create mode 100644 test/e2e/config/manifests.go diff --git a/pkg/config/config.go b/pkg/config/config.go index 9cd19c17a1..1f8a3dde65 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -15,11 +15,14 @@ package config import ( + "bytes" "encoding/json" "errors" "fmt" "io/ioutil" + "os" "strings" + "text/template" "time" "gopkg.in/yaml.v2" @@ -140,10 +143,27 @@ func NewConfigFromFile(filename string) (*Config, error) { return nil, err } + envVarMap := map[string]string{} + for _, e := range os.Environ() { + pair := strings.SplitN(e, "=", 2) + envVarMap[pair[0]] = pair[1] + } + + tpl := template.New("text").Option("missingkey=error") + tpl, err = tpl.Parse(string(data)) + if err != nil { + return nil, fmt.Errorf("error parsing configuration template %v", err) + } + buf := bytes.NewBufferString("") + err = tpl.Execute(buf, envVarMap) + if err != nil { + return nil, fmt.Errorf("error execute configuration template %v", err) + } + if strings.HasSuffix(filename, ".yaml") || strings.HasSuffix(filename, ".yml") { - err = yaml.Unmarshal(data, cfg) + err = yaml.Unmarshal(buf.Bytes(), cfg) } else { - err = json.Unmarshal(data, cfg) + err = json.Unmarshal(buf.Bytes(), cfg) } if err != nil { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index df0782140d..8877c7a52d 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -103,6 +103,113 @@ apisix: assert.Equal(t, cfg, newCfg, "bad configuration") } +func TestConfigWithEnvVar(t *testing.T) { + cfg := &Config{ + LogLevel: "warn", + LogOutput: "stdout", + HTTPListen: ":9090", + HTTPSListen: ":9443", + CertFilePath: "/etc/webhook/certs/cert.pem", + KeyFilePath: "/etc/webhook/certs/key.pem", + EnableProfiling: true, + Kubernetes: KubernetesConfig{ + ResyncInterval: types.TimeDuration{Duration: time.Hour}, + Kubeconfig: "", + AppNamespaces: []string{""}, + ElectionID: "my-election-id", + IngressClass: IngressClass, + IngressVersion: IngressNetworkingV1, + ApisixRouteVersion: ApisixRouteV2alpha1, + }, + APISIX: APISIXConfig{ + DefaultClusterName: "default", + DefaultClusterBaseURL: "http://127.0.0.1:8080/apisix", + DefaultClusterAdminKey: "123456", + }, + } + + defaultClusterBaseURLEnvName := "DEFAULT_CLUSTER_BASE_URL" + defaultClusterAdminKeyEnvName := "DEFAULT_CLUSTER_ADMIN_KEY" + kubeconfigEnvName := "KUBECONFIG" + + err := os.Setenv(defaultClusterBaseURLEnvName, "http://127.0.0.1:8080/apisix") + assert.Nil(t, err, "failed to set env variable: ", err) + _ = os.Setenv(defaultClusterAdminKeyEnvName, "123456") + _ = os.Setenv(kubeconfigEnvName, "") + + jsonData := ` +{ + "log_level": "warn", + "log_output": "stdout", + "http_listen": ":9090", + "https_listen": ":9443", + "enable_profiling": true, + "kubernetes": { + "kubeconfig": "{{.KUBECONFIG}}", + "resync_interval": "1h0m0s", + "election_id": "my-election-id", + "ingress_class": "apisix", + "ingress_version": "networking/v1" + }, + "apisix": { + "default_cluster_base_url": "{{.DEFAULT_CLUSTER_BASE_URL}}", + "default_cluster_admin_key": "{{.DEFAULT_CLUSTER_ADMIN_KEY}}" + } +} +` + tmpJSON, err := ioutil.TempFile("/tmp", "config-*.json") + assert.Nil(t, err, "failed to create temporary json configuration file: ", err) + defer os.Remove(tmpJSON.Name()) + + _, err = tmpJSON.Write([]byte(jsonData)) + assert.Nil(t, err, "failed to write json data: ", err) + tmpJSON.Close() + + newCfg, err := NewConfigFromFile(tmpJSON.Name()) + assert.Nil(t, err, "failed to new config from file: ", err) + assert.Nil(t, newCfg.Validate(), "failed to validate config") + + assert.Equal(t, cfg, newCfg, "bad configuration") + + yamlData := ` +log_level: warn +log_output: stdout +http_listen: :9090 +https_listen: :9443 +enable_profiling: true +kubernetes: + resync_interval: 1h0m0s + kubeconfig: "{{.KUBECONFIG}}" + election_id: my-election-id + ingress_class: apisix + ingress_version: networking/v1 +apisix: + default_cluster_base_url: {{.DEFAULT_CLUSTER_BASE_URL}} + default_cluster_admin_key: "{{.DEFAULT_CLUSTER_ADMIN_KEY}}" +` + tmpYAML, err := ioutil.TempFile("/tmp", "config-*.yaml") + assert.Nil(t, err, "failed to create temporary yaml configuration file: ", err) + defer os.Remove(tmpYAML.Name()) + + _, err = tmpYAML.Write([]byte(yamlData)) + assert.Nil(t, err, "failed to write yaml data: ", err) + tmpYAML.Close() + + newCfg, err = NewConfigFromFile(tmpYAML.Name()) + assert.Nil(t, err, "failed to new config from file: ", err) + assert.Nil(t, newCfg.Validate(), "failed to validate config") + + assert.Equal(t, cfg, newCfg, "bad configuration") + + _ = os.Unsetenv(defaultClusterBaseURLEnvName) + + _, err = NewConfigFromFile(tmpJSON.Name()) + assert.NotNil(t, err, "should failed because env variable missing") + + _, err = NewConfigFromFile(tmpYAML.Name()) + assert.NotNil(t, err, "should failed because env variable missing") +} + func TestConfigDefaultValue(t *testing.T) { yamlData := ` apisix: diff --git a/test/e2e/config/config.go b/test/e2e/config/config.go new file mode 100644 index 0000000000..51b7a98009 --- /dev/null +++ b/test/e2e/config/config.go @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +import ( + "context" + "fmt" + "time" + + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" + "github.com/onsi/ginkgo" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = ginkgo.Describe("deploy ingress controller with config", func() { + opts := &scaffold.Options{ + Name: "default", + Kubeconfig: scaffold.GetKubeconfig(), + APISIXConfigPath: "testdata/apisix-gw-config.yaml", + IngressAPISIXReplicas: 1, + HTTPBinServicePort: 80, + APISIXRouteVersion: "apisix.apache.org/v2beta2", + } + s := scaffold.NewScaffold(opts) + ginkgo.It("use configmap with env", func() { + label := fmt.Sprintf("apisix.ingress.watch=%s", s.Namespace()) + configMap := fmt.Sprintf(_ingressAPISIXConfigMapTemplate, label) + assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(configMap), "create configmap") + + client := s.GetKubernetesClient() + deployment, err := client.AppsV1().Deployments(s.Namespace()).Get(context.Background(), "ingress-apisix-controller-deployment-e2e-test", metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "get apisix ingress controller deployment") + + spec := &deployment.Spec.Template.Spec + spec.Containers[0].Command = []string{ + "/ingress-apisix/apisix-ingress-controller", + "ingress", + "--config-path", + "/ingress-apisix/conf/config.yaml", + } + spec.Volumes = append(spec.Volumes, v1.Volume{ + Name: "apisix-ingress-controller-config", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "ingress-apisix-controller-config", + }, + }, + }, + }) + spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, v1.VolumeMount{ + Name: "apisix-ingress-controller-config", + MountPath: "/ingress-apisix/conf/config.yaml", + SubPath: "config.yaml", + }) + spec.Containers[0].Env = append(spec.Containers[0].Env, v1.EnvVar{ + Name: "DEFAULT_CLUSTER_BASE_URL", + Value: "http://apisix-service-e2e-test:9180/apisix/admin", + }, v1.EnvVar{ + Name: "DEFAULT_CLUSTER_ADMIN_KEY", + Value: "edd1c9f034335f136f87ad84b625c8f1", + }) + + _, err = client.AppsV1().Deployments(s.Namespace()).Update(context.Background(), deployment, metav1.UpdateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "update apisix ingress controller deployment") + + time.Sleep(10 * time.Second) + assert.Nil(ginkgo.GinkgoT(), s.WaitAllIngressControllerPodsAvailable(), "wait all ingress controller pod available") + }) +}) diff --git a/test/e2e/config/manifests.go b/test/e2e/config/manifests.go new file mode 100644 index 0000000000..4010791cd6 --- /dev/null +++ b/test/e2e/config/manifests.go @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +const ( + _ingressAPISIXConfigMapTemplate = ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: ingress-apisix-controller-config +data: + config.yaml: | + apisix: + default_cluster_base_url: "{{.DEFAULT_CLUSTER_BASE_URL}}" + default_cluster_admin_key: "{{.DEFAULT_CLUSTER_ADMIN_KEY}}" + log_level: "debug" + log_output: "stdout" + http_listen: ":8080" + https_listen: ":8443" + enable_profiling: true + kubernetes: + namespace_selector: + - %s + apisix_route_version: "apisix.apache.org/v2beta2" + watch_endpoint_slices: true +` +) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 8c5c5975f3..07cf074b4d 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -16,6 +16,7 @@ package e2e import ( _ "github.com/apache/apisix-ingress-controller/test/e2e/annotations" + _ "github.com/apache/apisix-ingress-controller/test/e2e/config" _ "github.com/apache/apisix-ingress-controller/test/e2e/endpoints" _ "github.com/apache/apisix-ingress-controller/test/e2e/features" _ "github.com/apache/apisix-ingress-controller/test/e2e/ingress" diff --git a/test/e2e/go.mod b/test/e2e/go.mod index 4c48e5e3b5..643a62b9dd 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -11,6 +11,7 @@ require ( github.com/stretchr/testify v1.7.0 k8s.io/api v0.21.1 k8s.io/apimachinery v0.21.1 + k8s.io/client-go v0.21.1 ) replace github.com/apache/apisix-ingress-controller => ../../ diff --git a/test/e2e/scaffold/ingress.go b/test/e2e/scaffold/ingress.go index 8908602b4f..4de7a0cd08 100644 --- a/test/e2e/scaffold/ingress.go +++ b/test/e2e/scaffold/ingress.go @@ -441,7 +441,7 @@ func (s *Scaffold) newIngressAPISIXController() error { return nil } -func (s *Scaffold) waitAllIngressControllerPodsAvailable() error { +func (s *Scaffold) WaitAllIngressControllerPodsAvailable() error { opts := metav1.ListOptions{ LabelSelector: "app=ingress-apisix-controller-deployment-e2e-test", } diff --git a/test/e2e/scaffold/k8s.go b/test/e2e/scaffold/k8s.go index 8dbb70cc4a..842c343a9f 100644 --- a/test/e2e/scaffold/k8s.go +++ b/test/e2e/scaffold/k8s.go @@ -36,6 +36,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" ) type counter struct { @@ -504,3 +505,10 @@ func (s *Scaffold) EnsureNumEndpointsReady(t testing.TestingT, endpointsName str ) ginkgo.GinkgoT().Log(message) } + +// GetKubernetesClient get kubernetes client use by scaffold +func (s *Scaffold) GetKubernetesClient() *kubernetes.Clientset { + client, err := k8s.GetKubernetesClientFromOptionsE(s.t, s.kubectlOptions) + assert.Nil(ginkgo.GinkgoT(), err, "get kubernetes client") + return client +} diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index a330c86e4b..2007a9d725 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -324,7 +324,7 @@ func (s *Scaffold) beforeEach() { err = s.newIngressAPISIXController() assert.Nil(s.t, err, "initializing ingress apisix controller") - err = s.waitAllIngressControllerPodsAvailable() + err = s.WaitAllIngressControllerPodsAvailable() assert.Nil(s.t, err, "waiting for ingress apisix controller ready") }