feat: support environment variable in config file (#745)

pkg/config/config.go

@@ -15,11 +15,14 @@
 package config
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"strings"
+	"text/template"
 	"time"
 
 	"gopkg.in/yaml.v2"
@@ -140,10 +143,27 @@ func NewConfigFromFile(filename string) (*Config, error) {
 		return nil, err
 	}
 
+	envVarMap := map[string]string{}
+	for _, e := range os.Environ() {
+		pair := strings.SplitN(e, "=", 2)
+		envVarMap[pair[0]] = pair[1]
+	}
+
+	tpl := template.New("text").Option("missingkey=error")
+	tpl, err = tpl.Parse(string(data))
+	if err != nil {
+		return nil, fmt.Errorf("error parsing configuration template %v", err)
+	}
+	buf := bytes.NewBufferString("")
+	err = tpl.Execute(buf, envVarMap)
+	if err != nil {
+		return nil, fmt.Errorf("error execute configuration template %v", err)
+	}
+
 	if strings.HasSuffix(filename, ".yaml") || strings.HasSuffix(filename, ".yml") {
-		err = yaml.Unmarshal(data, cfg)
+		err = yaml.Unmarshal(buf.Bytes(), cfg)
 	} else {
-		err = json.Unmarshal(data, cfg)
+		err = json.Unmarshal(buf.Bytes(), cfg)
 	}
 
 	if err != nil {

pkg/config/config_test.go

@@ -103,6 +103,113 @@ apisix:
 	assert.Equal(t, cfg, newCfg, "bad configuration")
 }
 
+func TestConfigWithEnvVar(t *testing.T) {
+	cfg := &Config{
+		LogLevel:        "warn",
+		LogOutput:       "stdout",
+		HTTPListen:      ":9090",
+		HTTPSListen:     ":9443",
+		CertFilePath:    "/etc/webhook/certs/cert.pem",
+		KeyFilePath:     "/etc/webhook/certs/key.pem",
+		EnableProfiling: true,
+		Kubernetes: KubernetesConfig{
+			ResyncInterval:     types.TimeDuration{Duration: time.Hour},
+			Kubeconfig:         "",
+			AppNamespaces:      []string{""},
+			ElectionID:         "my-election-id",
+			IngressClass:       IngressClass,
+			IngressVersion:     IngressNetworkingV1,
+			ApisixRouteVersion: ApisixRouteV2alpha1,
+		},
+		APISIX: APISIXConfig{
+			DefaultClusterName:     "default",
+			DefaultClusterBaseURL:  "http://127.0.0.1:8080/apisix",
+			DefaultClusterAdminKey: "123456",
+		},
+	}
+
+	defaultClusterBaseURLEnvName := "DEFAULT_CLUSTER_BASE_URL"
+	defaultClusterAdminKeyEnvName := "DEFAULT_CLUSTER_ADMIN_KEY"
+	kubeconfigEnvName := "KUBECONFIG"
+
+	err := os.Setenv(defaultClusterBaseURLEnvName, "http://127.0.0.1:8080/apisix")
+	assert.Nil(t, err, "failed to set env variable: ", err)
+	_ = os.Setenv(defaultClusterAdminKeyEnvName, "123456")
+	_ = os.Setenv(kubeconfigEnvName, "")
+
+	jsonData := `
+{
+    "log_level": "warn",
+    "log_output": "stdout",
+    "http_listen": ":9090",
+    "https_listen": ":9443",
+    "enable_profiling": true,
+    "kubernetes": {
+        "kubeconfig": "{{.KUBECONFIG}}",
+        "resync_interval": "1h0m0s",
+        "election_id": "my-election-id",
+        "ingress_class": "apisix",
+        "ingress_version": "networking/v1"
+    },
+    "apisix": {
+        "default_cluster_base_url": "{{.DEFAULT_CLUSTER_BASE_URL}}",
+        "default_cluster_admin_key": "{{.DEFAULT_CLUSTER_ADMIN_KEY}}"
+    }
+}
+`
+	tmpJSON, err := ioutil.TempFile("/tmp", "config-*.json")
+	assert.Nil(t, err, "failed to create temporary json configuration file: ", err)
+	defer os.Remove(tmpJSON.Name())
+
+	_, err = tmpJSON.Write([]byte(jsonData))
+	assert.Nil(t, err, "failed to write json data: ", err)
+	tmpJSON.Close()
+
+	newCfg, err := NewConfigFromFile(tmpJSON.Name())
+	assert.Nil(t, err, "failed to new config from file: ", err)
+	assert.Nil(t, newCfg.Validate(), "failed to validate config")
+
+	assert.Equal(t, cfg, newCfg, "bad configuration")
+
+	yamlData := `
+log_level: warn
+log_output: stdout
+http_listen: :9090
+https_listen: :9443
+enable_profiling: true
+kubernetes:
+  resync_interval: 1h0m0s
+  kubeconfig: "{{.KUBECONFIG}}"
+  election_id: my-election-id
+  ingress_class: apisix
+  ingress_version: networking/v1
+apisix:
+  default_cluster_base_url: {{.DEFAULT_CLUSTER_BASE_URL}}
+  default_cluster_admin_key: "{{.DEFAULT_CLUSTER_ADMIN_KEY}}"
+`
+	tmpYAML, err := ioutil.TempFile("/tmp", "config-*.yaml")
+	assert.Nil(t, err, "failed to create temporary yaml configuration file: ", err)
+	defer os.Remove(tmpYAML.Name())
+
+	_, err = tmpYAML.Write([]byte(yamlData))
+	assert.Nil(t, err, "failed to write yaml data: ", err)
+	tmpYAML.Close()
+
+	newCfg, err = NewConfigFromFile(tmpYAML.Name())
+	assert.Nil(t, err, "failed to new config from file: ", err)
+	assert.Nil(t, newCfg.Validate(), "failed to validate config")
+
+	assert.Equal(t, cfg, newCfg, "bad configuration")
+
+	_ = os.Unsetenv(defaultClusterBaseURLEnvName)
+
+	_, err = NewConfigFromFile(tmpJSON.Name())
+	assert.NotNil(t, err, "should failed because env variable missing")
+
+	_, err = NewConfigFromFile(tmpYAML.Name())
+	assert.NotNil(t, err, "should failed because env variable missing")
+}
+
 func TestConfigDefaultValue(t *testing.T) {
 	yamlData := `
 apisix:

test/e2e/config/config.go

@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package config
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
+	"github.com/onsi/ginkgo"
+	"github.com/stretchr/testify/assert"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = ginkgo.Describe("deploy ingress controller with config", func() {
+	opts := &scaffold.Options{
+		Name:                  "default",
+		Kubeconfig:            scaffold.GetKubeconfig(),
+		APISIXConfigPath:      "testdata/apisix-gw-config.yaml",
+		IngressAPISIXReplicas: 1,
+		HTTPBinServicePort:    80,
+		APISIXRouteVersion:    "apisix.apache.org/v2beta2",
+	}
+	s := scaffold.NewScaffold(opts)
+	ginkgo.It("use configmap with env", func() {
+		label := fmt.Sprintf("apisix.ingress.watch=%s", s.Namespace())
+		configMap := fmt.Sprintf(_ingressAPISIXConfigMapTemplate, label)
+		assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(configMap), "create configmap")
+
+		client := s.GetKubernetesClient()
+		deployment, err := client.AppsV1().Deployments(s.Namespace()).Get(context.Background(), "ingress-apisix-controller-deployment-e2e-test", metav1.GetOptions{})
+		assert.Nil(ginkgo.GinkgoT(), err, "get apisix ingress controller deployment")
+
+		spec := &deployment.Spec.Template.Spec
+		spec.Containers[0].Command = []string{
+			"/ingress-apisix/apisix-ingress-controller",
+			"ingress",
+			"--config-path",
+			"/ingress-apisix/conf/config.yaml",
+		}
+		spec.Volumes = append(spec.Volumes, v1.Volume{
+			Name: "apisix-ingress-controller-config",
+			VolumeSource: v1.VolumeSource{
+				ConfigMap: &v1.ConfigMapVolumeSource{
+					LocalObjectReference: v1.LocalObjectReference{
+						Name: "ingress-apisix-controller-config",
+					},
+				},
+			},
+		})
+		spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, v1.VolumeMount{
+			Name:      "apisix-ingress-controller-config",
+			MountPath: "/ingress-apisix/conf/config.yaml",
+			SubPath:   "config.yaml",
+		})
+		spec.Containers[0].Env = append(spec.Containers[0].Env, v1.EnvVar{
+			Name:  "DEFAULT_CLUSTER_BASE_URL",
+			Value: "http://apisix-service-e2e-test:9180/apisix/admin",
+		}, v1.EnvVar{
+			Name:  "DEFAULT_CLUSTER_ADMIN_KEY",
+			Value: "edd1c9f034335f136f87ad84b625c8f1",
+		})
+
+		_, err = client.AppsV1().Deployments(s.Namespace()).Update(context.Background(), deployment, metav1.UpdateOptions{})
+		assert.Nil(ginkgo.GinkgoT(), err, "update apisix ingress controller deployment")
+
+		time.Sleep(10 * time.Second)
+		assert.Nil(ginkgo.GinkgoT(), s.WaitAllIngressControllerPodsAvailable(), "wait all ingress controller pod available")
+	})
+})

test/e2e/config/manifests.go

@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package config
+
+const (
+	_ingressAPISIXConfigMapTemplate = `
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ingress-apisix-controller-config
+data:
+  config.yaml: |
+    apisix:
+      default_cluster_base_url: "{{.DEFAULT_CLUSTER_BASE_URL}}"
+      default_cluster_admin_key: "{{.DEFAULT_CLUSTER_ADMIN_KEY}}"
+    log_level: "debug"
+    log_output: "stdout"
+    http_listen: ":8080"
+    https_listen: ":8443"
+    enable_profiling: true
+    kubernetes:
+      namespace_selector:
+      - %s
+      apisix_route_version: "apisix.apache.org/v2beta2"
+      watch_endpoint_slices: true
+`
+)

test/e2e/e2e.go

@@ -16,6 +16,7 @@ package e2e
 
 import (
 	_ "github.com/apache/apisix-ingress-controller/test/e2e/annotations"
+	_ "github.com/apache/apisix-ingress-controller/test/e2e/config"
 	_ "github.com/apache/apisix-ingress-controller/test/e2e/endpoints"
 	_ "github.com/apache/apisix-ingress-controller/test/e2e/features"
 	_ "github.com/apache/apisix-ingress-controller/test/e2e/ingress"

test/e2e/go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/stretchr/testify v1.7.0
 	k8s.io/api v0.21.1
 	k8s.io/apimachinery v0.21.1
+	k8s.io/client-go v0.21.1
 )
 
 replace github.com/apache/apisix-ingress-controller => ../../

test/e2e/scaffold/ingress.go

@@ -441,7 +441,7 @@ func (s *Scaffold) newIngressAPISIXController() error {
 	return nil
 }
 
-func (s *Scaffold) waitAllIngressControllerPodsAvailable() error {
+func (s *Scaffold) WaitAllIngressControllerPodsAvailable() error {
 	opts := metav1.ListOptions{
 		LabelSelector: "app=ingress-apisix-controller-deployment-e2e-test",
 	}

test/e2e/scaffold/k8s.go

@@ -36,6 +36,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
 )
 
 type counter struct {
@@ -504,3 +505,10 @@ func (s *Scaffold) EnsureNumEndpointsReady(t testing.TestingT, endpointsName str
 	)
 	ginkgo.GinkgoT().Log(message)
 }
+
+// GetKubernetesClient get kubernetes client use by scaffold
+func (s *Scaffold) GetKubernetesClient() *kubernetes.Clientset {
+	client, err := k8s.GetKubernetesClientFromOptionsE(s.t, s.kubectlOptions)
+	assert.Nil(ginkgo.GinkgoT(), err, "get kubernetes client")
+	return client
+}

test/e2e/scaffold/scaffold.go

@@ -324,7 +324,7 @@ func (s *Scaffold) beforeEach() {
 	err = s.newIngressAPISIXController()
 	assert.Nil(s.t, err, "initializing ingress apisix controller")
 
-	err = s.waitAllIngressControllerPodsAvailable()
+	err = s.WaitAllIngressControllerPodsAvailable()
 	assert.Nil(s.t, err, "waiting for ingress apisix controller ready")
 }