Add support for cfg via env vars and define external volumes

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/infracloudio/msbotbuilder-go v0.2.5
+	github.com/knadh/koanf v1.4.1
 	github.com/larksuite/oapi-sdk-go v1.1.44
 	github.com/mattermost/mattermost-server/v5 v5.39.3
 	github.com/olivere/elastic v6.2.37+incompatible
@@ -86,8 +87,11 @@ require (
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/minio/minio-go/v7 v7.0.11 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect

helm/botkube/README.md

@@ -85,7 +85,6 @@ serviceAccount:
 | containerSecurityContext.privileged | bool | `false` |  |
 | containerSecurityContext.readOnlyRootFilesystem | bool | `true` |  |
 | extraAnnotations | object | `{}` |  |
-| extraEnv | string | `nil` |  |
 | fullnameOverride | string | `""` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.registry | string | `"ghcr.io"` |  |
@@ -119,3 +118,6 @@ serviceAccount:
 | serviceMonitor.port | string | `"metrics"` |  |
 | tolerations | list | `[]` |  |
 | extraObjects | list | `[]` | Extra Kubernetes resources to create. Helm templating is allowed as it is evaluated before creating the resources. |
+| extraEnv          | list | `[]`    | Extra environment variables to pass to the BotKube container. For the syntax, see the [environment variable API](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#environment-variables) document.              |
+| extraVolumeMounts | list | `[]`    | Extra volume mounts to pass to the BotKube container. For the syntax, see the [Volumes API](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#volumes-1) document.                                               |
+| extraVolumes      | list | `[]`    | Extra volumes to pass to the BotKube container. Mount it later with extraVolumeMounts. For the syntax, see the [mount Volumes API](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/volume/#Volume) document. |

helm/botkube/templates/deployment.yaml

@@ -51,6 +51,9 @@ spec:
           volumeMounts:
             - name: config-volume
               mountPath: "/config"
+          {{- with .Values.extraVolumeMounts }}
+            {{ toYaml . | nindent 12 }}
+          {{- end }}
           {{- if .Values.config.ssl.enabled }}
             - name: certs
               mountPath: "/etc/ssl/certs"
@@ -73,12 +76,9 @@ spec:
             - name: KUBECONFIG
               value: "/.kube/config"
             {{- end }}
-        {{- if .Values.extraEnv }}
-          {{- range $name, $value := .Values.extraEnv }}
-            - name: {{ $name }}
-              value: {{ quote $value }}
+          {{- with .Values.extraEnv }}
+            {{ toYaml . | nindent 12 }}
           {{- end }}
-        {{- end }}
           {{- if .Values.resources }}
           resources:
 {{ toYaml .Values.resources | indent 12 }}
@@ -91,6 +91,9 @@ spec:
                 name: {{ include "botkube.fullname" . }}-configmap
             - secret:
                 name: {{ include "botkube.CommunicationsSecretName" . }}
+      {{- with .Values.extraVolumes }}
+        {{ toYaml . | nindent 8 }}
+      {{- end }}
       {{- if .Values.config.ssl.enabled }}
         - name: certs
           secret:

helm/botkube/values.yaml

@@ -378,7 +378,7 @@ service:
   port: 2112
   targetPort: 2112
 
-# Ingress settings to expose teams and lark endpoints
+## Ingress settings to expose teams and lark endpoints
 ingress:
   create: false
   annotations:
@@ -413,10 +413,50 @@ resources: {}
   #  cpu: 100m
   #  memory: 128Mi
 
+## Extra environment variables to pass to the BotKube container.
+## ref: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#environment-variables
 extraEnv: []
-## Extra environment variables to pass to the BotKube container, example HTTP_PROXY
-## extraEnv:
-##   HTTP_PROXY: <proxyURL>:<port>
+#  - name: <key>
+#    valueFrom:
+#      configMapKeyRef:
+#        name: configmap-name
+#        key: value_key
+#  - name: <key>
+#    value: value
+
+
+## Extra volumes to pass to the BotKube container. Mount it later with extraVolumeMounts.
+## ref: https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/volume/#Volume
+extraVolumes: []
+# - name: extra-volume-0
+#   secret:
+#     secretName: <secret-name>
+#
+## For CSI e.g. Vault:
+# - name: secrets-store-inline
+#   csi:
+#     driver: secrets-store.csi.k8s.io
+#     readOnly: true
+#     volumeAttributes:
+#       secretProviderClass: "vault-database"
+
+## Extra volume mounts to pass to the BotKube container.
+## ref: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#volumes-1
+extraVolumeMounts: []
+# - name: extra-volume-0
+#   mountPath: /mnt/volume0
+#   readOnly: true
+# - name: extra-volume-1
+#   mountPath: /mnt/volume1
+#   readOnly: true
+# - name: secret-files
+#   mountPath: /etc/secrets
+#   subPath: ""
+#
+## For CSI e.g. Vault:
+# - name: secrets-store-inline
+#   mountPath: "/mnt/secrets-store"
+#   readOnly: true
 
 nodeSelector: {}
 
@@ -439,12 +479,12 @@ serviceAccount:
   # annotations for the service account
   annotations: {}
 
-# Extra Kubernetes resources to create. Helm templating is allowed as it is evaluated before creating the resources.
+## Extra Kubernetes resources to create. Helm templating is allowed as it is evaluated before creating the resources.
 extraObjects: []
-# For example, to create a ClusterRoleBinding resource without creating a dedicated ClusterRole, uncomment the following snippet.
-# NOTE: While running Helm install/upgrade with this sample snippet uncommented, make sure to set the following values:
-#    1. `rbac.create: false`
-#    2.`extraClusterRoleName: {clusterRole}`, where {clusterRole} is a given ClusterRole name (e.g. `cluster-admin`).
+## For example, to create a ClusterRoleBinding resource without creating a dedicated ClusterRole, uncomment the following snippet.
+## NOTE: While running Helm install/upgrade with this sample snippet uncommented, make sure to set the following values:
+##    1. `rbac.create: false`
+##    2.`extraClusterRoleName: {clusterRole}`, where {clusterRole} is a given ClusterRole name (e.g. `cluster-admin`).
 #
 #    - apiVersion: rbac.authorization.k8s.io/v1
 #      kind: ClusterRoleBinding

pkg/config/config.go

@@ -22,7 +22,12 @@ package config
 import (
 	"os"
 	"path/filepath"
+	"strings"
 
+	"github.com/knadh/koanf"
+	koanfyaml "github.com/knadh/koanf/parsers/yaml"
+	"github.com/knadh/koanf/providers/env"
+	"github.com/knadh/koanf/providers/file"
 	"gopkg.in/yaml.v3"
 )
 
@@ -69,6 +74,9 @@ const (
 	DiscordBot BotPlatform = "discord"
 	// LarkBot bot platform
 	LarkBot BotPlatform = "lark"
+
+	communicationEnvVariablePrefix = "COMMUNICATIONS_"
+	communicationConfigDelimiter   = "."
 )
 
 // EventType to watch
@@ -253,19 +261,39 @@ func (eventType EventType) String() string {
 }
 
 // NewCommunicationsConfig return new communication config object
-func NewCommunicationsConfig() (c *Communications, err error) {
+func NewCommunicationsConfig() (*Communications, error) {
 	configPath := os.Getenv("CONFIG_PATH")
 	commCfgFilePath := filepath.Join(configPath, CommunicationConfigFileName)
-	rawCfg, err := os.ReadFile(filepath.Clean(commCfgFilePath))
+
+	k := koanf.New(communicationConfigDelimiter)
+
+	// Load base YAML config.
+	if err := k.Load(file.Provider(filepath.Clean(commCfgFilePath)), koanfyaml.Parser()); err != nil {
+		return nil, err
+	}
+
+	// Load environment variables and merge into the loaded config.
+	err := k.Load(env.Provider(
+		communicationEnvVariablePrefix,
+		communicationConfigDelimiter,
+		normalizeCommunicationConfigEnvName,
+	), nil)
 	if err != nil {
 		return nil, err
 	}
 
-	commCfg := &Communications{}
-	if err := yaml.Unmarshal(rawCfg, commCfg); err != nil {
+	var cfg Communications
+	err = k.UnmarshalWithConf("", &cfg, koanf.UnmarshalConf{Tag: "yaml"})
+	if err != nil {
 		return nil, err
 	}
-	return commCfg, nil
+
+	return &cfg, nil
+}
+
+func normalizeCommunicationConfigEnvName(name string) string {
+	name = strings.ToLower(name)
+	return strings.ReplaceAll(name, "_", ".")
 }
 
 // Load loads new configuration from file.

pkg/config/config_test.go

@@ -0,0 +1,62 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+const sampleCommunicationConfig = "testdata/comm_config.yaml"
+
+func TestCommunicationConfigSuccess(t *testing.T) {
+	t.Run("Load from file", func(t *testing.T) {
+		// given
+		t.Setenv("CONFIG_PATH", "testdata")
+
+		var expConfig Communications
+		loadYAMLFile(t, sampleCommunicationConfig, &expConfig)
+
+		// when
+		gotCfg, err := NewCommunicationsConfig()
+
+		//then
+		require.NoError(t, err)
+		require.NotNil(t, gotCfg)
+		assert.Equal(t, expConfig, *gotCfg)
+	})
+
+	t.Run("Load from file and override with environment variables", func(t *testing.T) {
+		// given
+		t.Setenv("CONFIG_PATH", "testdata")
+
+		fixToken := fmt.Sprintf("TOKEN_FROM_ENV_%d", time.Now().Unix())
+		t.Setenv("COMMUNICATIONS_SLACK_TOKEN", fixToken)
+		var expConfig Communications
+		loadYAMLFile(t, sampleCommunicationConfig, &expConfig)
+		expConfig.Communications.Slack.Token = fixToken
+
+		// when
+		gotCfg, err := NewCommunicationsConfig()
+
+		//then
+		require.NoError(t, err)
+		require.NotNil(t, gotCfg)
+		assert.Equal(t, expConfig, *gotCfg)
+	})
+}
+
+func loadYAMLFile(t *testing.T, path string, out interface{}) {
+	t.Helper()
+
+	raw, err := os.ReadFile(filepath.Clean(path))
+	require.NoError(t, err)
+
+	err = yaml.Unmarshal(raw, out)
+	require.NoError(t, err)
+}

pkg/config/testdata/comm_config.yaml

@@ -0,0 +1,57 @@
+# Channels configuration
+communications:
+  # Settings for Slack
+  slack:
+    enabled: false
+    channel: 'SLACK_CHANNEL'
+    token: 'SLACK_API_TOKEN'
+    notiftype: short                          # Change notification type short/long you want to receive. notiftype is optional and Default notification type is short (if not specified)
+
+  # Settings for Mattermost
+  mattermost:
+    enabled: false
+    url: 'MATTERMOST_SERVER_URL'              # URL where Mattermost is running. e.g https://example.com:9243
+    token: 'MATTERMOST_TOKEN'                 # Personal Access token generated by BotKube user
+    team: 'MATTERMOST_TEAM'                   # Mattermost Team to configure with BotKube 
+    channel: 'MATTERMOST_CHANNEL'             # Mattermost Channel for receiving BotKube alerts 
+    notiftype: short                          # Change notification type short/long you want to receive. notiftype is optional and Default notification type is short (if not specified)
+
+  # Settings for MS Teams
+  teams:
+    enabled: false
+    appID: 'APPLICATION_ID'
+    appPassword: 'APPLICATION_PASSWORD'
+    notiftype: short
+    port: 3978
+
+  # Settings for Discord
+  discord:
+    enabled: false
+    token: 'DISCORD_TOKEN'                    # BotKube Bot Token 
+    botid: 'DISCORD_BOT_ID'                   # BotKube Application Client ID 
+    channel: 'DISCORD_CHANNEL_ID'             # Discord Channel id for receiving BotKube alerts 
+    notiftype: short                          # Change notification type short/long you want to receive. notiftype is optional and Default notification type is short (if not specified)
+
+
+  # Settings for ELS
+  elasticsearch:
+    enabled: false
+    awsSigning:
+      enabled: false                          # enable awsSigning using IAM for Elastisearch hosted on AWS, if true make sure AWS environment variables are set. Refer https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
+      awsRegion: 'us-east-1'                  # AWS region where Elasticsearch is deployed
+      roleArn: ''                             # AWS IAM Role arn to assume for credentials, use this only if you dont want to use the EC2 instance role or not running on AWS instance
+    server: 'ELASTICSEARCH_ADDRESS'           # e.g https://example.com:9243
+    username: 'ELASTICSEARCH_USERNAME'        # Basic Auth
+    password: 'ELASTICSEARCH_PASSWORD'
+    skipTLSVerify: false                      # toggle verification of TLS certificate of the Elastic nodes. Verification is skipped when option is true. Enable to connect to clusters with self-signed certs
+    # ELS index settings
+    index:
+      name: botkube
+      type: botkube-event
+      shards: 1
+      replicas: 0
+
+  # Settings for Webhook
+  webhook:
+    enabled: false
+    url: 'WEBHOOK_URL'                        # e.g https://example.com:80