Add support for arbitrary [Secure]JsonData

Makefile

@@ -119,7 +119,7 @@ docker-buildx: test
 # Download controller-gen locally if necessary
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 controller-gen:
-	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1)
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.2)
 
 # Download kustomize locally if necessary
 KUSTOMIZE = $(shell pwd)/bin/kustomize

api/integreatly/v1alpha1/grafanadatasource_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"encoding/json"
 	"fmt"
 	"strings"
 
@@ -83,6 +84,18 @@ type GrafanaDataSourceFields struct {
 	SecureJsonData    GrafanaDataSourceSecureJsonData `json:"secureJsonData,omitempty"`
 	Version           int                             `json:"version,omitempty"`
 	Editable          bool                            `json:"editable,omitempty"`
+	// +kubebuilder:validation:Schemaless
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Type=object
+	// +optional
+	// CustomJsonData will be used in place of jsonData, if present, and supports arbitrary JSON, not just those of official datasources
+	CustomJsonData json.RawMessage `json:"customJsonData,omitempty"`
+	// +kubebuilder:validation:Schemaless
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Type=object
+	// +optional
+	// SecureCustomJsonData will be used in place of secureJsonData, if present, and supports arbitrary JSON, not just those of official datasources
+	CustomSecureJsonData json.RawMessage `json:"customSecureJsonData,omitempty"`
 }
 
 // GrafanaDataSourceJsonData contains the most common json options

config/crd/bases/integreatly.org_grafanadashboards.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.6.2
   creationTimestamp: null
   name: grafanadashboards.integreatly.org
 spec:

config/crd/bases/integreatly.org_grafanadatasources.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.6.2
   creationTimestamp: null
   name: grafanadatasources.integreatly.org
 spec:
@@ -47,6 +47,18 @@ spec:
                       type: string
                     basicAuthUser:
                       type: string
+                    customJsonData:
+                      description: CustomJsonData will be used in place of jsonData,
+                        if present, and supports arbitrary JSON, not just those of
+                        official datasources
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
+                    customSecureJsonData:
+                      description: SecureCustomJsonData will be used in place of secureJsonData,
+                        if present, and supports arbitrary JSON, not just those of
+                        official datasources
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
                     database:
                       type: string
                     editable:

config/crd/bases/integreatly.org_grafananotificationchannels.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.6.2
   creationTimestamp: null
   name: grafananotificationchannels.integreatly.org
 spec:

config/crd/bases/integreatly.org_grafanas.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.6.2
   creationTimestamp: null
   name: grafanas.integreatly.org
 spec:

controllers/grafanadatasource/datasource_pipeline.go

@@ -34,6 +34,20 @@ func (i *DatasourcePipelineImpl) ProcessDatasource(known *v1.ConfigMap) error {
 	return nil
 }
 
+func applyCustomJsonData(datasource v1alpha1.GrafanaDataSourceFields, unmarshaledDatasource map[string]interface{}) (needsRemarshaling bool) {
+	if datasource.CustomJsonData != nil {
+		needsRemarshaling = true
+		unmarshaledDatasource["jsonData"] = unmarshaledDatasource["customJsonData"]
+		delete(unmarshaledDatasource, "customJsonData")
+	}
+	if datasource.CustomSecureJsonData != nil {
+		needsRemarshaling = true
+		unmarshaledDatasource["secureJsonData"] = unmarshaledDatasource["customSecureJsonData"]
+		delete(unmarshaledDatasource, "customSecureJsonData")
+	}
+	return
+}
+
 func (i *DatasourcePipelineImpl) parse() error {
 	datasources := struct {
 		ApiVersion  int                                `json:"apiVersion"`
@@ -47,6 +61,27 @@ func (i *DatasourcePipelineImpl) parse() error {
 	if err != nil {
 		return err
 	}
+	unmarshaledDatasources := make(map[string]interface{})
+	if err = yaml.Unmarshal(bytes, &unmarshaledDatasources); err != nil {
+		return err
+	}
+	unmarshaledDatasourceList, ok := unmarshaledDatasources["datasources"].([]interface{})
+	needsRemarshaling := false
+	if ok {
+		for ix, datasource := range i.datasource.Spec.Datasources {
+			unmarshaledDatasource, ok := unmarshaledDatasourceList[ix].(map[string]interface{})
+			if !ok {
+				continue
+			}
+			needsRemarshaling = needsRemarshaling || applyCustomJsonData(datasource, unmarshaledDatasource)
+		}
+	}
+	if needsRemarshaling {
+		bytes, err = yaml.Marshal(unmarshaledDatasources)
+		if err != nil {
+			return err
+		}
+	}
 	i.contents = string(bytes)
 	return nil
 }

deploy/examples/GrafanaWithPlugin.yaml

@@ -0,0 +1,52 @@
+# For use with deploy/examples/datasources/Custom.yaml
+apiVersion: integreatly.org/v1alpha1
+kind: Grafana
+metadata:
+  name: example-grafana-with-plugin
+spec:
+  client:
+    preferService: true
+  # Plugin requires 8+
+  baseImage: docker.io/grafana/grafana:8.4.3
+  deployment:
+    envFrom:
+      - configMapRef:
+          name: grafana-plugins
+  ingress:
+    enabled: True
+    pathType: Prefix
+    path: "/"
+  config:
+    log:
+      mode: "console"
+      level: "error"
+    log.frontend:
+      enabled: true
+    auth:
+      disable_login_form: False
+      disable_signout_menu: True
+    auth.anonymous:
+      enabled: True
+  service:
+    name: "grafana-service"
+    labels:
+      app: "grafana"
+      type: "grafana-service"
+  dashboardLabelSelector:
+    - matchExpressions:
+        - { key: app, operator: In, values: [grafana] }
+  resources:
+    # Optionally specify container resources
+    limits:
+      cpu: 200m
+      memory: 200Mi
+    requests:
+      cpu: 100m
+      memory: 100Mi
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-plugins
+data:
+  GF_INSTALL_PLUGINS: grafadruid-druid-datasource

deploy/examples/datasources/Custom.yaml

@@ -0,0 +1,23 @@
+# For use with deploy/examples/GrafanaWithPlugin.yaml
+# Requires https://grafana.com/grafana/plugins/grafadruid-druid-datasource
+apiVersion: integreatly.org/v1alpha1
+kind: GrafanaDataSource
+metadata:
+  name: example-grafanadatasource
+spec:
+  name: middleware.yaml
+  datasources:
+    - name: Druid
+      type: grafadruid-druid-datasource
+      access: proxy
+      url: http://druid:8888
+      isDefault: false
+      version: 1
+      editable: true
+      customJsonData:
+        'connection.retryableRetryMax': 10
+        'connection.retryableRetryWaitMax': 5000
+        'connection.retryableRetryWaitMin': 750
+        'connection.url': 'http://druid:8888'
+      # for secureJsonData
+      # customSecureJsonData: {}

deploy/manifests/latest/crds.yaml

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.6.2
   name: grafanadashboards.integreatly.org
 spec:
   group: integreatly.org
@@ -166,7 +166,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.6.2
   name: grafanadatasources.integreatly.org
 spec:
   group: integreatly.org
@@ -209,6 +209,18 @@ spec:
                       type: string
                     basicAuthUser:
                       type: string
+                    customJsonData:
+                      description: CustomJsonData will be used in place of jsonData,
+                        if present, and supports arbitrary JSON, not just those of
+                        official datasources
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
+                    customSecureJsonData:
+                      description: SecureCustomJsonData will be used in place of secureJsonData,
+                        if present, and supports arbitrary JSON, not just those of
+                        official datasources
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
                     database:
                       type: string
                     editable:
@@ -579,7 +591,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.6.2
   creationTimestamp: null
   name: grafananotificationchannels.integreatly.org
 spec:
@@ -658,7 +670,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.6.2
   name: grafanas.integreatly.org
 spec:
   group: integreatly.org

documentation/api.md

@@ -594,6 +594,20 @@ GrafanaDataSourceSpec defines the desired state of GrafanaDataSource
           <br/>
         </td>
         <td>false</td>
+      </tr><tr>
+        <td><b>customJsonData</b></td>
+        <td>object</td>
+        <td>
+          CustomJsonData will be used in place of jsonData, if present, and supports arbitrary JSON, not just those of official datasources<br/>
+        </td>
+        <td>false</td>
+      </tr><tr>
+        <td><b>customSecureJsonData</b></td>
+        <td>object</td>
+        <td>
+          SecureCustomJsonData will be used in place of secureJsonData, if present, and supports arbitrary JSON, not just those of official datasources<br/>
+        </td>
+        <td>false</td>
       </tr><tr>
         <td><b>database</b></td>
         <td>string</td>

documentation/datasources.md

@@ -21,4 +21,6 @@ A data source accepts all properties
 listed [here](https://grafana.com/docs/administration/provisioning/#example-datasource-config-file), but does not
 support `apiVersion` and `deleteDatasources`.
 
+For custom datasources provided by plugins, use the `customJsonData` and `customSecureJsonData` fields instead of `jsonData` and `secureJsonData`.
+
 To see how to install datasource plugins, see [Plugins](./plugins.md).