extraConfigs

apis/druid/v1alpha1/druid_types.go

@@ -84,6 +84,10 @@ type DruidSpec struct {
 	// +required
 	CommonRuntimeProperties string `json:"common.runtime.properties"`
 
+	// References to ConfigMaps holding more files to mount to the CommonConfigMountPath.
+	// +optional
+	ExtraCommonConfig []*v1.ObjectReference `json:"extraCommonConfig"`
+
 	// Optional: Default is true, will delete the sts pod if sts is set to ordered ready to ensure
 	// issue: https://github.com/kubernetes/kubernetes/issues/67250
 	// doc: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#forced-rollback

chart/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.3.1
+version: 0.3.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

chart/templates/crds/druid.apache.org_druids.yaml

@@ -1653,6 +1653,71 @@ spec:
                       x-kubernetes-map-type: atomic
                   type: object
                 type: array
+              extraCommonConfig:
+                description: References to ConfigMaps holding more files to mount
+                  to the CommonConfigMountPath.
+                items:
+                  description: "ObjectReference contains enough information to let
+                    you inspect or modify the referred object. --- New uses of this
+                    type are discouraged because of difficulty describing its usage
+                    when embedded in APIs. 1. Ignored fields.  It includes many fields
+                    which are not generally honored.  For instance, ResourceVersion
+                    and FieldPath are both very rarely valid in actual usage. 2. Invalid
+                    usage help.  It is impossible to add specific help for individual
+                    usage.  In most embedded usages, there are particular restrictions
+                    like, \"must refer only to types A and B\" or \"UID not honored\"
+                    or \"name must be restricted\". Those cannot be well described
+                    when embedded. 3. Inconsistent validation.  Because the usages
+                    are different, the validation rules are different by usage, which
+                    makes it hard for users to predict what will happen. 4. The fields
+                    are both imprecise and overly precise.  Kind is not a precise
+                    mapping to a URL. This can produce ambiguity during interpretation
+                    and require a REST mapping.  In most cases, the dependency is
+                    on the group,resource tuple and the version of the actual struct
+                    is irrelevant. 5. We cannot easily change it.  Because this type
+                    is embedded in many locations, updates to this type will affect
+                    numerous schemas.  Don't make new APIs embed an underspecified
+                    API type they do not control. \n Instead of using this type, create
+                    a locally provided and used type that is well-focused on your
+                    reference. For example, ServiceReferences for admission registration:
+                    https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533
+                    ."
+                  properties:
+                    apiVersion:
+                      description: API version of the referent.
+                      type: string
+                    fieldPath:
+                      description: 'If referring to a piece of an object instead of
+                        an entire object, this string should contain a valid JSON/Go
+                        field access statement, such as desiredState.manifest.containers[2].
+                        For example, if the object reference is to a container within
+                        a pod, this would take on a value like: "spec.containers{name}"
+                        (where "name" refers to the name of the container that triggered
+                        the event) or if no container name is specified "spec.containers[2]"
+                        (container with index 2 in this pod). This syntax is chosen
+                        only to have some well-defined way of referencing a part of
+                        an object. TODO: this design is not final and this field is
+                        subject to change in the future.'
+                      type: string
+                    kind:
+                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                      type: string
+                    name:
+                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                      type: string
+                    namespace:
+                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                      type: string
+                    resourceVersion:
+                      description: 'Specific resourceVersion to which this reference
+                        is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                      type: string
+                    uid:
+                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                      type: string
+                  type: object
+                  x-kubernetes-map-type: atomic
+                type: array
               forceDeleteStsPodOnError:
                 type: boolean
               hdfs-site.xml:

config/crd/bases/druid.apache.org_druids.yaml

@@ -1653,6 +1653,71 @@ spec:
                       x-kubernetes-map-type: atomic
                   type: object
                 type: array
+              extraCommonConfig:
+                description: References to ConfigMaps holding more files to mount
+                  to the CommonConfigMountPath.
+                items:
+                  description: "ObjectReference contains enough information to let
+                    you inspect or modify the referred object. --- New uses of this
+                    type are discouraged because of difficulty describing its usage
+                    when embedded in APIs. 1. Ignored fields.  It includes many fields
+                    which are not generally honored.  For instance, ResourceVersion
+                    and FieldPath are both very rarely valid in actual usage. 2. Invalid
+                    usage help.  It is impossible to add specific help for individual
+                    usage.  In most embedded usages, there are particular restrictions
+                    like, \"must refer only to types A and B\" or \"UID not honored\"
+                    or \"name must be restricted\". Those cannot be well described
+                    when embedded. 3. Inconsistent validation.  Because the usages
+                    are different, the validation rules are different by usage, which
+                    makes it hard for users to predict what will happen. 4. The fields
+                    are both imprecise and overly precise.  Kind is not a precise
+                    mapping to a URL. This can produce ambiguity during interpretation
+                    and require a REST mapping.  In most cases, the dependency is
+                    on the group,resource tuple and the version of the actual struct
+                    is irrelevant. 5. We cannot easily change it.  Because this type
+                    is embedded in many locations, updates to this type will affect
+                    numerous schemas.  Don't make new APIs embed an underspecified
+                    API type they do not control. \n Instead of using this type, create
+                    a locally provided and used type that is well-focused on your
+                    reference. For example, ServiceReferences for admission registration:
+                    https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533
+                    ."
+                  properties:
+                    apiVersion:
+                      description: API version of the referent.
+                      type: string
+                    fieldPath:
+                      description: 'If referring to a piece of an object instead of
+                        an entire object, this string should contain a valid JSON/Go
+                        field access statement, such as desiredState.manifest.containers[2].
+                        For example, if the object reference is to a container within
+                        a pod, this would take on a value like: "spec.containers{name}"
+                        (where "name" refers to the name of the container that triggered
+                        the event) or if no container name is specified "spec.containers[2]"
+                        (container with index 2 in this pod). This syntax is chosen
+                        only to have some well-defined way of referencing a part of
+                        an object. TODO: this design is not final and this field is
+                        subject to change in the future.'
+                      type: string
+                    kind:
+                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                      type: string
+                    name:
+                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                      type: string
+                    namespace:
+                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                      type: string
+                    resourceVersion:
+                      description: 'Specific resourceVersion to which this reference
+                        is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                      type: string
+                    uid:
+                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                      type: string
+                  type: object
+                  x-kubernetes-map-type: atomic
+                type: array
               forceDeleteStsPodOnError:
                 type: boolean
               hdfs-site.xml:

controllers/druid/configuration.go

@@ -0,0 +1,124 @@
+package druid
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/datainfrahq/druid-operator/apis/druid/v1alpha1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func makeConfigMap(name string, namespace string, labels map[string]string, data map[string]string) (*v1.ConfigMap, error) {
+	return &v1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "ConfigMap",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+			Labels:    labels,
+		},
+		Data: data,
+	}, nil
+}
+
+func makeCommonConfigMap(ctx context.Context, sdk client.Client, m *v1alpha1.Druid, ls map[string]string) (*v1.ConfigMap, error) {
+	prop := m.Spec.CommonRuntimeProperties
+
+	if m.Spec.Zookeeper != nil {
+		if zm, err := createZookeeperManager(m.Spec.Zookeeper); err != nil {
+			return nil, err
+		} else {
+			prop = prop + "\n" + zm.Configuration() + "\n"
+		}
+	}
+
+	if m.Spec.MetadataStore != nil {
+		if msm, err := createMetadataStoreManager(m.Spec.MetadataStore); err != nil {
+			return nil, err
+		} else {
+			prop = prop + "\n" + msm.Configuration() + "\n"
+		}
+	}
+
+	if m.Spec.DeepStorage != nil {
+		if dsm, err := createDeepStorageManager(m.Spec.DeepStorage); err != nil {
+			return nil, err
+		} else {
+			prop = prop + "\n" + dsm.Configuration() + "\n"
+		}
+	}
+
+	data := map[string]string{
+		"common.runtime.properties": prop,
+	}
+
+	if m.Spec.DimensionsMapPath != "" {
+		data["metricDimensions.json"] = m.Spec.DimensionsMapPath
+	}
+	if m.Spec.HdfsSite != "" {
+		data["hdfs-site.xml"] = m.Spec.HdfsSite
+	}
+	if m.Spec.CoreSite != "" {
+		data["core-site.xml"] = m.Spec.CoreSite
+	}
+
+	if err := addExtraCommonConfig(ctx, sdk, m, data); err != nil {
+		return nil, err
+	}
+
+	cfg, err := makeConfigMap(
+		fmt.Sprintf("%s-druid-common-config", m.ObjectMeta.Name),
+		m.Namespace,
+		ls,
+		data)
+	return cfg, err
+}
+
+func addExtraCommonConfig(ctx context.Context, sdk client.Client, m *v1alpha1.Druid, data map[string]string) error {
+	if m.Spec.ExtraCommonConfig == nil {
+		return nil
+	}
+
+	for _, cmRef := range m.Spec.ExtraCommonConfig {
+		cm := &v1.ConfigMap{}
+		if err := sdk.Get(ctx, types.NamespacedName{
+			Name:      cmRef.Name,
+			Namespace: cmRef.Namespace}, cm); err != nil {
+			// If a configMap is not found - output error and keep reconciliation
+			continue
+		}
+
+		for fileName, fileContent := range cm.Data {
+			data[fileName] = fileContent
+		}
+	}
+
+	return nil
+}
+
+func makeConfigMapForNodeSpec(nodeSpec *v1alpha1.DruidNodeSpec, m *v1alpha1.Druid, lm map[string]string, nodeSpecUniqueStr string) (*v1.ConfigMap, error) {
+
+	data := map[string]string{
+		"runtime.properties": fmt.Sprintf("druid.port=%d\n%s", nodeSpec.DruidPort, nodeSpec.RuntimeProperties),
+		"jvm.config":         fmt.Sprintf("%s\n%s", firstNonEmptyStr(nodeSpec.JvmOptions, m.Spec.JvmOptions), nodeSpec.ExtraJvmOptions),
+	}
+	log4jconfig := firstNonEmptyStr(nodeSpec.Log4jConfig, m.Spec.Log4jConfig)
+	if log4jconfig != "" {
+		data["log4j2.xml"] = log4jconfig
+	}
+
+	return makeConfigMap(
+		fmt.Sprintf("%s-config", nodeSpecUniqueStr),
+		m.Namespace,
+		lm,
+		data)
+}
+
+func getNodeConfigMountPath(nodeSpec *v1alpha1.DruidNodeSpec) string {
+	return fmt.Sprintf("/druid/conf/druid/%s", nodeSpec.NodeType)
+}