Enable multiple instances for local HA

api/preview/documentdb_types.go

@@ -14,9 +14,9 @@ type DocumentDBSpec struct {
 	// +kubebuilder:validation:Maximum=1
 	NodeCount int `json:"nodeCount"`
 
-	// InstancesPerNode is the number of DocumentDB instances per node. Must be 1.
+	// InstancesPerNode is the number of DocumentDB instances per node. Range: 1-3.
 	// +kubebuilder:validation:Minimum=1
-	// +kubebuilder:validation:Maximum=1
+	// +kubebuilder:validation:Maximum=3
 	InstancesPerNode int `json:"instancesPerNode"`
 
 	// Resource specifies the storage resources for DocumentDB.

config/crd/bases/db.microsoft.com_documentdbs.yaml

@@ -109,9 +109,9 @@ spec:
                   If not specified, defaults to a version that matches the DocumentDB operator version.
                 type: string
               instancesPerNode:
-                description: InstancesPerNode is the number of DocumentDB instances
-                  per node. Must be 1.
-                maximum: 1
+                description: 'InstancesPerNode is the number of DocumentDB instances
+                  per node. Range: 1-3.'
+                maximum: 3
                 minimum: 1
                 type: integer
               nodeCount:

documentdb-chart/crds/db.microsoft.com_documentdbs.yaml

@@ -109,9 +109,9 @@ spec:
                   If not specified, defaults to a version that matches the DocumentDB operator version.
                 type: string
               instancesPerNode:
-                description: InstancesPerNode is the number of DocumentDB instances
-                  per node. Must be 1.
-                maximum: 1
+                description: 'InstancesPerNode is the number of DocumentDB instances
+                  per node. Range: 1-3.'
+                maximum: 3
                 minimum: 1
                 type: integer
               nodeCount:

documentdb-chart/templates/05_clusterrole.yaml

@@ -13,7 +13,7 @@ rules:
   resources: ["statefulsets", "deployments"]
   verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
 - apiGroups: [""]
-  resources: ["services", "pods", "endpoints", "leases", "serviceaccounts", "configmaps", "namespaces"]
+  resources: ["services", "pods", "endpoints", "leases", "serviceaccounts", "configmaps", "namespaces", "persistentvolumeclaims"]
   verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
 - apiGroups: ["metrics.k8s.io"]
   resources: ["pods"]
@@ -34,6 +34,6 @@ rules:
   resources: ["jobs"]
   verbs: ["create", "get", "list", "watch", "update", "delete"]
 - apiGroups: ["postgresql.cnpg.io"]
-  resources: ["clusters", "publications", "subscriptions"]
+  resources: ["clusters", "publications", "subscriptions", "clusters/status"]
   verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
 

internal/cnpg/cnpg_cluster.go

@@ -66,7 +66,7 @@ func GetCnpgClusterSpec(req ctrl.Request, documentdb dbpreview.DocumentDB, docum
 						"host replication all all trust",
 					},
 				},
-				Bootstrap: getBootstrapConfiguration(documentdb),
+				Bootstrap: getBootstrapConfiguration(),
 			}
 			spec.MaxStopDelay = getMaxStopDelayOrDefault(documentdb)
 			return spec
@@ -83,14 +83,17 @@ func getInheritedMetadataLabels(documentdb dbpreview.DocumentDB) *cnpgv1.Embedde
 	}
 }
 
-func getBootstrapConfiguration(documentdb dbpreview.DocumentDB) *cnpgv1.BootstrapConfiguration {
+func getBootstrapConfiguration() *cnpgv1.BootstrapConfiguration {
 	return &cnpgv1.BootstrapConfiguration{
 		InitDB: &cnpgv1.BootstrapInitDB{
 			PostInitSQL: []string{
 				"CREATE EXTENSION documentdb CASCADE",
 				"CREATE ROLE documentdb WITH LOGIN PASSWORD 'Admin100'",
 				"ALTER ROLE documentdb WITH SUPERUSER CREATEDB CREATEROLE REPLICATION BYPASSRLS",
 			},
+			PostInitApplicationSQL: []string{
+				"GRANT documentdb_admin_role TO streaming_replica",
+			},
 		},
 	}
 }

internal/controller/documentdb_controller.go

@@ -5,11 +5,14 @@ package controller
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"time"
 
 	cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -139,6 +142,17 @@ func (r *DocumentDBReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		}
 	}
 
+	// TODO make this only run once
+	// if currentCnpgCluster.Status.Phase == "Cluster in healthy state" && isPrimary {
+	if currentCnpgCluster.Status.Phase == "Cluster in healthy state" {
+		grantCommand := "GRANT documentdb_admin_role TO streaming_replica;"
+
+		if err := r.executeSQLCommand(ctx, documentdb.Name, req.Namespace, grantCommand, "grant-permissions"); err != nil {
+			log.Error(err, "Failed to grant permissions to streaming_replica")
+			return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil
+		}
+	}
+
 	return ctrl.Result{RequeueAfter: RequeueAfterLong}, nil
 }
 
@@ -223,3 +237,43 @@ func (r *DocumentDBReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Named("documentdb-controller").
 		Complete(r)
 }
+
+func (r *DocumentDBReconciler) executeSQLCommand(ctx context.Context, documentdbName, namespace, sqlCommand, uniqueName string) error {
+	zero := int32(0)
+	host := documentdbName + "-rw"
+	sqlPod := &batchv1.Job{
+		ObjectMeta: ctrl.ObjectMeta{
+			Name:      fmt.Sprintf("%s-%s-sql-executor", documentdbName, uniqueName),
+			Namespace: namespace,
+		},
+		Spec: batchv1.JobSpec{
+			Template: v1.PodTemplateSpec{
+				Spec: v1.PodSpec{
+					RestartPolicy: v1.RestartPolicyNever,
+					Containers: []v1.Container{
+						{
+							Name:  "sql-executor",
+							Image: "postgres:15",
+							Command: []string{
+								"psql",
+								"-h", host,
+								"-U", "postgres",
+								"-d", "postgres",
+								"-c", sqlCommand,
+							},
+						},
+					},
+				},
+			},
+			TTLSecondsAfterFinished: &zero,
+		},
+	}
+
+	if err := r.Client.Create(ctx, sqlPod); err != nil {
+		if !errors.IsAlreadyExists(err) {
+			return err
+		}
+	}
+
+	return nil
+}

plugins/sidecar-injector/internal/lifecycle/lifecycle.go

@@ -182,8 +182,11 @@ func (impl Implementation) reconcileMetadata(
 		},
 	}
 
-	// Check if the pod has the label replication_cluster_type=replica
-	if mutatedPod.Labels["replication_cluster_type"] == "replica" {
+	// Check if the pod has the label replication_cluster_type=replica or is not a primary by CNPG instanceRole
+	instanceRole := mutatedPod.Labels["cnpg.io/instanceRole"]
+	isLocalReplica := instanceRole == "replica"
+
+	if mutatedPod.Labels["replication_cluster_type"] == "replica" || isLocalReplica {
 		sidecar.Args = []string{"--create-user", "false", "--start-pg", "false", "--pg-port", "5432"}
 	} else {
 		sidecar.Args = []string{"--create-user", "true", "--start-pg", "false", "--pg-port", "5432"}