From 774dd9fd79ff9e6180127307241dd8a2b90f089b Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Wed, 1 Oct 2025 13:53:54 -0700 Subject: [PATCH] Enable multiple instances for local HA Signed-off-by: Rayhan Hossain --- api/preview/documentdb_types.go | 4 +- .../bases/db.microsoft.com_documentdbs.yaml | 6 +-- .../crds/db.microsoft.com_documentdbs.yaml | 6 +-- .../templates/05_clusterrole.yaml | 4 +- internal/cnpg/cnpg_cluster.go | 7 ++- internal/controller/documentdb_controller.go | 54 +++++++++++++++++++ .../internal/lifecycle/lifecycle.go | 7 ++- 7 files changed, 74 insertions(+), 14 deletions(-) diff --git a/api/preview/documentdb_types.go b/api/preview/documentdb_types.go index 3c0519fa..60581235 100644 --- a/api/preview/documentdb_types.go +++ b/api/preview/documentdb_types.go @@ -14,9 +14,9 @@ type DocumentDBSpec struct { // +kubebuilder:validation:Maximum=1 NodeCount int `json:"nodeCount"` - // InstancesPerNode is the number of DocumentDB instances per node. Must be 1. + // InstancesPerNode is the number of DocumentDB instances per node. Range: 1-3. // +kubebuilder:validation:Minimum=1 - // +kubebuilder:validation:Maximum=1 + // +kubebuilder:validation:Maximum=3 InstancesPerNode int `json:"instancesPerNode"` // Resource specifies the storage resources for DocumentDB. diff --git a/config/crd/bases/db.microsoft.com_documentdbs.yaml b/config/crd/bases/db.microsoft.com_documentdbs.yaml index 2d148818..d42650ac 100644 --- a/config/crd/bases/db.microsoft.com_documentdbs.yaml +++ b/config/crd/bases/db.microsoft.com_documentdbs.yaml @@ -109,9 +109,9 @@ spec: If not specified, defaults to a version that matches the DocumentDB operator version. type: string instancesPerNode: - description: InstancesPerNode is the number of DocumentDB instances - per node. Must be 1. - maximum: 1 + description: 'InstancesPerNode is the number of DocumentDB instances + per node. Range: 1-3.' + maximum: 3 minimum: 1 type: integer nodeCount: diff --git a/documentdb-chart/crds/db.microsoft.com_documentdbs.yaml b/documentdb-chart/crds/db.microsoft.com_documentdbs.yaml index 2d148818..d42650ac 100644 --- a/documentdb-chart/crds/db.microsoft.com_documentdbs.yaml +++ b/documentdb-chart/crds/db.microsoft.com_documentdbs.yaml @@ -109,9 +109,9 @@ spec: If not specified, defaults to a version that matches the DocumentDB operator version. type: string instancesPerNode: - description: InstancesPerNode is the number of DocumentDB instances - per node. Must be 1. - maximum: 1 + description: 'InstancesPerNode is the number of DocumentDB instances + per node. Range: 1-3.' + maximum: 3 minimum: 1 type: integer nodeCount: diff --git a/documentdb-chart/templates/05_clusterrole.yaml b/documentdb-chart/templates/05_clusterrole.yaml index d145d716..e5bae780 100644 --- a/documentdb-chart/templates/05_clusterrole.yaml +++ b/documentdb-chart/templates/05_clusterrole.yaml @@ -13,7 +13,7 @@ rules: resources: ["statefulsets", "deployments"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: [""] - resources: ["services", "pods", "endpoints", "leases", "serviceaccounts", "configmaps", "namespaces"] + resources: ["services", "pods", "endpoints", "leases", "serviceaccounts", "configmaps", "namespaces", "persistentvolumeclaims"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: ["metrics.k8s.io"] resources: ["pods"] @@ -34,6 +34,6 @@ rules: resources: ["jobs"] verbs: ["create", "get", "list", "watch", "update", "delete"] - apiGroups: ["postgresql.cnpg.io"] - resources: ["clusters", "publications", "subscriptions"] + resources: ["clusters", "publications", "subscriptions", "clusters/status"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] diff --git a/internal/cnpg/cnpg_cluster.go b/internal/cnpg/cnpg_cluster.go index 2a41a0f7..360308d4 100644 --- a/internal/cnpg/cnpg_cluster.go +++ b/internal/cnpg/cnpg_cluster.go @@ -66,7 +66,7 @@ func GetCnpgClusterSpec(req ctrl.Request, documentdb dbpreview.DocumentDB, docum "host replication all all trust", }, }, - Bootstrap: getBootstrapConfiguration(documentdb), + Bootstrap: getBootstrapConfiguration(), } spec.MaxStopDelay = getMaxStopDelayOrDefault(documentdb) return spec @@ -83,7 +83,7 @@ func getInheritedMetadataLabels(documentdb dbpreview.DocumentDB) *cnpgv1.Embedde } } -func getBootstrapConfiguration(documentdb dbpreview.DocumentDB) *cnpgv1.BootstrapConfiguration { +func getBootstrapConfiguration() *cnpgv1.BootstrapConfiguration { return &cnpgv1.BootstrapConfiguration{ InitDB: &cnpgv1.BootstrapInitDB{ PostInitSQL: []string{ @@ -91,6 +91,9 @@ func getBootstrapConfiguration(documentdb dbpreview.DocumentDB) *cnpgv1.Bootstra "CREATE ROLE documentdb WITH LOGIN PASSWORD 'Admin100'", "ALTER ROLE documentdb WITH SUPERUSER CREATEDB CREATEROLE REPLICATION BYPASSRLS", }, + PostInitApplicationSQL: []string{ + "GRANT documentdb_admin_role TO streaming_replica", + }, }, } } diff --git a/internal/controller/documentdb_controller.go b/internal/controller/documentdb_controller.go index 06056a88..ab8b7f90 100644 --- a/internal/controller/documentdb_controller.go +++ b/internal/controller/documentdb_controller.go @@ -5,11 +5,14 @@ package controller import ( "context" + "fmt" "sync" "time" cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -139,6 +142,17 @@ func (r *DocumentDBReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } + // TODO make this only run once + // if currentCnpgCluster.Status.Phase == "Cluster in healthy state" && isPrimary { + if currentCnpgCluster.Status.Phase == "Cluster in healthy state" { + grantCommand := "GRANT documentdb_admin_role TO streaming_replica;" + + if err := r.executeSQLCommand(ctx, documentdb.Name, req.Namespace, grantCommand, "grant-permissions"); err != nil { + log.Error(err, "Failed to grant permissions to streaming_replica") + return ctrl.Result{RequeueAfter: RequeueAfterShort}, nil + } + } + return ctrl.Result{RequeueAfter: RequeueAfterLong}, nil } @@ -223,3 +237,43 @@ func (r *DocumentDBReconciler) SetupWithManager(mgr ctrl.Manager) error { Named("documentdb-controller"). Complete(r) } + +func (r *DocumentDBReconciler) executeSQLCommand(ctx context.Context, documentdbName, namespace, sqlCommand, uniqueName string) error { + zero := int32(0) + host := documentdbName + "-rw" + sqlPod := &batchv1.Job{ + ObjectMeta: ctrl.ObjectMeta{ + Name: fmt.Sprintf("%s-%s-sql-executor", documentdbName, uniqueName), + Namespace: namespace, + }, + Spec: batchv1.JobSpec{ + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container{ + { + Name: "sql-executor", + Image: "postgres:15", + Command: []string{ + "psql", + "-h", host, + "-U", "postgres", + "-d", "postgres", + "-c", sqlCommand, + }, + }, + }, + }, + }, + TTLSecondsAfterFinished: &zero, + }, + } + + if err := r.Client.Create(ctx, sqlPod); err != nil { + if !errors.IsAlreadyExists(err) { + return err + } + } + + return nil +} diff --git a/plugins/sidecar-injector/internal/lifecycle/lifecycle.go b/plugins/sidecar-injector/internal/lifecycle/lifecycle.go index dad8916c..ae8a8546 100644 --- a/plugins/sidecar-injector/internal/lifecycle/lifecycle.go +++ b/plugins/sidecar-injector/internal/lifecycle/lifecycle.go @@ -182,8 +182,11 @@ func (impl Implementation) reconcileMetadata( }, } - // Check if the pod has the label replication_cluster_type=replica - if mutatedPod.Labels["replication_cluster_type"] == "replica" { + // Check if the pod has the label replication_cluster_type=replica or is not a primary by CNPG instanceRole + instanceRole := mutatedPod.Labels["cnpg.io/instanceRole"] + isLocalReplica := instanceRole == "replica" + + if mutatedPod.Labels["replication_cluster_type"] == "replica" || isLocalReplica { sidecar.Args = []string{"--create-user", "false", "--start-pg", "false", "--pg-port", "5432"} } else { sidecar.Args = []string{"--create-user", "true", "--start-pg", "false", "--pg-port", "5432"}