add multi-master option to galera

api/bases/mariadb.openstack.org_galeras.yaml

@@ -64,6 +64,10 @@ spec:
                 description: When TLS is configured, only allow connections to the
                   DB over TLS
                 type: boolean
+              enableMultiMaster:
+                default: false
+                description: Use multi-master service routing
+                type: boolean
               logToDisk:
                 description: Log Galera pod's output to disk
                 type: boolean
@@ -104,6 +108,7 @@ spec:
                 type: object
             required:
             - containerImage
+            - enableMultiMaster
             - replicas
             - secret
             - storageClass

api/bases/mariadb.openstack.org_mariadbdatabases.yaml

@@ -98,6 +98,9 @@ spec:
                   - type
                   type: object
                 type: array
+              enableMultiMaster:
+                description: whether the DB instance is using multi-master routing
+                type: boolean
               hash:
                 additionalProperties:
                   type: string

api/v1beta1/galera_types.go

@@ -89,6 +89,9 @@ type GaleraSpecCore struct {
 	// +kubebuilder:validation:Optional
 	// Log Galera pod's output to disk
 	LogToDisk bool `json:"logToDisk"`
+	// +kubebuilder:default=false
+	// Use multi-master service routing
+	EnableMultiMaster bool `json:"enableMultiMaster"`
 }
 
 // GaleraAttributes holds startup information for a Galera host

api/v1beta1/mariadbdatabase_types.go

@@ -55,6 +55,9 @@ type MariaDBDatabaseStatus struct {
 
 	// Whether TLS is supported by the DB instance
 	TLSSupport bool `json:"tlsSupport,omitempty"`
+
+	// whether the DB instance is using multi-master routing
+	EnableMultiMaster bool `json:"enableMultiMaster,omitempty"`
 }
 
 //+kubebuilder:object:root=true

config/crd/bases/mariadb.openstack.org_galeras.yaml

@@ -64,6 +64,10 @@ spec:
                 description: When TLS is configured, only allow connections to the
                   DB over TLS
                 type: boolean
+              enableMultiMaster:
+                default: false
+                description: Use multi-master service routing
+                type: boolean
               logToDisk:
                 description: Log Galera pod's output to disk
                 type: boolean
@@ -104,6 +108,7 @@ spec:
                 type: object
             required:
             - containerImage
+            - enableMultiMaster
             - replicas
             - secret
             - storageClass

config/crd/bases/mariadb.openstack.org_mariadbdatabases.yaml

@@ -98,6 +98,9 @@ spec:
                   - type
                   type: object
                 type: array
+              enableMultiMaster:
+                description: whether the DB instance is using multi-master routing
+                type: boolean
               hash:
                 additionalProperties:
                   type: string

controllers/galera_controller.go

@@ -509,16 +509,23 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 		// service get stuck.
 		controllerutil.AddFinalizer(service, helper.GetFinalizer())
 
-		// NOTE(dciabrin) We deploy Galera as an A/P service (i.e. no multi-master writes)
-		// by setting labels in the service's label selectors.
-		// This label is dynamically set based on the status of the Galera cluster,
-		// so in this CreateOrPatch block we must reuse whatever is present in
-		// the existing service CR in case we're patching it.
-		activePod, present := service.Spec.Selector[mariadb.ActivePodSelectorKey]
-		service.Spec = pkgsvc.Spec
-		if present {
-			service.Spec.Selector[mariadb.ActivePodSelectorKey] = activePod
+		if !instance.Spec.EnableMultiMaster {
+			// NOTE(dciabrin) We deploy Galera as an A/P service (i.e. no multi-master writes)
+			// by setting labels in the service's label selectors.
+			// This label is dynamically set based on the status of the Galera cluster,
+			// so in this CreateOrPatch block we must reuse whatever is present in
+			// the existing service CR in case we're patching it.
+			activePod, present := service.Spec.Selector[mariadb.ActivePodSelectorKey]
+			service.Spec = pkgsvc.Spec
+
+			if present {
+				service.Spec.Selector[mariadb.ActivePodSelectorKey] = activePod
+			}
+		} else {
+			service.Spec = pkgsvc.Spec
+			delete(service.Spec.Selector, mariadb.ActivePodSelectorKey)
 		}
+
 		err := controllerutil.SetControllerReference(instance, service, r.Client.Scheme())
 		if err != nil {
 			return err

controllers/mariadbdatabase_controller.go

@@ -27,6 +27,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
 	helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper"
@@ -145,7 +147,7 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 
 	// here we know that Galera exists so add a finalizer to ourselves and to the db CR. Before this point there is no reason to have a finalizer on ourselves as nothing to cleanup.
 	if instance.DeletionTimestamp.IsZero() || isNewInstance { // this condition can be removed if you wish as it is always true at this point otherwise we would returned earlier.
-		if controllerutil.AddFinalizer(dbGalera, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) {
+		if dbGalera.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(dbGalera, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) {
 			err := r.Update(ctx, dbGalera)
 			if err != nil {
 				return ctrl.Result{}, err
@@ -200,6 +202,9 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		return ctrl.Result{}, err
 	}
 
+	// DB instances setup for multi master
+	instance.Status.EnableMultiMaster = dbGalera.Spec.EnableMultiMaster
+
 	dbCreateHash := instance.Status.Hash[databasev1beta1.DbCreateHash]
 	dbCreateJob := job.NewJob(
 		jobDef,
@@ -245,8 +250,43 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 
 // SetupWithManager -
 func (r *MariaDBDatabaseReconciler) SetupWithManager(mgr ctrl.Manager) error {
+
+	updateStatusFn := func(ctx context.Context, o client.Object) []reconcile.Request {
+		log := GetLog(ctx, "MariaDBDatabase")
+
+		result := []reconcile.Request{}
+
+		mariaDBDatabases := &databasev1beta1.MariaDBDatabaseList{}
+
+		listOpts := []client.ListOption{
+			client.InNamespace(o.GetNamespace()),
+		}
+		if err := r.Client.List(ctx, mariaDBDatabases, listOpts...); err != nil {
+			log.Error(err, "Unable to retrieve MariaDBDatabase CRs %w")
+			return nil
+		}
+
+		for _, cr := range mariaDBDatabases.Items {
+
+			if o.GetName() == cr.GetLabels()["dbName"] {
+				name := client.ObjectKey{
+					Namespace: o.GetNamespace(),
+					Name:      cr.Name,
+				}
+				log.Info(fmt.Sprintf("Galera %s is used by MariaDBDatabase CR %s", o.GetName(), cr.Name))
+				result = append(result, reconcile.Request{NamespacedName: name})
+			}
+		}
+
+		if len(result) > 0 {
+			return result
+		}
+		return nil
+	}
+
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&databasev1beta1.MariaDBDatabase{}).
+		Watches(&databasev1beta1.Galera{}, handler.EnqueueRequestsFromMapFunc(updateStatusFn)).
 		Complete(r)
 }
 

tests/kuttl/tests/galera_cluster_restart/04-teardown.yaml

@@ -0,0 +1,10 @@
+apiVersion: kuttl.dev/v1beta
+kind: TestStep
+delete:
+- apiVersion: mariadb.openstack.org/v1beta1
+  kind: Galera
+  name: openstack
+commands:
+  - script: |
+      oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2
+      for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done

tests/kuttl/tests/galera_log_to_disk/03-teardown.yaml

@@ -0,0 +1,10 @@
+apiVersion: kuttl.dev/v1beta
+kind: TestStep
+delete:
+- apiVersion: mariadb.openstack.org/v1beta1
+  kind: Galera
+  name: openstack
+commands:
+  - script: |
+      oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2
+      for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done

tests/kuttl/tests/galera_multi_master/01-deploy_ap_galera.yaml

@@ -0,0 +1,19 @@
+apiVersion: mariadb.openstack.org/v1beta1
+kind: Galera
+metadata:
+  name: openstack
+spec:
+  secret: osp-secret
+  storageClass: local-storage
+  storageRequest: 500M
+  replicas: 3
+  enableMultiMaster: false
+---
+apiVersion: mariadb.openstack.org/v1beta1
+kind: MariaDBDatabase
+metadata:
+  name: mydatabase
+  labels:
+    dbName: openstack
+spec:
+  name: mydatabase

tests/kuttl/tests/galera_multi_master/02-assert.yaml

@@ -0,0 +1,131 @@
+#
+# Check for:
+#
+# - 1 MariaDB CR
+# - 1 Pod for MariaDB CR
+#
+
+apiVersion: mariadb.openstack.org/v1beta1
+kind: Galera
+metadata:
+  name: openstack
+spec:
+  replicas: 3
+  secret: osp-secret
+  storageRequest: 500M
+status:
+  bootstrapped: true
+  conditions:
+  - message: Setup complete
+    reason: Ready
+    status: "True"
+    type: Ready
+  - message: Deployment completed
+    reason: Ready
+    status: "True"
+    type: DeploymentReady
+  - message: Exposing service completed
+    reason: Ready
+    status: "True"
+    type: ExposeServiceReady
+  - message: Input data complete
+    reason: Ready
+    status: "True"
+    type: InputReady
+  - message: RoleBinding created
+    reason: Ready
+    status: "True"
+    type: RoleBindingReady
+  - message: Role created
+    reason: Ready
+    status: "True"
+    type: RoleReady
+  - message: ServiceAccount created
+    reason: Ready
+    status: "True"
+    type: ServiceAccountReady
+  - message: Service config create completed
+    reason: Ready
+    status: "True"
+    type: ServiceConfigReady
+  - message: Input data complete
+    reason: Ready
+    status: "True"
+    type: TLSInputReady
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: openstack-galera
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: galera
+      cr: galera-openstack
+      galera/name: openstack
+  serviceName: openstack-galera
+  template:
+    metadata:
+      labels:
+        app: galera
+        cr: galera-openstack
+        galera/name: openstack
+    spec:
+      containers:
+      - command:
+        - /usr/bin/dumb-init
+        - --
+        - /usr/local/bin/kolla_start
+        name: galera
+        ports:
+        - containerPort: 3306
+          name: mysql
+          protocol: TCP
+        - containerPort: 4567
+          name: galera
+          protocol: TCP
+      serviceAccount: galera-openstack
+      serviceAccountName: galera-openstack
+status:
+  availableReplicas: 3
+  readyReplicas: 3
+  replicas: 3
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: openstack-galera-0
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: openstack-galera-1
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: openstack-galera-2
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: openstack-galera
+spec:
+  ports:
+  - name: mysql
+    port: 3306
+    protocol: TCP
+    targetPort: 3306
+  selector:
+    app: galera
+    cr: galera-openstack
+---
+apiVersion: mariadb.openstack.org/v1beta1
+kind: MariaDBDatabase
+metadata:
+  labels:
+    dbName: openstack
+  name: mydatabase
+spec:
+  name: mydatabase

tests/kuttl/tests/galera_multi_master/03-assert.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: |
+      # ensure kubernetes pod key present
+      oc get -n ${NAMESPACE} service openstack -o yaml | grep "statefulset.kubernetes.io/pod-name: openstack-galera-"

tests/kuttl/tests/galera_multi_master/04-deploy_mm_galera.yaml

@@ -0,0 +1,10 @@
+apiVersion: mariadb.openstack.org/v1beta1
+kind: Galera
+metadata:
+  name: openstack
+spec:
+  secret: osp-secret
+  storageClass: local-storage
+  storageRequest: 500M
+  replicas: 3
+  enableMultiMaster: true

tests/kuttl/tests/galera_multi_master/05-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: |
+      # ensure kubernetes pod key *not* present
+      ! oc get -n ${NAMESPACE} service openstack -o yaml | grep "statefulset.kubernetes.io/pod-name: openstack-galera-"