Let operator set master writable and set cluster ready only when it's healty

.drone.yml

@@ -1,3 +1,7 @@
+branches:
+  - master
+  - 1.x
+
 workspace:
   base: /root/go
   path: src/github.com/presslabs/mysql-operator

hack/charts/mysql-operator/values.yaml

@@ -55,9 +55,8 @@ orchestrator:
     RecoverMasterClusterFilters: ['.*']
     RecoverIntermediateMasterClusterFilters: ['.*']
     # `reset slave all` and `set read_only=0` on promoted master
-    ApplyMySQLPromotionAfterMasterFailover: true
-    # set downtime on the failed master
-    MasterFailoverLostInstancesDowntimeMinutes: 10
+    ApplyMySQLPromotionAfterMasterFailover: false
+    MasterFailoverDetachReplicaMasterHost: true
     # https://github.com/github/orchestrator/blob/master/docs/configuration-recovery.md#promotion-actions
     # Safety! do not disable unless you know what you are doing
     FailMasterPromotionIfSQLThreadNotUpToDate: true

pkg/controller/internal/testutil/cluster.go

@@ -14,13 +14,22 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// nolint: errcheck,golint
 package testutil
 
 import (
+	"context"
 	"time"
 
+	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gstruct"
+	gomegatypes "github.com/onsi/gomega/types"
+
 	core "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	api "github.com/presslabs/mysql-operator/pkg/apis/mysql/v1alpha1"
 )
@@ -68,3 +77,35 @@ func NodeConditions(master, replicating, lagged, readOnly bool) []api.NodeCondit
 		},
 	}
 }
+
+// RefreshFn receives a client and a runtime.Objects and refreshes the object from k8s
+// example: Eventually(RefreshFn(c, cluster.Unwrap())).Should(HaveClusterStatusReadyNodes(2))
+func RefreshFn(c client.Client, obj runtime.Object) func() runtime.Object {
+	return func() runtime.Object {
+		objMeta, ok := obj.(metav1.Object)
+		if !ok {
+			return nil
+		}
+
+		objKey := types.NamespacedName{
+			Name:      objMeta.GetName(),
+			Namespace: objMeta.GetNamespace(),
+		}
+
+		if err := c.Get(context.TODO(), objKey, obj); err == nil {
+			return obj
+		}
+
+		// if the object is not updated then return nil, not the old object
+		return nil
+	}
+}
+
+// HaveClusterStatusReadyNodes a matcher that checks cluster ready nodes to equal the given value
+func HaveClusterStatusReadyNodes(nodes int) gomegatypes.GomegaMatcher {
+	return PointTo(MatchFields(IgnoreExtras, Fields{
+		"Status": MatchFields(IgnoreExtras, Fields{
+			"ReadyNodes": Equal(nodes),
+		}),
+	}))
+}

pkg/controller/mysqlcluster/internal/syncer/settings.go

@@ -28,25 +28,20 @@ const (
 	// MysqlPort is the default mysql port.
 	MysqlPort = constants.MysqlPort
 
-	// HelperXtrabackupPortName is name of the port on which we take backups
-	HelperXtrabackupPortName = "xtrabackup"
-	// HelperXtrabackupPort is the port on which we serve backups
-	HelperXtrabackupPort = constants.HelperXtrabackupPort
-
 	// OrcTopologyDir path where orc conf secret is mounted
 	OrcTopologyDir = constants.OrcTopologyDir
 
-	// HelperServerPort represents the port on which http server will run
-	HelperServerPort = constants.HelperServerPort
-	// HelperServerProbePath the probe path
-	HelperServerProbePath = constants.HelperServerProbePath
+	// SidecarServerPortName name of the port
+	SidecarServerPortName = "sidecar-http"
+	// SidecarServerPort represents the port on which http server will run
+	SidecarServerPort = constants.SidecarServerPort
+	// SidecarServerProbePath the probe path
+	SidecarServerProbePath = constants.SidecarServerProbePath
 
 	// ExporterPort is the port that metrics will be exported
 	ExporterPort = constants.ExporterPort
-
 	//ExporterPortName the name of the metrics exporter port
 	ExporterPortName = "prometheus"
-
 	// ExporterPath is the path on which metrics are expose
 	ExporterPath = constants.ExporterPath
 
@@ -56,13 +51,10 @@ const (
 
 	// ConfVolumeMountPath is the path where mysql configs will be mounted
 	ConfVolumeMountPath = constants.ConfVolumeMountPath
-
 	// DataVolumeMountPath is the path to mysql data
 	DataVolumeMountPath = constants.DataVolumeMountPath
-
 	// ConfMapVolumeMountPath represents the temp config mount path in init containers
 	ConfMapVolumeMountPath = constants.ConfMapVolumeMountPath
-
 	// ConfDPath is the path to extra mysql configs dir
 	ConfDPath = constants.ConfDPath
 )

pkg/controller/mysqlcluster/internal/syncer/statefullset.go

@@ -85,14 +85,6 @@ func NewStatefulSetSyncer(c client.Client, scheme *runtime.Scheme, cluster *mysq
 func (s *sfsSyncer) SyncFn(in runtime.Object) error {
 	out := in.(*apps.StatefulSet)
 
-	if out.Status.ReadyReplicas == *s.cluster.Spec.Replicas {
-		s.cluster.UpdateStatusCondition(api.ClusterConditionReady,
-			core.ConditionTrue, "statefulset ready", "Cluster is ready.")
-	} else {
-		s.cluster.UpdateStatusCondition(api.ClusterConditionReady,
-			core.ConditionFalse, "statefulset not ready", "Cluster is not ready.")
-	}
-
 	s.cluster.Status.ReadyNodes = int(out.Status.ReadyReplicas)
 
 	out.Spec.Replicas = s.cluster.Spec.Replicas
@@ -356,21 +348,20 @@ func (s *sfsSyncer) ensureContainersSpec() []core.Container {
 		},
 	})
 
-	helper := s.ensureContainer(containerSidecarName,
+	// SIDECAR container
+	sidecar := s.ensureContainer(containerSidecarName,
 		s.opt.HelperImage,
 		[]string{"config-and-serve"},
 	)
-	helper.Ports = ensurePorts(core.ContainerPort{
-		Name:          HelperXtrabackupPortName,
-		ContainerPort: HelperXtrabackupPort,
+	sidecar.Ports = ensurePorts(core.ContainerPort{
+		Name:          SidecarServerPortName,
+		ContainerPort: SidecarServerPort,
 	})
-	helper.Resources = ensureResources(containerSidecarName)
-
-	// HELPER container
-	helper.ReadinessProbe = ensureProbe(30, 5, 5, core.Handler{
+	sidecar.Resources = ensureResources(containerSidecarName)
+	sidecar.ReadinessProbe = ensureProbe(30, 5, 5, core.Handler{
 		HTTPGet: &core.HTTPGetAction{
-			Path:   HelperServerProbePath,
-			Port:   intstr.FromInt(HelperServerPort),
+			Path:   SidecarServerProbePath,
+			Port:   intstr.FromInt(SidecarServerPort),
 			Scheme: core.URISchemeHTTP,
 		},
 	})
@@ -417,7 +408,7 @@ func (s *sfsSyncer) ensureContainersSpec() []core.Container {
 
 	containers := []core.Container{
 		mysql,
-		helper,
+		sidecar,
 		exporter,
 		heartbeat,
 	}

pkg/controller/mysqlcluster/mysqlcluster_controller_test.go

@@ -40,6 +40,7 @@ import (
 
 	api "github.com/presslabs/mysql-operator/pkg/apis/mysql/v1alpha1"
 	"github.com/presslabs/mysql-operator/pkg/controller/internal/testutil"
+	util "github.com/presslabs/mysql-operator/pkg/controller/internal/testutil"
 	"github.com/presslabs/mysql-operator/pkg/internal/mysqlcluster"
 )
 
@@ -231,7 +232,7 @@ var _ = Describe("MysqlCluster controller", func() {
 				Expect(statefulSet.Spec.Template.ObjectMeta.Annotations["config_rev"]).To(Equal(cfgMap.ResourceVersion))
 				Expect(statefulSet.Spec.Template.ObjectMeta.Annotations["secret_rev"]).To(Equal(secret.ResourceVersion))
 			})
-			It("should update cluster ready condition", func() {
+			It("should update cluster ready nodes", func() {
 				// get statefulset
 				sfsKey := types.NamespacedName{
 					Name:      cluster.GetNameForResource(mysqlcluster.StatefulSet),
@@ -250,7 +251,9 @@ var _ = Describe("MysqlCluster controller", func() {
 				// expect a reconcile event
 				Eventually(requests, timeout).Should(Receive(Equal(expectedRequest)))
 				Eventually(requests, timeout).Should(Receive(Equal(expectedRequest)))
-				Eventually(getClusterConditions(c, cluster), timeout).Should(haveCondWithStatus(api.ClusterConditionReady, corev1.ConditionTrue))
+
+				// check ready nodes are updated
+				Eventually(util.RefreshFn(c, cluster.Unwrap())).Should(util.HaveClusterStatusReadyNodes(2))
 			})
 			It("should label pods as healthy and as master accordingly", func() {
 				pod0 := getPod(cluster, 0)
@@ -527,23 +530,6 @@ func nodeStatusForPod(cluster *mysqlcluster.MysqlCluster, pod *corev1.Pod, maste
 	}
 }
 
-// getClusterConditions is a helper func that returns a functions that returns cluster status conditions
-func getClusterConditions(c client.Client, cluster *mysqlcluster.MysqlCluster) func() []api.ClusterCondition {
-	return func() []api.ClusterCondition {
-		cl := &api.MysqlCluster{}
-		c.Get(context.TODO(), types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}, cl)
-		return cl.Status.Conditions
-	}
-}
-
-// haveCondWithStatus is a helper func that returns a matcher to check for an existing condition in a ClusterCondition list.
-func haveCondWithStatus(condType api.ClusterConditionType, status corev1.ConditionStatus) gomegatypes.GomegaMatcher {
-	return ContainElement(MatchFields(IgnoreExtras, Fields{
-		"Type":   Equal(condType),
-		"Status": Equal(status),
-	}))
-}
-
 func haveLabelWithValue(label, value string) gomegatypes.GomegaMatcher {
 	return PointTo(MatchFields(IgnoreExtras, Fields{
 		"ObjectMeta": MatchFields(IgnoreExtras, Fields{

pkg/controller/orchestrator/orchestrator_controller_test.go

@@ -41,8 +41,9 @@ import (
 )
 
 var (
-	one = int32(1)
-	two = int32(2)
+	one   = int32(1)
+	two   = int32(2)
+	three = int32(3)
 )
 
 var _ = Describe("Orchestrator controller", func() {