From b3b9e52f8b9d84e49981452dacc8922d9d71c4cb Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Mon, 31 May 2021 16:27:34 +0200 Subject: [PATCH 1/2] add basic retry around switchover --- pkg/cluster/pod.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index c8dfe2a13..a1cc0ffd4 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -304,9 +304,16 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error { } masterCandidateName := util.NameFromMeta(masterCandidatePod.ObjectMeta) - if err := c.Switchover(oldMaster, masterCandidateName); err != nil { - return fmt.Errorf("could not failover to pod %q: %v", masterCandidateName, err) - } + _ = retryutil.Retry(1*time.Minute, 5*time.Minute, + func() (bool, error) { + err := c.Switchover(oldMaster, masterCandidateName) + if err != nil { + c.logger.Errorf("could not failover to pod %q: %v", masterCandidateName, err) + return false, nil + } + return true, nil + }, + ) return nil } From 570f30b9307137c1b10c368e58bc636581bbe76d Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 4 Jun 2021 10:46:59 +0200 Subject: [PATCH 2/2] propagate the error --- pkg/cluster/pod.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/cluster/pod.go b/pkg/cluster/pod.go index a1cc0ffd4..229648dd1 100644 --- a/pkg/cluster/pod.go +++ b/pkg/cluster/pod.go @@ -304,7 +304,7 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error { } masterCandidateName := util.NameFromMeta(masterCandidatePod.ObjectMeta) - _ = retryutil.Retry(1*time.Minute, 5*time.Minute, + err = retryutil.Retry(1*time.Minute, 5*time.Minute, func() (bool, error) { err := c.Switchover(oldMaster, masterCandidateName) if err != nil { @@ -315,6 +315,10 @@ func (c *Cluster) MigrateMasterPod(podName spec.NamespacedName) error { }, ) + if err != nil { + return fmt.Errorf("could not migrate master pod: %v", err) + } + return nil }