Merge pull request #81 from hexfusion/fix-scaleup

Bug 1958913: discover-etcd-initial-cluster: retry if member is not part of member list and dataDir exists

openshift-tools/pkg/discover-etcd-initial-cluster/initial-cluster.go

@@ -198,15 +198,15 @@ func (o *DiscoverEtcdInitialClusterOptions) Run() error {
 		// this is the latter. Do not let etcd start and report the condition as an error.
 		// Result: return error and restart container
 		if found && !dataDirExists {
-			return fmt.Errorf("member %q dataDir has been destoyed and must be removed from the cluster", target.String())
+			return fmt.Errorf("member %q dataDir has been destroyed and must be removed from the cluster", target.String())
 		}
 
 		// Condition: member not found with dataDir
-		// The member has been removed from the cluster. etcd should not start and return error. The dataDir will be archived once the operator
+		// The member has been removed from the cluster. The dataDir will be archived once the operator
 		// scales up etcd.
-		// Result: return error and restart container
+		// Result: retry member check allowing operator time to scale up etcd again on this node.
 		if !found && dataDirExists {
-			return fmt.Errorf("member %q is no longer a member of the cluster and should not start", target.String())
+			fmt.Fprintf(os.Stderr, "      member %q not found in member list but dataDir exists, check operator logs for possible scaling problems\n", target.String())
 		}
 
 		// Condition: member not found, no dataDir

openshift-tools/pkg/discover-etcd-initial-cluster/initial-cluster_test.go

@@ -99,7 +99,7 @@ func TestDiscoverInitialCluster(t *testing.T) {
 			dataDir:            noDataDir,
 			wantInitialCluster: emptyInitialCluster,
 			wantErr:            true,
-			wantErrString:      fmt.Sprintf("member \"unixs://127.0.0.1:%s\" dataDir has been destoyed and must be removed from the cluster", targetPort),
+			wantErrString:      fmt.Sprintf("member \"unixs://127.0.0.1:%s\" dataDir has been destroyed and must be removed from the cluster", targetPort),
 		},
 		"started member found with dataDir": {
 			targetName:         clus.Members[0].Name,
@@ -132,7 +132,7 @@ func TestDiscoverInitialCluster(t *testing.T) {
 			dataDir:            clus.Members[0].DataDir,
 			wantInitialCluster: emptyInitialCluster,
 			wantErr:            true,
-			wantErrString:      fmt.Sprintf("member \"unixs://%s:%s\" is no longer a member of the cluster and should not start", targetHostNotFound, targetPort),
+			wantErrString:      "timed out",
 		},
 		"member not found no dataDir": {
 			targetName:         "not-a-member",