cluster: clear rack status when rack STS doesn't exist (#418)

Rack StatefulSet is created when rack is defined in Spec, and it's
Status is not available. Because Operator was not clearing Rack Status when
STS was missing, stale record blocked creating new one.

Fixes #418

pkg/controllers/cluster/status.go

@@ -57,8 +57,9 @@ func (cc *ClusterReconciler) updateStatus(ctx context.Context, cluster *scyllav1
 		}
 		// Get corresponding StatefulSet from lister
 		err := cc.Get(ctx, naming.NamespacedName(naming.StatefulSetNameForRack(rack, cluster), cluster.Namespace), sts)
-		// If it wasn't found, continue
+		// If it wasn't found, clear rack status and continue
 		if apierrors.IsNotFound(err) {
+			delete(cluster.Status.Racks, rack.Name)
 			continue
 		}
 		// If we got a different error, requeue and log it

pkg/controllers/cluster/status_test.go

@@ -0,0 +1,34 @@
+// Copyright (C) 2021 ScyllaDB
+
+package cluster
+
+import (
+	"context"
+	"testing"
+
+	"github.com/scylladb/scylla-operator/pkg/test/unit"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+func TestClusterStatus_RackStatusIsRemovedWhenStatefulSetNoLongerExists(t *testing.T) {
+	cluster := unit.NewSingleRackCluster(1)
+	rack := cluster.Spec.Datacenter.Racks[0]
+
+	scheme := runtime.NewScheme()
+	if err := clientgoscheme.AddToScheme(scheme); err != nil {
+		t.Fatal(err)
+	}
+	cc := &ClusterReconciler{
+		Client: fake.NewFakeClientWithScheme(scheme),
+	}
+
+	if err := cc.updateStatus(context.Background(), cluster); err != nil {
+		t.Errorf("expected nil, got %s", err)
+	}
+
+	if v, ok := cluster.Status.Racks[rack.Name]; ok {
+		t.Errorf("expected rack status to be cleared, got %v", v)
+	}
+}