Fix ScyllaDB version upgrade being stuck on Pod staying in maintenanc…

…e mode

Service controller copied maintenance mode label into required object
from cached informer when it was present.
Labels are also rewritten from existing objects in all Apply methods.
If maitenance mode label was removed in between when required object is
constructed, and Apply method is called, it would make it persitently
available in the Service object.
Instead adding it to required object, it should be carried over from
existing object only in Apply method, because this label is managed by
patches.

Fixes #1252

pkg/controller/scyllacluster/resource.go

@@ -80,12 +80,6 @@ func MemberService(sc *scyllav1.ScyllaCluster, rackName, name string, oldService
 		if hasReplaceLabel {
 			labels[naming.ReplaceLabel] = replaceAddr
 		}
-
-		// Copy the maintenance label, if present
-		oldMaintenanceLabel, oldMaintenanceLabelPresent := oldService.Labels[naming.NodeMaintenanceLabel]
-		if oldMaintenanceLabelPresent {
-			labels[naming.NodeMaintenanceLabel] = oldMaintenanceLabel
-		}
 	}
 
 	// Only new service should get the replace address, old service keeps "" until deleted.

pkg/controller/scyllacluster/resource_test.go

@@ -333,6 +333,32 @@ func TestMemberService(t *testing.T) {
 				},
 			},
 		},
+		{
+			name:          "existing service with maintenance mode label, it is not carried over into required object - #1252",
+			scyllaCluster: basicSC,
+			rackName:      basicRackName,
+			svcName:       basicSVCName,
+			oldService: &corev1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						naming.NodeMaintenanceLabel: "42",
+					},
+				},
+			},
+			expectedService: &corev1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:            basicSVCName,
+					Labels:          basicSVCLabels(),
+					OwnerReferences: basicSCOwnerRefs,
+				},
+				Spec: corev1.ServiceSpec{
+					Type:                     corev1.ServiceTypeClusterIP,
+					Selector:                 basicSVCSelector,
+					PublishNotReadyAddresses: true,
+					Ports:                    basicPorts,
+				},
+			},
+		},
 	}
 
 	for _, tc := range tt {