Setting machine instance state to unknown on machine failure

Bug 1808971: Machine status shows "running" when an instance was terminated

pkg/controller/machine/controller.go

@@ -87,6 +87,9 @@ const (
 	// Machine has a deletion timestamp
 	phaseDeleting = "Deleting"
 
+	// Hardcoded instance state set on machine failure
+	unknownInstanceState = "Unknown"
+
 	skipWaitForDeleteTimeoutSeconds = 60 * 5
 )
 
@@ -426,9 +429,17 @@ func (r *ReconcileMachine) setPhase(machine *machinev1.Machine, phase string, er
 		machine.Status.LastUpdated = &now
 		if phase == phaseFailed && errorMessage != "" {
 			machine.Status.ErrorMessage = &errorMessage
+			if machine.Annotations == nil {
+				machine.Annotations = map[string]string{}
+			}
+			machine.Annotations[MachineInstanceStateAnnotationName] = unknownInstanceState
+			if err := r.Client.Patch(context.Background(), machine, baseToPatch); err != nil {
+				klog.Errorf("Failed to update machine %q: %v", machine.GetName(), err)
+				return err
+			}
 		}
 		if err := r.Client.Status().Patch(context.Background(), machine, baseToPatch); err != nil {
-			klog.Errorf("Failed to update machine %q: %v", machine.GetName(), err)
+			klog.Errorf("Failed to update machine status %q: %v", machine.GetName(), err)
 			return err
 		}
 	}

pkg/controller/machine/controller_test.go

@@ -391,6 +391,8 @@ func TestSetPhase(t *testing.T) {
 
 	// Set phaseFailed with an errorMessage should store the message
 	expecterErrorMessage := "test"
+	// Set phaseFailed will result to force Unknown state for VM state display
+	expectedInstanceState := "Unknown"
 	if err := reconciler.setPhase(machine, phaseFailed, expecterErrorMessage); err != nil {
 		t.Fatal(err)
 	}
@@ -407,6 +409,15 @@ func TestSetPhase(t *testing.T) {
 	if expecterErrorMessage != *machine.Status.ErrorMessage {
 		t.Errorf("Expected: %v, got: %v", expecterErrorMessage, *machine.Status.ErrorMessage)
 	}
+	if got.Status.Phase == nil {
+		t.Fatal("Got phase nil")
+	}
+	if *got.Status.Phase != phaseFailed {
+		t.Errorf("Got: %v, expected: %v", *got.Status.Phase, phaseRunning)
+	}
+	if got.Annotations[MachineInstanceStateAnnotationName] != expectedInstanceState {
+		t.Errorf("Expected VM to go into: %v state, got: %v", expectedInstanceState, got.Annotations[MachineInstanceStateAnnotationName])
+	}
 }
 
 func TestMachineIsProvisioned(t *testing.T) {