[release-4.4] Bug 1843230: Read BMH status from annotation

pkg/apis/metal3/v1alpha1/baremetalhost_types.go

@@ -18,6 +18,9 @@ const (
 	// hosts to block delete operations until the physical host can be
 	// deprovisioned.
 	BareMetalHostFinalizer string = "baremetalhost.metal3.io"
+
+	// StatusAnnotation is the annotation that holds the Status of BMH
+	StatusAnnotation = "baremetalhost.metal3.io/status"
 )
 
 // OperationalStatus represents the state of the host

pkg/controller/baremetalhost/baremetalhost_controller.go

@@ -2,8 +2,10 @@ package baremetalhost
 
 import (
 	"context"
+	"encoding/json"
 	"flag"
 	"fmt"
+	"reflect"
 	"strings"
 	"time"
 
@@ -168,6 +170,29 @@ func (r *ReconcileBareMetalHost) Reconcile(request reconcile.Request) (result re
 		return reconcile.Result{}, errors.Wrap(err, "could not load host data")
 	}
 
+	// Check if Status is empty and status annotation is present
+	// Manually restore data.
+	if !r.hostHasStatus(host) {
+		reqLogger.Info("Fetching Status from Annotation")
+		objStatus, err := r.getHostStatusFromAnnotation(host)
+		if err == nil && objStatus != nil {
+			host.Status = *objStatus
+			if host.Status.LastUpdated.IsZero() {
+				// Ensure the LastUpdated timestamp in set to avoid
+				// infinite loops if the annotation only contained
+				// part of the status information.
+				t := metav1.Now()
+				host.Status.LastUpdated = &t
+			}
+			errStatus := r.client.Status().Update(context.TODO(), host)
+			if errStatus != nil {
+				return reconcile.Result{}, errors.Wrap(err, "Could not restore status from annotation")
+			}
+			return reconcile.Result{Requeue: true}, nil
+		}
+		reqLogger.Info("No status cache found")
+	}
+
 	// NOTE(dhellmann): Handle a few steps outside of the phase
 	// structure because they require extra data lookup (like the
 	// credential checks) or have to be done "first" (like delete
@@ -229,10 +254,12 @@ func (r *ReconcileBareMetalHost) Reconcile(request reconcile.Request) (result re
 	// over when there is an unrecoverable error (tracked through the
 	// error state of the host).
 	if actResult.Dirty() {
+
+		// Save Host
 		info.log.Info("saving host status",
 			"operational status", host.OperationalStatus(),
 			"provisioning state", host.Status.Provisioning.State)
-		if err = r.saveStatus(host); err != nil {
+		if err = r.saveHostStatus(host); err != nil {
 			return reconcile.Result{}, errors.Wrap(err,
 				fmt.Sprintf("failed to save host status after %q", initialState))
 		}
@@ -297,7 +324,7 @@ func (r *ReconcileBareMetalHost) credentialsErrorResult(err error, request recon
 			// overwrites our discovered state
 			host.Status.ErrorMessage = err.Error()
 			host.Status.ErrorType = ""
-			saveErr := r.saveStatus(host)
+			saveErr := r.saveHostStatus(host)
 			if saveErr != nil {
 				return reconcile.Result{Requeue: true}, saveErr
 			}
@@ -361,7 +388,7 @@ func (r *ReconcileBareMetalHost) actionDeleting(prov provisioner.Provisioner, in
 		return actionError{errors.Wrap(err, "failed to delete")}
 	}
 	if provResult.Dirty {
-		err = r.saveStatus(info.host)
+		err = r.saveHostStatus(info.host)
 		if err != nil {
 			return actionError{errors.Wrap(err, "failed to save host after deleting")}
 		}
@@ -690,10 +717,86 @@ func (r *ReconcileBareMetalHost) actionManageReady(prov provisioner.Provisioner,
 	return r.manageHostPower(prov, info)
 }
 
-func (r *ReconcileBareMetalHost) saveStatus(host *metal3v1alpha1.BareMetalHost) error {
+func (r *ReconcileBareMetalHost) saveHostStatus(host *metal3v1alpha1.BareMetalHost) error {
 	t := metav1.Now()
 	host.Status.LastUpdated = &t
-	return r.client.Status().Update(context.TODO(), host)
+
+	if err := r.saveHostAnnotation(host); err != nil {
+		return err
+	}
+
+	//Refetch host again
+	obj := host.Status.DeepCopy()
+	err := r.client.Get(context.TODO(),
+		client.ObjectKey{
+			Name:      host.Name,
+			Namespace: host.Namespace,
+		},
+		host,
+	)
+	if err != nil {
+		return errors.Wrap(err, "Failed to update Status annotation")
+	}
+	host.Status = *obj
+	err = r.client.Status().Update(context.TODO(), host)
+	return err
+}
+
+func (r *ReconcileBareMetalHost) saveHostAnnotation(host *metal3v1alpha1.BareMetalHost) error {
+	//Repopulate annotation again
+	objStatus, err := r.getHostStatusFromAnnotation(host)
+	if err != nil {
+		return err
+	}
+
+	if objStatus != nil {
+		// This value is copied to avoid continually updating the annotation
+		objStatus.LastUpdated = host.Status.LastUpdated
+		if reflect.DeepEqual(host.Status, *objStatus) {
+			return nil
+		}
+	}
+
+	delete(host.Annotations, metal3v1alpha1.StatusAnnotation)
+	newAnnotation, err := marshalStatusAnnotation(&host.Status)
+	if err != nil {
+		return err
+	}
+	if host.Annotations == nil {
+		host.Annotations = make(map[string]string)
+	}
+	host.Annotations[metal3v1alpha1.StatusAnnotation] = string(newAnnotation)
+	return r.client.Update(context.TODO(), host.DeepCopy())
+}
+
+func marshalStatusAnnotation(status *metal3v1alpha1.BareMetalHostStatus) ([]byte, error) {
+	newAnnotation, err := json.Marshal(status)
+	if err != nil {
+		return []byte{}, errors.Wrap(err, "failed to marshall status annotation")
+	}
+	return newAnnotation, nil
+}
+
+func unmarshalStatusAnnotation(content []byte) (*metal3v1alpha1.BareMetalHostStatus, error) {
+	objStatus := &metal3v1alpha1.BareMetalHostStatus{}
+	if err := json.Unmarshal(content, objStatus); err != nil {
+		return nil, errors.Wrap(err, "Failed to fetch Status from annotation")
+	}
+	return objStatus, nil
+}
+
+// extract host from Status annotation
+func (r *ReconcileBareMetalHost) getHostStatusFromAnnotation(host *metal3v1alpha1.BareMetalHost) (*metal3v1alpha1.BareMetalHostStatus, error) {
+	annotations := host.GetAnnotations()
+	content := []byte(annotations[metal3v1alpha1.StatusAnnotation])
+	if annotations[metal3v1alpha1.StatusAnnotation] == "" {
+		return nil, nil
+	}
+	objStatus, err := unmarshalStatusAnnotation(content)
+	if err != nil {
+		return nil, err
+	}
+	return objStatus, nil
 }
 
 func (r *ReconcileBareMetalHost) setErrorCondition(request reconcile.Request, host *metal3v1alpha1.BareMetalHost, errType metal3v1alpha1.ErrorType, message string) (changed bool, err error) {
@@ -706,7 +809,7 @@ func (r *ReconcileBareMetalHost) setErrorCondition(request reconcile.Request, ho
 			"adding error message",
 			"message", message,
 		)
-		err = r.saveStatus(host)
+		err = r.saveHostStatus(host)
 		if err != nil {
 			err = errors.Wrap(err, "failed to update error message")
 		}
@@ -813,6 +916,10 @@ func (r *ReconcileBareMetalHost) publishEvent(request reconcile.Request, event c
 	return
 }
 
+func (r *ReconcileBareMetalHost) hostHasStatus(host *metal3v1alpha1.BareMetalHost) bool {
+	return !host.Status.LastUpdated.IsZero()
+}
+
 func hostHasFinalizer(host *metal3v1alpha1.BareMetalHost) bool {
 	return utils.StringInList(host.Finalizers, metal3v1alpha1.BareMetalHostFinalizer)
 }

pkg/controller/baremetalhost/baremetalhost_controller_test.go

@@ -4,6 +4,7 @@ import (
 	goctx "context"
 	"encoding/base64"
 	"fmt"
+	"reflect"
 	"testing"
 
 	corev1 "k8s.io/api/core/v1"
@@ -27,6 +28,7 @@ import (
 const (
 	namespace         string = "test-namespace"
 	defaultSecretName string = "bmc-creds-valid"
+	statusAnnotation  string = `{"operationalStatus":"OK","lastUpdated":"2020-04-15T15:00:50Z","hardwareProfile":"StatusProfile","hardware":{"systemVendor":{"manufacturer":"QEMU","productName":"Standard PC (Q35 + ICH9, 2009)","serialNumber":""},"firmware":{"bios":{"date":"","vendor":"","version":""}},"ramMebibytes":4096,"nics":[{"name":"eth0","model":"0x1af4 0x0001","mac":"00:b7:8b:bb:3d:f6","ip":"172.22.0.64","speedGbps":0,"vlanId":0,"pxe":true},{"name":"eth1","model":"0x1af4  0x0001","mac":"00:b7:8b:bb:3d:f8","ip":"192.168.111.20","speedGbps":0,"vlanId":0,"pxe":false}],"storage":[{"name":"/dev/sda","rotational":true,"sizeBytes":53687091200,"vendor":"QEMU","model":"QEMU HARDDISK","serialNumber":"drive-scsi0-0-0-0","hctl":"6:0:0:0"}],"cpu":{"arch":"x86_64","model":"Intel Xeon E3-12xx v2 (IvyBridge)","clockMegahertz":2494.224,"flags":["aes","apic","arat","avx","clflush","cmov","constant_tsc","cx16","cx8","de","eagerfpu","ept","erms","f16c","flexpriority","fpu","fsgsbase","fxsr","hypervisor","lahf_lm","lm","mca","mce","mmx","msr","mtrr","nopl","nx","pae","pat","pclmulqdq","pge","pni","popcnt","pse","pse36","rdrand","rdtscp","rep_good","sep","smep","sse","sse2","sse4_1","sse4_2","ssse3","syscall","tpr_shadow","tsc","tsc_adjust","tsc_deadline_timer","vme","vmx","vnmi","vpid","x2apic","xsave","xsaveopt","xtopology"],"count":4},"hostname":"node-0"},"provisioning":{"state":"provisioned","ID":"8a0ede17-7b87-44ac-9293-5b7d50b94b08","image":{"url":"bar","checksum":""}},"goodCredentials":{"credentials":{"name":"node-0-bmc-secret","namespace":"metal3"},"credentialsVersion":"879"},"triedCredentials":{"credentials":{"name":"node-0-bmc-secret","namespace":"metal3"},"credentialsVersion":"879"},"errorMessage":"","poweredOn":true,"operationHistory":{"register":{"start":"2020-04-15T12:06:26Z","end":"2020-04-15T12:07:12Z"},"inspect":{"start":"2020-04-15T12:07:12Z","end":"2020-04-15T12:09:29Z"},"provision":{"start":null,"end":null},"deprovision":{"start":null,"end":null}}}`
 )
 
 func init() {
@@ -121,6 +123,7 @@ func tryReconcile(t *testing.T, r *ReconcileBareMetalHost, host *metal3v1alpha1.
 		}
 
 		result, err := r.Reconcile(request)
+
 		if err != nil {
 			t.Fatal(err)
 			break
@@ -184,6 +187,119 @@ func waitForProvisioningState(t *testing.T, r *ReconcileBareMetalHost, host *met
 	)
 }
 
+// TestStatusAnnotation_EmptyStatus ensures that status is manually populated
+// when status annotation is present and status field is empty.
+func TestStatusAnnotation_EmptyStatus(t *testing.T) {
+	host := newDefaultHost(t)
+	host.Annotations = map[string]string{
+		metal3v1alpha1.StatusAnnotation: statusAnnotation,
+	}
+	host.Spec.Online = true
+	host.Spec.Image = &metal3v1alpha1.Image{URL: "foo", Checksum: "123"}
+
+	r := newTestReconciler(host)
+
+	tryReconcile(t, r, host,
+		func(host *metal3v1alpha1.BareMetalHost, result reconcile.Result) bool {
+			if host.Status.HardwareProfile == "StatusProfile" && host.Status.Provisioning.Image.URL == "bar" {
+				return true
+			}
+			return false
+		},
+	)
+}
+
+// TestStatusAnnotation_StatusPresent tests that if status is present
+// status annotation is ignored.
+func TestStatusAnnotation_StatusPresent(t *testing.T) {
+	host := newDefaultHost(t)
+	host.Annotations = map[string]string{
+		metal3v1alpha1.StatusAnnotation: statusAnnotation,
+	}
+	host.Spec.Online = true
+	time := metav1.Now()
+	host.Status.LastUpdated = &time
+	host.Status.Provisioning.Image = metal3v1alpha1.Image{URL: "foo", Checksum: "123"}
+	r := newTestReconciler(host)
+
+	tryReconcile(t, r, host,
+		func(host *metal3v1alpha1.BareMetalHost, result reconcile.Result) bool {
+			if host.Status.HardwareProfile != "StatusProfile" && host.Status.Provisioning.Image.URL == "foo" {
+				return true
+			}
+			return false
+		},
+	)
+}
+
+// TestStatusAnnotation_Partial ensures that if the status annotation
+// does not include the LastUpdated value reconciliation does not go
+// into an infinite loop.
+func TestStatusAnnotation_Partial(t *testing.T) {
+	// Build a version of the annotation text that does not include
+	// a LastUpdated value.
+	unpackedStatus, err := unmarshalStatusAnnotation([]byte(statusAnnotation))
+	if err != nil {
+		t.Fatal(err)
+		return
+	}
+	unpackedStatus.LastUpdated = nil
+	packedStatus, err := marshalStatusAnnotation(unpackedStatus)
+	if err != nil {
+		t.Fatal(err)
+		return
+	}
+
+	host := newDefaultHost(t)
+	host.Annotations = map[string]string{
+		metal3v1alpha1.StatusAnnotation: string(packedStatus),
+	}
+	host.Spec.Online = true
+	host.Spec.Image = &metal3v1alpha1.Image{URL: "foo", Checksum: "123"}
+
+	r := newTestReconciler(host)
+
+	tryReconcile(t, r, host,
+		func(host *metal3v1alpha1.BareMetalHost, result reconcile.Result) bool {
+			if host.Status.HardwareProfile == "StatusProfile" && host.Status.Provisioning.Image.URL == "bar" {
+				return true
+			}
+			return false
+		},
+	)
+}
+
+// TestStatusAnnotation tests if statusAnnotation is populated correctly
+func TestStatusAnnotation(t *testing.T) {
+	host := newDefaultHost(t)
+	host.Spec.Online = true
+	host.Spec.Image = &metal3v1alpha1.Image{URL: "foo", Checksum: "123"}
+	bmcSecret := newSecret(defaultSecretName, "User", "Pass")
+	r := newTestReconciler(host, bmcSecret)
+
+	tryReconcile(t, r, host,
+		func(host *metal3v1alpha1.BareMetalHost, result reconcile.Result) bool {
+			if utils.StringInList(host.Finalizers, metal3v1alpha1.BareMetalHostFinalizer) {
+				return true
+			}
+			return false
+		},
+	)
+
+	tryReconcile(t, r, host,
+		func(host *metal3v1alpha1.BareMetalHost, result reconcile.Result) bool {
+			objStatus, _ := r.getHostStatusFromAnnotation(host)
+			objStatus.LastUpdated = host.Status.LastUpdated
+
+			if reflect.DeepEqual(host.Status, *objStatus) {
+				return true
+			}
+			return false
+		},
+	)
+
+}
+
 // TestAddFinalizers ensures that the finalizers for the host are
 // updated as part of reconciling it.
 func TestAddFinalizers(t *testing.T) {