MGMT-17365: Add support for setting agent labels in BMH as annotations

BMH is the custom resource that is used to add a new host. Agent on the
other hand is created automatically when a host registers. Since there
is a need to control agent labels the following agent label support was
added:

In order to add an entry that controls agent label, a new BMH annotation
needs to be added.
The annotation key is prefixed with the string
'bmac.agent-install.openshift.io.agent-label.'.  The remainder of the
annotation is considered the label key.
The value of the annotation is identical to the agent label value.

Note: agent labels cannot be deleted by usage of BMH annotations.

docs/hive-integration/README.md

@@ -171,6 +171,8 @@ In case that the Bare Metal Operator is installed, the Baremetal Agent Controlle
       - `bmac.agent-install.openshift.io/installer-args`
     - IgnitionConfigOverrides (optional for user to set)
       - `bmac.agent-install.openshift.io/ignition-config-overrides`
+    - AgentLabels (optional for user to set)
+      -  `bmac.agent-install.openshift.io.agent-label.` (prefix)
 - Reconcile the BareMetalHost hardware details by copying the Agent's inventory data to the BMH's `hardwaredetails` annotation.
 - Disable ironic inspection
 
@@ -341,6 +343,28 @@ metadata:
   namespace: mynamespace
 spec:
 ```
+### Creating agent labels from BMH
+
+There is an option to add agent labels from BMH.  In order to add agent label, a BMH annotation is added.  The annotation key
+has a prefix __bmac.agent-install.openshift.io.agent-label.__. The suffix of the annotation is regarded as the agent label key.
+The annotation value is the agent label value.
+
+Note: Agent labels are just added from BMH annotations. They are not removed of there is no corresponding BMH annotation.
+
+Here is an example of such BMH annotation representing an agent label:
+
+```yaml
+apiVersion: metal3.io/v1alpha1
+kind: BareMetalHost
+metadata:
+  annotations:
+    bmac.agent-install.openshift.io.agent-label.agent-label: 'label-value'
+  creationTimestamp: "2021-04-14T10:46:57Z"
+  generation: 1
+  name: openshift-worker-0
+  namespace: mynamespace
+spec:
+```
 
 ### Creating Additional manifests
 

internal/controller/controllers/bmh_agent_controller.go

@@ -95,6 +95,7 @@ const (
 	OPENSHIFT_MACHINE_API_NAMESPACE     = "openshift-machine-api"
 	ASSISTED_DEPLOY_METHOD              = "start_assisted_install"
 	NODE_LABEL_PREFIX                   = "bmac.agent-install.openshift.io.node-label."
+	AGENT_LABEL_PREFIX                  = "bmac.agent-install.openshift.io.agent-label."
 
 	BMH_NODE_DRAIN_START_ANNOTATION   = "bmac.agent-install.openshift.io/drain-started-at"
 	BMH_NODE_DRAIN_TIMEOUT_ANNOTATION = "bmac.agent-install.openshift.io/drain-timeout" // in time.Duration format
@@ -490,6 +491,15 @@ func (r *BMACReconciler) reconcileAgentSpec(log logrus.FieldLogger, bmh *bmh_v1a
 		dirty = true
 	}
 
+	setDirty, err = r.reconcileAgentLabels(bmh, agent)
+	if err != nil {
+		log.WithError(err).Errorf("failed to reconcile bmh agent labels %s/%s", bmh.Namespace, bmh.Name)
+		return reconcileError{err: err}
+	}
+	if setDirty {
+		dirty = true
+	}
+
 	log.Debugf("Agent spec reconcile finished:  %v", agent)
 
 	return reconcileComplete{dirty: dirty}
@@ -513,6 +523,29 @@ func (r *BMACReconciler) reconcileNodeLabels(bmh *bmh_v1alpha1.BareMetalHost, ag
 	return false
 }
 
+func (r *BMACReconciler) reconcileAgentLabels(bmh *bmh_v1alpha1.BareMetalHost, agent *aiv1beta1.Agent) (bool, error) {
+	agentLabels := make(map[string]string)
+
+	for key, value := range bmh.ObjectMeta.GetAnnotations() {
+		if strings.HasPrefix(key, AGENT_LABEL_PREFIX) {
+			agentLabels[key[len(AGENT_LABEL_PREFIX):]] = value
+		}
+	}
+	var ret bool
+	labels := agent.GetLabels()
+	for key, value := range agentLabels {
+		existingValue, exists := getLabel(labels, key)
+		if !exists || existingValue != value {
+			labels = AddLabel(labels, key, value)
+			ret = true
+		}
+	}
+	if ret {
+		agent.SetLabels(labels)
+	}
+	return ret, nil
+}
+
 func (r *BMACReconciler) reconcileClusterReference(bmh *bmh_v1alpha1.BareMetalHost, agent *aiv1beta1.Agent, infraEnv *aiv1beta1.InfraEnv) (bool, error) {
 	clusterReferenceStr, annotationExists := bmh.Annotations[BMH_CLUSTER_REFERENCE]
 

internal/controller/controllers/bmh_agent_controller_test.go

@@ -658,6 +658,67 @@ var _ = Describe("bmac reconcile", func() {
 					Expect(updatedAgent.Spec.NodeLabels).To(BeEmpty())
 				})
 			})
+			Context("should set the agent labels based on the corresponding BMH annotations", func() {
+				addAnnotation := func(bmh *bmh_v1alpha1.BareMetalHost, key, value string) {
+					if bmh.Annotations == nil {
+						bmh.Annotations = make(map[string]string)
+					}
+					bmh.Annotations[key] = value
+					Expect(c.Update(ctx, bmh)).ToNot(HaveOccurred())
+				}
+				addLabel := func(bmh *bmh_v1alpha1.BareMetalHost, key, value string) {
+					addAnnotation(bmh, AGENT_LABEL_PREFIX+key, value)
+				}
+				expectToContainKeyValue := func(agent *v1beta1.Agent, key, value string) {
+					v, exists := getLabel(agent.Labels, key)
+					Expect(exists).To(BeTrue())
+					Expect(v).To(Equal(value))
+				}
+				expectToNotContainKey := func(agent *v1beta1.Agent, key string) {
+					_, exists := getLabel(agent.Labels, key)
+					Expect(exists).To(BeFalse())
+				}
+				BeforeEach(func() {
+					addLabel(host, "initial-key", "initial-value")
+				})
+				It("set initial agent labels", func() {
+					result, err := bmhr.Reconcile(ctx, newBMHRequest(host))
+					Expect(err).To(BeNil())
+					Expect(result).To(Equal(ctrl.Result{}))
+
+					updatedAgent := &v1beta1.Agent{}
+					err = c.Get(ctx, types.NamespacedName{Name: agent.Name, Namespace: agent.Namespace}, updatedAgent)
+					Expect(err).To(BeNil())
+					expectToContainKeyValue(updatedAgent, "initial-key", "initial-value")
+				})
+				It("modify agent labels", func() {
+					h2 := &bmh_v1alpha1.BareMetalHost{}
+					Expect(c.Get(ctx, types.NamespacedName{Name: host.Name, Namespace: host.Namespace}, h2)).ToNot(HaveOccurred())
+					addLabel(h2, "third-label", "blah")
+					addLabel(h2, "forth-label", "forth")
+					Expect(c.Update(ctx, h2)).ToNot(HaveOccurred())
+					result, err := bmhr.Reconcile(ctx, newBMHRequest(host))
+					Expect(err).To(BeNil())
+					Expect(result).To(Equal(ctrl.Result{}))
+
+					updatedAgent := &v1beta1.Agent{}
+					Expect(c.Get(ctx, types.NamespacedName{Name: agent.Name, Namespace: agent.Namespace}, updatedAgent)).ToNot(HaveOccurred())
+					expectToContainKeyValue(updatedAgent, "third-label", "blah")
+					expectToContainKeyValue(updatedAgent, "forth-label", "forth")
+					expectToContainKeyValue(updatedAgent, "initial-key", "initial-value")
+					expectToNotContainKey(updatedAgent, "initial-key1")
+
+					Expect(c.Get(ctx, types.NamespacedName{Name: host.Name, Namespace: host.Namespace}, h2)).ToNot(HaveOccurred())
+					addLabel(h2, "forth-label", "forth-label-value")
+					Expect(c.Update(ctx, updatedAgent)).ToNot(HaveOccurred())
+					result, err = bmhr.Reconcile(ctx, newBMHRequest(h2))
+					Expect(err).To(BeNil())
+					Expect(result).To(Equal(ctrl.Result{}))
+					err = c.Get(ctx, types.NamespacedName{Name: agent.Name, Namespace: agent.Namespace}, updatedAgent)
+					Expect(err).To(BeNil())
+					expectToContainKeyValue(updatedAgent, "forth-label", "forth-label-value")
+				})
+			})
 			Context("reconcile cluster reference", func() {
 				const (
 					clusterName              = "cluster-name"

internal/controller/controllers/crd_utils.go

@@ -185,3 +185,11 @@ func AddLabel(labels map[string]string, labelKey, labelValue string) map[string]
 	labels[labelKey] = labelValue
 	return labels
 }
+
+func getLabel(labels map[string]string, labelKey string) (value string, exists bool) {
+	if labels == nil {
+		return "", false
+	}
+	value, exists = labels[labelKey]
+	return
+}