tag metrics with pod labels if configured to do so

README.adoc

@@ -663,6 +663,15 @@ kubernetes:
 ----
 |Restricts the number of metrics that will be stored for each pod being monitored.
 
+|K8S_POD_LABEL_TAGS_PREFIX
+|
+[source,yaml]
+----
+kubernetes:
+  pod_label_tags_prefix: VALUE
+----
+| Pods might have one or more labels (name/value pairs). You can tag each metric with these pod labels if "pod_label_tags_prefix" is not an empty string. If it is an empty string or not specified, these tags will not be created. When not an empty string, for each label on a pod a tag will be placed on each pod metric with this string prefixing the pod label name. For example, if the prefix string is `labels.` and a pod has a label `something=foo` then that pod's metrics will have a tag named `labels.something` with value of `foo`. If you wish to create these tags with no prefix (that is, you want the tag names to be exactly the same as the label names) set the prefix value to `\_empty_`.
+
 |COLLECTOR_MINIMUM_COLLECTION_INTERVAL
 |
 [source,yaml]
@@ -875,3 +884,17 @@ tags:
   my-pod-name: ${POD:name}
   some-env-tag: var is ${SOME_ENV_VAR}
 ----
+
+=== Pod Label Tags
+
+There is a special setting in the Kubernetes section of the agent global configuration called `pod_label_tags_prefix`. When specified and not an empty string, this enables pod label tags to be created. This means every metric from every pod will get one tag per pod label with the name of the tag that of the pod label prefixed with the string defined in this `pod_label_tags_prefix` setting. If you wish to create these tags with no prefix (that is, you want the tag names to be exactly the same as the label names) set the prefix value to `\_empty_`.
+
+For example, if the agent global configuration has this:
+
+[source,yaml]
+----
+kubernetes:
+  pod_label_tags_prefix: labels.
+----
+
+and a pod has a label `something=foo` then that pod's metrics will have a tag named `labels.something` with a value of `foo`. If the prefix string was set to `\_empty_`, the tag will be named the same as the label name which in this example is `something`.

config/config.go

@@ -44,13 +44,14 @@ const (
 	ENV_IDENTITY_CERT_FILE        = "HAWKULAR_OPENSHIFT_AGENT_CERT_FILE"
 	ENV_IDENTITY_PRIVATE_KEY_FILE = "HAWKULAR_OPENSHIFT_AGENT_PRIVATE_KEY_FILE"
 
-	ENV_K8S_MASTER_URL          = "K8S_MASTER_URL"
-	ENV_K8S_POD_NAMESPACE       = "K8S_POD_NAMESPACE"
-	ENV_K8S_POD_NAME            = "K8S_POD_NAME"
-	ENV_K8S_TOKEN               = "K8S_TOKEN"
-	ENV_K8S_CA_CERT_FILE        = "K8S_CA_CERT_FILE"
-	ENV_K8S_TENANT              = "K8S_TENANT"
-	ENV_K8S_MAX_METRICS_PER_POD = "K8S_MAX_METRICS_PER_POD"
+	ENV_K8S_MASTER_URL            = "K8S_MASTER_URL"
+	ENV_K8S_POD_NAMESPACE         = "K8S_POD_NAMESPACE"
+	ENV_K8S_POD_NAME              = "K8S_POD_NAME"
+	ENV_K8S_TOKEN                 = "K8S_TOKEN"
+	ENV_K8S_CA_CERT_FILE          = "K8S_CA_CERT_FILE"
+	ENV_K8S_TENANT                = "K8S_TENANT"
+	ENV_K8S_MAX_METRICS_PER_POD   = "K8S_MAX_METRICS_PER_POD"
+	ENV_K8S_POD_LABEL_TAGS_PREFIX = "K8S_POD_LABEL_TAGS_PREFIX"
 
 	ENV_EMITTER_ADDRESS                      = "EMITTER_ADDRESS"
 	ENV_EMITTER_METRICS_ENABLED              = "EMITTER_METRICS_ENABLED"
@@ -91,25 +92,39 @@ type Collector struct {
 }
 
 // Kubernetes provides all the details necessary to communicate with the Kubernetes system.
+//
 // Master_Url should be an empty string if the agent is deployed in OpenShift.
+//
 // Pod_Namespace and Pod_Name should identify the pod where the agent is running (if it is
 // running in OpenShift) or should identify any pod in the node to be monitored by the agent
 // (if the agent is not running in OpenShift). Pod_Namespace should be empty if you do not wish
 // for the agent to monitor anything in OpenShift.
+//
 // If Tenant is supplied, all metrics collected from all pods will have this tenant.
 // You can specify ${x} tokens in the value for Tenant, such as ${some_env} or one of the POD
 // tokens such as ${POD:namespace_name} which means all metrics will be stored under a tenant
 // that is the same name of the pod namespace where the metric was collected.
 // If Tenant is not supplied, the default is the Tenant defined in the Hawkular_Server section.
+//
+// Kubernetes pods might have one or more labels (name/value pairs). You can tag each metric
+// with these pod labels if "Pod_Label_Tags_Prefix" is not an empty string. If it is an empty
+// string or not specified, these tags will not be created. When not an empty string, for each
+// label on a pod a tag will be placed on each pod metric with this string prefixing the
+// pod label name. For example, if the prefix string is "labels." and a pod has a
+// label "something=foo" then that pod's metrics will have a tag named "labels.something"
+// with value of "foo". If you wish to create these tags with no prefix (that is, you want the
+// tag names to be exactly the same as the label names) set the prefix value to "_empty_".
+//
 // USED FOR YAML
 type Kubernetes struct {
-	Master_URL          string ",omitempty"
-	Token               string ",omitempty"
-	CA_Cert_File        string ",omitempty"
-	Pod_Namespace       string ",omitempty"
-	Pod_Name            string ",omitempty"
-	Tenant              string ",omitempty"
-	Max_Metrics_Per_Pod int    ",omitempty"
+	Master_URL            string ",omitempty"
+	Token                 string ",omitempty"
+	CA_Cert_File          string ",omitempty"
+	Pod_Namespace         string ",omitempty"
+	Pod_Name              string ",omitempty"
+	Tenant                string ",omitempty"
+	Max_Metrics_Per_Pod   int    ",omitempty"
+	Pod_Label_Tags_Prefix string ",omitempty"
 }
 
 // Emitter defines the behavior of the emitter which is responsible for
@@ -160,6 +175,7 @@ func NewConfig() (c *Config) {
 	c.Kubernetes.CA_Cert_File = getDefaultString(ENV_K8S_CA_CERT_FILE, "")
 	c.Kubernetes.Tenant = getDefaultString(ENV_K8S_TENANT, "")
 	c.Kubernetes.Max_Metrics_Per_Pod = getDefaultInt(ENV_K8S_MAX_METRICS_PER_POD, 50)
+	c.Kubernetes.Pod_Label_Tags_Prefix = getDefaultString(ENV_K8S_POD_LABEL_TAGS_PREFIX, "")
 
 	c.Emitter.Metrics_Enabled = getDefaultString(ENV_EMITTER_METRICS_ENABLED, "true")
 	c.Emitter.Status_Enabled = getDefaultString(ENV_EMITTER_STATUS_ENABLED, "false")

config/config_test.go

@@ -138,6 +138,9 @@ func TestDefaults(t *testing.T) {
 	if conf.Kubernetes.Max_Metrics_Per_Pod != 50 {
 		t.Error("Max metrics per pod default is wrong")
 	}
+	if conf.Kubernetes.Pod_Label_Tags_Prefix != "" {
+		t.Error("Pod label tags prefix default is wrong")
+	}
 	if len(conf.Endpoints) != 0 {
 		t.Error("There should be no endpoints by default")
 	}
@@ -174,9 +177,10 @@ func TestMarshalUnmarshal(t *testing.T) {
 			URL: "http://server:80",
 		},
 		Kubernetes: Kubernetes{
-			Pod_Namespace:       "TestMarshalUnmarshal namespace",
-			Pod_Name:            "TestMarshalUnmarshal name",
-			Max_Metrics_Per_Pod: 123,
+			Pod_Namespace:         "TestMarshalUnmarshal namespace",
+			Pod_Name:              "TestMarshalUnmarshal name",
+			Max_Metrics_Per_Pod:   123,
+			Pod_Label_Tags_Prefix: "labels.",
 		},
 		Emitter: Emitter{
 			Metrics_Enabled: "false",
@@ -246,6 +250,9 @@ func TestMarshalUnmarshal(t *testing.T) {
 	if conf.Kubernetes.Max_Metrics_Per_Pod != 123 {
 		t.Errorf("Failed to unmarshal max metrics per pod:\n%v", conf)
 	}
+	if conf.Kubernetes.Pod_Label_Tags_Prefix != "labels." {
+		t.Error("Pod Label Tags Prefix is wrong")
+	}
 	if conf.Endpoints[0].Collection_Interval != "123s" {
 		t.Error("First endpoint is not correct")
 	}

deploy/openshift/hawkular-openshift-agent-configmap.yaml

@@ -10,6 +10,7 @@ data:
   config.yaml: |
     kubernetes:
       tenant: ${POD:namespace_name}
+      pod_label_tags_prefix: labels.
     hawkular_server:
       url: https://hawkular-metrics.openshift-infra.svc.cluster.local       
       credentials:

k8s/node_event_consumer.go

@@ -26,6 +26,7 @@ import (
 	"github.com/hawkular/hawkular-openshift-agent/collector/manager"
 	"github.com/hawkular/hawkular-openshift-agent/config"
 	"github.com/hawkular/hawkular-openshift-agent/config/security"
+	"github.com/hawkular/hawkular-openshift-agent/config/tags"
 	"github.com/hawkular/hawkular-openshift-agent/log"
 	"github.com/hawkular/hawkular-openshift-agent/util/expand"
 )
@@ -242,6 +243,21 @@ func (nec *NodeEventConsumer) startCollecting(ne *NodeEvent) {
 			continue
 		}
 
+		// if all pod labels should be used as tags on all metrics, create those tags from the pod labels.
+		endpointTags := tags.Tags{}
+		endpointTags.AppendTags(cmeEndpoint.Tags)
+		if nec.Config.Kubernetes.Pod_Label_Tags_Prefix != "" {
+			// each label will be prefixed with the configured prefix string unless that prefix
+			// was "_empty_" in which case the user is telling us to use the label name as-is with no prefix.
+			prefix := nec.Config.Kubernetes.Pod_Label_Tags_Prefix
+			if prefix == "_empty_" {
+				prefix = ""
+			}
+			for n, v := range ne.Pod.Labels {
+				endpointTags[fmt.Sprintf("%v%v", prefix, n)] = v
+			}
+		}
+
 		// We need to convert the k8s endpoint to the generic endpoint struct.
 		newEndpoint := &collector.Endpoint{
 			URL:                 url.String(),
@@ -252,7 +268,7 @@ func (nec *NodeEventConsumer) startCollecting(ne *NodeEvent) {
 			Credentials:         endpointCredentials,
 			Collection_Interval: cmeEndpoint.Collection_Interval,
 			Metrics:             cmeEndpoint.Metrics,
-			Tags:                cmeEndpoint.Tags,
+			Tags:                endpointTags,
 		}
 
 		// make sure the endpoint is configured correctly