Carry #780: Export env variables as prometheus labels

container/docker/factory.go

@@ -49,6 +49,8 @@ var dockerCgroupRegexp = regexp.MustCompile(`.+-([a-z0-9]{64})\.scope$`)
 
 var noSystemd = flag.Bool("nosystemd", false, "Explicitly disable systemd support for Docker containers")
 
+var dockerEnvWhitelist = flag.String("docker_env_metadata_whitelist", "", "a comma-separated list of environment variable keys that needs to be collected for docker containers")
+
 // TODO(vmarmol): Export run dir too for newer Dockers.
 // Directory holding Docker container state information.
 func DockerStateDir() string {
@@ -117,6 +119,9 @@ func (self *dockerFactory) NewContainerHandler(name string, inHostNamespace bool
 	if err != nil {
 		return
 	}
+
+	metadataEnvs := strings.Split(*dockerEnvWhitelist, ",")
+
 	handler, err = newDockerContainerHandler(
 		client,
 		name,
@@ -125,6 +130,7 @@ func (self *dockerFactory) NewContainerHandler(name string, inHostNamespace bool
 		self.storageDriver,
 		&self.cgroupSubsystems,
 		inHostNamespace,
+		metadataEnvs,
 	)
 	return
 }

container/docker/handler.go

@@ -66,8 +66,9 @@ type dockerContainerHandler struct {
 	// Time at which this container was created.
 	creationTime time.Time
 
-	// Metadata labels associated with the container.
+	// Metadata associated with the container.
 	labels map[string]string
+	envs   map[string]string
 
 	// The container PID used to switch namespaces as required
 	pid int
@@ -93,6 +94,7 @@ func newDockerContainerHandler(
 	storageDriver storageDriver,
 	cgroupSubsystems *containerlibcontainer.CgroupSubsystems,
 	inHostNamespace bool,
+	metadataEnvs []string,
 ) (container.ContainerHandler, error) {
 	// Create the cgroup paths.
 	cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints))
@@ -157,6 +159,16 @@ func newDockerContainerHandler(
 	handler.image = ctnr.Config.Image
 	handler.networkMode = ctnr.HostConfig.NetworkMode
 
+	// split env vars to get metadata map.
+	for _, exposedEnv := range metadataEnvs {
+		for _, envVar := range ctnr.Config.Env {
+			splits := strings.SplitN(envVar, "=", 2)
+			if splits[0] == exposedEnv {
+				handler.envs[strings.ToLower(exposedEnv)] = splits[1]
+			}
+		}
+	}
+
 	return handler, nil
 }
 
@@ -246,6 +258,7 @@ func (self *dockerContainerHandler) GetSpec() (info.ContainerSpec, error) {
 	}
 
 	spec.Labels = self.labels
+	spec.Envs = self.envs
 	spec.Image = self.image
 	spec.HasNetwork = hasNet(self.networkMode)
 

info/v1/container.go

@@ -45,6 +45,8 @@ type ContainerSpec struct {
 
 	// Metadata labels associated with this container.
 	Labels map[string]string `json:"labels,omitempty"`
+	// Metadata envs associated with this container. Only whitelisted envs are added.
+	Envs map[string]string `json:"envs,omitempty"`
 
 	HasCpu bool    `json:"has_cpu"`
 	Cpu    CpuSpec `json:"cpu,omitempty"`

info/v2/container.go

@@ -66,6 +66,8 @@ type ContainerSpec struct {
 
 	// Metadata labels associated with this container.
 	Labels map[string]string `json:"labels,omitempty"`
+	// Metadata envs associated with this container. Only whitelisted envs are added.
+	Envs map[string]string `json:"envs,omitempty"`
 
 	HasCpu bool    `json:"has_cpu"`
 	Cpu    CpuSpec `json:"cpu,omitempty"`

metrics/prometheus.go

@@ -16,6 +16,7 @@ package metrics
 
 import (
 	"fmt"
+	"regexp"
 	"time"
 
 	info "github.com/google/cadvisor/info/v1"
@@ -508,11 +509,20 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric)
 		if c.containerNameToLabels != nil {
 			newLabels := c.containerNameToLabels(name)
 			for k, v := range newLabels {
-				baseLabels = append(baseLabels, k)
+				baseLabels = append(baseLabels, sanitizeLabelName(k))
 				baseLabelValues = append(baseLabelValues, v)
 			}
 		}
 
+		for k, v := range container.Spec.Labels {
+			baseLabels = append(baseLabels, sanitizeLabelName(k))
+			baseLabelValues = append(baseLabelValues, v)
+		}
+		for k, v := range container.Spec.Envs {
+			baseLabels = append(baseLabels, sanitizeLabelName(k))
+			baseLabelValues = append(baseLabelValues, v)
+		}
+
 		// Container spec
 		desc := prometheus.NewDesc("container_start_time_seconds", "Start time of the container since unix epoch in seconds.", baseLabels, nil)
 		ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.CreationTime.Unix()), baseLabelValues...)
@@ -571,3 +581,11 @@ func specMemoryValue(v uint64) float64 {
 	}
 	return float64(v)
 }
+
+var invalidLabelCharRE = regexp.MustCompile(`[^a-zA-Z0-9_]`)
+
+// sanitizeLabelName replaces anything that doesn't match
+// client_label.LabelNameRE with an underscore.
+func sanitizeLabelName(name string) string {
+	return invalidLabelCharRE.ReplaceAllString(name, "_")
+}

metrics/prometheus_test.go

@@ -57,6 +57,12 @@ func (p testSubcontainersInfoProvider) SubcontainersInfo(string, *info.Container
 			Spec: info.ContainerSpec{
 				Image:        "test",
 				CreationTime: time.Unix(1257894000, 0),
+				Labels: map[string]string{
+					"foo.label": "bar",
+				},
+				Envs: map[string]string{
+					"foo+env": "prod",
+				},
 			},
 			Stats: []*info.ContainerStats{
 				{
@@ -154,12 +160,18 @@ func (p testSubcontainersInfoProvider) SubcontainersInfo(string, *info.Container
 }
 
 var (
-	includeRe = regexp.MustCompile(`^(?:(?:# HELP |# TYPE)container_|cadvisor_version_info\{)`)
+	includeRe = regexp.MustCompile(`^(?:(?:# HELP |# TYPE )?container_|cadvisor_version_info\{)`)
 	ignoreRe  = regexp.MustCompile(`^container_last_seen\{`)
 )
 
 func TestPrometheusCollector(t *testing.T) {
-	prometheus.MustRegister(NewPrometheusCollector(testSubcontainersInfoProvider{}, nil))
+	c := NewPrometheusCollector(testSubcontainersInfoProvider{}, func(name string) map[string]string {
+		return map[string]string{
+			"zone.name": "hello",
+		}
+	})
+	prometheus.MustRegister(c)
+	defer prometheus.Unregister(c)
 
 	rw := httptest.NewRecorder()
 	prometheus.Handler().ServeHTTP(rw, &http.Request{})