From 009761cb53962a55fe8297be51a89ccc2f40a840 Mon Sep 17 00:00:00 2001 From: Florian Pfitzer Date: Tue, 4 Aug 2015 14:49:42 +0200 Subject: [PATCH 1/4] Export image name and env variables as prometheus labels --- container/docker/factory.go | 6 ++ container/docker/handler.go | 13 ++++ metrics/prometheus.go | 5 ++ metrics/prometheus_test.go | 3 + metrics/testdata/prometheus_metrics | 108 ++++++++++++++-------------- 5 files changed, 81 insertions(+), 54 deletions(-) diff --git a/container/docker/factory.go b/container/docker/factory.go index 69c981fa81..672c9c54aa 100644 --- a/container/docker/factory.go +++ b/container/docker/factory.go @@ -49,6 +49,8 @@ var dockerCgroupRegexp = regexp.MustCompile(`.+-([a-z0-9]{64})\.scope$`) var noSystemd = flag.Bool("nosystemd", false, "Explicitly disable systemd support for Docker containers") +var dockerMetadataEnvs = flag.String("docker_metadata_env", "", "Comma seperated list with names of env variables, which will be exported as metadata (default: empty)") + // TODO(vmarmol): Export run dir too for newer Dockers. // Directory holding Docker container state information. func DockerStateDir() string { @@ -117,6 +119,9 @@ func (self *dockerFactory) NewContainerHandler(name string, inHostNamespace bool if err != nil { return } + + exposedMetadata := strings.Split(*dockerMetadataEnvs, ",") + handler, err = newDockerContainerHandler( client, name, @@ -125,6 +130,7 @@ func (self *dockerFactory) NewContainerHandler(name string, inHostNamespace bool self.storageDriver, &self.cgroupSubsystems, inHostNamespace, + exposedMetadata, ) return } diff --git a/container/docker/handler.go b/container/docker/handler.go index 2a1b3661ce..431766027b 100644 --- a/container/docker/handler.go +++ b/container/docker/handler.go @@ -93,6 +93,7 @@ func newDockerContainerHandler( storageDriver storageDriver, cgroupSubsystems *containerlibcontainer.CgroupSubsystems, inHostNamespace bool, + exposedMetadata []string, ) (container.ContainerHandler, error) { // Create the cgroup paths. cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints)) @@ -157,6 +158,18 @@ func newDockerContainerHandler( handler.image = ctnr.Config.Image handler.networkMode = ctnr.HostConfig.NetworkMode + // split env vars to get metadata map. + if len(exposedMetadata) > 0 { + for _, envVar := range ctnr.Config.Env { + splits := strings.SplitN(envVar, "=", 2) + for _, exposedVar := range exposedMetadata { + if splits[0] == exposedVar { + handler.labels[strings.ToLower(exposedVar)] = splits[1] + } + } + } + } + return handler, nil } diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 79e09d73a6..a8e6abb293 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -513,6 +513,11 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) } } + for labelKey, labelValue := range container.Spec.Labels { + baseLabels = append(baseLabels, labelKey) + baseLabelValues = append(baseLabelValues, labelValue) + } + // Container spec desc := prometheus.NewDesc("container_start_time_seconds", "Start time of the container since unix epoch in seconds.", baseLabels, nil) ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.CreationTime.Unix()), baseLabelValues...) diff --git a/metrics/prometheus_test.go b/metrics/prometheus_test.go index 9c2299fce7..e59caacf4f 100644 --- a/metrics/prometheus_test.go +++ b/metrics/prometheus_test.go @@ -57,6 +57,9 @@ func (p testSubcontainersInfoProvider) SubcontainersInfo(string, *info.Container Spec: info.ContainerSpec{ Image: "test", CreationTime: time.Unix(1257894000, 0), + Labels: map[string]string{ + "foo": "bar", + }, }, Stats: []*info.ContainerStats{ { diff --git a/metrics/testdata/prometheus_metrics b/metrics/testdata/prometheus_metrics index 5c953232be..70e598c8c9 100644 --- a/metrics/testdata/prometheus_metrics +++ b/metrics/testdata/prometheus_metrics @@ -3,123 +3,123 @@ cadvisor_version_info{cadvisorRevision="abcdef",cadvisorVersion="0.16.0",dockerVersion="1.8.1",kernelVersion="4.1.6-200.fc22.x86_64",osVersion="Fedora 22 (Twenty Two)"} 1 # HELP container_cpu_system_seconds_total Cumulative system cpu time consumed in seconds. # TYPE container_cpu_system_seconds_total counter -container_cpu_system_seconds_total{id="testcontainer",image="test",name="testcontaineralias"} 7e-09 +container_cpu_system_seconds_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 7e-09 # HELP container_cpu_usage_seconds_total Cumulative cpu time consumed per cpu in seconds. # TYPE container_cpu_usage_seconds_total counter -container_cpu_usage_seconds_total{cpu="cpu00",id="testcontainer",image="test",name="testcontaineralias"} 2e-09 -container_cpu_usage_seconds_total{cpu="cpu01",id="testcontainer",image="test",name="testcontaineralias"} 3e-09 -container_cpu_usage_seconds_total{cpu="cpu02",id="testcontainer",image="test",name="testcontaineralias"} 4e-09 -container_cpu_usage_seconds_total{cpu="cpu03",id="testcontainer",image="test",name="testcontaineralias"} 5e-09 +container_cpu_usage_seconds_total{cpu="cpu00",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 2e-09 +container_cpu_usage_seconds_total{cpu="cpu01",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 3e-09 +container_cpu_usage_seconds_total{cpu="cpu02",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4e-09 +container_cpu_usage_seconds_total{cpu="cpu03",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 5e-09 # HELP container_cpu_user_seconds_total Cumulative user cpu time consumed in seconds. # TYPE container_cpu_user_seconds_total counter -container_cpu_user_seconds_total{id="testcontainer",image="test",name="testcontaineralias"} 6e-09 +container_cpu_user_seconds_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 6e-09 # HELP container_fs_io_current Number of I/Os currently in progress # TYPE container_fs_io_current gauge -container_fs_io_current{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 42 -container_fs_io_current{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 47 +container_fs_io_current{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 42 +container_fs_io_current{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 47 # HELP container_fs_io_time_seconds_total Cumulative count of seconds spent doing I/Os # TYPE container_fs_io_time_seconds_total counter -container_fs_io_time_seconds_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 4.3e-08 -container_fs_io_time_seconds_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 4.8e-08 +container_fs_io_time_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.3e-08 +container_fs_io_time_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.8e-08 # HELP container_fs_io_time_weighted_seconds_total Cumulative weighted I/O time in seconds # TYPE container_fs_io_time_weighted_seconds_total counter -container_fs_io_time_weighted_seconds_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 4.4e-08 -container_fs_io_time_weighted_seconds_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 4.9e-08 +container_fs_io_time_weighted_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.4e-08 +container_fs_io_time_weighted_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.9e-08 # HELP container_fs_limit_bytes Number of bytes that can be consumed by the container on this filesystem. # TYPE container_fs_limit_bytes gauge -container_fs_limit_bytes{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 22 -container_fs_limit_bytes{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 37 +container_fs_limit_bytes{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 22 +container_fs_limit_bytes{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 37 # HELP container_fs_read_seconds_total Cumulative count of seconds spent reading # TYPE container_fs_read_seconds_total counter -container_fs_read_seconds_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 2.7e-08 -container_fs_read_seconds_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 4.2e-08 +container_fs_read_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 2.7e-08 +container_fs_read_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.2e-08 # HELP container_fs_reads_merged_total Cumulative count of reads merged # TYPE container_fs_reads_merged_total counter -container_fs_reads_merged_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 25 -container_fs_reads_merged_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 40 +container_fs_reads_merged_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 25 +container_fs_reads_merged_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 40 # HELP container_fs_reads_total Cumulative count of reads completed # TYPE container_fs_reads_total counter -container_fs_reads_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 24 -container_fs_reads_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 39 +container_fs_reads_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 24 +container_fs_reads_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 39 # HELP container_fs_sector_reads_total Cumulative count of sector reads completed # TYPE container_fs_sector_reads_total counter -container_fs_sector_reads_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 26 -container_fs_sector_reads_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 41 +container_fs_sector_reads_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 26 +container_fs_sector_reads_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 41 # HELP container_fs_sector_writes_total Cumulative count of sector writes completed # TYPE container_fs_sector_writes_total counter -container_fs_sector_writes_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 40 -container_fs_sector_writes_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 45 +container_fs_sector_writes_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 40 +container_fs_sector_writes_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 45 # HELP container_fs_usage_bytes Number of bytes that are consumed by the container on this filesystem. # TYPE container_fs_usage_bytes gauge -container_fs_usage_bytes{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 23 -container_fs_usage_bytes{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 38 +container_fs_usage_bytes{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 23 +container_fs_usage_bytes{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 38 # HELP container_fs_write_seconds_total Cumulative count of seconds spent writing # TYPE container_fs_write_seconds_total counter -container_fs_write_seconds_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 4.1e-08 -container_fs_write_seconds_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 4.6e-08 +container_fs_write_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.1e-08 +container_fs_write_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.6e-08 # HELP container_fs_writes_merged_total Cumulative count of writes merged # TYPE container_fs_writes_merged_total counter -container_fs_writes_merged_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 39 -container_fs_writes_merged_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 44 +container_fs_writes_merged_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 39 +container_fs_writes_merged_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 44 # HELP container_fs_writes_total Cumulative count of writes completed # TYPE container_fs_writes_total counter -container_fs_writes_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 28 -container_fs_writes_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 43 +container_fs_writes_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 28 +container_fs_writes_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 43 # HELP container_last_seen Last time a container was seen by the exporter # TYPE container_last_seen gauge -container_last_seen{id="testcontainer",image="test",name="testcontaineralias"} 1.426203694e+09 +container_last_seen{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 1.426203694e+09 # HELP container_memory_failcnt Number of memory usage hits limits # TYPE container_memory_failcnt counter -container_memory_failcnt{id="testcontainer",image="test",name="testcontaineralias"} 0 +container_memory_failcnt{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 0 # HELP container_memory_failures_total Cumulative count of memory allocation failures. # TYPE container_memory_failures_total counter -container_memory_failures_total{id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgfault"} 10 -container_memory_failures_total{id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgmajfault"} 11 -container_memory_failures_total{id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgfault"} 12 -container_memory_failures_total{id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgmajfault"} 13 +container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgfault"} 10 +container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgmajfault"} 11 +container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgfault"} 12 +container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgmajfault"} 13 # HELP container_memory_usage_bytes Current memory usage in bytes. # TYPE container_memory_usage_bytes gauge -container_memory_usage_bytes{id="testcontainer",image="test",name="testcontaineralias"} 8 +container_memory_usage_bytes{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 8 # HELP container_memory_working_set_bytes Current working set in bytes. # TYPE container_memory_working_set_bytes gauge -container_memory_working_set_bytes{id="testcontainer",image="test",name="testcontaineralias"} 9 +container_memory_working_set_bytes{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 9 # HELP container_network_receive_bytes_total Cumulative count of bytes received # TYPE container_network_receive_bytes_total counter -container_network_receive_bytes_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 14 +container_network_receive_bytes_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 14 # HELP container_network_receive_errors_total Cumulative count of errors encountered while receiving # TYPE container_network_receive_errors_total counter -container_network_receive_errors_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 16 +container_network_receive_errors_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 16 # HELP container_network_receive_packets_dropped_total Cumulative count of packets dropped while receiving # TYPE container_network_receive_packets_dropped_total counter -container_network_receive_packets_dropped_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 17 +container_network_receive_packets_dropped_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 17 # HELP container_network_receive_packets_total Cumulative count of packets received # TYPE container_network_receive_packets_total counter -container_network_receive_packets_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 15 +container_network_receive_packets_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 15 # HELP container_network_transmit_bytes_total Cumulative count of bytes transmitted # TYPE container_network_transmit_bytes_total counter -container_network_transmit_bytes_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 18 +container_network_transmit_bytes_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 18 # HELP container_network_transmit_errors_total Cumulative count of errors encountered while transmitting # TYPE container_network_transmit_errors_total counter -container_network_transmit_errors_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 20 +container_network_transmit_errors_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 20 # HELP container_network_transmit_packets_dropped_total Cumulative count of packets dropped while transmitting # TYPE container_network_transmit_packets_dropped_total counter -container_network_transmit_packets_dropped_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 21 +container_network_transmit_packets_dropped_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 21 # HELP container_network_transmit_packets_total Cumulative count of packets transmitted # TYPE container_network_transmit_packets_total counter -container_network_transmit_packets_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 19 +container_network_transmit_packets_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 19 # HELP container_scrape_error 1 if there was an error while getting container metrics, 0 otherwise # TYPE container_scrape_error gauge container_scrape_error 0 # HELP container_start_time_seconds Start time of the container since unix epoch in seconds. # TYPE container_start_time_seconds gauge -container_start_time_seconds{id="testcontainer",image="test",name="testcontaineralias"} 1.257894e+09 +container_start_time_seconds{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 1.257894e+09 # HELP container_tasks_state Number of tasks in given state # TYPE container_tasks_state gauge -container_tasks_state{id="testcontainer",image="test",name="testcontaineralias",state="iowaiting"} 54 -container_tasks_state{id="testcontainer",image="test",name="testcontaineralias",state="running"} 51 -container_tasks_state{id="testcontainer",image="test",name="testcontaineralias",state="sleeping"} 50 -container_tasks_state{id="testcontainer",image="test",name="testcontaineralias",state="stopped"} 52 -container_tasks_state{id="testcontainer",image="test",name="testcontaineralias",state="uninterruptible"} 53 +container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="iowaiting"} 54 +container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="running"} 51 +container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="sleeping"} 50 +container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="stopped"} 52 +container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="uninterruptible"} 53 # HELP http_request_duration_microseconds The HTTP request latencies in microseconds. # TYPE http_request_duration_microseconds summary http_request_duration_microseconds{handler="prometheus",quantile="0.5"} 0 From 9131bcb315354198610253769fb93103b84f0eec Mon Sep 17 00:00:00 2001 From: Daniel Dao Date: Wed, 16 Dec 2015 14:09:25 +0000 Subject: [PATCH 2/4] fix prometheus regex matching the previous regex wasnt able to match anything. This regex should hopefully do better. Signed-off-by: Daniel Dao --- metrics/prometheus_test.go | 2 +- metrics/testdata/prometheus_metrics | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metrics/prometheus_test.go b/metrics/prometheus_test.go index e59caacf4f..cf45f510fc 100644 --- a/metrics/prometheus_test.go +++ b/metrics/prometheus_test.go @@ -157,7 +157,7 @@ func (p testSubcontainersInfoProvider) SubcontainersInfo(string, *info.Container } var ( - includeRe = regexp.MustCompile(`^(?:(?:# HELP |# TYPE)container_|cadvisor_version_info\{)`) + includeRe = regexp.MustCompile(`^(?:(?:# HELP |# TYPE )?container_|cadvisor_version_info\{)`) ignoreRe = regexp.MustCompile(`^container_last_seen\{`) ) diff --git a/metrics/testdata/prometheus_metrics b/metrics/testdata/prometheus_metrics index 70e598c8c9..54877ec291 100644 --- a/metrics/testdata/prometheus_metrics +++ b/metrics/testdata/prometheus_metrics @@ -1,4 +1,4 @@ -# HELP cadvisor_version_info A metric with a constant '1' value labeled by kernel version, OS version, docker version & cadvisor version. +# HELP cadvisor_version_info A metric with a constant '1' value labeled by kernel version, OS version, docker version, cadvisor version & cadvisor revision. # TYPE cadvisor_version_info gauge cadvisor_version_info{cadvisorRevision="abcdef",cadvisorVersion="0.16.0",dockerVersion="1.8.1",kernelVersion="4.1.6-200.fc22.x86_64",osVersion="Fedora 22 (Twenty Two)"} 1 # HELP container_cpu_system_seconds_total Cumulative system cpu time consumed in seconds. From 544b852a3b5efdabd63e451bab0f042539964524 Mon Sep 17 00:00:00 2001 From: Daniel Dao Date: Wed, 16 Dec 2015 15:01:19 +0000 Subject: [PATCH 3/4] satitize label name before setting them as prometheus label Signed-off-by: Daniel Dao --- metrics/prometheus.go | 13 +++- metrics/prometheus_test.go | 10 ++- metrics/testdata/prometheus_metrics | 108 ++++++++++++++-------------- 3 files changed, 73 insertions(+), 58 deletions(-) diff --git a/metrics/prometheus.go b/metrics/prometheus.go index a8e6abb293..a1908285c9 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -16,6 +16,7 @@ package metrics import ( "fmt" + "regexp" "time" info "github.com/google/cadvisor/info/v1" @@ -508,13 +509,13 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) if c.containerNameToLabels != nil { newLabels := c.containerNameToLabels(name) for k, v := range newLabels { - baseLabels = append(baseLabels, k) + baseLabels = append(baseLabels, sanitizeLabelName(k)) baseLabelValues = append(baseLabelValues, v) } } for labelKey, labelValue := range container.Spec.Labels { - baseLabels = append(baseLabels, labelKey) + baseLabels = append(baseLabels, sanitizeLabelName(labelKey)) baseLabelValues = append(baseLabelValues, labelValue) } @@ -576,3 +577,11 @@ func specMemoryValue(v uint64) float64 { } return float64(v) } + +var invalidLabelCharRE = regexp.MustCompile(`[^a-zA-Z0-9_]`) + +// sanitizeLabelName replaces anything that doesn't match +// client_label.LabelNameRE with an underscore. +func sanitizeLabelName(name string) string { + return invalidLabelCharRE.ReplaceAllString(name, "_") +} diff --git a/metrics/prometheus_test.go b/metrics/prometheus_test.go index cf45f510fc..e22b8d5d02 100644 --- a/metrics/prometheus_test.go +++ b/metrics/prometheus_test.go @@ -58,7 +58,7 @@ func (p testSubcontainersInfoProvider) SubcontainersInfo(string, *info.Container Image: "test", CreationTime: time.Unix(1257894000, 0), Labels: map[string]string{ - "foo": "bar", + "foo.metric": "bar", }, }, Stats: []*info.ContainerStats{ @@ -162,7 +162,13 @@ var ( ) func TestPrometheusCollector(t *testing.T) { - prometheus.MustRegister(NewPrometheusCollector(testSubcontainersInfoProvider{}, nil)) + c := NewPrometheusCollector(testSubcontainersInfoProvider{}, func(name string) map[string]string { + return map[string]string{ + "zone.name": "hello", + } + }) + prometheus.MustRegister(c) + defer prometheus.Unregister(c) rw := httptest.NewRecorder() prometheus.Handler().ServeHTTP(rw, &http.Request{}) diff --git a/metrics/testdata/prometheus_metrics b/metrics/testdata/prometheus_metrics index 54877ec291..3202025748 100644 --- a/metrics/testdata/prometheus_metrics +++ b/metrics/testdata/prometheus_metrics @@ -3,123 +3,123 @@ cadvisor_version_info{cadvisorRevision="abcdef",cadvisorVersion="0.16.0",dockerVersion="1.8.1",kernelVersion="4.1.6-200.fc22.x86_64",osVersion="Fedora 22 (Twenty Two)"} 1 # HELP container_cpu_system_seconds_total Cumulative system cpu time consumed in seconds. # TYPE container_cpu_system_seconds_total counter -container_cpu_system_seconds_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 7e-09 +container_cpu_system_seconds_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 7e-09 # HELP container_cpu_usage_seconds_total Cumulative cpu time consumed per cpu in seconds. # TYPE container_cpu_usage_seconds_total counter -container_cpu_usage_seconds_total{cpu="cpu00",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 2e-09 -container_cpu_usage_seconds_total{cpu="cpu01",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 3e-09 -container_cpu_usage_seconds_total{cpu="cpu02",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4e-09 -container_cpu_usage_seconds_total{cpu="cpu03",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 5e-09 +container_cpu_usage_seconds_total{cpu="cpu00",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2e-09 +container_cpu_usage_seconds_total{cpu="cpu01",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 3e-09 +container_cpu_usage_seconds_total{cpu="cpu02",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4e-09 +container_cpu_usage_seconds_total{cpu="cpu03",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5e-09 # HELP container_cpu_user_seconds_total Cumulative user cpu time consumed in seconds. # TYPE container_cpu_user_seconds_total counter -container_cpu_user_seconds_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 6e-09 +container_cpu_user_seconds_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 6e-09 # HELP container_fs_io_current Number of I/Os currently in progress # TYPE container_fs_io_current gauge -container_fs_io_current{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 42 -container_fs_io_current{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 47 +container_fs_io_current{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 42 +container_fs_io_current{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 47 # HELP container_fs_io_time_seconds_total Cumulative count of seconds spent doing I/Os # TYPE container_fs_io_time_seconds_total counter -container_fs_io_time_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.3e-08 -container_fs_io_time_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.8e-08 +container_fs_io_time_seconds_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.3e-08 +container_fs_io_time_seconds_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.8e-08 # HELP container_fs_io_time_weighted_seconds_total Cumulative weighted I/O time in seconds # TYPE container_fs_io_time_weighted_seconds_total counter -container_fs_io_time_weighted_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.4e-08 -container_fs_io_time_weighted_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.9e-08 +container_fs_io_time_weighted_seconds_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.4e-08 +container_fs_io_time_weighted_seconds_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.9e-08 # HELP container_fs_limit_bytes Number of bytes that can be consumed by the container on this filesystem. # TYPE container_fs_limit_bytes gauge -container_fs_limit_bytes{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 22 -container_fs_limit_bytes{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 37 +container_fs_limit_bytes{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 22 +container_fs_limit_bytes{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 37 # HELP container_fs_read_seconds_total Cumulative count of seconds spent reading # TYPE container_fs_read_seconds_total counter -container_fs_read_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 2.7e-08 -container_fs_read_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.2e-08 +container_fs_read_seconds_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2.7e-08 +container_fs_read_seconds_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.2e-08 # HELP container_fs_reads_merged_total Cumulative count of reads merged # TYPE container_fs_reads_merged_total counter -container_fs_reads_merged_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 25 -container_fs_reads_merged_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 40 +container_fs_reads_merged_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 25 +container_fs_reads_merged_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 40 # HELP container_fs_reads_total Cumulative count of reads completed # TYPE container_fs_reads_total counter -container_fs_reads_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 24 -container_fs_reads_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 39 +container_fs_reads_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 24 +container_fs_reads_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 39 # HELP container_fs_sector_reads_total Cumulative count of sector reads completed # TYPE container_fs_sector_reads_total counter -container_fs_sector_reads_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 26 -container_fs_sector_reads_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 41 +container_fs_sector_reads_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 26 +container_fs_sector_reads_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 41 # HELP container_fs_sector_writes_total Cumulative count of sector writes completed # TYPE container_fs_sector_writes_total counter -container_fs_sector_writes_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 40 -container_fs_sector_writes_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 45 +container_fs_sector_writes_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 40 +container_fs_sector_writes_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 45 # HELP container_fs_usage_bytes Number of bytes that are consumed by the container on this filesystem. # TYPE container_fs_usage_bytes gauge -container_fs_usage_bytes{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 23 -container_fs_usage_bytes{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 38 +container_fs_usage_bytes{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 23 +container_fs_usage_bytes{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 38 # HELP container_fs_write_seconds_total Cumulative count of seconds spent writing # TYPE container_fs_write_seconds_total counter -container_fs_write_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.1e-08 -container_fs_write_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.6e-08 +container_fs_write_seconds_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.1e-08 +container_fs_write_seconds_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.6e-08 # HELP container_fs_writes_merged_total Cumulative count of writes merged # TYPE container_fs_writes_merged_total counter -container_fs_writes_merged_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 39 -container_fs_writes_merged_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 44 +container_fs_writes_merged_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 39 +container_fs_writes_merged_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 44 # HELP container_fs_writes_total Cumulative count of writes completed # TYPE container_fs_writes_total counter -container_fs_writes_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 28 -container_fs_writes_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 43 +container_fs_writes_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 28 +container_fs_writes_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 43 # HELP container_last_seen Last time a container was seen by the exporter # TYPE container_last_seen gauge -container_last_seen{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 1.426203694e+09 +container_last_seen{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1.426203694e+09 # HELP container_memory_failcnt Number of memory usage hits limits # TYPE container_memory_failcnt counter -container_memory_failcnt{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 0 +container_memory_failcnt{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0 # HELP container_memory_failures_total Cumulative count of memory allocation failures. # TYPE container_memory_failures_total counter -container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgfault"} 10 -container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgmajfault"} 11 -container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgfault"} 12 -container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgmajfault"} 13 +container_memory_failures_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgfault",zone_name="hello"} 10 +container_memory_failures_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgmajfault",zone_name="hello"} 11 +container_memory_failures_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgfault",zone_name="hello"} 12 +container_memory_failures_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgmajfault",zone_name="hello"} 13 # HELP container_memory_usage_bytes Current memory usage in bytes. # TYPE container_memory_usage_bytes gauge -container_memory_usage_bytes{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 8 +container_memory_usage_bytes{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 8 # HELP container_memory_working_set_bytes Current working set in bytes. # TYPE container_memory_working_set_bytes gauge -container_memory_working_set_bytes{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 9 +container_memory_working_set_bytes{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 9 # HELP container_network_receive_bytes_total Cumulative count of bytes received # TYPE container_network_receive_bytes_total counter -container_network_receive_bytes_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 14 +container_network_receive_bytes_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 14 # HELP container_network_receive_errors_total Cumulative count of errors encountered while receiving # TYPE container_network_receive_errors_total counter -container_network_receive_errors_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 16 +container_network_receive_errors_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 16 # HELP container_network_receive_packets_dropped_total Cumulative count of packets dropped while receiving # TYPE container_network_receive_packets_dropped_total counter -container_network_receive_packets_dropped_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 17 +container_network_receive_packets_dropped_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 17 # HELP container_network_receive_packets_total Cumulative count of packets received # TYPE container_network_receive_packets_total counter -container_network_receive_packets_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 15 +container_network_receive_packets_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 15 # HELP container_network_transmit_bytes_total Cumulative count of bytes transmitted # TYPE container_network_transmit_bytes_total counter -container_network_transmit_bytes_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 18 +container_network_transmit_bytes_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 18 # HELP container_network_transmit_errors_total Cumulative count of errors encountered while transmitting # TYPE container_network_transmit_errors_total counter -container_network_transmit_errors_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 20 +container_network_transmit_errors_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 20 # HELP container_network_transmit_packets_dropped_total Cumulative count of packets dropped while transmitting # TYPE container_network_transmit_packets_dropped_total counter -container_network_transmit_packets_dropped_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 21 +container_network_transmit_packets_dropped_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 21 # HELP container_network_transmit_packets_total Cumulative count of packets transmitted # TYPE container_network_transmit_packets_total counter -container_network_transmit_packets_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 19 +container_network_transmit_packets_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 19 # HELP container_scrape_error 1 if there was an error while getting container metrics, 0 otherwise # TYPE container_scrape_error gauge container_scrape_error 0 # HELP container_start_time_seconds Start time of the container since unix epoch in seconds. # TYPE container_start_time_seconds gauge -container_start_time_seconds{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 1.257894e+09 +container_start_time_seconds{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1.257894e+09 # HELP container_tasks_state Number of tasks in given state # TYPE container_tasks_state gauge -container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="iowaiting"} 54 -container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="running"} 51 -container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="sleeping"} 50 -container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="stopped"} 52 -container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="uninterruptible"} 53 +container_tasks_state{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",state="iowaiting",zone_name="hello"} 54 +container_tasks_state{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",state="running",zone_name="hello"} 51 +container_tasks_state{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",state="sleeping",zone_name="hello"} 50 +container_tasks_state{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",state="stopped",zone_name="hello"} 52 +container_tasks_state{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",state="uninterruptible",zone_name="hello"} 53 # HELP http_request_duration_microseconds The HTTP request latencies in microseconds. # TYPE http_request_duration_microseconds summary http_request_duration_microseconds{handler="prometheus",quantile="0.5"} 0 From e5b6bfa94f4f8d492c96f4ada64db325eccc9c29 Mon Sep 17 00:00:00 2001 From: Daniel Dao Date: Tue, 5 Jan 2016 12:42:48 +0000 Subject: [PATCH 4/4] add whitelisted env as container metadata This add Envs to container spec as a metadata source. When using prometheus exposition format, they will be merged into the list of metrics' labels. Also changed the cli flag to docker_env_metadata_whitelist, and add refenrences of whitelist envs to API Signed-off-by: Daniel Dao --- container/docker/factory.go | 6 +- container/docker/handler.go | 14 ++-- info/v1/container.go | 2 + info/v2/container.go | 2 + metrics/prometheus.go | 10 ++- metrics/prometheus_test.go | 5 +- metrics/testdata/prometheus_metrics | 108 ++++++++++++++-------------- 7 files changed, 79 insertions(+), 68 deletions(-) diff --git a/container/docker/factory.go b/container/docker/factory.go index 672c9c54aa..a917831e97 100644 --- a/container/docker/factory.go +++ b/container/docker/factory.go @@ -49,7 +49,7 @@ var dockerCgroupRegexp = regexp.MustCompile(`.+-([a-z0-9]{64})\.scope$`) var noSystemd = flag.Bool("nosystemd", false, "Explicitly disable systemd support for Docker containers") -var dockerMetadataEnvs = flag.String("docker_metadata_env", "", "Comma seperated list with names of env variables, which will be exported as metadata (default: empty)") +var dockerEnvWhitelist = flag.String("docker_env_metadata_whitelist", "", "a comma-separated list of environment variable keys that needs to be collected for docker containers") // TODO(vmarmol): Export run dir too for newer Dockers. // Directory holding Docker container state information. @@ -120,7 +120,7 @@ func (self *dockerFactory) NewContainerHandler(name string, inHostNamespace bool return } - exposedMetadata := strings.Split(*dockerMetadataEnvs, ",") + metadataEnvs := strings.Split(*dockerEnvWhitelist, ",") handler, err = newDockerContainerHandler( client, @@ -130,7 +130,7 @@ func (self *dockerFactory) NewContainerHandler(name string, inHostNamespace bool self.storageDriver, &self.cgroupSubsystems, inHostNamespace, - exposedMetadata, + metadataEnvs, ) return } diff --git a/container/docker/handler.go b/container/docker/handler.go index 431766027b..1221b8be02 100644 --- a/container/docker/handler.go +++ b/container/docker/handler.go @@ -66,8 +66,9 @@ type dockerContainerHandler struct { // Time at which this container was created. creationTime time.Time - // Metadata labels associated with the container. + // Metadata associated with the container. labels map[string]string + envs map[string]string // The container PID used to switch namespaces as required pid int @@ -93,7 +94,7 @@ func newDockerContainerHandler( storageDriver storageDriver, cgroupSubsystems *containerlibcontainer.CgroupSubsystems, inHostNamespace bool, - exposedMetadata []string, + metadataEnvs []string, ) (container.ContainerHandler, error) { // Create the cgroup paths. cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints)) @@ -159,13 +160,11 @@ func newDockerContainerHandler( handler.networkMode = ctnr.HostConfig.NetworkMode // split env vars to get metadata map. - if len(exposedMetadata) > 0 { + for _, exposedEnv := range metadataEnvs { for _, envVar := range ctnr.Config.Env { splits := strings.SplitN(envVar, "=", 2) - for _, exposedVar := range exposedMetadata { - if splits[0] == exposedVar { - handler.labels[strings.ToLower(exposedVar)] = splits[1] - } + if splits[0] == exposedEnv { + handler.envs[strings.ToLower(exposedEnv)] = splits[1] } } } @@ -259,6 +258,7 @@ func (self *dockerContainerHandler) GetSpec() (info.ContainerSpec, error) { } spec.Labels = self.labels + spec.Envs = self.envs spec.Image = self.image spec.HasNetwork = hasNet(self.networkMode) diff --git a/info/v1/container.go b/info/v1/container.go index 53b8b5d21f..11c03c3101 100644 --- a/info/v1/container.go +++ b/info/v1/container.go @@ -45,6 +45,8 @@ type ContainerSpec struct { // Metadata labels associated with this container. Labels map[string]string `json:"labels,omitempty"` + // Metadata envs associated with this container. Only whitelisted envs are added. + Envs map[string]string `json:"envs,omitempty"` HasCpu bool `json:"has_cpu"` Cpu CpuSpec `json:"cpu,omitempty"` diff --git a/info/v2/container.go b/info/v2/container.go index 3c854260bd..3063ea4463 100644 --- a/info/v2/container.go +++ b/info/v2/container.go @@ -66,6 +66,8 @@ type ContainerSpec struct { // Metadata labels associated with this container. Labels map[string]string `json:"labels,omitempty"` + // Metadata envs associated with this container. Only whitelisted envs are added. + Envs map[string]string `json:"envs,omitempty"` HasCpu bool `json:"has_cpu"` Cpu CpuSpec `json:"cpu,omitempty"` diff --git a/metrics/prometheus.go b/metrics/prometheus.go index a1908285c9..ff4e5d89b5 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -514,9 +514,13 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) } } - for labelKey, labelValue := range container.Spec.Labels { - baseLabels = append(baseLabels, sanitizeLabelName(labelKey)) - baseLabelValues = append(baseLabelValues, labelValue) + for k, v := range container.Spec.Labels { + baseLabels = append(baseLabels, sanitizeLabelName(k)) + baseLabelValues = append(baseLabelValues, v) + } + for k, v := range container.Spec.Envs { + baseLabels = append(baseLabels, sanitizeLabelName(k)) + baseLabelValues = append(baseLabelValues, v) } // Container spec diff --git a/metrics/prometheus_test.go b/metrics/prometheus_test.go index e22b8d5d02..5410c8d437 100644 --- a/metrics/prometheus_test.go +++ b/metrics/prometheus_test.go @@ -58,7 +58,10 @@ func (p testSubcontainersInfoProvider) SubcontainersInfo(string, *info.Container Image: "test", CreationTime: time.Unix(1257894000, 0), Labels: map[string]string{ - "foo.metric": "bar", + "foo.label": "bar", + }, + Envs: map[string]string{ + "foo+env": "prod", }, }, Stats: []*info.ContainerStats{ diff --git a/metrics/testdata/prometheus_metrics b/metrics/testdata/prometheus_metrics index 3202025748..18390c20d9 100644 --- a/metrics/testdata/prometheus_metrics +++ b/metrics/testdata/prometheus_metrics @@ -3,123 +3,123 @@ cadvisor_version_info{cadvisorRevision="abcdef",cadvisorVersion="0.16.0",dockerVersion="1.8.1",kernelVersion="4.1.6-200.fc22.x86_64",osVersion="Fedora 22 (Twenty Two)"} 1 # HELP container_cpu_system_seconds_total Cumulative system cpu time consumed in seconds. # TYPE container_cpu_system_seconds_total counter -container_cpu_system_seconds_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 7e-09 +container_cpu_system_seconds_total{foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 7e-09 # HELP container_cpu_usage_seconds_total Cumulative cpu time consumed per cpu in seconds. # TYPE container_cpu_usage_seconds_total counter -container_cpu_usage_seconds_total{cpu="cpu00",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2e-09 -container_cpu_usage_seconds_total{cpu="cpu01",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 3e-09 -container_cpu_usage_seconds_total{cpu="cpu02",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4e-09 -container_cpu_usage_seconds_total{cpu="cpu03",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5e-09 +container_cpu_usage_seconds_total{cpu="cpu00",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2e-09 +container_cpu_usage_seconds_total{cpu="cpu01",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 3e-09 +container_cpu_usage_seconds_total{cpu="cpu02",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4e-09 +container_cpu_usage_seconds_total{cpu="cpu03",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5e-09 # HELP container_cpu_user_seconds_total Cumulative user cpu time consumed in seconds. # TYPE container_cpu_user_seconds_total counter -container_cpu_user_seconds_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 6e-09 +container_cpu_user_seconds_total{foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 6e-09 # HELP container_fs_io_current Number of I/Os currently in progress # TYPE container_fs_io_current gauge -container_fs_io_current{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 42 -container_fs_io_current{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 47 +container_fs_io_current{device="sda1",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 42 +container_fs_io_current{device="sda2",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 47 # HELP container_fs_io_time_seconds_total Cumulative count of seconds spent doing I/Os # TYPE container_fs_io_time_seconds_total counter -container_fs_io_time_seconds_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.3e-08 -container_fs_io_time_seconds_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.8e-08 +container_fs_io_time_seconds_total{device="sda1",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.3e-08 +container_fs_io_time_seconds_total{device="sda2",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.8e-08 # HELP container_fs_io_time_weighted_seconds_total Cumulative weighted I/O time in seconds # TYPE container_fs_io_time_weighted_seconds_total counter -container_fs_io_time_weighted_seconds_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.4e-08 -container_fs_io_time_weighted_seconds_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.9e-08 +container_fs_io_time_weighted_seconds_total{device="sda1",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.4e-08 +container_fs_io_time_weighted_seconds_total{device="sda2",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.9e-08 # HELP container_fs_limit_bytes Number of bytes that can be consumed by the container on this filesystem. # TYPE container_fs_limit_bytes gauge -container_fs_limit_bytes{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 22 -container_fs_limit_bytes{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 37 +container_fs_limit_bytes{device="sda1",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 22 +container_fs_limit_bytes{device="sda2",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 37 # HELP container_fs_read_seconds_total Cumulative count of seconds spent reading # TYPE container_fs_read_seconds_total counter -container_fs_read_seconds_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2.7e-08 -container_fs_read_seconds_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.2e-08 +container_fs_read_seconds_total{device="sda1",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2.7e-08 +container_fs_read_seconds_total{device="sda2",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.2e-08 # HELP container_fs_reads_merged_total Cumulative count of reads merged # TYPE container_fs_reads_merged_total counter -container_fs_reads_merged_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 25 -container_fs_reads_merged_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 40 +container_fs_reads_merged_total{device="sda1",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 25 +container_fs_reads_merged_total{device="sda2",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 40 # HELP container_fs_reads_total Cumulative count of reads completed # TYPE container_fs_reads_total counter -container_fs_reads_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 24 -container_fs_reads_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 39 +container_fs_reads_total{device="sda1",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 24 +container_fs_reads_total{device="sda2",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 39 # HELP container_fs_sector_reads_total Cumulative count of sector reads completed # TYPE container_fs_sector_reads_total counter -container_fs_sector_reads_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 26 -container_fs_sector_reads_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 41 +container_fs_sector_reads_total{device="sda1",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 26 +container_fs_sector_reads_total{device="sda2",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 41 # HELP container_fs_sector_writes_total Cumulative count of sector writes completed # TYPE container_fs_sector_writes_total counter -container_fs_sector_writes_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 40 -container_fs_sector_writes_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 45 +container_fs_sector_writes_total{device="sda1",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 40 +container_fs_sector_writes_total{device="sda2",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 45 # HELP container_fs_usage_bytes Number of bytes that are consumed by the container on this filesystem. # TYPE container_fs_usage_bytes gauge -container_fs_usage_bytes{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 23 -container_fs_usage_bytes{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 38 +container_fs_usage_bytes{device="sda1",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 23 +container_fs_usage_bytes{device="sda2",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 38 # HELP container_fs_write_seconds_total Cumulative count of seconds spent writing # TYPE container_fs_write_seconds_total counter -container_fs_write_seconds_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.1e-08 -container_fs_write_seconds_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.6e-08 +container_fs_write_seconds_total{device="sda1",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.1e-08 +container_fs_write_seconds_total{device="sda2",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.6e-08 # HELP container_fs_writes_merged_total Cumulative count of writes merged # TYPE container_fs_writes_merged_total counter -container_fs_writes_merged_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 39 -container_fs_writes_merged_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 44 +container_fs_writes_merged_total{device="sda1",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 39 +container_fs_writes_merged_total{device="sda2",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 44 # HELP container_fs_writes_total Cumulative count of writes completed # TYPE container_fs_writes_total counter -container_fs_writes_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 28 -container_fs_writes_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 43 +container_fs_writes_total{device="sda1",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 28 +container_fs_writes_total{device="sda2",foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 43 # HELP container_last_seen Last time a container was seen by the exporter # TYPE container_last_seen gauge -container_last_seen{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1.426203694e+09 +container_last_seen{foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1.426203694e+09 # HELP container_memory_failcnt Number of memory usage hits limits # TYPE container_memory_failcnt counter -container_memory_failcnt{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0 +container_memory_failcnt{foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0 # HELP container_memory_failures_total Cumulative count of memory allocation failures. # TYPE container_memory_failures_total counter -container_memory_failures_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgfault",zone_name="hello"} 10 -container_memory_failures_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgmajfault",zone_name="hello"} 11 -container_memory_failures_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgfault",zone_name="hello"} 12 -container_memory_failures_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgmajfault",zone_name="hello"} 13 +container_memory_failures_total{foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgfault",zone_name="hello"} 10 +container_memory_failures_total{foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgmajfault",zone_name="hello"} 11 +container_memory_failures_total{foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgfault",zone_name="hello"} 12 +container_memory_failures_total{foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgmajfault",zone_name="hello"} 13 # HELP container_memory_usage_bytes Current memory usage in bytes. # TYPE container_memory_usage_bytes gauge -container_memory_usage_bytes{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 8 +container_memory_usage_bytes{foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 8 # HELP container_memory_working_set_bytes Current working set in bytes. # TYPE container_memory_working_set_bytes gauge -container_memory_working_set_bytes{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 9 +container_memory_working_set_bytes{foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 9 # HELP container_network_receive_bytes_total Cumulative count of bytes received # TYPE container_network_receive_bytes_total counter -container_network_receive_bytes_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 14 +container_network_receive_bytes_total{foo_env="prod",foo_label="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 14 # HELP container_network_receive_errors_total Cumulative count of errors encountered while receiving # TYPE container_network_receive_errors_total counter -container_network_receive_errors_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 16 +container_network_receive_errors_total{foo_env="prod",foo_label="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 16 # HELP container_network_receive_packets_dropped_total Cumulative count of packets dropped while receiving # TYPE container_network_receive_packets_dropped_total counter -container_network_receive_packets_dropped_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 17 +container_network_receive_packets_dropped_total{foo_env="prod",foo_label="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 17 # HELP container_network_receive_packets_total Cumulative count of packets received # TYPE container_network_receive_packets_total counter -container_network_receive_packets_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 15 +container_network_receive_packets_total{foo_env="prod",foo_label="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 15 # HELP container_network_transmit_bytes_total Cumulative count of bytes transmitted # TYPE container_network_transmit_bytes_total counter -container_network_transmit_bytes_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 18 +container_network_transmit_bytes_total{foo_env="prod",foo_label="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 18 # HELP container_network_transmit_errors_total Cumulative count of errors encountered while transmitting # TYPE container_network_transmit_errors_total counter -container_network_transmit_errors_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 20 +container_network_transmit_errors_total{foo_env="prod",foo_label="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 20 # HELP container_network_transmit_packets_dropped_total Cumulative count of packets dropped while transmitting # TYPE container_network_transmit_packets_dropped_total counter -container_network_transmit_packets_dropped_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 21 +container_network_transmit_packets_dropped_total{foo_env="prod",foo_label="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 21 # HELP container_network_transmit_packets_total Cumulative count of packets transmitted # TYPE container_network_transmit_packets_total counter -container_network_transmit_packets_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 19 +container_network_transmit_packets_total{foo_env="prod",foo_label="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 19 # HELP container_scrape_error 1 if there was an error while getting container metrics, 0 otherwise # TYPE container_scrape_error gauge container_scrape_error 0 # HELP container_start_time_seconds Start time of the container since unix epoch in seconds. # TYPE container_start_time_seconds gauge -container_start_time_seconds{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1.257894e+09 +container_start_time_seconds{foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1.257894e+09 # HELP container_tasks_state Number of tasks in given state # TYPE container_tasks_state gauge -container_tasks_state{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",state="iowaiting",zone_name="hello"} 54 -container_tasks_state{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",state="running",zone_name="hello"} 51 -container_tasks_state{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",state="sleeping",zone_name="hello"} 50 -container_tasks_state{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",state="stopped",zone_name="hello"} 52 -container_tasks_state{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",state="uninterruptible",zone_name="hello"} 53 +container_tasks_state{foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",state="iowaiting",zone_name="hello"} 54 +container_tasks_state{foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",state="running",zone_name="hello"} 51 +container_tasks_state{foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",state="sleeping",zone_name="hello"} 50 +container_tasks_state{foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",state="stopped",zone_name="hello"} 52 +container_tasks_state{foo_env="prod",foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",state="uninterruptible",zone_name="hello"} 53 # HELP http_request_duration_microseconds The HTTP request latencies in microseconds. # TYPE http_request_duration_microseconds summary http_request_duration_microseconds{handler="prometheus",quantile="0.5"} 0