From 8940433542b549305f4e140d96bd2093a5be3ad2 Mon Sep 17 00:00:00 2001 From: Florian Pfitzer Date: Tue, 4 Aug 2015 14:49:42 +0200 Subject: [PATCH] Export image name and env variables as prometheus labels --- container/docker/factory.go | 6 ++ container/docker/handler.go | 13 ++++ metrics/prometheus.go | 5 ++ metrics/prometheus_test.go | 3 + metrics/testdata/prometheus_metrics | 106 ++++++++++++++-------------- 5 files changed, 80 insertions(+), 53 deletions(-) diff --git a/container/docker/factory.go b/container/docker/factory.go index e449586edf..bd59fc25ae 100644 --- a/container/docker/factory.go +++ b/container/docker/factory.go @@ -42,6 +42,8 @@ var DockerNamespace = "docker" var dockerRootDir = flag.String("docker_root", "/var/lib/docker", "Absolute path to the Docker state root directory (default: /var/lib/docker)") var dockerRunDir = flag.String("docker_run", "/var/run/docker", "Absolute path to the Docker run directory (default: /var/run/docker)") +var dockerMetadataEnvs = flag.String("docker_metadata_env", "", "Comma seperated list with names of env variables, which will be exported as metadata (default: empty)") + // TODO(vmarmol): Export run dir too for newer Dockers. // Directory holding Docker container state information. func DockerStateDir() string { @@ -101,6 +103,9 @@ func (self *dockerFactory) NewContainerHandler(name string, inHostNamespace bool if err != nil { return } + + exposedMetadata := strings.Split(*dockerMetadataEnvs, ",") + handler, err = newDockerContainerHandler( client, name, @@ -109,6 +114,7 @@ func (self *dockerFactory) NewContainerHandler(name string, inHostNamespace bool self.usesAufsDriver, &self.cgroupSubsystems, inHostNamespace, + exposedMetadata, ) return } diff --git a/container/docker/handler.go b/container/docker/handler.go index 7bf57e46a8..e90c235640 100644 --- a/container/docker/handler.go +++ b/container/docker/handler.go @@ -93,6 +93,7 @@ func newDockerContainerHandler( usesAufsDriver bool, cgroupSubsystems *containerLibcontainer.CgroupSubsystems, inHostNamespace bool, + exposedMetadata []string, ) (container.ContainerHandler, error) { // Create the cgroup paths. cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints)) @@ -141,6 +142,18 @@ func newDockerContainerHandler( handler.image = ctnr.Config.Image handler.networkMode = ctnr.HostConfig.NetworkMode + // split env vars to get metadata map. + if len(exposedMetadata) > 0 { + for _, envVar := range ctnr.Config.Env { + splits := strings.SplitN(envVar, "=", 2) + for _, exposedVar := range exposedMetadata { + if splits[0] == exposedVar { + handler.labels[strings.ToLower(exposedVar)] = splits[1] + } + } + } + } + return handler, nil } diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 2e8ec06a6f..636ba249ef 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -493,6 +493,11 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) } baseLabelValues := []string{id, name, image}[:len(baseLabels)] + for labelKey, labelValue := range container.Spec.Labels { + baseLabels = append(baseLabels, labelKey) + baseLabelValues = append(baseLabelValues, labelValue) + } + // Container spec desc := prometheus.NewDesc("container_start_time_seconds", "Start time of the container since unix epoch in seconds.", baseLabels, nil) ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.CreationTime.Unix()), baseLabelValues...) diff --git a/metrics/prometheus_test.go b/metrics/prometheus_test.go index 2ca5fa8133..a96c1474b6 100644 --- a/metrics/prometheus_test.go +++ b/metrics/prometheus_test.go @@ -55,6 +55,9 @@ func (p testSubcontainersInfoProvider) SubcontainersInfo(string, *info.Container Spec: info.ContainerSpec{ Image: "test", CreationTime: time.Unix(1257894000, 0), + Labels: map[string]string{ + "foo": "bar", + }, }, Stats: []*info.ContainerStats{ { diff --git a/metrics/testdata/prometheus_metrics b/metrics/testdata/prometheus_metrics index b4d089bfa4..cfc25ccd0f 100644 --- a/metrics/testdata/prometheus_metrics +++ b/metrics/testdata/prometheus_metrics @@ -3,120 +3,120 @@ cadvisor_version_info{cadvisorVersion="0.16.0",dockerVersion="1.8.1",kernelVersion="4.1.6-200.fc22.x86_64",osVersion="Fedora 22 (Twenty Two)"} 1 # HELP container_cpu_system_seconds_total Cumulative system cpu time consumed in seconds. # TYPE container_cpu_system_seconds_total counter -container_cpu_system_seconds_total{id="testcontainer",image="test",name="testcontaineralias"} 7e-09 +container_cpu_system_seconds_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 7e-09 # HELP container_cpu_usage_seconds_total Cumulative cpu time consumed per cpu in seconds. # TYPE container_cpu_usage_seconds_total counter -container_cpu_usage_seconds_total{cpu="cpu00",id="testcontainer",image="test",name="testcontaineralias"} 2e-09 -container_cpu_usage_seconds_total{cpu="cpu01",id="testcontainer",image="test",name="testcontaineralias"} 3e-09 -container_cpu_usage_seconds_total{cpu="cpu02",id="testcontainer",image="test",name="testcontaineralias"} 4e-09 -container_cpu_usage_seconds_total{cpu="cpu03",id="testcontainer",image="test",name="testcontaineralias"} 5e-09 +container_cpu_usage_seconds_total{cpu="cpu00",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 2e-09 +container_cpu_usage_seconds_total{cpu="cpu01",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 3e-09 +container_cpu_usage_seconds_total{cpu="cpu02",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4e-09 +container_cpu_usage_seconds_total{cpu="cpu03",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 5e-09 # HELP container_cpu_user_seconds_total Cumulative user cpu time consumed in seconds. # TYPE container_cpu_user_seconds_total counter -container_cpu_user_seconds_total{id="testcontainer",image="test",name="testcontaineralias"} 6e-09 +container_cpu_user_seconds_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 6e-09 # HELP container_fs_io_current Number of I/Os currently in progress # TYPE container_fs_io_current gauge -container_fs_io_current{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 42 -container_fs_io_current{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 47 +container_fs_io_current{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 42 +container_fs_io_current{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 47 # HELP container_fs_io_time_seconds_total Cumulative count of seconds spent doing I/Os # TYPE container_fs_io_time_seconds_total counter -container_fs_io_time_seconds_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 4.3e-08 -container_fs_io_time_seconds_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 4.8e-08 +container_fs_io_time_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.3e-08 +container_fs_io_time_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.8e-08 # HELP container_fs_io_time_weighted_seconds_total Cumulative weighted I/O time in seconds # TYPE container_fs_io_time_weighted_seconds_total counter -container_fs_io_time_weighted_seconds_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 4.4e-08 -container_fs_io_time_weighted_seconds_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 4.9e-08 +container_fs_io_time_weighted_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.4e-08 +container_fs_io_time_weighted_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.9e-08 # HELP container_fs_limit_bytes Number of bytes that can be consumed by the container on this filesystem. # TYPE container_fs_limit_bytes gauge -container_fs_limit_bytes{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 22 -container_fs_limit_bytes{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 37 +container_fs_limit_bytes{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 22 +container_fs_limit_bytes{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 37 # HELP container_fs_read_seconds_total Cumulative count of seconds spent reading # TYPE container_fs_read_seconds_total counter -container_fs_read_seconds_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 2.7e-08 -container_fs_read_seconds_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 4.2e-08 +container_fs_read_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 2.7e-08 +container_fs_read_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.2e-08 # HELP container_fs_reads_merged_total Cumulative count of reads merged # TYPE container_fs_reads_merged_total counter -container_fs_reads_merged_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 25 -container_fs_reads_merged_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 40 +container_fs_reads_merged_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 25 +container_fs_reads_merged_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 40 # HELP container_fs_reads_total Cumulative count of reads completed # TYPE container_fs_reads_total counter -container_fs_reads_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 24 -container_fs_reads_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 39 +container_fs_reads_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 24 +container_fs_reads_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 39 # HELP container_fs_sector_reads_total Cumulative count of sector reads completed # TYPE container_fs_sector_reads_total counter -container_fs_sector_reads_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 26 -container_fs_sector_reads_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 41 +container_fs_sector_reads_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 26 +container_fs_sector_reads_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 41 # HELP container_fs_sector_writes_total Cumulative count of sector writes completed # TYPE container_fs_sector_writes_total counter -container_fs_sector_writes_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 40 -container_fs_sector_writes_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 45 +container_fs_sector_writes_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 40 +container_fs_sector_writes_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 45 # HELP container_fs_usage_bytes Number of bytes that are consumed by the container on this filesystem. # TYPE container_fs_usage_bytes gauge -container_fs_usage_bytes{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 23 -container_fs_usage_bytes{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 38 +container_fs_usage_bytes{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 23 +container_fs_usage_bytes{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 38 # HELP container_fs_write_seconds_total Cumulative count of seconds spent writing # TYPE container_fs_write_seconds_total counter -container_fs_write_seconds_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 4.1e-08 -container_fs_write_seconds_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 4.6e-08 +container_fs_write_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.1e-08 +container_fs_write_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.6e-08 # HELP container_fs_writes_merged_total Cumulative count of writes merged # TYPE container_fs_writes_merged_total counter -container_fs_writes_merged_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 39 -container_fs_writes_merged_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 44 +container_fs_writes_merged_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 39 +container_fs_writes_merged_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 44 # HELP container_fs_writes_total Cumulative count of writes completed # TYPE container_fs_writes_total counter -container_fs_writes_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 28 -container_fs_writes_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 43 +container_fs_writes_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 28 +container_fs_writes_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 43 # HELP container_last_seen Last time a container was seen by the exporter # TYPE container_last_seen gauge -container_last_seen{id="testcontainer",image="test",name="testcontaineralias"} 1.426203694e+09 +container_last_seen{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 1.426203694e+09 # HELP container_memory_failures_total Cumulative count of memory allocation failures. # TYPE container_memory_failures_total counter -container_memory_failures_total{id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgfault"} 10 -container_memory_failures_total{id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgmajfault"} 11 -container_memory_failures_total{id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgfault"} 12 -container_memory_failures_total{id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgmajfault"} 13 +container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgfault"} 10 +container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgmajfault"} 11 +container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgfault"} 12 +container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgmajfault"} 13 # HELP container_memory_usage_bytes Current memory usage in bytes. # TYPE container_memory_usage_bytes gauge -container_memory_usage_bytes{id="testcontainer",image="test",name="testcontaineralias"} 8 +container_memory_usage_bytes{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 8 # HELP container_memory_working_set_bytes Current working set in bytes. # TYPE container_memory_working_set_bytes gauge -container_memory_working_set_bytes{id="testcontainer",image="test",name="testcontaineralias"} 9 +container_memory_working_set_bytes{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 9 # HELP container_network_receive_bytes_total Cumulative count of bytes received # TYPE container_network_receive_bytes_total counter -container_network_receive_bytes_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 14 +container_network_receive_bytes_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 14 # HELP container_network_receive_errors_total Cumulative count of errors encountered while receiving # TYPE container_network_receive_errors_total counter -container_network_receive_errors_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 16 +container_network_receive_errors_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 16 # HELP container_network_receive_packets_dropped_total Cumulative count of packets dropped while receiving # TYPE container_network_receive_packets_dropped_total counter -container_network_receive_packets_dropped_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 17 +container_network_receive_packets_dropped_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 17 # HELP container_network_receive_packets_total Cumulative count of packets received # TYPE container_network_receive_packets_total counter -container_network_receive_packets_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 15 +container_network_receive_packets_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 15 # HELP container_network_transmit_bytes_total Cumulative count of bytes transmitted # TYPE container_network_transmit_bytes_total counter -container_network_transmit_bytes_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 18 +container_network_transmit_bytes_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 18 # HELP container_network_transmit_errors_total Cumulative count of errors encountered while transmitting # TYPE container_network_transmit_errors_total counter -container_network_transmit_errors_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 20 +container_network_transmit_errors_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 20 # HELP container_network_transmit_packets_dropped_total Cumulative count of packets dropped while transmitting # TYPE container_network_transmit_packets_dropped_total counter -container_network_transmit_packets_dropped_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 21 +container_network_transmit_packets_dropped_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 21 # HELP container_network_transmit_packets_total Cumulative count of packets transmitted # TYPE container_network_transmit_packets_total counter -container_network_transmit_packets_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 19 +container_network_transmit_packets_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 19 # HELP container_scrape_error 1 if there was an error while getting container metrics, 0 otherwise # TYPE container_scrape_error gauge container_scrape_error 0 # HELP container_start_time_seconds Start time of the container since unix epoch in seconds. # TYPE container_start_time_seconds gauge -container_start_time_seconds{id="testcontainer",image="test",name="testcontaineralias"} 1.257894e+09 +container_start_time_seconds{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 1.257894e+09 # HELP container_tasks_state Number of tasks in given state # TYPE container_tasks_state gauge -container_tasks_state{id="testcontainer",image="test",name="testcontaineralias",state="iowaiting"} 54 -container_tasks_state{id="testcontainer",image="test",name="testcontaineralias",state="running"} 51 -container_tasks_state{id="testcontainer",image="test",name="testcontaineralias",state="sleeping"} 50 -container_tasks_state{id="testcontainer",image="test",name="testcontaineralias",state="stopped"} 52 -container_tasks_state{id="testcontainer",image="test",name="testcontaineralias",state="uninterruptible"} 53 +container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="iowaiting"} 54 +container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="running"} 51 +container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="sleeping"} 50 +container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="stopped"} 52 +container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="uninterruptible"} 53 # HELP http_request_duration_microseconds The HTTP request latencies in microseconds. # TYPE http_request_duration_microseconds summary http_request_duration_microseconds{handler="prometheus",quantile="0.5"} 0