Skip to content

Export image name and env variables as prometheus labels #780

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Export image name and env variables as prometheus labels #780

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions container/docker/factory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ var DockerNamespace = "docker"
var dockerRootDir = flag.String("docker_root", "/var/lib/docker", "Absolute path to the Docker state root directory (default: /var/lib/docker)")
var dockerRunDir = flag.String("docker_run", "/var/run/docker", "Absolute path to the Docker run directory (default: /var/run/docker)")

var dockerMetadataEnvs = flag.String("docker_metadata_env", "", "Comma seperated list with names of env variables, which will be exported as metadata (default: empty)")

// TODO(vmarmol): Export run dir too for newer Dockers.
// Directory holding Docker container state information.
func DockerStateDir() string {
Expand Down Expand Up @@ -101,6 +103,9 @@ func (self *dockerFactory) NewContainerHandler(name string, inHostNamespace bool
if err != nil {
return
}

exposedMetadata := strings.Split(*dockerMetadataEnvs, ",")

handler, err = newDockerContainerHandler(
client,
name,
Expand All @@ -109,6 +114,7 @@ func (self *dockerFactory) NewContainerHandler(name string, inHostNamespace bool
self.usesAufsDriver,
&self.cgroupSubsystems,
inHostNamespace,
exposedMetadata,
)
return
}
Expand Down
13 changes: 13 additions & 0 deletions container/docker/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ func newDockerContainerHandler(
usesAufsDriver bool,
cgroupSubsystems *containerLibcontainer.CgroupSubsystems,
inHostNamespace bool,
exposedMetadata []string,
) (container.ContainerHandler, error) {
// Create the cgroup paths.
cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints))
Expand Down Expand Up @@ -141,6 +142,18 @@ func newDockerContainerHandler(
handler.image = ctnr.Config.Image
handler.networkMode = ctnr.HostConfig.NetworkMode

// split env vars to get metadata map.
if len(exposedMetadata) > 0 {
for _, envVar := range ctnr.Config.Env {
splits := strings.SplitN(envVar, "=", 2)
for _, exposedVar := range exposedMetadata {
if splits[0] == exposedVar {
handler.labels[strings.ToLower(exposedVar)] = splits[1]
}
}
}
}

return handler, nil
}

Expand Down
5 changes: 5 additions & 0 deletions metrics/prometheus.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -493,6 +493,11 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric)
}
baseLabelValues := []string{id, name, image}[:len(baseLabels)]

for labelKey, labelValue := range container.Spec.Labels {
baseLabels = append(baseLabels, labelKey)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

labelKey will need to be sanitized as Prometheus labels need to match this regexp.

You can copy the sanitize method from here & please add a test for this.

baseLabelValues = append(baseLabelValues, labelValue)
}

// Container spec
desc := prometheus.NewDesc("container_start_time_seconds", "Start time of the container since unix epoch in seconds.", baseLabels, nil)
ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.CreationTime.Unix()), baseLabelValues...)
Expand Down
3 changes: 3 additions & 0 deletions metrics/prometheus_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ func (p testSubcontainersInfoProvider) SubcontainersInfo(string, *info.Container
Spec: info.ContainerSpec{
Image: "test",
CreationTime: time.Unix(1257894000, 0),
Labels: map[string]string{
"foo": "bar",
},
},
Stats: []*info.ContainerStats{
{
Expand Down
106 changes: 53 additions & 53 deletions metrics/testdata/prometheus_metrics
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,120 +3,120 @@
cadvisor_version_info{cadvisorVersion="0.16.0",dockerVersion="1.8.1",kernelVersion="4.1.6-200.fc22.x86_64",osVersion="Fedora 22 (Twenty Two)"} 1
# HELP container_cpu_system_seconds_total Cumulative system cpu time consumed in seconds.
# TYPE container_cpu_system_seconds_total counter
container_cpu_system_seconds_total{id="testcontainer",image="test",name="testcontaineralias"} 7e-09
container_cpu_system_seconds_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 7e-09
# HELP container_cpu_usage_seconds_total Cumulative cpu time consumed per cpu in seconds.
# TYPE container_cpu_usage_seconds_total counter
container_cpu_usage_seconds_total{cpu="cpu00",id="testcontainer",image="test",name="testcontaineralias"} 2e-09
container_cpu_usage_seconds_total{cpu="cpu01",id="testcontainer",image="test",name="testcontaineralias"} 3e-09
container_cpu_usage_seconds_total{cpu="cpu02",id="testcontainer",image="test",name="testcontaineralias"} 4e-09
container_cpu_usage_seconds_total{cpu="cpu03",id="testcontainer",image="test",name="testcontaineralias"} 5e-09
container_cpu_usage_seconds_total{cpu="cpu00",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 2e-09
container_cpu_usage_seconds_total{cpu="cpu01",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 3e-09
container_cpu_usage_seconds_total{cpu="cpu02",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4e-09
container_cpu_usage_seconds_total{cpu="cpu03",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 5e-09
# HELP container_cpu_user_seconds_total Cumulative user cpu time consumed in seconds.
# TYPE container_cpu_user_seconds_total counter
container_cpu_user_seconds_total{id="testcontainer",image="test",name="testcontaineralias"} 6e-09
container_cpu_user_seconds_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 6e-09
# HELP container_fs_io_current Number of I/Os currently in progress
# TYPE container_fs_io_current gauge
container_fs_io_current{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 42
container_fs_io_current{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 47
container_fs_io_current{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 42
container_fs_io_current{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 47
# HELP container_fs_io_time_seconds_total Cumulative count of seconds spent doing I/Os
# TYPE container_fs_io_time_seconds_total counter
container_fs_io_time_seconds_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 4.3e-08
container_fs_io_time_seconds_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 4.8e-08
container_fs_io_time_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.3e-08
container_fs_io_time_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.8e-08
# HELP container_fs_io_time_weighted_seconds_total Cumulative weighted I/O time in seconds
# TYPE container_fs_io_time_weighted_seconds_total counter
container_fs_io_time_weighted_seconds_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 4.4e-08
container_fs_io_time_weighted_seconds_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 4.9e-08
container_fs_io_time_weighted_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.4e-08
container_fs_io_time_weighted_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.9e-08
# HELP container_fs_limit_bytes Number of bytes that can be consumed by the container on this filesystem.
# TYPE container_fs_limit_bytes gauge
container_fs_limit_bytes{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 22
container_fs_limit_bytes{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 37
container_fs_limit_bytes{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 22
container_fs_limit_bytes{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 37
# HELP container_fs_read_seconds_total Cumulative count of seconds spent reading
# TYPE container_fs_read_seconds_total counter
container_fs_read_seconds_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 2.7e-08
container_fs_read_seconds_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 4.2e-08
container_fs_read_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 2.7e-08
container_fs_read_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.2e-08
# HELP container_fs_reads_merged_total Cumulative count of reads merged
# TYPE container_fs_reads_merged_total counter
container_fs_reads_merged_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 25
container_fs_reads_merged_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 40
container_fs_reads_merged_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 25
container_fs_reads_merged_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 40
# HELP container_fs_reads_total Cumulative count of reads completed
# TYPE container_fs_reads_total counter
container_fs_reads_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 24
container_fs_reads_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 39
container_fs_reads_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 24
container_fs_reads_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 39
# HELP container_fs_sector_reads_total Cumulative count of sector reads completed
# TYPE container_fs_sector_reads_total counter
container_fs_sector_reads_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 26
container_fs_sector_reads_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 41
container_fs_sector_reads_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 26
container_fs_sector_reads_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 41
# HELP container_fs_sector_writes_total Cumulative count of sector writes completed
# TYPE container_fs_sector_writes_total counter
container_fs_sector_writes_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 40
container_fs_sector_writes_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 45
container_fs_sector_writes_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 40
container_fs_sector_writes_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 45
# HELP container_fs_usage_bytes Number of bytes that are consumed by the container on this filesystem.
# TYPE container_fs_usage_bytes gauge
container_fs_usage_bytes{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 23
container_fs_usage_bytes{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 38
container_fs_usage_bytes{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 23
container_fs_usage_bytes{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 38
# HELP container_fs_write_seconds_total Cumulative count of seconds spent writing
# TYPE container_fs_write_seconds_total counter
container_fs_write_seconds_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 4.1e-08
container_fs_write_seconds_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 4.6e-08
container_fs_write_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.1e-08
container_fs_write_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.6e-08
# HELP container_fs_writes_merged_total Cumulative count of writes merged
# TYPE container_fs_writes_merged_total counter
container_fs_writes_merged_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 39
container_fs_writes_merged_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 44
container_fs_writes_merged_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 39
container_fs_writes_merged_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 44
# HELP container_fs_writes_total Cumulative count of writes completed
# TYPE container_fs_writes_total counter
container_fs_writes_total{device="sda1",id="testcontainer",image="test",name="testcontaineralias"} 28
container_fs_writes_total{device="sda2",id="testcontainer",image="test",name="testcontaineralias"} 43
container_fs_writes_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 28
container_fs_writes_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 43
# HELP container_last_seen Last time a container was seen by the exporter
# TYPE container_last_seen gauge
container_last_seen{id="testcontainer",image="test",name="testcontaineralias"} 1.426203694e+09
container_last_seen{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 1.426203694e+09
# HELP container_memory_failures_total Cumulative count of memory allocation failures.
# TYPE container_memory_failures_total counter
container_memory_failures_total{id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgfault"} 10
container_memory_failures_total{id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgmajfault"} 11
container_memory_failures_total{id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgfault"} 12
container_memory_failures_total{id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgmajfault"} 13
container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgfault"} 10
container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgmajfault"} 11
container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgfault"} 12
container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgmajfault"} 13
# HELP container_memory_usage_bytes Current memory usage in bytes.
# TYPE container_memory_usage_bytes gauge
container_memory_usage_bytes{id="testcontainer",image="test",name="testcontaineralias"} 8
container_memory_usage_bytes{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 8
# HELP container_memory_working_set_bytes Current working set in bytes.
# TYPE container_memory_working_set_bytes gauge
container_memory_working_set_bytes{id="testcontainer",image="test",name="testcontaineralias"} 9
container_memory_working_set_bytes{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 9
# HELP container_network_receive_bytes_total Cumulative count of bytes received
# TYPE container_network_receive_bytes_total counter
container_network_receive_bytes_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 14
container_network_receive_bytes_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 14
# HELP container_network_receive_errors_total Cumulative count of errors encountered while receiving
# TYPE container_network_receive_errors_total counter
container_network_receive_errors_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 16
container_network_receive_errors_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 16
# HELP container_network_receive_packets_dropped_total Cumulative count of packets dropped while receiving
# TYPE container_network_receive_packets_dropped_total counter
container_network_receive_packets_dropped_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 17
container_network_receive_packets_dropped_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 17
# HELP container_network_receive_packets_total Cumulative count of packets received
# TYPE container_network_receive_packets_total counter
container_network_receive_packets_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 15
container_network_receive_packets_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 15
# HELP container_network_transmit_bytes_total Cumulative count of bytes transmitted
# TYPE container_network_transmit_bytes_total counter
container_network_transmit_bytes_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 18
container_network_transmit_bytes_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 18
# HELP container_network_transmit_errors_total Cumulative count of errors encountered while transmitting
# TYPE container_network_transmit_errors_total counter
container_network_transmit_errors_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 20
container_network_transmit_errors_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 20
# HELP container_network_transmit_packets_dropped_total Cumulative count of packets dropped while transmitting
# TYPE container_network_transmit_packets_dropped_total counter
container_network_transmit_packets_dropped_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 21
container_network_transmit_packets_dropped_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 21
# HELP container_network_transmit_packets_total Cumulative count of packets transmitted
# TYPE container_network_transmit_packets_total counter
container_network_transmit_packets_total{id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 19
container_network_transmit_packets_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 19
# HELP container_scrape_error 1 if there was an error while getting container metrics, 0 otherwise
# TYPE container_scrape_error gauge
container_scrape_error 0
# HELP container_start_time_seconds Start time of the container since unix epoch in seconds.
# TYPE container_start_time_seconds gauge
container_start_time_seconds{id="testcontainer",image="test",name="testcontaineralias"} 1.257894e+09
container_start_time_seconds{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 1.257894e+09
# HELP container_tasks_state Number of tasks in given state
# TYPE container_tasks_state gauge
container_tasks_state{id="testcontainer",image="test",name="testcontaineralias",state="iowaiting"} 54
container_tasks_state{id="testcontainer",image="test",name="testcontaineralias",state="running"} 51
container_tasks_state{id="testcontainer",image="test",name="testcontaineralias",state="sleeping"} 50
container_tasks_state{id="testcontainer",image="test",name="testcontaineralias",state="stopped"} 52
container_tasks_state{id="testcontainer",image="test",name="testcontaineralias",state="uninterruptible"} 53
container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="iowaiting"} 54
container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="running"} 51
container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="sleeping"} 50
container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="stopped"} 52
container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="uninterruptible"} 53
# HELP http_request_duration_microseconds The HTTP request latencies in microseconds.
# TYPE http_request_duration_microseconds summary
http_request_duration_microseconds{handler="prometheus",quantile="0.5"} 0
Expand Down