Merge pull request #2 from groove-x/feature/support-docker-container-…

…metrics

support docker metrics

.gitignore

@@ -15,3 +15,5 @@
 pkg-build
 build
 *.deb
+.idea
+/cgroup-exporter

README.md

@@ -19,6 +19,7 @@ container_cpu_user_seconds_total{id="/system.slice/wpa_supplicant.service"} 524.
 container_cpu_user_seconds_total{id="/system.slice/ssh.service"} 1.30
 container_cpu_user_seconds_total{id="/system.slice/docker.service"} 2219.16
 container_cpu_user_seconds_total{id="/system.slice/NetworkManager.service"} 4283.36
+container_cpu_user_seconds_total{id="/docker/grafana"} 0.70
 :
 
 # HELP container_memory_usage_bytes Current memory usage in bytes, including all memory regardless of when it was accessed
@@ -27,6 +28,7 @@ container_memory_usage_bytes{id="/system.slice/wpa_supplicant.service"} 1871872
 container_memory_usage_bytes{id="/system.slice/ssh.service"} 61440
 container_memory_usage_bytes{id="/system.slice/docker.service"} 37171200
 container_memory_usage_bytes{id="/system.slice/NetworkManager.service"} 18305024
+container_memory_usage_bytes{id="/docker/grafana"} 61407232
 :
 
 # HELP container_memory_rss Size of RSS in bytes.
@@ -35,5 +37,27 @@ container_memory_rss{id="/system.slice/wpa_supplicant.service"} 331776
 container_memory_rss{id="/system.slice/ssh.service"} 110592
 container_memory_rss{id="/system.slice/docker.service"} 24072192
 container_memory_rss{id="/system.slice/NetworkManager.service"} 5066752
+container_memory_rss{id="/docker/grafana"} 16224256
 :
 ```
+
+## options
+
+| arg | description |
+| --- | --- |
+| `--metrics.docker` | enable docker container metrics |
+
+
+## customize systemd
+
+You can customize systemd setting with options.
+
+`/etc/systemd/system/prometheus-cgroup-exporter.service.d/local.conf`:
+
+```
+[Service]
+ExecStart=
+ExecStart=/usr/local/bin/cgroup-exporter --metrics.docker
+```
+
+see [systemd.unit / Example 2. Overriding vendor settings](https://www.freedesktop.org/software/systemd/man/systemd.unit.html#id-1.14.3).

deb.json

@@ -1,6 +1,6 @@
 {
   "name": "prometheus-cgroup-exporter",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "maintainer": "GROOVE X Development Team <dev@groove-x.com>",
   "description": "prometheus cgroup exporter",
   "changelog-cmd": "git log --pretty='format:%cd %h %s %d [%an]' --date=iso --merges",

go.mod

@@ -3,12 +3,17 @@ module github.com/groove-x/cgroup-exporter
 go 1.11
 
 require (
+	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/containerd/cgroups v0.0.0-20181001140508-d2400726cfa7
 	github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v1.13.1
+	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.3.4-0.20181018102642-03db2b60b820 // indirect
 	github.com/godbus/dbus v4.1.1-0.20180905195443-5f1bd775722e+incompatible // indirect
 	github.com/gogo/protobuf v1.2.0 // indirect
+	github.com/moby/moby v1.13.1
+	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/runtime-spec v1.0.2-0.20180913141938-5806c3563733 // indirect
-	github.com/pkg/errors v0.8.1-0.20181014145847-6ed0a2e59ebe // indirect
-	golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06 // indirect
+	golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect
 )

go.sum

@@ -1,16 +1,47 @@
+github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
+github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
 github.com/containerd/cgroups v0.0.0-20181001140508-d2400726cfa7 h1:HpTs4G3F32Jue3Hu3Tc4hD4d9AhALxYE4+vCO0SkSvw=
 github.com/containerd/cgroups v0.0.0-20181001140508-d2400726cfa7/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI=
 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d h1:t5Wuyh53qYyg9eqn4BbnlIT+vmhyww0TatL+zT3uWgI=
 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
+github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
+github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.3.4-0.20181018102642-03db2b60b820 h1:EQSiSOsNbDAIw+kdd4NPThBl+510eNrB58SRYcKwUK4=
 github.com/docker/go-units v0.3.4-0.20181018102642-03db2b60b820/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/godbus/dbus v4.1.1-0.20180905195443-5f1bd775722e+incompatible h1:1EAOqHEDzmRZ+SWNSMZ3nnczBDhMT1jIfE3QZ2iHZ9s=
 github.com/godbus/dbus v4.1.1-0.20180905195443-5f1bd775722e+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/moby/moby v1.13.1 h1:mC5WwQwCXt/dYxZ1cIrRsnJAWw7VdtcTZUIGr4tXzOM=
+github.com/moby/moby v1.13.1/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/runtime-spec v1.0.2-0.20180913141938-5806c3563733 h1:M9SVV7xezQ8PC/0NPAek6TUr1IdwVx5wGkdojnXwalM=
 github.com/opencontainers/runtime-spec v1.0.2-0.20180913141938-5806c3563733/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/pkg/errors v0.8.1-0.20181014145847-6ed0a2e59ebe h1:iN/Ye31z0uoRJNk0Cd8EtLFjKmkl59NtvEw2zt6nbBA=
-github.com/pkg/errors v0.8.1-0.20181014145847-6ed0a2e59ebe/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06 h1:0oC8rFnE+74kEmuHZ46F6KHsMr5Gx2gUQPuNz28iQZM=
-golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"log"
@@ -14,14 +15,17 @@ import (
 	"time"
 
 	"github.com/containerd/cgroups"
+	"github.com/docker/docker/api/types"
+	"github.com/moby/moby/client"
 )
 
 var (
 	versionFlag = flag.Bool("version", false, "version")
 	version     string
 	git         string
 
-	address = flag.String("address", ":48900", "address")
+	address      = flag.String("address", ":48900", "address")
+	enableDocker = flag.Bool("metrics.docker", false, "docker container metrics")
 )
 
 func main() {
@@ -39,7 +43,16 @@ func main() {
 		log.Fatalf("cgroups load: %s", err)
 	}
 
-	http.HandleFunc("/metrics", exportMetrics(system))
+	var dockerClient *client.Client
+	if *enableDocker {
+		dockerClient, err = client.NewEnvClient()
+		if err != nil {
+			log.Fatalf("%v", err)
+		}
+		defer dockerClient.Close()
+	}
+
+	http.HandleFunc("/metrics", exportMetrics(system, dockerClient))
 
 	server := &http.Server{
 		Addr: *address,
@@ -69,35 +82,89 @@ func subsystem() ([]cgroups.Subsystem, error) {
 	return s, nil
 }
 
-func exportMetrics(system cgroups.Cgroup) func(w http.ResponseWriter, r *http.Request) {
+func statsCgroups(ctx context.Context, system cgroups.Cgroup) (map[string]*cgroups.Metrics, error) {
+	processes, err := system.Processes(cgroups.Devices, true)
+	if err != nil {
+		return nil, fmt.Errorf("cgroups load: %s", err)
+	}
+
+	groups := make(map[string]*cgroups.Metrics, len(processes))
+	for _, p := range processes {
+		name := strings.TrimPrefix(p.Path, "/sys/fs/cgroup/devices")
+		name = strings.TrimSuffix(name, "/")
+		if _, ok := groups[name]; ok {
+			continue
+		}
+
+		control, err := cgroups.Load(subsystem, func(subsystem cgroups.Name) (string, error) {
+			return name, nil
+		})
+		if err != nil {
+			log.Printf("cgroups load: %s", err)
+			continue
+		}
+		stats, err := control.Stat(cgroups.IgnoreNotExist)
+		if err != nil {
+			log.Printf("control stat: %s", err)
+			continue
+		}
+		groups[name] = stats
+	}
+	return groups, nil
+}
+
+type dockerStats struct {
+	CPU    types.CPUStats    `json:"cpu_stats,omitempty"`
+	PreCPU types.CPUStats    `json:"precpu_stats,omitempty"` // "Pre"="Previous"
+	Memory types.MemoryStats `json:"memory_stats,omitempty"`
+}
+
+func statsDockerContainers(ctx context.Context, dockerClient *client.Client) (map[string]dockerStats, error) {
+	if dockerClient == nil {
+		return nil, nil
+	}
+	containers, err := dockerClient.ContainerList(ctx, types.ContainerListOptions{
+		All:   true,
+		Limit: 0,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("list docker containers: %s", err)
+	}
+
+	dockerContainers := make(map[string]dockerStats, len(containers))
+	for _, container := range containers {
+		res, err := dockerClient.ContainerStats(ctx, container.ID, false)
+		if err != nil {
+			log.Printf("failed to stats docker container %s: %s", container.ID, err)
+			continue
+		}
+		var stats dockerStats
+		if err := json.NewDecoder(res.Body).Decode(&stats); err != nil {
+			res.Body.Close()
+			return nil, fmt.Errorf("failed to decode stats json: %s", err)
+		}
+		res.Body.Close()
+		name := fmt.Sprintf("/docker%s", strings.Join(container.Names, "/"))
+		dockerContainers[name] = stats
+	}
+
+	return dockerContainers, nil
+}
+
+func exportMetrics(system cgroups.Cgroup, dockerClient *client.Client) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
-		processes, err := system.Processes(cgroups.Devices, true)
+		ctx := r.Context()
+
+		groups, err := statsCgroups(ctx, system)
 		if err != nil {
-			msg := fmt.Sprintf("cgroups load: %s", err)
-			http.Error(w, msg, http.StatusInternalServerError)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
 		}
 
-		groups := make(map[string]*cgroups.Metrics, len(processes))
-		for _, p := range processes {
-			name := strings.TrimPrefix(p.Path, "/sys/fs/cgroup/devices")
-			name = strings.TrimSuffix(name, "/")
-			if _, ok := groups[name]; ok {
-				continue
-			}
-
-			control, err := cgroups.Load(subsystem, func(subsystem cgroups.Name) (string, error) {
-				return name, nil
-			})
-			if err != nil {
-				log.Printf("cgroups load: %s", err)
-				continue
-			}
-			stats, err := control.Stat(cgroups.IgnoreNotExist)
-			if err != nil {
-				log.Printf("control stat: %s", err)
-				continue
-			}
-			groups[name] = stats
+		dockerContainers, err := statsDockerContainers(ctx, dockerClient)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
 		}
 
 		fmt.Fprintln(w, `# HELP container_cpu_user_seconds_total Cumulative user cpu time consumed in seconds.
@@ -106,20 +173,32 @@ func exportMetrics(system cgroups.Cgroup) func(w http.ResponseWriter, r *http.Re
 			fmt.Fprintf(w, `container_cpu_user_seconds_total{id=%s} %.2f`, strconv.Quote(name), float64(stats.CPU.Usage.User)/1000000000.0)
 			fmt.Fprintln(w)
 		}
+		for name, stats := range dockerContainers {
+			fmt.Fprintf(w, `container_cpu_user_seconds_total{id=%s} %.2f`, strconv.Quote(name), float64(stats.CPU.CPUUsage.UsageInUsermode)/1000000000.0)
+			fmt.Fprintln(w)
+		}
 
 		fmt.Fprintln(w, `# HELP container_memory_usage_bytes Current memory usage in bytes, including all memory regardless of when it was accessed
 # TYPE container_memory_usage_bytes gauge`)
 		for name, stats := range groups {
 			fmt.Fprintf(w, `container_memory_usage_bytes{id=%s} %d`, strconv.Quote(name), stats.Memory.Usage.Usage)
 			fmt.Fprintln(w)
 		}
+		for name, stats := range dockerContainers {
+			fmt.Fprintf(w, `container_memory_usage_bytes{id=%s} %d`, strconv.Quote(name), stats.Memory.Usage)
+			fmt.Fprintln(w)
+		}
 
 		fmt.Fprintln(w, `# HELP container_memory_rss Size of RSS in bytes.
 # TYPE container_memory_rss gauge`)
 		for name, stats := range groups {
 			fmt.Fprintf(w, `container_memory_rss{id=%s} %d`, strconv.Quote(name), stats.Memory.RSS)
 			fmt.Fprintln(w)
 		}
+		for name, stats := range dockerContainers {
+			fmt.Fprintf(w, `container_memory_rss{id=%s} %d`, strconv.Quote(name), stats.Memory.Stats["rss"])
+			fmt.Fprintln(w)
+		}
 
 		return
 	}