Docker Compose(metrics): Metric Ingestion via Labels

docker-compose/common/compose-include/minio.yaml

@@ -1,8 +1,15 @@
 
 services:
   minio:
+    # https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md
     labels:
       - logs.agent.grafana.com/scrape=false
+      - metrics.agent.grafana.com/scrape=true
+      - metrics.agent.grafana.com/job=minio-job
+      - metrics.agent.grafana.com/path=/minio/v2/metrics/cluster
+      - metrics.agent.grafana.com/port=9000
+      - metrics.agent.grafana.com/interval=15s
+      - metrics.agent.grafana.com/timeout=10s
     image: ${MINIO_IMAGE:-docker.io/minio/minio:latest}
     entrypoint:
       - sh

docker-compose/common/config/agent-flow/logs.river

@@ -1,7 +1,7 @@
 // https://github.com/grafana/agent-configurator
 
 logging {
-	level  = "info"
+	level  = coalesce(env("AGENT_LOG_LEVEL"), "info")
 	format = "logfmt"
 }
 
@@ -10,7 +10,7 @@ logging {
  ********************************************/
 
 module.file "docker_compose" {
-	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker_compose.river"
+	filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker_compose.river"
 
 	arguments {
 		logs_endpoint = "http://gateway:3100"

docker-compose/common/config/agent-flow/metrics.river

@@ -1,10 +1,14 @@
 // https://github.com/grafana/agent-configurator
 
 logging {
-	level  = "warn"
+	level  = coalesce(env("AGENT_LOG_LEVEL"), "info")
 	format = "logfmt"
 }
 
+/********************************************
+ * LGTMP Receiver provider
+ ********************************************/
+
 module.file "docker_compose" {
 	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker_compose.river"
 
@@ -13,117 +17,15 @@ module.file "docker_compose" {
 	}
 }
 
-discovery.relabel "containers" {
-	targets = module.file.docker_compose.exports.relabelings_common.output
-}
-
 /********************************************
  * Metrics
  ********************************************/
 
-prometheus.exporter.unix "containers" {
-	set_collectors     = ["cpu"]
-	disable_collectors = ["diskstats", "mdadm", "textfile", "hwmon"]
-}
-
-prometheus.scrape "integrations" {
-	targets = concat(
-		prometheus.exporter.unix.containers.targets,
-	)
-
-	enable_protobuf_negotiation = true
-	scrape_classic_histograms   = true
-
-	scrape_interval = "15s"
-
-	clustering {
-		enabled = true
-	}
-
-	forward_to = [prometheus.relabel.integrations.receiver]
-}
+module.file "metrics_primary" {
+	filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/metrics/all.river"
 
-prometheus.scrape "containers" {
-	targets         = discovery.relabel.containers.output
-	scrape_interval = "15s"
-
-	enable_protobuf_negotiation = true
-	scrape_classic_histograms   = true
-
-	clustering {
-		enabled = true
-	}
-
-	forward_to = [module.file.docker_compose.exports.metrics_receiver]
-}
-
-prometheus.scrape "minio" {
-	targets = [{"__address__" = "minio:9000", "job" = "minio-job"}]
-
-	scrape_interval = "15s"
-
-	enable_protobuf_negotiation = true
-	scrape_classic_histograms   = true
-
-	clustering {
-		enabled = true
-	}
-	metrics_path = "/minio/v2/metrics/cluster"
-
-	forward_to = [prometheus.relabel.integrations.receiver]
-}
-
-prometheus.relabel "integrations" {
-	rule {
-		source_labels = ["job"]
-		regex         = "(integrations|monitoring-system)/(.*)"
-		target_label  = "pod"
-		replacement   = "${2}"
-	}
-
-	rule {
-		source_labels = ["job"]
-		regex         = "(integrations|monitoring-system)/(.*)"
-		target_label  = "container"
-		replacement   = "${2}"
-	}
-
-	forward_to = [module.file.docker_compose.exports.metrics_receiver]
-}
-
-/********************************************
- * Otelcol for metrics
- ********************************************/
-
-otelcol.receiver.otlp "containers" {
-	grpc {
-		endpoint = "0.0.0.0:4317"
-	}
-
-	http {
-		endpoint = "0.0.0.0:4318"
-	}
-
-	output {
-		metrics = [otelcol.processor.batch.containers.input]
-	}
-}
-
-otelcol.processor.batch "containers" {
-	output {
-		metrics = [otelcol.processor.memory_limiter.containers.input]
-	}
-}
-
-otelcol.processor.memory_limiter "containers" {
-	check_interval = "1s"
-	limit          = "256MiB"
-
-	output {
-		metrics = [otelcol.exporter.prometheus.containers.input]
+	arguments {
+		forward_to = [module.file.docker_compose.exports.metrics_receiver]
+		clustering = true
 	}
 }
-
-otelcol.exporter.prometheus "containers" {
-	forward_to = [module.file.docker_compose.exports.metrics_receiver]
-}

docker-compose/common/config/agent-flow/modules/docker/README.md

@@ -6,7 +6,7 @@ The following service labels are supported:
 
 | Label            | Description |
 | :--------------- | :-----------|
-| `logs.agent.grafana.com/scrape` | Allow a service to declare it's logs should be dropped. |
+| `logs.agent.grafana.com/scrape` | Allow a service to declare it's logs should be ingested (default is `true`). |
 | `logs.agent.grafana.com/tenant` | Allow a service to override the tenant for its logs. |
 | `logs.agent.grafana.com/log-format` | If specified additional processing is performed to extract details based on the specified format.  This value can be a comma-delimited list, in the instances a pod may have multiple containers.  The following formats are currently supported: <ul><li>common-log<li>donet<li>istio<li>json<li>klog<li>log4j-json<li>logfmt<li>otel<li>postgres<li>python<li>spring-boot<li>syslog<li>zerolog</ul> |
 | `logs.agent.grafana.com/scrub-level` | Boolean whether or not the level should be dropped from the log message (as it is a label). |
@@ -22,3 +22,20 @@ The following service labels are supported:
 | `logs.agent.grafana.com/mask-ipv4` | Boolean whether or not to mask IPv4 addresses in the log line, if true the data will be masked as`*ipv4*salt*` |
 | `logs.agent.grafana.com/mask-ipv6` | Boolean whether or not to mask IPv6 addresses in the log line, if true the data will be masked as `*ipv6*salt*` |
 | `logs.agent.grafana.com/mask-phone` | Boolean whether or not to mask phone numbers in the log line, if true the data will be masked as `*phone*salt*` |
+
+---
+
+## Metrics
+
+The following service labels are supported for gathering of metrics for docker compose services:
+
+| Label            | Description |
+| :--------------- | :-----------|
+| `metrics.agent.grafana.com/scrape` <br>or<br> `prometheus.io/scrape` | Boolean whether or not to scrape the service for metrics (default is `true`).|
+| `metrics.agent.grafana.com/scheme` <br>or<br> `prometheus.io/scheme` | The default scraping scheme is `http`, only support `http` now. |
+| `metrics.agent.grafana.com/path` <br>or<br> `prometheus.io/path` | the default path to scrape is `/metrics`, this can be specified as a single value which would override, the scrape path being used for all ports attached to the target |
+| `metrics.agent.grafana.com/port` <br>or<br> `prometheus.io/port` | the default `port` to scrape is the target port, this can be specified as a single value which would override the scrape port being used for all ports attached to the target, note that even if an target had multiple targets, the relabel_config targets are deduped before scraping |
+| `metrics.agent.grafana.com/tenant` | The tenant their metrics should be sent to, this does not necessarily have to be the actual tenantId, it can be a friendly name as well that is simply used to determine if the metrics should be gathered for the current tenant |
+| `metrics.agent.grafana.com/job` <br>or<br> `prometheus.io/job` | The job label value to use when collecting their metrics.  However, it is common to use an integration or community project where rules / dashboards are provided for you.  Oftentimes, this provided assets use hard-coded values for a job label i.e. `...{job="integrations/kubernetes/cadvisor"...}` or `...{job="minio-job"...}` setting this annotation to that value will allow the provided asset to work out of the box. |
+| `metrics.agent.grafana.com/interval` <br>or<br> `prometheus.io/interval` | The default interval to scrape is `15s`, this can be override. |
+| `metrics.agent.grafana.com/timeout` <br>or<br> `prometheus.io/timeout` | The default timeout for scraping is `10s`, this can be override. |

docker-compose/common/config/agent-flow/modules/docker/metrics/all.river

@@ -0,0 +1,67 @@
+/*
+Module: metrics-all
+Description: Wrapper module to include all Docker containers metric modules
+*/
+argument "forward_to" {
+	comment = "Must be a list(MetricssReceiver) where collected metrics should be forwarded to"
+}
+
+argument "tenant" {
+	comment  = "The tenant to filter logs to.  This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this can be a regex."
+	optional = true
+}
+
+argument "clustering" {
+	// Docs: https://grafana.com/docs/agent/latest/flow/concepts/clustering/
+	comment = "Whether or not clustering should be enabled (default: true)"
+}
+
+module.file "mf_metrics_auto_scrape" {
+	filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/metrics/metrics-auto-scrape.river"
+
+	arguments {
+		forward_to = argument.forward_to.value
+		tenant     = coalesce(argument.tenant.value, ".*")
+		clustering = coalesce(argument.clustering.value, "true")
+	}
+}
+
+prometheus.exporter.unix "peu_containers" {
+	set_collectors     = ["cpu"]
+	disable_collectors = ["diskstats", "mdadm", "textfile", "hwmon"]
+}
+
+prometheus.scrape "pc_integrations" {
+	targets = concat(
+		prometheus.exporter.unix.peu_containers.targets,
+	)
+
+	enable_protobuf_negotiation = true
+	scrape_classic_histograms   = true
+
+	scrape_interval = "15s"
+
+	clustering {
+		enabled = true
+	}
+
+	forward_to = [prometheus.relabel.pr_integrations.receiver]
+}
+
+prometheus.relabel "pr_integrations" {
+	rule {
+		source_labels = ["job"]
+		regex         = "integrations/(.*)"
+		target_label  = "pod"
+		replacement   = "${2}"
+	}
+
+	rule {
+		source_labels = ["job"]
+		regex         = "integrations/(.*)"
+		target_label  = "container"
+		replacement   = "${2}"
+	}
+
+	forward_to = argument.forward_to.value
+}