Skip to content

Commit

Permalink
Merge pull request #49 from qclaogui:metrics-ingestion-via-labels
Browse files Browse the repository at this point in the history
Docker Compose(metrics): Metric Ingestion via Labels
  • Loading branch information
qclaogui committed Mar 14, 2024
2 parents 4eaaf25 + 45dcbf4 commit 090a3ba
Show file tree
Hide file tree
Showing 13 changed files with 520 additions and 257 deletions.
7 changes: 7 additions & 0 deletions docker-compose/common/compose-include/minio.yaml
Original file line number Diff line number Diff line change
@@ -1,8 +1,15 @@

services:
minio:
# https://github.com/qclaogui/codelab-monitoring/blob/main/docker-compose/common/config/agent-flow/modules/docker/README.md
labels:
- logs.agent.grafana.com/scrape=false
- metrics.agent.grafana.com/scrape=true
- metrics.agent.grafana.com/job=minio-job
- metrics.agent.grafana.com/path=/minio/v2/metrics/cluster
- metrics.agent.grafana.com/port=9000
- metrics.agent.grafana.com/interval=15s
- metrics.agent.grafana.com/timeout=10s
image: ${MINIO_IMAGE:-docker.io/minio/minio:latest}
entrypoint:
- sh
Expand Down
4 changes: 2 additions & 2 deletions docker-compose/common/config/agent-flow/logs.river
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// https://github.com/grafana/agent-configurator

logging {
level = "info"
level = coalesce(env("AGENT_LOG_LEVEL"), "info")
format = "logfmt"
}

Expand All @@ -10,7 +10,7 @@ logging {
********************************************/

module.file "docker_compose" {
filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker_compose.river"
filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker_compose.river"

arguments {
logs_endpoint = "http://gateway:3100"
Expand Down
118 changes: 10 additions & 108 deletions docker-compose/common/config/agent-flow/metrics.river
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
// https://github.com/grafana/agent-configurator

logging {
level = "warn"
level = coalesce(env("AGENT_LOG_LEVEL"), "info")
format = "logfmt"
}

/********************************************
* LGTMP Receiver provider
********************************************/

module.file "docker_compose" {
filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker_compose.river"

Expand All @@ -13,117 +17,15 @@ module.file "docker_compose" {
}
}

discovery.relabel "containers" {
targets = module.file.docker_compose.exports.relabelings_common.output
}

/********************************************
* Metrics
********************************************/

prometheus.exporter.unix "containers" {
set_collectors = ["cpu"]
disable_collectors = ["diskstats", "mdadm", "textfile", "hwmon"]
}

prometheus.scrape "integrations" {
targets = concat(
prometheus.exporter.unix.containers.targets,
)

enable_protobuf_negotiation = true
scrape_classic_histograms = true

scrape_interval = "15s"

clustering {
enabled = true
}

forward_to = [prometheus.relabel.integrations.receiver]
}
module.file "metrics_primary" {
filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/metrics/all.river"

prometheus.scrape "containers" {
targets = discovery.relabel.containers.output
scrape_interval = "15s"

enable_protobuf_negotiation = true
scrape_classic_histograms = true

clustering {
enabled = true
}

forward_to = [module.file.docker_compose.exports.metrics_receiver]
}

prometheus.scrape "minio" {
targets = [{"__address__" = "minio:9000", "job" = "minio-job"}]

scrape_interval = "15s"

enable_protobuf_negotiation = true
scrape_classic_histograms = true

clustering {
enabled = true
}
metrics_path = "/minio/v2/metrics/cluster"

forward_to = [prometheus.relabel.integrations.receiver]
}

prometheus.relabel "integrations" {
rule {
source_labels = ["job"]
regex = "(integrations|monitoring-system)/(.*)"
target_label = "pod"
replacement = "${2}"
}

rule {
source_labels = ["job"]
regex = "(integrations|monitoring-system)/(.*)"
target_label = "container"
replacement = "${2}"
}

forward_to = [module.file.docker_compose.exports.metrics_receiver]
}

/********************************************
* Otelcol for metrics
********************************************/

otelcol.receiver.otlp "containers" {
grpc {
endpoint = "0.0.0.0:4317"
}

http {
endpoint = "0.0.0.0:4318"
}

output {
metrics = [otelcol.processor.batch.containers.input]
}
}

otelcol.processor.batch "containers" {
output {
metrics = [otelcol.processor.memory_limiter.containers.input]
}
}

otelcol.processor.memory_limiter "containers" {
check_interval = "1s"
limit = "256MiB"

output {
metrics = [otelcol.exporter.prometheus.containers.input]
arguments {
forward_to = [module.file.docker_compose.exports.metrics_receiver]
clustering = true
}
}

otelcol.exporter.prometheus "containers" {
forward_to = [module.file.docker_compose.exports.metrics_receiver]
}
19 changes: 18 additions & 1 deletion docker-compose/common/config/agent-flow/modules/docker/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ The following service labels are supported:

| Label | Description |
| :--------------- | :-----------|
| `logs.agent.grafana.com/scrape` | Allow a service to declare it's logs should be dropped. |
| `logs.agent.grafana.com/scrape` | Allow a service to declare it's logs should be ingested (default is `true`). |
| `logs.agent.grafana.com/tenant` | Allow a service to override the tenant for its logs. |
| `logs.agent.grafana.com/log-format` | If specified additional processing is performed to extract details based on the specified format. This value can be a comma-delimited list, in the instances a pod may have multiple containers. The following formats are currently supported: <ul><li>common-log<li>donet<li>istio<li>json<li>klog<li>log4j-json<li>logfmt<li>otel<li>postgres<li>python<li>spring-boot<li>syslog<li>zerolog</ul> |
| `logs.agent.grafana.com/scrub-level` | Boolean whether or not the level should be dropped from the log message (as it is a label). |
Expand All @@ -22,3 +22,20 @@ The following service labels are supported:
| `logs.agent.grafana.com/mask-ipv4` | Boolean whether or not to mask IPv4 addresses in the log line, if true the data will be masked as`*ipv4*salt*` |
| `logs.agent.grafana.com/mask-ipv6` | Boolean whether or not to mask IPv6 addresses in the log line, if true the data will be masked as `*ipv6*salt*` |
| `logs.agent.grafana.com/mask-phone` | Boolean whether or not to mask phone numbers in the log line, if true the data will be masked as `*phone*salt*` |

---

## Metrics

The following service labels are supported for gathering of metrics for docker compose services:

| Label | Description |
| :--------------- | :-----------|
| `metrics.agent.grafana.com/scrape` <br>or<br> `prometheus.io/scrape` | Boolean whether or not to scrape the service for metrics (default is `true`).|
| `metrics.agent.grafana.com/scheme` <br>or<br> `prometheus.io/scheme` | The default scraping scheme is `http`, only support `http` now. |
| `metrics.agent.grafana.com/path` <br>or<br> `prometheus.io/path` | the default path to scrape is `/metrics`, this can be specified as a single value which would override, the scrape path being used for all ports attached to the target |
| `metrics.agent.grafana.com/port` <br>or<br> `prometheus.io/port` | the default `port` to scrape is the target port, this can be specified as a single value which would override the scrape port being used for all ports attached to the target, note that even if an target had multiple targets, the relabel_config targets are deduped before scraping |
| `metrics.agent.grafana.com/tenant` | The tenant their metrics should be sent to, this does not necessarily have to be the actual tenantId, it can be a friendly name as well that is simply used to determine if the metrics should be gathered for the current tenant |
| `metrics.agent.grafana.com/job` <br>or<br> `prometheus.io/job` | The job label value to use when collecting their metrics. However, it is common to use an integration or community project where rules / dashboards are provided for you. Oftentimes, this provided assets use hard-coded values for a job label i.e. `...{job="integrations/kubernetes/cadvisor"...}` or `...{job="minio-job"...}` setting this annotation to that value will allow the provided asset to work out of the box. |
| `metrics.agent.grafana.com/interval` <br>or<br> `prometheus.io/interval` | The default interval to scrape is `15s`, this can be override. |
| `metrics.agent.grafana.com/timeout` <br>or<br> `prometheus.io/timeout` | The default timeout for scraping is `10s`, this can be override. |
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
Module: metrics-all
Description: Wrapper module to include all Docker containers metric modules
*/
argument "forward_to" {
comment = "Must be a list(MetricssReceiver) where collected metrics should be forwarded to"
}

argument "tenant" {
comment = "The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this can be a regex."
optional = true
}

argument "clustering" {
// Docs: https://grafana.com/docs/agent/latest/flow/concepts/clustering/
comment = "Whether or not clustering should be enabled (default: true)"
}

module.file "mf_metrics_auto_scrape" {
filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-config") + "/modules/docker/metrics/metrics-auto-scrape.river"

arguments {
forward_to = argument.forward_to.value
tenant = coalesce(argument.tenant.value, ".*")
clustering = coalesce(argument.clustering.value, "true")
}
}

prometheus.exporter.unix "peu_containers" {
set_collectors = ["cpu"]
disable_collectors = ["diskstats", "mdadm", "textfile", "hwmon"]
}

prometheus.scrape "pc_integrations" {
targets = concat(
prometheus.exporter.unix.peu_containers.targets,
)

enable_protobuf_negotiation = true
scrape_classic_histograms = true

scrape_interval = "15s"

clustering {
enabled = true
}

forward_to = [prometheus.relabel.pr_integrations.receiver]
}

prometheus.relabel "pr_integrations" {
rule {
source_labels = ["job"]
regex = "integrations/(.*)"
target_label = "pod"
replacement = "${2}"
}

rule {
source_labels = ["job"]
regex = "integrations/(.*)"
target_label = "container"
replacement = "${2}"
}

forward_to = argument.forward_to.value
}
Loading

0 comments on commit 090a3ba

Please sign in to comment.