Metric collection compatibility enhancement

.github/workflows/test-grafana.yml

@@ -38,8 +38,6 @@ jobs:
         run: ./.github/check-service.sh blackbox-exporter
       - name: Test json exporter
         run: ./.github/check-service.sh json-exporter
-      - name: Test cryptowat exporter
-        run: ./.github/check-service.sh cryptowat-exporter
       - name: Test cadvisor
         run: ./.github/check-service.sh cadvisor
       - name: Test promtail

.gitignore

@@ -9,8 +9,6 @@ ext-network.yml.original
 !.eth/ethdo/README.md
 !.eth/ethdo/create-withdrawal-change.sh
 *.swp
-alertmanager/config.yml
-prometheus/custom-prom.yml
 ssv-config.yaml
 ssv-config.yaml.bak
 blox-ssv-config.yaml

grafana-cloud.yml

@@ -1,6 +1,5 @@
-# Custom Prometheus rules. Use prometheus/custom-prom.yml which will be appended to
-# /etc/prometheus/prometheus.yml inside the prometheus container. It will be appended after
-# the last scrape target that the entrypoint script adds.
+# Custom Prometheus rules. Use prometheus/custom-prom.yml which will be merged intoto
+# /etc/prometheus/prometheus.yml inside the prometheus container.
 #
 # You could add additional scrape targets as well as additional global sections such as remote-write.
 # This file is called grafana-cloud.yml as adding a remote write to Grafana Cloud is likely a
@@ -28,10 +27,12 @@ services:
       - /etc/localtime:/etc/localtime:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - ./prometheus/custom-prom.yml:/etc/prometheus/custom-prom.yml:ro
-    environment:
-      - CLIENT=${COMPOSE_FILE}
-    entrypoint: make-config.sh
-    command: ["/bin/prometheus", "--storage.tsdb.path=/prometheus", "--web.console.libraries=/usr/share/prometheus/console_libraries", "--web.console.templates=/usr/share/prometheus/consoles"]
+      - ./prometheus/conf.d/:/etc/prometheus/conf.d/:ro
+    command:
+      - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time=40d
+      - --web.console.libraries=/usr/share/prometheus/console_libraries
+      - --web.console.templates=/usr/share/prometheus/consoles
     <<: *logging
     labels:
       - metrics.scrape=true
@@ -102,22 +103,6 @@ services:
       - metrics.path=/metrics
       - metrics.port=7979
 
-  cryptowat-exporter:
-    restart: "unless-stopped"
-    image: nbarrientos/cryptowat_exporter:latest
-    volumes:
-      - /etc/localtime:/etc/localtime:ro
-    environment:
-      - CRYPTOWAT_EXCHANGES=kraken
-      - CRYPTOWAT_PAIRS=etheur,ethusd,ethgbp,ethcad,ethchf,ethjpy,ethbtc
-      - CRYPTOWAT_CACHESECONDS=300
-      - TZ=Europe/Zurich
-    <<: *logging
-    labels:
-      - metrics.scrape=true
-      - metrics.path=/metrics
-      - metrics.port=9745
-
   cadvisor:
     restart: "unless-stopped"
     image: gcr.io/cadvisor/cadvisor:v0.47.2

grafana-rootless.yml

@@ -17,16 +17,19 @@ services:
     volumes:
       - prom-data:/prometheus
       - /etc/localtime:/etc/localtime:ro
+      - ./prometheus/conf.d/:/etc/prometheus/conf.d/:ro
       - ./prometheus/custom-prom.yml:/etc/prometheus/custom-prom.yml:ro
     environment:
       - CLIENT=${COMPOSE_FILE}
-    entrypoint: choose-config.sh
-    command: ["/bin/prometheus", "--storage.tsdb.path=/prometheus", "--web.console.libraries=/usr/share/prometheus/console_libraries", "--web.console.templates=/usr/share/prometheus/consoles"]
+    command:
+      - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time=40d
+      - --web.console.libraries=/usr/share/prometheus/console_libraries
+      - --web.console.templates=/usr/share/prometheus/consoles
     <<: *logging
     depends_on:
       - blackbox-exporter
       - json-exporter
-      - cryptowat-exporter
       - ethereum-metrics-exporter
 
   ethereum-metrics-exporter:
@@ -62,18 +65,6 @@ services:
       - --config.file
       - /config/json.yml
 
-  cryptowat-exporter:
-    restart: "unless-stopped"
-    image: nbarrientos/cryptowat_exporter
-    volumes:
-      - /etc/localtime:/etc/localtime:ro
-    environment:
-      - CRYPTOWAT_EXCHANGES=kraken
-      - CRYPTOWAT_PAIRS=ethusd,ethchf
-      - CRYPTOWAT_CACHESECONDS=300
-      - TZ=Europe/Zurich
-    <<: *logging
-
   grafana:
     restart: "unless-stopped"
     build:

grafana.yml

@@ -18,12 +18,9 @@ services:
       - prom-data:/prometheus
       - /etc/localtime:/etc/localtime:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ./prometheus/conf.d/:/etc/prometheus/conf.d/:ro
       - ./prometheus/custom-prom.yml:/etc/prometheus/custom-prom.yml:ro
-    environment:
-      - CLIENT=${COMPOSE_FILE}
-    entrypoint: make-config.sh
     command:
-      - /bin/prometheus
       - --storage.tsdb.path=/prometheus
       - --storage.tsdb.retention.time=40d
       - --web.console.libraries=/usr/share/prometheus/console_libraries
@@ -98,22 +95,6 @@ services:
       - metrics.path=/metrics
       - metrics.port=7979
 
-  cryptowat-exporter:
-    restart: "unless-stopped"
-    image: nbarrientos/cryptowat_exporter:latest
-    volumes:
-      - /etc/localtime:/etc/localtime:ro
-    environment:
-      - CRYPTOWAT_EXCHANGES=kraken
-      - CRYPTOWAT_PAIRS=etheur,ethusd,ethgbp,ethcad,ethchf,ethjpy,ethbtc
-      - CRYPTOWAT_CACHESECONDS=300
-      - TZ=Europe/Zurich
-    <<: *logging
-    labels:
-      - metrics.scrape=true
-      - metrics.path=/metrics
-      - metrics.port=9745
-
   cadvisor:
     restart: "unless-stopped"
     image: gcr.io/cadvisor/cadvisor:v0.47.2

grafana/provision-dashboards.sh

@@ -9,6 +9,7 @@ if [ "$(id -u)" = '0' ]; then
   exec su-exec grafana "$0" "$@"
 fi
 
+shopt -s extglob
 case "$CLIENT" in
   *prysm* )
     #  prysm_small
@@ -118,6 +119,20 @@ case "$CLIENT" in
         | sed 's/{instance=~\\"\$instance\.\*\\"}//g' | sed 's/instance=~\\"\$instance\.\*\\",//g' \
         | sed 's/eXfXfqH7z/Prometheus/g' >"${__file}"
     ;;&
+  !(*grafana-rootless*) )
+      # cadvisor and node exporter dashboard
+      __id=10619
+      __revision=$(wget -t 3 -T 10 -qO - https://grafana.com/api/dashboards/${__id} | jq .revision)
+      __url="https://grafana.com/api/dashboards/${__id}/revisions/${__revision}/download"
+      __file='/etc/grafana/provisioning/dashboards/docker-host-container-overview.json'
+      wget -t 3 -T 10 -qcO - "${__url}" | jq 'walk(if . == "${DS_PROMETHEUS}" then "Prometheus" else . end)' >"${__file}"
+      # Log file dashboard (via loki)
+      __id=18700
+      __revision=$(wget -t 3 -T 10 -qO - https://grafana.com/api/dashboards/${__id} | jq .revision)
+      __url="https://grafana.com/api/dashboards/${__id}/revisions/${__revision}/download"
+      __file='/etc/grafana/provisioning/dashboards/eth-docker-logs.json'
+      wget -t 3 -T 10 -qcO - "${__url}" | jq 'walk(if . == "${DS_LOKI}" then "Loki" else . end)' >"${__file}"
+    ;;&
   * )
     # Home staking dashboard
     __id=17846
@@ -131,18 +146,6 @@ case "$CLIENT" in
     __url="https://grafana.com/api/dashboards/${__id}/revisions/${__revision}/download"
     __file='/etc/grafana/provisioning/dashboards/ethereum-metrics-exporter-single.json'
     wget -t 3 -T 10 -qcO - "${__url}" | jq 'walk(if . == "${DS_PROMETHEUS}" then "Prometheus" else . end)' >"${__file}"
-    # cadvisor and node exporter dashboard
-    __id=10619
-    __revision=$(wget -t 3 -T 10 -qO - https://grafana.com/api/dashboards/${__id} | jq .revision)
-    __url="https://grafana.com/api/dashboards/${__id}/revisions/${__revision}/download"
-    __file='/etc/grafana/provisioning/dashboards/docker-host-container-overview.json'
-    wget -t 3 -T 10 -qcO - "${__url}" | jq 'walk(if . == "${DS_PROMETHEUS}" then "Prometheus" else . end)' >"${__file}"
-    # Log file dashboard (via loki)
-    __id=18700
-    __revision=$(wget -t 3 -T 10 -qO - https://grafana.com/api/dashboards/${__id} | jq .revision)
-    __url="https://grafana.com/api/dashboards/${__id}/revisions/${__revision}/download"
-    __file='/etc/grafana/provisioning/dashboards/eth-docker-logs.json'
-    wget -t 3 -T 10 -qcO - "${__url}" | jq 'walk(if . == "${DS_LOKI}" then "Loki" else . end)' >"${__file}"
     ;;
 esac
 

prometheus/.gitignore

@@ -0,0 +1,5 @@
+# Ensure user-added configs don't get mixed up in git
+.conf.d/*
+!.conf.d/internet-scraper.yml
+!.conf.d/coingecko-scraper.yml
+custom-prom.yml

prometheus/Dockerfile

@@ -1,15 +1,13 @@
+ARG YQ_TAG=4.35.2@sha256:54e3310f27cb1776b2d2a8c66e4644fe5691a066c62583c82e6eadeb531a3c6c
+ARG YQ_REPO=mikefarah/yq
+FROM $YQ_REPO:$YQ_TAG AS yq
+
 FROM prom/prometheus:latest
 
-COPY ./*-prom.yml /etc/prometheus/
+COPY --from=yq /usr/bin/yq /usr/local/bin/yq
+
+COPY base-config.yml rootless-base-config.yml /etc/prometheus/
 COPY ./rootless/*.yml /etc/prometheus/rootless/
-COPY --chown=nobody:nobody ./make-config.sh /usr/local/bin/
-COPY --chown=nobody:nobody ./rootless/choose-config.sh /usr/local/bin/
-# Belt and suspenders
-RUN chmod -R 755 /usr/local/bin/*
-USER root
-RUN chown nobody:nobody /etc/prometheus/*.yml
-RUN chown nobody:nobody /etc/prometheus/rootless/*.yml
-USER nobody
-
-ENTRYPOINT make-config.sh
-CMD ["/bin/prometheus", "--storage.tsdb.path=/prometheus", "--web.console.libraries=/usr/share/prometheus/console_libraries", "--web.console.templates=/usr/share/prometheus/consoles"]
+COPY ./docker-entrypoint.sh /usr/local/bin/
+
+ENTRYPOINT [ "docker-entrypoint.sh" ]

prometheus/base-config.yml

@@ -0,0 +1,58 @@
+# This file is not intended to be modified - any changes here could be overwritten next time you update.
+# If you just need to add some new scrape targets to Prometheus, place these in the conf.d folder.
+# You can use the existing files as inspiration, and the docs are here:
+# https://prometheus.io/docs/prometheus/latest/configuration/configuration/
+#
+# If you need to override the settings in this file, or add other top-level configuration
+# (e.g.) remote_write config, please use custom-prom.yml.
+# The modifications in that file will supercede the settings here (or extend, in the case of lists)
+
+global:
+  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  scrape_timeout:      12s # Timeout must be shorter than the interval
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+
+# Service detection from the Docker socket.
+#
+# If you have extra containers you'd like to monitor, you will need to add the following labels:
+#   - `metrics.scrape=true`
+#   - `metrics.port=<the metrics port>`
+# And you will have to ensure that the container is connected to the same network by including
+# `ext-network.yml` in your .env file.
+scrape_configs:
+  - job_name: 'docker'
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+    relabel_configs:
+        - source_labels: [__meta_docker_container_label_metrics_scrape]
+          regex: true
+          action: keep
+        - source_labels: [__meta_docker_container_label_com_docker_compose_service]
+          target_label: job
+        - source_labels: [__meta_docker_container_label_com_docker_compose_project]
+          target_label: project
+        - action: replace  # Optional: override metrics_path using metrics.path (default is /metrics)
+          regex: (.+)
+          source_labels:
+          - __meta_docker_container_label_metrics_path
+          target_label: __metrics_path__
+        - action: replace  # Required: Configure the HTTP host and port to scrape using metrics.port
+          regex: ([^:]+)(?::\d+)?;(\d+)
+          replacement: $1:$2
+          source_labels:
+          - __address__
+          - __meta_docker_container_label_metrics_port
+          target_label: __address__
+        - action: replace  # Optional: Sets Prometheus 'job' label using metrics.job, otherwise uses the compose service name
+          regex: (.+)
+          source_labels:
+          - __meta_docker_container_label_metrics_job
+          target_label: job
+        - action: replace  # Optional: override the global scrape interval using metrics.interval
+          regex: (.+)
+          source_labels:
+          - __meta_docker_container_label_metrics_interval
+          target_label: __scrape_interval__
+
+scrape_config_files:
+  - /etc/prometheus/conf.d/*.yml

prometheus/conf.d/coingecko.yml

@@ -0,0 +1,18 @@
+# Scrape Coingeck price feeds
+
+scrape_configs:
+  - job_name: 'json'
+    metrics_path: /probe
+    scrape_interval: 300s
+    params:
+      module: [default]
+    static_configs:
+    - targets:
+      - https://api.coingecko.com/api/v3/simple/price?ids=ethereum&vs_currencies=usd
+    relabel_configs:
+    - source_labels: [__address__]
+      target_label: __param_target
+    - source_labels: [__param_target]
+      target_label: instance
+    - target_label: __address__
+      replacement: json-exporter:7979

prometheus/conf.d/ping.yml

@@ -0,0 +1,31 @@
+# Checking internet connectivity
+
+scrape_configs:
+  - job_name: 'ping_google'
+    metrics_path: /probe
+    params:
+      module: [icmp]
+    static_configs:
+      - targets:
+        - 8.8.8.8
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-exporter:9115
+  - job_name: 'ping_cloudflare'
+    metrics_path: /probe
+    params:
+      module: [icmp]
+    static_configs:
+      - targets:
+        - 1.1.1.1
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-exporter:9115

prometheus/custom-prom.yml.sample

@@ -1,5 +1,18 @@
+# Additional config for Prometheus. Modifications here will override the global settings, however
+# in the case of lists (e.g. scrape_configs) new items will be added.
+# This means that it's not possible to override existing scrape target configurations.
+
+global:
+  label_limit: 2000
+  scrape_interval: 10s
+  scrape_timeout: 8s
+  evaluation_interval: 10s
+
 remote_write:
   - url: "<Your Metrics instance remote_write endpoint>"
     basic_auth:
       username: "your Grafana Cloud username"
       password: "your Grafana Cloud API key"
+
+scrape_configs:
+  - job_name: Another job