Use Grafana Alloy to simplify Exporter usage

observability/build-docker.sh

@@ -0,0 +1,8 @@
+
+docker build -t cogstacksystems/cogstack-observability-prometheus:latest -f prometheus/Dockerfile.prometheus ./prometheus
+
+docker build -t cogstacksystems/cogstack-observability-blackbox-exporter:latest -f prometheus/Dockerfile.blackbox ./prometheus
+
+docker build -t cogstacksystems/cogstack-observability-grafana:latest -f grafana/Dockerfile ./grafana
+
+docker build -t cogstacksystems/cogstack-observability-traefik:latest -f traefik/Dockerfile ./traefik

observability/examples/alloy/docker-compose.yml

@@ -0,0 +1,78 @@
+# Observability main stack. Prometheus and Grafana. 
+# Depends on docker-compose.exporters.yml for the network
+name: "cogstack-observability"
+services:
+  prometheus:
+    image: cogstacksystems/cogstack-observability-prometheus:latest
+    restart: unless-stopped
+    ports:
+      - "9090:9090"
+    volumes:
+      - ${BASE_DIR-.}/prometheus:/etc/prometheus/cogstack/site/
+      - prometheus-data:/prometheus
+    networks:
+      - observability
+    command: 
+    - "--config.file=/etc/prometheus/cogstack/defaults/prometheus.yml"
+    - "--storage.tsdb.path=/prometheus"
+    - "--storage.tsdb.retention.time=30d"
+    - "--web.external-url=/prometheus"
+    - "--web.route-prefix=/prometheus"
+    - "--web.enable-remote-write-receiver"
+  grafana:
+    image: cogstacksystems/cogstack-observability-grafana:latest
+    restart: unless-stopped
+    volumes:
+      - grafana-data:/var/lib/grafana
+    networks:
+      - observability
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true # Allows use of grafana without sign in 
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
+  traefik:
+    image: cogstacksystems/cogstack-observability-traefik:latest
+    networks:
+     - observability
+    restart: unless-stopped
+    ports:
+      - "80:80"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
+  blackbox-exporter:
+    image: cogstacksystems/cogstack-observability-blackbox-exporter:latest
+    restart: unless-stopped
+    networks:
+      - observability
+  alloy:
+    image: grafana/alloy:latest
+    command:
+      - run
+      - --server.http.listen-addr=0.0.0.0:12345
+      - --storage.path=/var/lib/alloy/data
+      - --server.http.ui-path-prefix=/alloy
+      - /etc/alloy
+    ports:
+      - "12345:12345"
+    volumes:
+      - ${BASE_DIR-.}/grafana-alloy/config.alloy:/etc/alloy/config.alloy
+      # CAdvisor
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.alloy.rule=PathPrefix(`/alloy`)"
+    environment:
+      - PROMETHEUS_URL=${PROMETHEUS_URL-http://cogstack-observability-prometheus-1:9090/prometheus/api/v1/write}
+      - ALLOY_HOSTNAME=${ALLOY_HOSTNAME} # Used to add a label to metrics
+      - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS} # Used to add a label to metrics
+    networks:
+      - observability
+networks:
+  observability:
+    driver: bridge
+
+volumes:
+  prometheus-data:
+  grafana-data:

observability/examples/alloy/grafana-alloy/config.alloy

@@ -0,0 +1,38 @@
+logging {
+  level  = "debug"
+  format = "logfmt"
+}
+
+prometheus.remote_write "default" {
+  endpoint {
+    url = sys.env("PROMETHEUS_URL")
+  }
+  external_labels = {
+    host = sys.env("ALLOY_HOSTNAME"),
+    ip_address = sys.env("ALLOY_IP_ADDRESS"),
+  }
+}
+
+prometheus.scrape "exporter" {
+    scrape_interval = "15s"
+    targets    = array.concat(
+        prometheus.exporter.self.alloy.targets,
+        prometheus.exporter.cadvisor.local_cadvisor.targets,
+        prometheus.exporter.unix.local_node_exporter.targets,
+    )
+    forward_to = [prometheus.remote_write.default.receiver]
+}
+
+// Alloys internal metrics
+prometheus.exporter.self "alloy" {
+}
+
+// CAdvisor
+prometheus.exporter.cadvisor "local_cadvisor" {
+  docker_host = "unix:///var/run/docker.sock"
+  storage_duration = "5m"
+}
+
+// Node exporter
+prometheus.exporter.unix "local_node_exporter" {
+}

observability/examples/alloy/prometheus/scrape-configs/probers/probe-internal.yml

@@ -0,0 +1,6 @@
+# Example of probe targets
+- targets:
+    - https://cogstack.org
+  labels:
+    name: cogstack-homepage
+    job: probe-services

observability/examples/alloy/prometheus/scrape-configs/recording-rules/slo.yml

@@ -0,0 +1,8 @@
+groups:
+  - name:  slo-target-rules
+    rules:
+      # What SLO am I targeting
+      - record: slo_target_over_30_days
+        expr: 0.95 # We target 95% uptime over 30 days
+        labels:
+          job: "probe-external-demo-apps" #Job here must match the job in the probe targets

observability/examples/full/docker-compose.yml

@@ -29,6 +29,11 @@ services:
       - "80:80"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
+  blackbox-exporter:
+    image: cogstacksystems/cogstack-observability-blackbox-exporter:latest
+    restart: unless-stopped
+    networks:
+      - observability
 networks:
   observability:
     driver: bridge

observability/grafana/Dockerfile.alloy

@@ -0,0 +1 @@
+# TODO

observability/grafana/alloy/default-config.alloy

@@ -0,0 +1,38 @@
+logging {
+  level  = "debug"
+  format = "logfmt"
+}
+
+prometheus.remote_write "default" {
+  endpoint {
+    url = sys.env("PROMETHEUS_URL")
+  }
+  external_labels = {
+    host = sys.env("ALLOY_HOSTNAME"),
+    ip_address = sys.env("ALLOY_IP_ADDRESS"),
+  }
+}
+
+prometheus.scrape "exporter" {
+    scrape_interval = "15s"
+    targets    = array.concat(
+        prometheus.exporter.self.alloy.targets,
+        prometheus.exporter.cadvisor.local_cadvisor.targets,
+        prometheus.exporter.unix.local_node_exporter.targets,
+    )
+    forward_to = [prometheus.remote_write.default.receiver]
+}
+
+// Alloys internal metrics
+prometheus.exporter.self "alloy" {
+}
+
+// CAdvisor
+prometheus.exporter.cadvisor "local_cadvisor" {
+  docker_host = "unix:///var/run/docker.sock"
+  storage_duration = "5m"
+}
+
+// Node exporter
+prometheus.exporter.unix "local_node_exporter" {
+}

observability/prometheus/Dockerfile.prometheus

@@ -10,5 +10,6 @@ CMD [ \
     "--storage.tsdb.path=/prometheus", \
     "--storage.tsdb.retention.time=30d", \
     "--web.external-url=/prometheus", \
-    "--web.route-prefix=/prometheus" \
+    "--web.route-prefix=/prometheus", \
+    "--web.enable-remote-write-receiver" \
     ]