diff --git a/observability/build-docker.sh b/observability/build-docker.sh new file mode 100644 index 0000000..d3521cc --- /dev/null +++ b/observability/build-docker.sh @@ -0,0 +1,8 @@ + +docker build -t cogstacksystems/cogstack-observability-prometheus:latest -f prometheus/Dockerfile.prometheus ./prometheus + +docker build -t cogstacksystems/cogstack-observability-blackbox-exporter:latest -f prometheus/Dockerfile.blackbox ./prometheus + +docker build -t cogstacksystems/cogstack-observability-grafana:latest -f grafana/Dockerfile ./grafana + +docker build -t cogstacksystems/cogstack-observability-traefik:latest -f traefik/Dockerfile ./traefik \ No newline at end of file diff --git a/observability/examples/alloy/docker-compose.yml b/observability/examples/alloy/docker-compose.yml new file mode 100755 index 0000000..4795443 --- /dev/null +++ b/observability/examples/alloy/docker-compose.yml @@ -0,0 +1,78 @@ +# Observability main stack. Prometheus and Grafana. +# Depends on docker-compose.exporters.yml for the network +name: "cogstack-observability" +services: + prometheus: + image: cogstacksystems/cogstack-observability-prometheus:latest + restart: unless-stopped + ports: + - "9090:9090" + volumes: + - ${BASE_DIR-.}/prometheus:/etc/prometheus/cogstack/site/ + - prometheus-data:/prometheus + networks: + - observability + command: + - "--config.file=/etc/prometheus/cogstack/defaults/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=30d" + - "--web.external-url=/prometheus" + - "--web.route-prefix=/prometheus" + - "--web.enable-remote-write-receiver" + grafana: + image: cogstacksystems/cogstack-observability-grafana:latest + restart: unless-stopped + volumes: + - grafana-data:/var/lib/grafana + networks: + - observability + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true # Allows use of grafana without sign in + - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer + traefik: + image: cogstacksystems/cogstack-observability-traefik:latest + networks: + - observability + restart: unless-stopped + ports: + - "80:80" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro # So that Traefik can listen to the Docker events + blackbox-exporter: + image: cogstacksystems/cogstack-observability-blackbox-exporter:latest + restart: unless-stopped + networks: + - observability + alloy: + image: grafana/alloy:latest + command: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + - --server.http.ui-path-prefix=/alloy + - /etc/alloy + ports: + - "12345:12345" + volumes: + - ${BASE_DIR-.}/grafana-alloy/config.alloy:/etc/alloy/config.alloy + # CAdvisor + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + labels: + - "traefik.enable=true" + - "traefik.http.routers.alloy.rule=PathPrefix(`/alloy`)" + environment: + - PROMETHEUS_URL=${PROMETHEUS_URL-http://cogstack-observability-prometheus-1:9090/prometheus/api/v1/write} + - ALLOY_HOSTNAME=${ALLOY_HOSTNAME} # Used to add a label to metrics + - ALLOY_IP_ADDRESS=${ALLOY_IP_ADDRESS} # Used to add a label to metrics + networks: + - observability +networks: + observability: + driver: bridge + +volumes: + prometheus-data: + grafana-data: \ No newline at end of file diff --git a/observability/examples/alloy/grafana-alloy/config.alloy b/observability/examples/alloy/grafana-alloy/config.alloy new file mode 100644 index 0000000..eaa2c5c --- /dev/null +++ b/observability/examples/alloy/grafana-alloy/config.alloy @@ -0,0 +1,38 @@ +logging { + level = "debug" + format = "logfmt" +} + +prometheus.remote_write "default" { + endpoint { + url = sys.env("PROMETHEUS_URL") + } + external_labels = { + host = sys.env("ALLOY_HOSTNAME"), + ip_address = sys.env("ALLOY_IP_ADDRESS"), + } +} + +prometheus.scrape "exporter" { + scrape_interval = "15s" + targets = array.concat( + prometheus.exporter.self.alloy.targets, + prometheus.exporter.cadvisor.local_cadvisor.targets, + prometheus.exporter.unix.local_node_exporter.targets, + ) + forward_to = [prometheus.remote_write.default.receiver] +} + +// Alloys internal metrics +prometheus.exporter.self "alloy" { +} + +// CAdvisor +prometheus.exporter.cadvisor "local_cadvisor" { + docker_host = "unix:///var/run/docker.sock" + storage_duration = "5m" +} + +// Node exporter +prometheus.exporter.unix "local_node_exporter" { +} diff --git a/observability/examples/alloy/prometheus/scrape-configs/probers/probe-internal.yml b/observability/examples/alloy/prometheus/scrape-configs/probers/probe-internal.yml new file mode 100644 index 0000000..b3d7353 --- /dev/null +++ b/observability/examples/alloy/prometheus/scrape-configs/probers/probe-internal.yml @@ -0,0 +1,6 @@ +# Example of probe targets +- targets: + - https://cogstack.org + labels: + name: cogstack-homepage + job: probe-services \ No newline at end of file diff --git a/observability/examples/alloy/prometheus/scrape-configs/recording-rules/slo.yml b/observability/examples/alloy/prometheus/scrape-configs/recording-rules/slo.yml new file mode 100644 index 0000000..440913c --- /dev/null +++ b/observability/examples/alloy/prometheus/scrape-configs/recording-rules/slo.yml @@ -0,0 +1,8 @@ +groups: + - name: slo-target-rules + rules: + # What SLO am I targeting + - record: slo_target_over_30_days + expr: 0.95 # We target 95% uptime over 30 days + labels: + job: "probe-external-demo-apps" #Job here must match the job in the probe targets \ No newline at end of file diff --git a/observability/examples/full/docker-compose.yml b/observability/examples/full/docker-compose.yml index 8fb9362..ab994fa 100755 --- a/observability/examples/full/docker-compose.yml +++ b/observability/examples/full/docker-compose.yml @@ -29,6 +29,11 @@ services: - "80:80" volumes: - /var/run/docker.sock:/var/run/docker.sock:ro # So that Traefik can listen to the Docker events + blackbox-exporter: + image: cogstacksystems/cogstack-observability-blackbox-exporter:latest + restart: unless-stopped + networks: + - observability networks: observability: driver: bridge diff --git a/observability/grafana/Dockerfile.alloy b/observability/grafana/Dockerfile.alloy new file mode 100644 index 0000000..f87f5c1 --- /dev/null +++ b/observability/grafana/Dockerfile.alloy @@ -0,0 +1 @@ +# TODO \ No newline at end of file diff --git a/observability/grafana/alloy/default-config.alloy b/observability/grafana/alloy/default-config.alloy new file mode 100644 index 0000000..eaa2c5c --- /dev/null +++ b/observability/grafana/alloy/default-config.alloy @@ -0,0 +1,38 @@ +logging { + level = "debug" + format = "logfmt" +} + +prometheus.remote_write "default" { + endpoint { + url = sys.env("PROMETHEUS_URL") + } + external_labels = { + host = sys.env("ALLOY_HOSTNAME"), + ip_address = sys.env("ALLOY_IP_ADDRESS"), + } +} + +prometheus.scrape "exporter" { + scrape_interval = "15s" + targets = array.concat( + prometheus.exporter.self.alloy.targets, + prometheus.exporter.cadvisor.local_cadvisor.targets, + prometheus.exporter.unix.local_node_exporter.targets, + ) + forward_to = [prometheus.remote_write.default.receiver] +} + +// Alloys internal metrics +prometheus.exporter.self "alloy" { +} + +// CAdvisor +prometheus.exporter.cadvisor "local_cadvisor" { + docker_host = "unix:///var/run/docker.sock" + storage_duration = "5m" +} + +// Node exporter +prometheus.exporter.unix "local_node_exporter" { +} diff --git a/observability/prometheus/Dockerfile.prometheus b/observability/prometheus/Dockerfile.prometheus index 0db9a47..a743fcb 100644 --- a/observability/prometheus/Dockerfile.prometheus +++ b/observability/prometheus/Dockerfile.prometheus @@ -10,5 +10,6 @@ CMD [ \ "--storage.tsdb.path=/prometheus", \ "--storage.tsdb.retention.time=30d", \ "--web.external-url=/prometheus", \ - "--web.route-prefix=/prometheus" \ + "--web.route-prefix=/prometheus", \ + "--web.enable-remote-write-receiver" \ ] \ No newline at end of file