From 776085b8d9dca63e900678162874fd3e342d5c39 Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Fri, 30 May 2025 16:15:43 +0000
Subject: [PATCH 1/3] Start documentation. Created Quickstart steps

---
 observability/docs/get-started/quickstart.md  | 86 +++++++++++++++++++
 .../docs/get-started/setup-tutorial.md        | 17 ++++
 .../docs/get-started/userguide-tutorial.md    |  0
 observability/docs/overview.md                | 18 ++++
 .../examples/simple/docker-compose.yml        | 11 +--
 .../examples/simple/probe-simple.yml          | 11 +++
 .../scrape-configs/probers/probe-simple.yml   |  6 --
 7 files changed, 135 insertions(+), 14 deletions(-)
 create mode 100644 observability/docs/get-started/quickstart.md
 create mode 100644 observability/docs/get-started/setup-tutorial.md
 create mode 100644 observability/docs/get-started/userguide-tutorial.md
 create mode 100644 observability/docs/overview.md
 create mode 100644 observability/examples/simple/probe-simple.yml
 delete mode 100644 observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml

diff --git a/observability/docs/get-started/quickstart.md b/observability/docs/get-started/quickstart.md
new file mode 100644
index 0000000..bebdd61
--- /dev/null
+++ b/observability/docs/get-started/quickstart.md
@@ -0,0 +1,86 @@
+## QuickStart
+
+This tutorial guides you through running the simplest setup of the observability stack using example configuration files and Docker Compose.
+
+After completing these steps, you will have a full observability stack running locally, showing the availability of web pages you want to target
+
+### Requirements
+
+- Docker installed ([install Docker](https://docs.docker.com/get-docker/))
+- Docker Compose installed ([install Docker Compose](https://docs.docker.com/compose/install/))
+- A terminal with network access
+
+### Step 1: Download the sample files
+
+Create a folder on your machine, then download these two files from GitHub into it:
+
+- `docker-compose.yml`
+- `prometheus-config.yml`
+
+You can do this manually or by running:
+
+```bash
+mkdir observability-stack
+cd observability-stack
+curl -O https://github.com/CogStack/cogstack-platform-toolkit/blob/main/observability/examples/simple/docker-compose.yml
+curl -O https://github.com/CogStack/cogstack-platform-toolkit/blob/main/observability/examples/simple/probe-simple.yml
+```
+
+### Step 2: Start the stack
+
+In the folder containing the downloaded files, run:
+
+```bash
+docker compose up -d
+```
+
+### Step 3: Access the dashboards
+Open your web browser and go to:
+
+`localhost/grafana`
+
+You should see the Grafana dashboard displaying the availability of the sample web page.
+
+There it is, you can now see the availability of the cogstack homepage, as well as the observability stack
+
+### Step 4: Probe your own web page
+Now you can look at getting monitoring on your own page.
+
+In your current folder, edit the file `probe-simple.yml` that you downloaded from git.
+
+Add the following yml to the bottom of the file:
+
+```yaml
+- targets:
+    - https://google.com
+  labels:
+    name: google
+    job: probe-my-own-site
+```
+
+
+The change should get applied automatically, but if you dont want to wait then run
+
+```
+docker compose restart
+```
+
+Now refresh the grafana dashboard, and you can see the availability of google.com, it's probably 100%!
+
+
+## Next steps
+This is the end of this quickstart tutorial, that enables probing availability of endpoints.
+
+For the next steps we can:
+- Productionise our deployment to enable further features
+- Enable *Telemetry* like VM memory usage, and Elasticsearch index size, by running Exporters
+- Enable *Alerting* based on our availability and a defined Service Level Objective (SLO)
+- Look further into the available dashboards
+- Fully customize the stack with our own dashboards, recording rules and metrics
+-
+
+
+
+
+
+
diff --git a/observability/docs/get-started/setup-tutorial.md b/observability/docs/get-started/setup-tutorial.md
new file mode 100644
index 0000000..8748972
--- /dev/null
+++ b/observability/docs/get-started/setup-tutorial.md
@@ -0,0 +1,17 @@
+
+This page 
+
+
+
+## Run the Observability Stack using Docker Compose
+
+See the /examples folder for a working example of running this
+
+To setup the stack for your deployment:
+- Create prometheus configurations as listed below
+- Copy this docker-compose.yml file
+- Mount your site config files into `/etc/prometheus/cogstack/site` 
+- Run with docker compose
+
+To collect metrics from VMs to fill out the dashboards
+- Run the Exporters on each VM as detailed below
\ No newline at end of file
diff --git a/observability/docs/get-started/userguide-tutorial.md b/observability/docs/get-started/userguide-tutorial.md
new file mode 100644
index 0000000..e69de29
diff --git a/observability/docs/overview.md b/observability/docs/overview.md
new file mode 100644
index 0000000..84158f2
--- /dev/null
+++ b/observability/docs/overview.md
@@ -0,0 +1,18 @@
+# Cogstack Observability Stack
+
+This project provides observability of a cogstack deployment.
+
+It provides the following features:
+- Dashboards for availability
+- Telemetry of infrastructure such as host memory usage, and elasticsearch index size
+- Alerting based on a defined Service Level Objective (SLO) and burn rates
+- Blackbox Probing of services to find service level indicators of uptime and latency
+- A working inventory of what is running where
+
+## Contents
+
+See the [Quickstart](./get-started/quickstart.md) to see how to easily run this stack.
+
+
+
+
diff --git a/observability/examples/simple/docker-compose.yml b/observability/examples/simple/docker-compose.yml
index 8be8a9f..a0ff472 100755
--- a/observability/examples/simple/docker-compose.yml
+++ b/observability/examples/simple/docker-compose.yml
@@ -6,7 +6,7 @@ services:
     image: cogstacksystems/cogstack-observability-prometheus:latest
     restart: unless-stopped
     volumes:
-      - ./prometheus:/etc/prometheus/cogstack/site
+      - ./probe-simple.yml:/etc/prometheus/cogstack/site/scrape-configs/probers/probe-simple.yml
       - prometheus-data:/prometheus
     networks:
       - observability
@@ -17,8 +17,6 @@ services:
       - grafana-data:/var/lib/grafana
     networks:
       - observability
-    ports:
-     - '3000:3000'
     environment:
       - GF_AUTH_ANONYMOUS_ENABLED=true # Allows use of grafana without sign in 
       - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
@@ -28,17 +26,14 @@ services:
      - observability
     restart: unless-stopped
     ports:
-      - "80:80"
-      - "8080:8080"
+      - "180:80"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
   blackbox-exporter:
     image: cogstacksystems/cogstack-observability-blackbox-exporter:latest
     restart: unless-stopped
-    ports:
-    - "9115:9115"
     networks:
-      - observability-exporters
+      - observability
 networks:
   observability:
     driver: bridge
diff --git a/observability/examples/simple/probe-simple.yml b/observability/examples/simple/probe-simple.yml
new file mode 100644
index 0000000..4a2ac3e
--- /dev/null
+++ b/observability/examples/simple/probe-simple.yml
@@ -0,0 +1,11 @@
+# Example of probe targets
+- targets:
+    - https://cogstack.org
+  labels:
+    name: cogstack-homepage
+    job: probe-services
+- targets:
+    - https://google.com
+  labels:
+    name: google
+    job: probe-services
\ No newline at end of file
diff --git a/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml b/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml
deleted file mode 100644
index a350490..0000000
--- a/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-# Example of probe targets
-- targets:
-    - https://cogstack.org
-  labels:
-    name: cogstack-homepage
-    job: "probe-simple-example"
\ No newline at end of file

From bfd26e0b8bdaef2a3ffc350eeaca69412838349b Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Mon, 2 Jun 2025 11:14:37 +0000
Subject: [PATCH 2/3] Use quickstart script to run it in one line

---
 observability/docs/get-started/quickstart.md  | 21 +++++++------------
 .../scrape-configs/probers}/probe-simple.yml  |  0
 observability/examples/simple/quickstart.sh   | 15 +++++++++++++
 3 files changed, 23 insertions(+), 13 deletions(-)
 rename observability/examples/simple/{ => prometheus/scrape-configs/probers}/probe-simple.yml (100%)
 create mode 100644 observability/examples/simple/quickstart.sh

diff --git a/observability/docs/get-started/quickstart.md b/observability/docs/get-started/quickstart.md
index bebdd61..7e061b6 100644
--- a/observability/docs/get-started/quickstart.md
+++ b/observability/docs/get-started/quickstart.md
@@ -12,19 +12,16 @@ After completing these steps, you will have a full observability stack running l
 
 ### Step 1: Download the sample files
 
-Create a folder on your machine, then download these two files from GitHub into it:
+Run this quickstart script to setup the project
+```bash
+curl https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/quickstart.sh | bash
+```
 
-- `docker-compose.yml`
-- `prometheus-config.yml`
+Alternatively, create this manually by downloading these two files from github, and creating the folder structure.
 
-You can do this manually or by running:
+- [docker-compose.yml](https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/docker-compose.yml)
+- [prometheus/scrape-configs/probers/probe-simple.yml](https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml)
 
-```bash
-mkdir observability-stack
-cd observability-stack
-curl -O https://github.com/CogStack/cogstack-platform-toolkit/blob/main/observability/examples/simple/docker-compose.yml
-curl -O https://github.com/CogStack/cogstack-platform-toolkit/blob/main/observability/examples/simple/probe-simple.yml
-```
 
 ### Step 2: Start the stack
 
@@ -46,7 +43,7 @@ There it is, you can now see the availability of the cogstack homepage, as well
 ### Step 4: Probe your own web page
 Now you can look at getting monitoring on your own page.
 
-In your current folder, edit the file `probe-simple.yml` that you downloaded from git.
+In your current folder, edit the file `prometheus/scrape-configs/probers/probe-simple.yml` that you downloaded from git.
 
 Add the following yml to the bottom of the file:
 
@@ -77,8 +74,6 @@ For the next steps we can:
 - Enable *Alerting* based on our availability and a defined Service Level Objective (SLO)
 - Look further into the available dashboards
 - Fully customize the stack with our own dashboards, recording rules and metrics
--
-
 
 
 
diff --git a/observability/examples/simple/probe-simple.yml b/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml
similarity index 100%
rename from observability/examples/simple/probe-simple.yml
rename to observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml
diff --git a/observability/examples/simple/quickstart.sh b/observability/examples/simple/quickstart.sh
new file mode 100644
index 0000000..adabd13
--- /dev/null
+++ b/observability/examples/simple/quickstart.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+mkdir -p observability-stack/prometheus/scrape-configs/probers
+cd observability-stack
+
+echo "Downloading docker-compose.yml..."
+curl -fsSL -o docker-compose.yml \
+  https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/docker-compose.yml
+
+echo "Downloading probe-simple.yml into prometheus/scrape-configs/probers/..."
+curl -fsSL -o prometheus/scrape-configs/probers/probe-simple.yml \
+  https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml
+
+echo "Setup complete in observability-stack/"

From a17c6eea0888793404a3d8364dcc9777befe4f19 Mon Sep 17 00:00:00 2001
From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.>
Date: Mon, 2 Jun 2025 12:33:41 +0000
Subject: [PATCH 3/3] Setup quickstart with script. Add Node exporter to
 quickstart

---
 .gitignore                                    |  5 +++
 .../docs/get-started/quickstart-manual.md     | 36 +++++++++++++++++++
 observability/docs/get-started/quickstart.md  | 30 +++-------------
 .../examples/simple/docker-compose.yml        | 14 ++++++--
 .../exporters/exporters-simple.yml            |  5 +++
 .../scrape-configs/probers/probe-simple.yml   |  5 ---
 observability/examples/simple/quickstart.sh   | 17 +++++++--
 7 files changed, 77 insertions(+), 35 deletions(-)
 create mode 100644 observability/docs/get-started/quickstart-manual.md
 create mode 100644 observability/examples/simple/prometheus/scrape-configs/exporters/exporters-simple.yml

diff --git a/.gitignore b/.gitignore
index b63db6c..66019d4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,8 @@
+# Custom ignores
+observability/examples/simple/observability-simple
+
+
+# Python ignores
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
diff --git a/observability/docs/get-started/quickstart-manual.md b/observability/docs/get-started/quickstart-manual.md
new file mode 100644
index 0000000..71d009c
--- /dev/null
+++ b/observability/docs/get-started/quickstart-manual.md
@@ -0,0 +1,36 @@
+
+## Manual Quickstart
+
+The quickstart page uses a script to setup the folders for you.
+
+This page instead details how to do it manually, to provide clarity.
+
+## Step 1: Setup directory
+Create the necessary directory structure
+```
+mkdir -p observability-simple/prometheus/scrape-configs/probers
+mkdir -p observability-simple/prometheus/scrape-configs/exporters
+```
+
+
+Download these two files from github, and place in the right folder
+
+- [docker-compose.yml](https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/docker-compose.yml) in observability-simple/
+- [prometheus/scrape-configs/probers/probe-simple.yml](https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml)
+
+### Step 2: Start the stack
+
+In the folder containing the downloaded files, run:
+
+```bash
+docker compose up -d
+```
+
+### Step 3: Access the dashboards
+Open your web browser and go to:
+
+`localhost/grafana`
+
+You should see the Grafana dashboard displaying the availability of the sample web page.
+
+There it is, you can now see the availability of the cogstack homepage, as well as the observability stack
diff --git a/observability/docs/get-started/quickstart.md b/observability/docs/get-started/quickstart.md
index 7e061b6..565ff5c 100644
--- a/observability/docs/get-started/quickstart.md
+++ b/observability/docs/get-started/quickstart.md
@@ -10,38 +10,19 @@ After completing these steps, you will have a full observability stack running l
 - Docker Compose installed ([install Docker Compose](https://docs.docker.com/compose/install/))
 - A terminal with network access
 
-### Step 1: Download the sample files
+### Step 1: Run the Quickstart script
 
 Run this quickstart script to setup the project
 ```bash
 curl https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/quickstart.sh | bash
 ```
+Now go to "http://localhost/grafana" to see the dashboards
 
-Alternatively, create this manually by downloading these two files from github, and creating the folder structure.
+Thats everything. The stack is running and you can see the availability.
 
-- [docker-compose.yml](https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/docker-compose.yml)
-- [prometheus/scrape-configs/probers/probe-simple.yml](https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml)
 
-
-### Step 2: Start the stack
-
-In the folder containing the downloaded files, run:
-
-```bash
-docker compose up -d
-```
-
-### Step 3: Access the dashboards
-Open your web browser and go to:
-
-`localhost/grafana`
-
-You should see the Grafana dashboard displaying the availability of the sample web page.
-
-There it is, you can now see the availability of the cogstack homepage, as well as the observability stack
-
-### Step 4: Probe your own web page
-Now you can look at getting monitoring on your own page.
+### Optional Step: Probe your own web page
+Now you can look at getting monitoring of your own page
 
 In your current folder, edit the file `prometheus/scrape-configs/probers/probe-simple.yml` that you downloaded from git.
 
@@ -57,7 +38,6 @@ Add the following yml to the bottom of the file:
 
 
 The change should get applied automatically, but if you dont want to wait then run
-
 ```
 docker compose restart
 ```
diff --git a/observability/examples/simple/docker-compose.yml b/observability/examples/simple/docker-compose.yml
index a0ff472..659a067 100755
--- a/observability/examples/simple/docker-compose.yml
+++ b/observability/examples/simple/docker-compose.yml
@@ -6,7 +6,7 @@ services:
     image: cogstacksystems/cogstack-observability-prometheus:latest
     restart: unless-stopped
     volumes:
-      - ./probe-simple.yml:/etc/prometheus/cogstack/site/scrape-configs/probers/probe-simple.yml
+      - ./prometheus:/etc/prometheus/cogstack/site/
       - prometheus-data:/prometheus
     networks:
       - observability
@@ -26,7 +26,7 @@ services:
      - observability
     restart: unless-stopped
     ports:
-      - "180:80"
+      - "80:80"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro       # So that Traefik can listen to the Docker events
   blackbox-exporter:
@@ -34,6 +34,16 @@ services:
     restart: unless-stopped
     networks:
       - observability
+  node-exporter:
+    image: prom/node-exporter
+    restart: unless-stopped
+    networks:
+      - observability
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.node-exporter.rule=PathPrefix(`/node-exporter`)"
+      - "traefik.http.middlewares.node-exporter-stripprefix.stripprefix.prefixes=/node-exporter"
+      - "traefik.http.routers.node-exporter.middlewares=node-exporter-stripprefix@docker"
 networks:
   observability:
     driver: bridge
diff --git a/observability/examples/simple/prometheus/scrape-configs/exporters/exporters-simple.yml b/observability/examples/simple/prometheus/scrape-configs/exporters/exporters-simple.yml
new file mode 100644
index 0000000..8036d08
--- /dev/null
+++ b/observability/examples/simple/prometheus/scrape-configs/exporters/exporters-simple.yml
@@ -0,0 +1,5 @@
+- targets:
+    - cogstack-observability-node-exporter-1:9100
+  labels:
+    job: node_exporter
+    host: localhost
\ No newline at end of file
diff --git a/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml b/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml
index 4a2ac3e..b3d7353 100644
--- a/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml
+++ b/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml
@@ -3,9 +3,4 @@
     - https://cogstack.org
   labels:
     name: cogstack-homepage
-    job: probe-services
-- targets:
-    - https://google.com
-  labels:
-    name: google
     job: probe-services
\ No newline at end of file
diff --git a/observability/examples/simple/quickstart.sh b/observability/examples/simple/quickstart.sh
index adabd13..45b67b7 100644
--- a/observability/examples/simple/quickstart.sh
+++ b/observability/examples/simple/quickstart.sh
@@ -1,8 +1,9 @@
 #!/bin/bash
 set -e
 
-mkdir -p observability-stack/prometheus/scrape-configs/probers
-cd observability-stack
+mkdir -p observability-simple/prometheus/scrape-configs/probers
+mkdir -p observability-simple/prometheus/scrape-configs/exporters
+cd observability-simple
 
 echo "Downloading docker-compose.yml..."
 curl -fsSL -o docker-compose.yml \
@@ -12,4 +13,14 @@ echo "Downloading probe-simple.yml into prometheus/scrape-configs/probers/..."
 curl -fsSL -o prometheus/scrape-configs/probers/probe-simple.yml \
   https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/prometheus/scrape-configs/probers/probe-simple.yml
 
-echo "Setup complete in observability-stack/"
+echo "Downloading exporters-simple.yml into prometheus/scrape-configs/exporters/..."
+curl -fsSL -o prometheus/scrape-configs/exporters/exporters-simple.yml \
+  https://raw.githubusercontent.com/CogStack/cogstack-platform-toolkit/main/observability/examples/simple/prometheus/scrape-configs/exporters/exporters-simple.yaml
+
+
+echo "Setup complete in observability-simple/"
+
+echo "Starting the observability stack"
+docker compose up -d
+
+ echo "Please open http://localhost/grafana in your browser"
\ No newline at end of file