chore: Metrics

.env.example

@@ -34,5 +34,13 @@ ENV_DOCKER_USER_GROUP="ggroup"
 ENV_PING_USERNAME=
 ENV_PING_PASSWORD=
 
+# --- HTTP Server
+ENV_HTTP_PORT=8080
+
 # --- SEO: SPA application directory
 ENV_SPA_DIR=
+ENV_SPA_IMAGES_DIR=
+
+# --- Monitoring: Grafana admin password
+#     REQUIRED for Grafana dashboard access
+GRAFANA_ADMIN_PASSWORD=

.gitignore

@@ -10,8 +10,8 @@ storage/seo/*.*
 !storage/seo/.gitkeep
 
 # --- [Caddy]: mtls
-caddy/mtls/*.*
-!caddy/mtls/.gitkeep
+infra/caddy/mtls/*.*
+!infra/caddy/mtls/.gitkeep
 
 # --- [API]: Bin
 bin/*

Makefile

@@ -34,14 +34,15 @@ CGO_ENABLED           := 1
 # -------------------------------------------------------------------------------------------------------------------- #
 # -------------------------------------------------------------------------------------------------------------------- #
 
-include ./metal/makefile/helpers.mk
-include ./metal/makefile/env.mk
-include ./metal/makefile/db.mk
-include ./metal/makefile/app.mk
-include ./metal/makefile/logs.mk
-include ./metal/makefile/build.mk
-include ./metal/makefile/infra.mk
-include ./metal/makefile/caddy.mk
+include ./infra/makefile/helpers.mk
+include ./infra/makefile/env.mk
+include ./infra/makefile/db.mk
+include ./infra/makefile/app.mk
+include ./infra/makefile/logs.mk
+include ./infra/makefile/build.mk
+include ./infra/makefile/infra.mk
+include ./infra/makefile/caddy.mk
+include ./infra/makefile/monitor.mk
 
 # -------------------------------------------------------------------------------------------------------------------- #
 # -------------------------------------------------------------------------------------------------------------------- #
@@ -104,6 +105,14 @@ help:
 	@printf "$(BOLD)$(BLUE)Caddy Commands:$(NC)\n"
 	@printf "  $(BOLD)$(GREEN)caddy-gen-cert$(NC)   : Generate the caddy's mtls certificates.\n"
 	@printf "  $(BOLD)$(GREEN)caddy-del-cert$(NC)   : Remove the caddy's mtls certificates.\n"
-	@printf "  $(BOLD)$(GREEN)caddy-validate$(NC)   : Validates caddy's files syntax.\n"
+	@printf "  $(BOLD)$(GREEN)caddy-validate$(NC)   : Validates caddy's files syntax.\n\n"
+
+	@printf "$(BOLD)$(BLUE)Monitoring Commands:$(NC)\n"
+	@printf "  $(BOLD)$(GREEN)monitor-up$(NC)       : Start the monitoring stack (Prometheus, Grafana).\n"
+	@printf "  $(BOLD)$(GREEN)monitor-down$(NC)     : Stop the monitoring stack.\n"
+	@printf "  $(BOLD)$(GREEN)monitor-status$(NC)   : Show status of monitoring services.\n"
+	@printf "  $(BOLD)$(GREEN)monitor-test$(NC)     : Run monitoring stack test suite.\n"
+	@printf "  $(BOLD)$(GREEN)monitor-grafana$(NC)  : Open Grafana dashboards in browser.\n"
+	@printf "  $(BOLD)$(GREEN)monitor-help$(NC)     : Show detailed monitoring commands.\n"
 
 	@printf "$(NC)\n"

docker-compose.yml

@@ -12,6 +12,14 @@ volumes:
     caddy_config:
     go_mod_cache:
         driver: local
+    prometheus_data_prod:
+        driver: local
+    prometheus_data_local:
+        driver: local
+    grafana_data_prod:
+        driver: local
+    grafana_data_local:
+        driver: local
 
     # --- DB: Define a named volume at the top level.
     #     Docker will manage its lifecycle.
@@ -30,7 +38,7 @@ services:
         caddy_prod:
             image: api-caddy_prod
             build:
-                context: ./caddy
+                context: ./infra/caddy
                 dockerfile: Dockerfile
                 args:
                     - CADDY_VERSION=2.10.2
@@ -40,24 +48,35 @@ services:
             restart: unless-stopped
             depends_on:
                 - api
+
+            # --- The 443:443/udp is required for HTTP/3
+            #     NOTES:
+            #           - Admin API (2019) listens on all interfaces but is NOT published to host
+            #           - Prometheus scrapes metrics from dedicated endpoint (9180) via Docker internal DNS
             ports:
                 - "80:80"
                 - "443:443"
                 - "443:443/udp" # Required for HTTP/3
+                # NOTE: Admin API (2019) is NOT published to host (internal Docker network only)
+                # Prometheus scrapes Caddy metrics from :9180 via Docker internal DNS
+
+            # --- Dedicated /metrics endpoint for Prometheus (internal network only)
+            expose:
+                - "9180"
             volumes:
                 - caddy_data:/data
                 - caddy_config:/config
-                - ./caddy/Caddyfile.prod:/etc/caddy/Caddyfile
+                - ./infra/caddy/Caddyfile.prod:/etc/caddy/Caddyfile
                 - ${CADDY_LOGS_PATH}:/var/log/caddy
-                - ./caddy/mtls:/etc/caddy/mtls:ro
+                - ./infra/caddy/mtls:/etc/caddy/mtls:ro
             networks:
                 caddy_net:
                   aliases:
                     - proxy
 
         caddy_local:
             build:
-                context: ./caddy
+                context: ./infra/caddy
                 dockerfile: Dockerfile
                 args:
                     - CADDY_VERSION=2.10.2
@@ -68,15 +87,247 @@ services:
             depends_on:
                 - api
             ports:
-                - "8080:80"
+                - "18080:80"
                 - "8443:443"
+                - "127.0.0.1:2019:2019" # Admin API - localhost only for debugging
+
+            # --- Dedicated /metrics endpoint for Prometheus (internal network only)
+            expose:
+                - "9180"
+
             volumes:
                 - caddy_data:/data
                 - caddy_config:/config
-                - ./caddy/mtls:/etc/caddy/mtls:ro
-                - ./caddy/Caddyfile.local:/etc/caddy/Caddyfile
+                - ./infra/caddy/mtls:/etc/caddy/mtls:ro
+                - ./infra/caddy/Caddyfile.local:/etc/caddy/Caddyfile
+            networks:
+                - caddy_net
+
+        prometheus:
+            image: prom/prometheus:v3.0.1
+            profiles: ["prod"]
+            container_name: oullin_prometheus
+            restart: unless-stopped
+            command:
+                - '--config.file=/etc/prometheus/prometheus.yml'
+                - '--storage.tsdb.path=/prometheus'
+                - '--storage.tsdb.retention.time=30d'
+                - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+                - '--web.console.templates=/usr/share/prometheus/consoles'
+            ports:
+                - "127.0.0.1:9090:9090"
+            volumes:
+                - ./infra/metrics/prometheus/provisioning/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+                - prometheus_data_prod:/prometheus
+            networks:
+                - caddy_net
+                - oullin_net
+            depends_on:
+                caddy_prod:
+                    condition: service_started
+                postgres_exporter:
+                    condition: service_healthy
+            healthcheck:
+                test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:9090/-/healthy"]
+                interval: 10s
+                timeout: 5s
+                retries: 5
+                start_period: 10s
+            deploy:
+                resources:
+                    limits:
+                        cpus: '1.0'
+                        memory: 1G
+                    reservations:
+                        cpus: '0.25'
+                        memory: 256M
+
+        prometheus_local:
+            image: prom/prometheus:v3.0.1
+            profiles: ["local"]
+            container_name: oullin_prometheus_local
+            restart: unless-stopped
+            command:
+                - '--config.file=/etc/prometheus/prometheus.yml'
+                - '--storage.tsdb.path=/prometheus'
+                - '--storage.tsdb.retention.time=7d'
+                - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+                - '--web.console.templates=/usr/share/prometheus/consoles'
+            ports:
+                - "9090:9090"
+            volumes:
+                - ./infra/metrics/prometheus/provisioning/prometheus.local.yml:/etc/prometheus/prometheus.yml:ro
+                - prometheus_data_local:/prometheus
+            networks:
+                - caddy_net
+                - oullin_net
+            depends_on:
+                caddy_local:
+                    condition: service_started
+                postgres_exporter_local:
+                    condition: service_healthy
+            healthcheck:
+                test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:9090/-/healthy"]
+                interval: 10s
+                timeout: 5s
+                retries: 5
+                start_period: 10s
+            deploy:
+                resources:
+                    limits:
+                        cpus: '1.0'
+                        memory: 1G
+                    reservations:
+                        cpus: '0.25'
+                        memory: 256M
+
+        postgres_exporter:
+            image: prometheuscommunity/postgres-exporter:v0.15.0
+            profiles: ["prod"]
+            container_name: oullin_postgres_exporter
+            restart: unless-stopped
+            entrypoint: ["/postgres-exporter-entrypoint.sh"]
+            volumes:
+                - ./infra/metrics/prometheus/scripts/postgres-exporter-entrypoint.sh:/postgres-exporter-entrypoint.sh:ro
+            secrets:
+                - pg_username
+                - pg_password
+                - pg_dbname
+            networks:
+                - oullin_net
+                - caddy_net
+            depends_on:
+                api-db:
+                    condition: service_healthy
+            expose:
+                - "9187"
+            healthcheck:
+                test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:9187/"]
+                interval: 10s
+                timeout: 5s
+                retries: 5
+                start_period: 10s
+            deploy:
+                resources:
+                    limits:
+                        cpus: '0.25'
+                        memory: 128M
+                    reservations:
+                        cpus: '0.05'
+                        memory: 32M
+
+        postgres_exporter_local:
+            image: prometheuscommunity/postgres-exporter:v0.15.0
+            profiles: ["local"]
+            container_name: oullin_postgres_exporter_local
+            restart: unless-stopped
+            entrypoint: ["/postgres-exporter-entrypoint.sh"]
+            volumes:
+                - ./infra/metrics/prometheus/scripts/postgres-exporter-entrypoint.sh:/postgres-exporter-entrypoint.sh:ro
+            secrets:
+                - pg_username
+                - pg_password
+                - pg_dbname
+            networks:
+                - oullin_net
+                - caddy_net
+            depends_on:
+                api-db:
+                    condition: service_healthy
+            expose:
+                - "9187"
+            healthcheck:
+                test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:9187/"]
+                interval: 10s
+                timeout: 5s
+                retries: 5
+                start_period: 10s
+            deploy:
+                resources:
+                    limits:
+                        cpus: '0.25'
+                        memory: 128M
+                    reservations:
+                        cpus: '0.05'
+                        memory: 32M
+
+        grafana:
+            image: grafana/grafana:11.4.0
+            profiles: ["prod"]
+            container_name: oullin_grafana
+            restart: unless-stopped
+            ports:
+                - "127.0.0.1:3000:3000"
+            environment:
+                - GF_SERVER_ROOT_URL=http://localhost:3000
+                - GF_SECURITY_ADMIN_USER=admin
+                - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:?GRAFANA_ADMIN_PASSWORD must be set in .env file}
+                - GF_USERS_ALLOW_SIGN_UP=false
+                - GF_AUTH_ANONYMOUS_ENABLED=false
+                - GF_INSTALL_PLUGINS=
+                - GF_DATASOURCE_PROMETHEUS_URL=http://oullin_prometheus:9090
+            volumes:
+                - grafana_data_prod:/var/lib/grafana
+                - ./infra/metrics/grafana/provisioning:/etc/grafana/provisioning:ro
+                - ./infra/metrics/grafana/dashboards:/var/lib/grafana/dashboards:ro
+            networks:
+                - caddy_net
+            depends_on:
+                prometheus:
+                    condition: service_healthy
+            healthcheck:
+                test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3000/"]
+                interval: 10s
+                timeout: 5s
+                retries: 5
+                start_period: 30s
+            deploy:
+                resources:
+                    limits:
+                        cpus: '0.5'
+                        memory: 512M
+                    reservations:
+                        cpus: '0.1'
+                        memory: 128M
+
+        grafana_local:
+            image: grafana/grafana:11.4.0
+            profiles: ["local"]
+            container_name: oullin_grafana_local
+            restart: unless-stopped
+            ports:
+                - "3000:3000"
+            environment:
+                - GF_SERVER_ROOT_URL=http://localhost:3000
+                - GF_SECURITY_ADMIN_USER=admin
+                - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:?GRAFANA_ADMIN_PASSWORD must be set in .env file}
+                - GF_USERS_ALLOW_SIGN_UP=false
+                - GF_AUTH_ANONYMOUS_ENABLED=false
+                - GF_INSTALL_PLUGINS=
+                - GF_DATASOURCE_PROMETHEUS_URL=http://oullin_prometheus_local:9090
+            volumes:
+                - grafana_data_local:/var/lib/grafana
+                - ./infra/metrics/grafana/provisioning:/etc/grafana/provisioning:ro
+                - ./infra/metrics/grafana/dashboards:/var/lib/grafana/dashboards:ro
             networks:
                 - caddy_net
+            depends_on:
+                prometheus_local:
+                    condition: service_healthy
+            healthcheck:
+                test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3000/"]
+                interval: 10s
+                timeout: 5s
+                retries: 5
+                start_period: 30s
+            deploy:
+                resources:
+                    limits:
+                        cpus: '0.5'
+                        memory: 512M
+                    reservations:
+                        cpus: '0.1'
+                        memory: 128M
 
         # A dedicated service for running one-off Go commands
         api-runner:
@@ -86,7 +337,7 @@ services:
             - ./.env
           build:
             context: .
-            dockerfile: ./docker/dockerfile-api
+            dockerfile: ./infra/docker/dockerfile-api
             target: builder
           volumes:
             - .:/app
@@ -128,7 +379,7 @@ services:
                 ENV_HTTP_HOST: 0.0.0.0
             build:
                 context: .
-                dockerfile: ./docker/dockerfile-api
+                dockerfile: ./infra/docker/dockerfile-api
                 args:
                     - APP_VERSION=0.0.0.1
                     - APP_HOST_PORT=${ENV_HTTP_PORT}