Merge pull request #6 from prosenjitjoy/feat-prometheus

added metrics and alerting

.env

@@ -10,4 +10,7 @@ RATING_PORT=8082
 MOVIE_PORT=8083
 CONSUL_URL=localhost:8500
 JAEGER_URL=localhost:4317
+METADATA_METRICS_PORT=8091
+RATING_METRICS_PORT=8092
+MOVIE_METRICS_PORT=8093
 ENVIRONMENT=dev

Makefile

@@ -1,7 +1,7 @@
 include .env
 
 create-consul:
-	podman run --name consul -p 8500:8500 -p 8600:8600/udp -d hashicorp/consul:latest agent -server -ui -node=server-1 -bootstrap-expect=1 -client=0.0.0.0
+	podman run --name consul --hostname consul -p 8500:8500 -p 8600:8600/udp -d hashicorp/consul:latest agent -server -ui -node=server-1 -bootstrap-expect=1 -client=0.0.0.0
 
 delete-consul:
 	podman rm -f consul
@@ -12,26 +12,40 @@ proto-generate:
 	protoc --proto_path=proto --go_out=rpc --go_opt=paths=source_relative --go-grpc_out=rpc --go-grpc_opt=paths=source_relative proto/*.proto
 
 create-pulsar:
-	podman run --name pulsar -p 6650:6650 -p 8080:8080 -d apachepulsar/pulsar:3.1.1 bin/pulsar standalone
+	podman run --name pulsar --hostname pulsar -p 6650:6650 -p 8080:8080 -d apachepulsar/pulsar:3.1.1 bin/pulsar standalone
 
 delete-pulsar:
 	podman rm -f pulsar
 	podman volume prune
 
 create-postgres:
-	podman run --name postgres -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=moviedb -p 5432:5432 -d postgres:latest
+	podman run --name postgres --hostname postgres -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=moviedb -p 5432:5432 -d postgres:latest
 
 delete-postgres:
 	podman rm -f postgres
 	podman volume prune
 
 create-jaeger:
-	podman run --name jaeger -e COLLECTOR_OTLP_ENABLED=true -p 6831:6831/udp -p 6832:6832/udp -p 5778:5778 -p 16686:16686 -p 4317:4317 -p 4318:4318 -p 14250:14250 -p 14268:14268 -p 14269:14269 -p 9411:9411 -d jaegertracing/all-in-one:1.52
+	podman run --name jaeger --hostname jaeger -e COLLECTOR_OTLP_ENABLED=true -p 6831:6831/udp -p 6832:6832/udp -p 5778:5778 -p 16686:16686 -p 4317:4317 -p 4318:4318 -p 14250:14250 -p 14268:14268 -p 14269:14269 -p 9411:9411 -d jaegertracing/all-in-one:1.52
 
 delete-jaeger:
 	podman rm -f jaeger
 	podman volume prune
 
+create-prometheus:
+	podman run --name prometheus --hostname prometheus --network slirp4netns:allow_host_loopback=true -p 9090:9090 -v ./configs:/etc/prometheus -d prom/prometheus:latest
+
+delete-prometheus:
+	podman rm -f prometheus
+	podman volume prune
+
+create-alertmanager:
+	podman run --name alertmanager --hostname alertmanager -p 9093:9093 -v ./configs:/etc/alertmanager -d prom/alertmanager:latest --config.file=/etc/alertmanager/alertmanager.yml
+
+delete-alertmanager:
+	podman rm -f alertmanager
+	podman volume prune
+
 create-migration:
 	migrate create -ext sql -dir database/migration -seq $(name)
 
@@ -61,4 +75,4 @@ generate-mock:
 run-test:
 	go test ./...
 
-.PHONY: create-consul delete-consul proto-generate create-pulsar delete-pulsar create-postgres delete-postgres create-jaeger delete-jaeger create-migration migrate-up migrate-down generate-dbdocs generate-schema generate-sqlc generate-image generate-mock run-test
+.PHONY: create-consul delete-consul proto-generate create-pulsar delete-pulsar create-postgres delete-postgres create-migration migrate-up migrate-down create-jaeger delete-jaeger create-prometheus delete-prometheus create-alertmanager delete-alertmanager generate-dbdocs generate-schema generate-sqlc generate-image generate-mock run-test

configs/alertmanager.yml

@@ -0,0 +1,15 @@
+global:
+  resolve_timeout: 5m
+route:
+  group_by: ['alertname']
+  repeat_interval: 1m
+  receiver: 'email'
+receivers:
+- name: 'email'
+  email_configs:
+  - to: 'test@gmail.com'
+    from: 'test@gmail.com'
+    smarthost: smtp.gmail.com:587
+    auth_username: 'test@gmail.com'
+    auth_identity: 'test@gmail.com'
+    auth_password: 'test'

configs/alerts.rules

@@ -0,0 +1,24 @@
+groups:
+- name: Service availability
+  rules:
+  - alert: Metadata service down
+    expr: up{service="metadata"} == 0
+    labels:
+      severity: warning
+    annotations:
+      title: Metadata service is down
+      description: Failed to scrape {{ $labels.service }} service on {{ $labels.instance }}. Service possibly down.
+  - alert: Rating service down
+    expr: up{service="rating"} == 0
+    labels:
+      severity: warning
+    annotations:
+      title: Metadata service is down
+      description: Failed to scrape {{ $labels.service }} service on {{ $labels.instance }}. Service possibly down.
+  - alert: Movie service down
+    expr: up{service="movie"} == 0
+    labels:
+      severity: warning
+    annotations:
+      title: Metadata service is down
+      description: Failed to scrape {{ $labels.service }} service on {{ $labels.instance }}. Service possibly down.

configs/prometheus.yml

@@ -0,0 +1,40 @@
+global:
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  evaluation_interval: 15s
+alerting:
+  alertmanagers:
+  - follow_redirects: true
+    enable_http2: true
+    scheme: http
+    timeout: 10s
+    api_version: v2
+    static_configs:
+    - targets:
+      - host.containers.internal:9093 
+rule_files:
+- alerts.rules
+scrape_configs:
+- job_name: prometheus
+  honor_timestamps: true
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  metrics_path: /metrics
+  scheme: http
+  follow_redirects: true
+  enable_http2: true
+  static_configs:
+  - targets:  
+    - localhost:9090
+  - targets:    
+    - host.containers.internal:8091
+    labels:
+      service: metadata
+  - targets:    
+    - host.containers.internal:8092
+    labels:
+      service: rating
+  - targets:    
+    - host.containers.internal:8093
+    labels:
+      service: movie

go.mod

@@ -6,11 +6,12 @@ require (
 	github.com/apache/pulsar-client-go v0.11.1
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.4.0
-	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
 	github.com/hashicorp/consul/api v1.26.1
 	github.com/ilyakaznacheev/cleanenv v1.5.0
-	github.com/jackc/pgx/v5 v5.5.0
+	github.com/jackc/pgx/v5 v5.5.1
+	github.com/prometheus/client_golang v1.11.1
 	github.com/stretchr/testify v1.8.4
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1
 	go.opentelemetry.io/otel v1.21.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0
 	go.opentelemetry.io/otel/sdk v1.21.0
@@ -21,6 +22,7 @@ require (
 )
 
 require (
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
 	github.com/99designs/keyring v1.2.1 // indirect
 	github.com/AthenZ/athenz v1.10.39 // indirect
@@ -68,31 +70,28 @@ require (
 	github.com/pierrec/lz4 v2.0.5+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.11.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.26.0 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/sirupsen/logrus v1.6.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
-	go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
 	go.opentelemetry.io/otel/metric v1.21.0 // indirect
 	go.opentelemetry.io/otel/trace v1.21.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
-	golang.org/x/crypto v0.16.0 // indirect
+	golang.org/x/crypto v0.15.0 // indirect
 	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
 	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/net v0.18.0 // indirect
 	golang.org/x/oauth2 v0.11.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
-	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/term v0.14.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect
 )