grafana-oncall: init

Signed-off-by: Victor Login <batazor111@gmail.com>
shortlink-org · Nov 10, 2022 · 92dd2de · 92dd2de
1 parent 20d10cb
commit 92dd2de
Show file tree

Hide file tree

Showing 7 changed files with 112 additions and 18 deletions.
diff --git a/README.md b/README.md
@@ -155,33 +155,37 @@ make skaffold-debug
 + Development
   + [Skaffold](https://skaffold.dev/)
   + Telepresence
-+ Logger
-  + [Loki](./docs/logger.md)
-+ Prometheus
-  + prometheus-operator
-    + notify: slack, email, telegram
 + HealthCheck
 + Support K8S
   + Helm Chart
     + [pingcap/chaos-meshh](https://github.com/pingcap/chaos-mesh)
   + Minikube
   + Backup/Restore [(Velero)](https://velero.io/)
   + Custom CSI driver (fork [csi-driver-host-pat](https://github.com/kubernetes-csi/csi-driver-host-path))
-+ Ingress
++ Ingress (Gateway)
   + Istio
   + Nginx
+  + Traefik
 + MetalLB
 + [kyverno](https://kyverno.io/) - Kubernetes Native Policy Management
 + Storage
   + [rook-ceph](https://rook.io/)
     + ceph cluster (3 node)
     + grafana dashboard
     + prometheus metrics
-
-### Gateway
-
-+ Traefik
-+ Nginx
++ Observability
+  + [Jaeger](https://www.jaegertracing.io/) - Distributed tracing
+  + **Prometheus** - Monitoring
+    + [Prometheus](https://prometheus.io/)
+    + prometheus-operator
+      + notify: slack, email, telegram
+  + **Grafana stack (LGTM)**
+    * [Grafana](https://github.com/grafana/grafana), the open and composable observability and data visualization platform.
+    * [Loki](https://github.com/grafana/loki), like Prometheus, but for logs.
+      + [docs](./docs/logger.md)
+    * [Grafana](https://github.com/grafana/grafana), the open and composable observability and data visualization platform.
+    + [OnCall](https://grafana.com/oss/oncall/) - On-call scheduling
+    + [Phlare](https://grafana.com/oss/phlare/) - Profiling
 
 ### UI
 

diff --git a/data/oncall/oncall.db b/data/oncall/oncall.db
diff --git a/ops/Makefile/common.mk b/ops/Makefile/common.mk
@@ -27,6 +27,7 @@ up: ## Run for specific job
 		-f docker-compose.yaml \
 		-f ops/docker-compose/tooling/services/coredns.yaml \
 		-f ops/docker-compose/mq/rabbitmq.yaml \
+		-f ops/docker-compose/database/redis.yaml \
 		-f ops/docker-compose/tooling/observability/prometheus.yaml \
 		-f ops/docker-compose/tooling/observability/grafana.yaml \
 		-f ops/docker-compose/tooling/observability/grafana-phlare.yaml \
@@ -49,6 +50,7 @@ run: ## Run this project in docker compose
 		-f ops/docker-compose/tooling/observability/grafana-loki.yaml \
 		-f ops/docker-compose/tooling/observability/grafana-tempo.yaml \
 		-f ops/docker-compose/tooling/observability/grafana-phlare.yaml \
+		-f ops/docker-compose/tooling/observability/grafana-oncall.yaml \
 		-f ops/docker-compose/mq/rabbitmq.yaml \
 		up -d --remove-orphans
 

diff --git a/ops/docker-compose/database/redis.yaml b/ops/docker-compose/database/redis.yaml
@@ -1,8 +1,17 @@
 version: '2.4'
 
+x-network: &network
+#  networks:
+#    simple:
+#      ipv4_address: ${DATABASE_IP}
+  dns:
+    - ${DNS_IP}
+  dns_search: ${DNS_SEARCH}
+
 services:
 
   redis:
+    <<: *network
     depends_on:
       - coredns
     image: redis:7.0-alpine
@@ -11,12 +20,11 @@ services:
     container_name: redis
     ports:
       - "6379:6379"
-#    networks:
-#      simple:
-#        ipv4_address: ${DATABASE_IP}
-    dns:
-      - ${DNS_IP}
-    dns_search: ${DNS_SEARCH}
+    healthcheck:
+      test: [ "CMD", "redis-cli", "ping" ]
+      timeout: 5s
+      interval: 5s
+      retries: 10
 #    logging:
 #      driver: fluent-bit
 #      options:

diff --git a/ops/docker-compose/mq/rabbitmq.yaml b/ops/docker-compose/mq/rabbitmq.yaml
@@ -19,6 +19,12 @@ services:
     ports:
       - "5672:5672"
       - "15672:15672"
+    healthcheck:
+      test: rabbitmq-diagnostics -q ping
+      start_period: 1m
+      interval: 30s
+      timeout: 10s
+      retries: 3
 #    logging:
 #      driver: ${LOGGER_DRIVER}
 #      options:

diff --git a/ops/docker-compose/tooling/observability/grafana-oncall.yaml b/ops/docker-compose/tooling/observability/grafana-oncall.yaml
@@ -0,0 +1,73 @@
+version: '2.4'
+
+x-network: &network
+  networks:
+    - simple
+  dns:
+    - ${DNS_IP}
+  dns_search: ${DNS_SEARCH}
+
+x-environment: &oncall-environment
+  BASE_URL: host.docker.internal
+  DATABASE_TYPE: sqlite3
+  BROKER_TYPE: redis
+  DOMAIN: http://localhost:8080
+  SECRET_KEY: b3613a53-8e63-4c71-8fc6-349ced8d13dd
+  REDIS_URI: redis://redis:6379/4
+  DJANGO_SETTINGS_MODULE: settings.hobby
+  CELERY_WORKER_QUEUE: "default,critical,long,slack,telegram,webhook,retry,celery"
+  CELERY_WORKER_CONCURRENCY: "1"
+  CELERY_WORKER_MAX_TASKS_PER_CHILD: "100"
+  CELERY_WORKER_SHUTDOWN_INTERVAL: "65m"
+  CELERY_WORKER_BEAT_ENABLED: "True"
+
+services:
+
+  oncall:
+    <<: *network
+    depends_on:
+      coredns:
+        condition: service_started
+      oncall_db_migration:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
+    image: grafana/oncall:latest
+    container_name: oncall
+    command: sh -c "uwsgi --ini uwsgi.ini"
+    environment: *oncall-environment
+    restart: on-failure
+    volumes:
+      - ./data/oncall:/var/lib/oncall
+
+  oncall_db_migration:
+    <<: *network
+    depends_on:
+      coredns:
+        condition: service_started
+      redis:
+        condition: service_healthy
+    image: grafana/oncall:latest
+    container_name: oncall_db_migration
+    command: python manage.py migrate --noinput
+    environment: *oncall-environment
+    volumes:
+      - ./data/oncall:/var/lib/oncall
+
+  celery:
+    <<: *network
+    image: grafana/oncall:latest
+    container_name: oncall_celery
+    restart: always
+    command: sh -c "./celery_with_exporter.sh"
+    environment: *oncall-environment
+    volumes:
+      - ./data/oncall:/var/lib/oncall
+    depends_on:
+      coredns:
+        condition: service_started
+      oncall_db_migration:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
+
diff --git a/ops/docker-compose/tooling/observability/grafana.yaml b/ops/docker-compose/tooling/observability/grafana.yaml
@@ -10,7 +10,6 @@ x-network: &network
 services:
 
   grafana:
-    <<: *network
     depends_on:
       - coredns
     image: grafana/grafana:main
@@ -29,6 +28,8 @@ services:
       - GF_DIAGNOSTICS_PROFILING_ENABLED=true
       - GF_DIAGNOSTICS_PROFILING_ADDR=0.0.0.0
       - GF_DIAGNOSTICS_PROFILING_PORT=6060
+      - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=grafana-oncall-app
+      - GF_INSTALL_PLUGINS=grafana-oncall-app
     ports:
       - 3030:3000
     networks: