Observability

deploy/observability/grafana/config.alloy

@@ -0,0 +1,63 @@
+otelcol.receiver.otlp "otlp_receiver" {
+  grpc {
+    endpoint = "0.0.0.0:4317"
+  }
+
+  http {
+    endpoint = "0.0.0.0:4318"
+  }
+
+  output {
+    metrics = [otelcol.processor.batch.metrics.input]
+    traces  = [otelcol.processor.batch.traces.input]
+  }
+}
+
+
+otelcol.processor.batch "metrics" {
+  timeout = "5s"
+
+  send_batch_size = 1024
+  send_batch_max_size = 2048
+
+  output {
+    metrics = [otelcol.exporter.prometheus.metrics.input]
+  }
+}
+
+
+otelcol.processor.batch "traces" {
+  timeout = "5s"
+
+  send_batch_size = 1024
+  send_batch_max_size = 2048
+
+  output {
+    traces = [otelcol.exporter.otlp.tempo.input]
+  }
+}
+
+
+otelcol.exporter.prometheus "metrics" {
+  forward_to = [
+    prometheus.remote_write.default.receiver,
+  ]
+}
+
+
+prometheus.remote_write "default" {
+  endpoint {
+    url = "http://prometheus:9090/api/v1/write"
+  }
+}
+
+
+otelcol.exporter.otlp "tempo" {
+  client {
+    endpoint = "tempo:4317"
+
+    tls {
+      insecure = true
+    }
+  }
+}

deploy/observability/grafana/prometheus.yml

@@ -0,0 +1,9 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: "prometheus"
+
+    static_configs:
+      - targets:
+          - "prometheus:9090"

deploy/observability/grafana/tempo.yaml

@@ -0,0 +1,24 @@
+stream_over_http_enabled: true
+
+server:
+  http_listen_port: 3200
+  log_level: info
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: "0.0.0.0:4317"
+
+storage:
+  trace:
+    backend: local
+    wal:
+      path: /var/tempo/wal
+    local:
+      path: /var/tempo/blocks
+
+
+usage_report:
+  reporting_enabled: false

deploy/observability/jaeger/collector.yaml

@@ -0,0 +1,51 @@
+service:
+  extensions: [jaeger_storage, jaeger_query, healthcheckv2]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [jaeger_storage_exporter]
+  telemetry:
+    resource:
+      service.name: jaeger
+
+    logs:
+      level: debug
+
+extensions:
+  healthcheckv2:
+    use_v2: true
+    http:
+
+  jaeger_query:
+    storage:
+      traces: some-storage
+    ui:
+      config_file: /jaeger/config-ui.json
+  jaeger_storage:
+    backends:
+      some-storage:
+        clickhouse:
+          addresses:
+            - clickhouse:9000
+          database: jaeger
+          auth:
+            basic:
+              username: jaeger
+              password: jaegerSecretPassword
+          create_schema: true
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  batch:
+
+exporters:
+  jaeger_storage_exporter:
+    trace_storage: some-storage

deploy/observability/jaeger/config-ui.json

@@ -0,0 +1,3 @@
+{
+    "archiveEnabled": true
+}

deploy/observability/jaeger/query.yaml

@@ -0,0 +1,44 @@
+service:
+  extensions: [jaeger_storage, jaeger_query, healthcheckv2]
+  pipelines:
+    traces:
+      receivers: [nop]
+      processors: [batch]
+      exporters: [nop]
+  telemetry:
+    resource:
+      service.name: jaeger-query
+    logs:
+      level: info
+
+extensions:
+  healthcheckv2:
+    use_v2: true
+    http:
+
+  jaeger_query:
+    storage:
+      traces: query_storage
+    ui:
+      config_file: /jaeger/config-ui.json
+
+  jaeger_storage:
+    backends:
+      query_storage:
+        clickhouse:
+          addresses:
+            - clickhouse:9000
+          database: jaeger
+          auth:
+            basic:
+              username: jaeger
+              password: jaegerSecretPassword
+
+receivers:
+  nop:
+
+processors:
+  batch:
+
+exporters:
+  nop:

deploy/persistence/clickhouse.env

@@ -0,0 +1,3 @@
+CLICKHOUSE_DB=jaeger
+CLICKHOUSE_USER=jaeger
+CLICKHOUSE_PASSWORD=jaegerSecretPassword

docker-compose.base.yaml

@@ -29,13 +29,26 @@ services:
     networks:
       - conode-network
 
+  grafana:
+    image: grafana/grafana:12.1.1
+
+    labels:
+      - "traefik.enable=true"
+
+      - "traefik.http.routers.grafana.rule=Host(`grafana.conode.com`)"
+      - "traefik.http.routers.grafana.entrypoints=web"
+      - "traefik.http.services.grafana.loadbalancer.server.port=3000"
+
+    networks:
+      - conode-network
+
   oauth:
     image: quay.io/oauth2-proxy/oauth2-proxy:latest
     command: ["--config=/etc/oauth2-proxy/oauth2-proxy.toml"]
 
     environment:
       OAUTH2_PROXY_CLIENT_ID: conode
-      OAUTH2_PROXY_CLIENT_SECRET: HxZkeEWaKX0J1rXNPpkshkhojgm6roqrNxWWtTBtUiwSm4kN1QzIH1cCcTulKPlYqCEFVELVuFu94tqFGDyjuo
+      OAUTH2_PROXY_CLIENT_SECRET: y9QISRU4LktDcAOvrXmw5q8t5s44JshMrnc4PIqPfq1vIEGipEtYgmUxvgiG9nb0PsxVwvlVQQ4FPOlhl9tiZ3
       OAUTH2_PROXY_COOKIE_SECRET: x3CpXUQxfDgXzDDxIdST3mTbTDosKOlx
 
     volumes:
@@ -83,7 +96,7 @@ services:
         condition: service_healthy
 
     env_file:
-      - "./deploy/keycloak.env"
+      - "./deploy/keycloak/keycloak.env"
 
     healthcheck:
       test:
@@ -104,7 +117,7 @@ services:
 
     volumes:
       - keycloak-data:/opt/keycloak/
-      - ./deploy/keycloak.json:/opt/keycloak/data/import/realm-export.json
+      - ./deploy/keycloak/keycloak.json:/opt/keycloak/data/import/realm-export.json
 
     networks:
       - conode-network
@@ -114,25 +127,78 @@ services:
 
     volumes:
       - postgres-data:/var/lib/postgresql/data
-      - ./deploy:/docker-entrypoint-initdb.d:ro
+      - ./deploy/persistence:/docker-entrypoint-initdb.d:ro
 
     networks:
       conode-network:
         aliases:
           - persistence.literally.io
 
     env_file:
-      - "./deploy/postgres.env"
+      - "./deploy/persistence/postgres.env"
 
     healthcheck:
       test: ["CMD", "pg_isready", "-U", "conode", "-d", "conode"]
       interval: 5s
       timeout: 15s
       retries: 5
 
+  alloy:
+    image: grafana/alloy:latest
+
+    volumes:
+      - ./deploy/observability/grafana/config.alloy:/etc/alloy/config.alloy
+      - alloy-data:/var/lib/alloy/data
+
+    command:
+      - run
+      - /etc/alloy/config.alloy
+      - --storage.path=/var/lib/alloy/data
+      - --server.http.listen-addr=0.0.0.0:12345
+
+    networks:
+      - conode-network
+
+
+  prometheus:
+    image: prom/prometheus
+
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.enable-remote-write-receiver"
+
+    volumes:
+      - prometheus-data:/prometheus
+      - ./deploy/observability/grafana/prometheus.yml:/etc/prometheus/prometheus.yml
+
+    networks:
+      - conode-network
+
+
+  tempo:
+    image: grafana/tempo:latest
+
+    command:
+      - "-target=all"
+      - "-config.file=/etc/tempo.yaml"
+
+    restart: always
+
+    volumes:
+      - ./deploy/observability/grafana/tempo.yaml:/etc/tempo.yaml
+      - tempo-data:/var/tempo
+
+    networks:
+      - conode-network
+
+
 volumes:
+  prometheus-data:
   postgres-data:
   keycloak-data:
+  tempo-data:
+  alloy-data:
 
 networks:
   conode-network:

example.config.toml

@@ -15,3 +15,7 @@ expires_in = 3600
 jwks_url = "http://keycloak:8080/realms/conode/protocol/openid-connect/certs"
 issuer = "http://keycloak.conode.com/realms/conode"
 audience = "conode"
+
+[telemetry]
+endpoint = "collector:4317"
+enabled = true

pyproject.toml

@@ -11,6 +11,12 @@ dependencies = [
     "cryptography==46.0.5",
     "dishka==1.7.2",
     "fastapi==0.128.0",
+    "opentelemetry-api>=1.44.0",
+    "opentelemetry-exporter-otlp-proto-grpc>=1.44.0",
+    "opentelemetry-instrumentation-fastapi>=0.65b0",
+    "opentelemetry-instrumentation-logging>=0.65b0",
+    "opentelemetry-instrumentation-sqlalchemy>=0.65b0",
+    "opentelemetry-sdk>=1.44.0",
     "pydantic[email]==2.13.2",
     "pyjwt==2.11.0",
     "sqlalchemy==2.0.46",