feat: Add OpenTelemetry tracing

changes/5295.added

@@ -0,0 +1 @@
+Added OpenTelemetry tracing and metrics.

development/configs/grafana/datasources.yaml

@@ -0,0 +1,57 @@
+---
+apiVersion: 1
+datasources:
+  - name: "Logs"
+    uid: "loki"
+    type: "loki"
+    url: "http://loki:3100"
+    editable: true
+    access: "proxy"
+    orgId: 1
+    jsonData:
+      derivedFields:
+        - datasourceUid: "tempo"
+          matcherRegex: "trace_id=([\\S]+)"
+          url: '$${__value.raw}'
+          name: "trace_id"
+  - name: "Metrics"
+    uid: "mimir"
+    type: "prometheus"
+    url: "http://mimir:9009/prometheus"
+    access: "proxy"
+    orgId: 1
+    version: 1
+    editable: true
+  - name: "Tempo"
+    uid: "tempo"
+    type: "tempo"
+    url: "http://tempo:3200"
+    access: "proxy"
+    orgId: 1
+    version: 1
+    editable: true
+    basicAuth: false
+    jsonData:
+      tracesToLogsV2:
+        # Field with an internal link pointing to a logs data source in Grafana.
+        # datasourceUid value must match the uid value of the logs data source.
+        datasourceUid: 'loki'
+        spanStartTimeShift: '-5s'
+        spanEndTimeShift: '5s'
+        filterByTraceID: true
+        filterBySpanID: false
+      serviceMap:
+        datasourceUid: "mimir"
+      lokiSearch:
+        datasourceUid: 'loki'
+  - name: "Nautobot"
+    type: "yesoreyeram-infinity-datasource"
+    orgId: 1
+    jsonData:
+      allowedHosts:
+        - "http://nautobot:8080"
+      apiKeyKey: "Authorization"
+      auth_method: "apiKey"
+    editable: true
+    secureJsonData:
+      apiKeyValue: "0123456789abcdef0123456789abcdef01234567"

development/configs/grafana/grafana.ini

@@ -0,0 +1,7 @@
+[feature_toggles]
+enable = nestedFolders,correlations,vizAndWidgetSplit
+
+[auth.anonymous]
+enabled = true
+org_name = Main Org.
+org_role = Viewer

development/configs/mimir-config.yaml

@@ -0,0 +1,47 @@
+# Do not use this configuration in production.
+# It is for demonstration purposes only.
+multitenancy_enabled: false
+
+blocks_storage:
+  backend: filesystem
+  bucket_store:
+    sync_dir: /tmp/mimir/tsdb-sync
+  filesystem:
+    dir: /tmp/mimir/data/tsdb
+  tsdb:
+    dir: /tmp/mimir/tsdb
+
+compactor:
+  data_dir: /tmp/mimir/compactor
+  sharding_ring:
+    kvstore:
+      store: memberlist
+
+distributor:
+  ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: memberlist
+
+ingester:
+  ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: memberlist
+    replication_factor: 1
+
+ruler_storage:
+  backend: local
+  local:
+    directory: /tmp/mimir/rules
+
+server:
+  http_listen_port: 9009
+  log_level: error
+
+store_gateway:
+  sharding_ring:
+    replication_factor: 1
+
+limits:
+  max_global_exemplars_per_user: 100000

development/configs/otel-collector.yaml

@@ -0,0 +1,26 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+
+exporters:
+  otlp:
+    endpoint: "tempo:4317"
+    tls:
+      insecure: true
+
+  prometheusremotewrite:
+    endpoint: "http://mimir:9009/api/v1/push"
+    # When using the official Prometheus (running via Docker)
+    # endpoint: 'http://prometheus:9090/api/v1/write', add:
+    tls:
+      insecure: true
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [otlp]
+    metrics:
+      receivers: [otlp]
+      exporters: [prometheusremotewrite]

development/configs/promtail.yaml

@@ -0,0 +1,29 @@
+---
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: "/tmp/positions.yaml"
+
+clients:
+  - url: "http://loki:3100/loki/api/v1/push"
+
+scrape_configs:
+  - job_name: "flog_scrape"
+    docker_sd_configs:
+      - host: "unix:///var/run/docker.sock"
+        refresh_interval: "5s"
+    relabel_configs:
+      - source_labels: ["__meta_docker_container_name"]
+        regex: "/(.*)"
+        target_label: "container"
+      - source_labels: ["__meta_docker_container_log_stream"]
+        target_label: "logstream"
+      - source_labels: ["__meta_docker_container_label_logging_jobname"]
+        target_label: "job"
+    pipeline_stages:
+    - regex:
+        expression: '^.*resource\.service\.name=(?P<service_name>[\S]+).*$'
+    - labels:
+        service_name:

development/configs/tempo-config.yaml

@@ -0,0 +1,40 @@
+# Do not use this configuration in production.
+# It is for demonstration purposes only.
+# metrics_generator_enabled: true
+# search_enabled: true
+
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+        grpc:
+
+metrics_generator:
+  ring:
+    kvstore: {}
+  processor:
+    service_graphs: {}
+    span_metrics: {}
+  registry:
+    external_labels:
+      source: tempo
+  storage:
+    path: /tmp/tempo/generator/wal
+    remote_write:
+      - url: http://mimir:9009/api/v1/push
+        send_exemplars: true
+
+storage:
+  trace:
+    backend: local
+    wal:
+      path: /tmp/tempo/wal
+    local:
+      path: /tmp/tempo/blocks
+
+overrides:
+  metrics_generator_processors: ["service-graphs", "span-metrics"]

development/dev.env

@@ -48,3 +48,12 @@ NAUTOBOT_SUPERUSER_NAME=admin
 NAUTOBOT_SUPERUSER_EMAIL=admin@example.com
 NAUTOBOT_SUPERUSER_PASSWORD=admin
 NAUTOBOT_SUPERUSER_API_TOKEN=0123456789abcdef0123456789abcdef01234567
+
+OTEL_PYTHON_DJANGO_INSTRUMENT=True
+OTEL_PYTHON_LOG_CORRELATION=True
+OTEL_TRACES_EXPORTER="otlp"
+OTEL_METRICS_EXPORTER="otlp"
+
+OTEL_EXPORTER_OTLP_ENDPOINT="http://otel:4317"
+OTEL_EXPORTER_OTLP_PROTOCOL="grpc"
+OTEL_EXPORTER_OTLP_INSECURE=True

development/docker-compose.observability.yml

@@ -0,0 +1,66 @@
+---
+version: "3.8"
+
+volumes:
+  grafana_data:
+
+services:
+  loki:
+    image: "grafana/loki:2.9.2"
+    ports:
+      - "3100:3100"
+    command: "-config.file=/etc/loki/local-config.yaml"
+
+  otel:
+    image: otel/opentelemetry-collector-contrib:latest
+    command: ["--config=/etc/otel-collector.yaml"]
+    volumes:
+      - ./configs/otel-collector.yaml:/etc/otel-collector.yaml
+    depends_on:
+      - "tempo"
+      - "mimir"
+      - "loki"
+
+  tempo:
+    image: grafana/tempo
+    command:
+      - -config.file
+      - /etc/tempo/tempo-config.yaml
+    volumes:
+      - ./configs/:/etc/tempo
+    ports:
+      - "3200:3200"
+      - "4317:4317"
+      - "4318:4318"
+      - "55680:55680"
+
+  mimir:
+    image: grafana/mimir
+    command:
+      - -config.file=/etc/mimir/mimir-config.yaml
+      - -server.grpc-listen-port=9096
+    volumes:
+      - ./configs/:/etc/mimir
+    ports:
+      - "9009:9009"
+      - "9096:9096"
+
+  promtail:
+    image: "grafana/promtail:latest"
+    volumes:
+      - "./configs/promtail.yaml:/etc/promtail/docker-config.yaml"
+      - "/var/lib/docker/containers:/var/lib/docker/containers:ro"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+    command: "-config.file=/etc/promtail/docker-config.yaml"
+
+  grafana:
+    image: "grafana/grafana:latest"
+    ports:
+      - "3000:3000"
+    volumes:
+      - "grafana_data:/var/lib/grafana"
+      - "./configs/grafana/datasources.yaml:/etc/grafana/provisioning/datasources/all.yml"
+      - "./configs/grafana/grafana.ini:/etc/grafana/grafana.ini"
+    environment:
+      - "GF_INSTALL_PLUGINS=yesoreyeram-infinity-datasource,volkovlabs-echarts-panel,marcusolsson-dynamictext-panel"
+      - "GF_PANELS_DISABLE_SANITIZE_HTML=true"

invoke.yml.example

@@ -10,6 +10,7 @@ nautobot:
     - "docker-compose.postgres.yml"
     - "docker-compose.dev.yml"
     - "docker-compose.override.yml"
+    # - "docker-compose.observability.yml"
   docker_image_names_main:
     - "networktocode/nautobot"
     - "ghcr.io/nautobot/nautobot"

mkdocs.yml

@@ -302,6 +302,7 @@ nav:
         - "user-guide/administration/configuration/index.md"
         - Required Settings: "user-guide/administration/configuration/required-settings.md"
         - Optional Settings: "user-guide/administration/configuration/optional-settings.md"
+        - OpenTelemetry: "user-guide/administration/configuration/opentelemetry.md"
         - Time Zones: "user-guide/administration/configuration/time-zones.md"
         - External Authentication:
           - LDAP: "user-guide/administration/configuration/authentication/ldap.md"
@@ -314,6 +315,7 @@ nav:
         - Caching: "user-guide/administration/guides/caching.md"
         - Health Check: "user-guide/administration/guides/healthcheck.md"
         - Permissions: "user-guide/administration/guides/permissions.md"
+        - OpenTelemetry: "user-guide/administration/guides/opentelemetry.md"
         - Prometheus Metrics: "user-guide/administration/guides/prometheus-metrics.md"
         - Replicating Nautobot: "user-guide/administration/guides/replicating-nautobot.md"
         - Request Profiling: "user-guide/administration/guides/request-profiling.md"

nautobot/core/cli/__init__.py

@@ -14,6 +14,8 @@
 from jinja2 import BaseLoader, Environment
 
 from nautobot import __version__
+from nautobot.core.cli.opentelemetry import instrument
+from nautobot.core.settings import OTEL_PYTHON_DJANGO_INSTRUMENT
 from nautobot.core.settings_funcs import is_truthy
 from nautobot.extras.plugins.utils import load_plugins
 
@@ -283,6 +285,11 @@ def main():
 
     # If we get here, it's a regular Django management command - so load in the nautobot_config.py then hand off
     _load_settings(args.config_path)
+
+    if OTEL_PYTHON_DJANGO_INSTRUMENT:
+        # Enable OpenTelemetry Instrumentation
+        instrument()
+
     execute_from_command_line([sys.argv[0], *unparsed_args])