kong example and dashboard

collector/kong/dashboards/overview/main.tf

@@ -0,0 +1,187 @@
+terraform {
+  required_providers {
+    lightstep = {
+      source  = "lightstep/lightstep"
+      version = "~> 1.76.0"
+    }
+  }
+  required_version = ">= v1.0.11"
+}
+
+variable "lightstep_project" {
+  description = "Lightstep Project Name"
+  type        = string
+}
+
+output "dashboard_url" {
+  value       = "https://app.lightstep.com/${var.lightstep_project}/dashboard/${lightstep_dashboard.otel_collector_dashboard.id}"
+  description = "OpenTelemetry Collector Kong Dashboard URL"
+}
+
+resource "lightstep_dashboard" "otel_collector_dashboard" {
+  project_name   = var.lightstep_project
+  dashboard_name = "Kong Metrics"
+  dashboard_description = "Monitor Kong with this metrics overview dashboard."
+
+  chart {
+    name = "Requests Total"
+    rank = "0"
+    type = "timeseries"
+
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = <<EOT
+metric kong_nginx_requests_total | filter ((service.name == $service_name) && (net.host.name == $net_host_name) && (service.instance.id == $service_instance_id) && (net.host.port == $net_host_port) && (http.scheme == $http_scheme)) | delta | group_by [], sum
+EOT
+    }
+  }
+
+  chart {
+    name = "Datastore Reachable"
+    rank = "1"
+    type = "timeseries"
+
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = <<EOT
+metric kong_datastore_reachable | filter ((service.name == $service_name) && (net.host.name == $net_host_name) && (service.instance.id == $service_instance_id) && (net.host.port == $net_host_port) && (http.scheme == $http_scheme)) | delta | group_by [], sum
+EOT
+    }
+  }
+
+  chart {
+    name = "Total Capacity in Bytes"
+    rank = "2"
+    type = "timeseries"
+
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = <<EOT
+metric kong_memory_lua_shared_dict_total_bytes | filter ((service.name == $service_name) && (net.host.name == $net_host_name) && (service.instance.id == $service_instance_id) && (net.host.port == $net_host_port) && (http.scheme == $http_scheme) && (kong_subsystem == $kong_subsystem) && (node_id == $node_id)) | delta | group_by [], sum
+EOT
+    }
+  }
+
+  chart {
+    name = "Allocated Bytes"
+    rank = "3"
+    type = "timeseries"
+
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = <<EOT
+metric kong_memory_workers_lua_vms_bytes | filter ((service.name == $service_name) && (net.host.name == $net_host_name) && (service.instance.id == $service_instance_id) && (net.host.port == $net_host_port) && (http.scheme == $http_scheme) && (kong_subsystem == $kong_subsystem) && (node_id == $node_id)) | delta | group_by [], sum
+EOT
+    }
+  }
+
+  chart {
+    name = "Node Metadata"
+    rank = "4"
+    type = "timeseries"
+
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = <<EOT
+metric kong_node_info | filter ((service.name == $service_name) && (net.host.name == $net_host_name) && (service.instance.id == $service_instance_id) && (net.host.port == $net_host_port) && (http.scheme == $http_scheme)) | delta | group_by [], sum
+EOT
+    }
+  }
+
+  chart {
+    name = "Allocated Slabs in Bytes"
+    rank = "5"
+    type = "timeseries"
+
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = <<EOT
+metric kong_memory_lua_shared_dict_bytes | filter ((service.name == $service_name) && (net.host.name == $net_host_name) && (service.instance.id == $service_instance_id) && (net.host.port == $net_host_port) && (http.scheme == $http_scheme) && (kong_subsystem == $kong_subsystem) && (node_id == $node_id)) | delta | group_by [], sum
+EOT
+    }
+  }
+
+  chart {
+    name = "Connections"
+    rank = "6"
+    type = "timeseries"
+
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = <<EOT
+metric kong_nginx_connections_total | filter ((service.name == $service_name) && (net.host.name == $net_host_name) && (service.instance.id == $service_instance_id) && (net.host.port == $net_host_port) && (http.scheme == $http_scheme) && (node_id == $node_id)) | delta | group_by [], sum
+EOT
+    }
+  }
+
+  chart {
+    name = "Nginx Timers"
+    rank = "7"
+    type = "timeseries"
+
+    query {
+      query_name   = "a"
+      display      = "line"
+      hidden       = false
+      query_string = <<EOT
+metric kong_nginx_timers | filter ((service.name == $service_name) && (net.host.name == $net_host_name) && (service.instance.id == $service_instance_id) && (net.host.port == $net_host_port) && (http.scheme == $http_scheme)) | delta | group_by [], sum
+EOT
+    }
+  }
+
+  template_variable {
+    name                     = "http_scheme"
+    default_values           = []
+    suggestion_attribute_key = "http.scheme"
+  }
+
+  template_variable {
+    name                     = "kong_subsystem"
+    default_values           = []
+    suggestion_attribute_key = "kong_subsystem"
+  }
+
+  template_variable {
+    name                     = "net_host_name"
+    default_values           = []
+    suggestion_attribute_key = "net.host.name"
+  }
+
+  template_variable {
+    name                     = "net_host_port"
+    default_values           = []
+    suggestion_attribute_key = "net.host.port"
+  }
+
+  template_variable {
+    name                     = "node_id"
+    default_values           = []
+    suggestion_attribute_key = "node_id"
+  }
+
+  template_variable {
+    name                     = "service_instance_id"
+    default_values           = []
+    suggestion_attribute_key = "service.instance.id"
+  }
+
+  template_variable {
+    name                     = "service_name"
+    default_values           = []
+    suggestion_attribute_key = "service.name"
+  }
+}

collector/kong/examples/kube/Makefile

@@ -0,0 +1,20 @@
+
+all: setup install-all
+
+setup: create-cluster
+install-all: install-kong install-collector
+
+create-cluster:
+	kind create cluster --name kong --config kind-config.yaml
+
+delete-cluster:
+	kind delete cluster --name kong
+
+install-kong:
+	kubectl create namespace kong
+	kubectl create secret generic kong-enterprise-license --from-literal=license="'{}'" -n kong --dry-run=client -o yaml | kubectl apply -f -
+	kubectl apply -f https://raw.githubusercontent.com/Kong/kubernetes-ingress-controller/v2.11.0/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml
+	kubectl apply -f metrics-service.yaml
+
+install-collector:
+	./install_collector.sh

collector/kong/examples/kube/README.md

@@ -0,0 +1,82 @@
+---
+# Ingest metrics using the Kong integration
+
+The OTel Collector has a variety of [third party receivers](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/master/receiver) that provide integration with a wide variety of metric sources.
+
+Please note that not all metrics receivers available for the OpenTelemetry Collector have been tested by ServiceNow Cloud Observability, and there may be bugs or unexpected issues in using these contributed receivers with ServiceNow Cloud Observability metrics. File any issues with the appropriate OpenTelemetry community.
+{: .callout}
+
+## Prerequisites for local installation
+
+You must have a ServiceNow Cloud Observability [access token](/docs/create-and-manage-access-tokens) for the project to report metrics to.
+
+## Running the Example
+
+### Run the example by running:
+```bash
+make all
+```
+That will create local kubernetes cluster and install kong and collector.
+
+### Otherwise you can use following commands:
+
+#### 1. Create local kind cluster
+
+```bash
+kind create cluster --config kind-config.yaml
+```
+
+#### 2. Run kong
+
+Next create kong namespace, add kong secret(empty test secret in this case), install kong and expose metrics endpoint
+
+```bash
+kubectl create namespace kong
+kubectl create secret generic kong-enterprise-license --from-literal=license="'{}'" -n kong --dry-run=client -o yaml | kubectl apply -f -
+kubectl apply -f https://raw.githubusercontent.com/Kong/kubernetes-ingress-controller/v2.11.0/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml
+kubectl apply -f metrics-service.yaml
+```
+#### 3. Run collector
+
+Next install collector and point it to the kong metrics endpoint, "kong-metrics.kong:8100" in this case.
+
+```bash
+./install_collector.sh
+```
+
+## Configuration
+
+Installation procedures for the OpenTelemetry Collector varies by deployment mode. Please refer to the [collector documentation](https://opentelemetry.io/docs/collector/) for more information.
+
+Detailed description of available [Kong metrics](https://docs.konghq.com/hub/kong-inc/prometheus/#available-metrics).
+
+The following example configuration collects metrics from Kong and sends them into ServiceNow Cloud Observability:
+
+```yaml
+receivers:
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: kong
+          static_configs:
+            - targets: [kong-metrics.kong:8100]
+
+exporters:
+  logging:
+    loglevel: debug
+  otlp/public:
+    endpoint: ingest.lightstep.com:443
+    headers:
+        "lightstep-access-token": "${LS_ACCESS_TOKEN}"
+
+processors:
+  batch:
+
+service:
+  pipelines:
+    metrics/iis:
+      receivers: [prometheus]
+      processors: [batch]
+      exporters: [logging, otlp/public]
+```
+

collector/kong/examples/kube/install_collector.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+helm install my-collector open-telemetry/opentelemetry-collector -f values-collector.yaml  -f - <<EOF
+config:
+  exporters:
+    otlp/public:
+      headers:
+        - "lightstep-access-token": "${LS_ACCESS_TOKEN}"
+EOF

collector/kong/examples/kube/kind-config.yaml

@@ -0,0 +1,17 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: InitConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "ingress-ready=true"
+  extraPortMappings:
+  - containerPort: 8000
+    hostPort: 80
+    protocol: TCP
+  - containerPort: 8443
+    hostPort: 443
+    protocol: TCP

collector/kong/examples/kube/metrics-service.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kong-metrics
+  namespace: kong
+spec:
+  ports:
+  - name: metrics
+    port: 8100
+    protocol: TCP
+    targetPort: 8100
+  selector:
+    app: proxy-kong

collector/kong/examples/kube/values-collector.yaml

@@ -0,0 +1,50 @@
+mode: "deployment"
+
+config:
+  exporters:
+    logging:
+      loglevel: debug
+    otlp/public:
+      endpoint: ingest.lightstep.com:443
+      headers:
+        - "lightstep-access-token": "LS_ACCESS_TOKEN"
+  processors:
+    batch: {}
+    memory_limiter: null
+  receivers:
+    prometheus:
+      config:
+        scrape_configs:
+          - job_name: opentelemetry-collector
+            scrape_interval: 10s
+            static_configs:
+              - targets:
+                  - kong-metrics.kong:8100
+  service:
+    telemetry:
+      metrics:
+        address: 0.0.0.0:8888
+    pipelines:
+      logs:
+        exporters:
+          - logging
+          - otlp/public
+        processors:
+          - memory_limiter
+          - batch
+        receivers:
+          - otlp
+      metrics:
+        exporters:
+          - logging
+          - otlp/public
+        processors:
+          - memory_limiter
+          - batch
+        receivers:
+          - prometheus
+
+image:
+  repository: otel/opentelemetry-collector-contrib
+  pullPolicy: IfNotPresent
+  tag: "0.60.0"