From 6372eb868948b230e185d8dbeb7c711ec16641bd Mon Sep 17 00:00:00 2001 From: heydbut Date: Mon, 18 Sep 2023 18:23:53 -0400 Subject: [PATCH 1/4] Add opa server dashboard --- .../dashboards/overview/main.tf | 529 ++++++++++++++++++ .../examples/compose/docker-compose.yaml | 8 + 2 files changed, 537 insertions(+) create mode 100644 collector/open-policy-agent/dashboards/overview/main.tf diff --git a/collector/open-policy-agent/dashboards/overview/main.tf b/collector/open-policy-agent/dashboards/overview/main.tf new file mode 100644 index 0000000..af81b2c --- /dev/null +++ b/collector/open-policy-agent/dashboards/overview/main.tf @@ -0,0 +1,529 @@ +terraform { + required_providers { + lightstep = { + source = "lightstep/lightstep" + version = "~> 1.76.0" + } + } + required_version = ">= v1.0.11" +} + +variable "lightstep_project" { + description = "Cloud Observability Project Name" + type = string +} + +output "dashboard_url" { + value = "https://app.lightstep.com/${var.lightstep_project}/dashboard/${lightstep_dashboard.otel_collector_opa_dashboard.id}" + description = "OpenTelemetry Collector Open Policy Agent Dashboard URL" +} + +resource "lightstep_dashboard" "otel_collector_opa_dashboard" { + project_name = var.lightstep_project + dashboard_name = "Open Policy Agent" + dashboard_description = "Monitoring dashboard for Open Policy Agent (OPA), allowing real-time tracking of resource utilization." + + group { + rank = 0 + title = "" + visibility_type = "implicit" + + chart { + name = "HTTP Response Time Quantile" + description = "" + type = "timeseries" + rank = 0 + x_pos = 0 + y_pos = 0 + width = 24 + height = 9 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric http_request_duration_seconds | filter (\"service.instance.id\" == $service) | delta | group_by [], sum | point percentile(value, 50.0), percentile(value, 95.0), percentile(value, 99.0), percentile(value, 99.9)" + } + } + chart { + name = "Total Used Memory" + description = "" + type = "timeseries" + rank = 1 + x_pos = 24 + y_pos = 0 + width = 24 + height = 9 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_sys_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + } + chart { + name = "Memory In Off-Heap" + description = "" + type = "timeseries" + rank = 2 + x_pos = 0 + y_pos = 9 + width = 48 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_mspan_inuse_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + query { + query_name = "b" + display = "line" + hidden = false + query_string = "metric go_memstats_mspan_sys_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + query { + query_name = "c" + display = "line" + hidden = false + query_string = "metric go_memstats_mcache_inuse_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + query { + query_name = "d" + display = "line" + hidden = false + query_string = "metric go_memstats_mcache_sys_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + query { + query_name = "e" + display = "line" + hidden = false + query_string = "metric go_memstats_buck_hash_sys_bytes | filter (\"service.name\" == $service) | latest | group_by [], sum" + } + query { + query_name = "f" + display = "line" + hidden = false + query_string = "metric go_memstats_gc_sys_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + query { + query_name = "g" + display = "line" + hidden = false + query_string = "metric go_memstats_next_gc_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + } + } + group { + rank = 1 + title = "Object & Pointers" + visibility_type = "explicit" + + chart { + name = "Memory In Stack" + description = "" + type = "timeseries" + rank = 0 + x_pos = 0 + y_pos = 0 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_stack_sys_bytes | filter (\"service.name\" == $service) | latest | group_by [], sum" + } + } + chart { + name = "Pointer Dereferences Rate" + description = "Counts rate of how many pointer dereferences happened" + type = "timeseries" + rank = 1 + x_pos = 24 + y_pos = 0 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_lookups_total | filter (\"service.instance.id\" == $service) | rate | group_by [], sum" + } + } + chart { + name = "Objects Allocation Rate" + description = "" + type = "timeseries" + rank = 2 + x_pos = 0 + y_pos = 8 + width = 16 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_mallocs_total | filter (\"service.instance.id\" == $service) | rate | group_by [], sum" + } + } + chart { + name = "Live Objects Rate" + description = "" + type = "timeseries" + rank = 3 + x_pos = 16 + y_pos = 8 + width = 16 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "with\n a = metric go_memstats_mallocs_total | filter (\"service.instance.id\" == $service) | rate | group_by [], sum;\n b = metric go_memstats_frees_total | filter (\"service.instance.id\" == $service) | rate | group_by [], sum;\njoin ((a - b)), a=0, b=0" + } + } + chart { + name = "Bytes Allocation Rate" + description = "" + type = "timeseries" + rank = 4 + x_pos = 32 + y_pos = 8 + width = 16 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_alloc_bytes_total | filter (\"service.instance.id\" == $service) | rate | group_by [], sum" + } + } + } + group { + rank = 2 + title = "Memory In Heap" + visibility_type = "explicit" + + chart { + name = "Bytes Obtained From System" + description = "" + type = "timeseries" + rank = 0 + x_pos = 0 + y_pos = 0 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_heap_sys_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + } + chart { + name = "Bytes Allocated & In-use" + description = "" + type = "timeseries" + rank = 1 + x_pos = 24 + y_pos = 0 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_heap_alloc_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + } + chart { + name = "Bytes Waiting To Be Used" + description = "" + type = "timeseries" + rank = 2 + x_pos = 0 + y_pos = 8 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_heap_idle_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + } + chart { + name = "Bytes In Use" + description = "" + type = "timeseries" + rank = 3 + x_pos = 24 + y_pos = 8 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_heap_inuse_bytes | filter (\"service.instance.id\" == $service) | latest 2m | group_by [], sum" + } + } + chart { + name = "Bytes Released To OS" + description = "" + type = "timeseries" + rank = 4 + x_pos = 0 + y_pos = 16 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_heap_released_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + } + chart { + name = "Objects Allocated" + description = "" + type = "timeseries" + rank = 5 + x_pos = 24 + y_pos = 16 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_heap_objects | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + } + } + group { + rank = 3 + title = "Memory Off-Heap" + visibility_type = "explicit" + + chart { + name = "Mspan Bytes Usage" + description = "" + type = "timeseries" + rank = 0 + x_pos = 0 + y_pos = 0 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_mspan_inuse_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + } + chart { + name = "Bytes Used For Mspan Obtained From System" + description = "" + type = "timeseries" + rank = 1 + x_pos = 24 + y_pos = 0 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_mspan_sys_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + } + chart { + name = "Bytes Used For Mcache Obtained From System" + description = "" + type = "timeseries" + rank = 2 + x_pos = 0 + y_pos = 8 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_mcache_sys_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + } + chart { + name = "Bytes In Use By Mcache" + description = "" + type = "timeseries" + rank = 3 + x_pos = 24 + y_pos = 8 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_mcache_inuse_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + } + chart { + name = "Bytes Used For Garbage Collection System Metadata" + description = "" + type = "timeseries" + rank = 4 + x_pos = 0 + y_pos = 16 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_gc_sys_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + } + chart { + name = "Bytes Used By The Profiling Bucket Hash Table" + description = "" + type = "timeseries" + rank = 5 + x_pos = 24 + y_pos = 16 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_buck_hash_sys_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + } + chart { + name = "Bytes When Next Garbage Collection Will Take Place" + description = "" + type = "timeseries" + rank = 6 + x_pos = 0 + y_pos = 24 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_memstats_next_gc_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + } + chart { + name = "Bytes Used For Other System Allocations" + description = "" + type = "timeseries" + rank = 7 + x_pos = 24 + y_pos = 24 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "with\n a = metric go_memstats_other_sys_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum;\n b = metric go_memstats_other_sys_bytes | filter (\"service.instance.id\" == $service) | latest | group_by [], sum;\njoin ((a - b)), a=0, b=0" + } + } + chart { + name = "Goroutines" + description = "" + type = "timeseries" + rank = 8 + x_pos = 0 + y_pos = 32 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_goroutines | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + } + chart { + name = "GC Duration Quantile" + description = "" + type = "timeseries" + rank = 9 + x_pos = 24 + y_pos = 32 + width = 24 + height = 8 + + query { + query_name = "a" + display = "line" + hidden = false + query_string = "metric go_gc_duration_seconds_p25 | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + query { + query_name = "b" + display = "line" + hidden = false + query_string = "metric go_gc_duration_seconds_p50 | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + query { + query_name = "c" + display = "line" + hidden = false + query_string = "metric go_gc_duration_seconds_p75 | filter (\"service.instance.id\" == $service) | latest | group_by [], sum" + } + } + } + + template_variable { + name = "service" + default_values = [] + suggestion_attribute_key = "service.instance.id" + } + + label { + key = "" + value = "policy" + } + label { + key = "" + value = "memorymanagement" + } + label { + key = "" + value = "opa" + } + label { + key = "" + value = "openpolicyagent" + } +} diff --git a/collector/open-policy-agent/examples/compose/docker-compose.yaml b/collector/open-policy-agent/examples/compose/docker-compose.yaml index 629eb77..beb4ecb 100644 --- a/collector/open-policy-agent/examples/compose/docker-compose.yaml +++ b/collector/open-policy-agent/examples/compose/docker-compose.yaml @@ -16,6 +16,7 @@ services: depends_on: - bundle_server networks: + - policy - integrations api_server: @@ -27,13 +28,18 @@ services: - POLICY_PATH=/v1/data/httpapi/authz depends_on: - opa + networks: + - policy bundle_server: image: nginx:1.20.0-alpine + hostname: bundle_server ports: - 8888:80 volumes: - ./bundle.tar.gz:/usr/share/nginx/html/bundle.tar.gz:ro + networks: + - policy otel-collector: container_name: otel-collector @@ -52,3 +58,5 @@ services: networks: integrations: + policy: + From 91425f4452c9305ce4a18b66c4cedfd38d924818 Mon Sep 17 00:00:00 2001 From: Daniel <116079873+heydbut@users.noreply.github.com> Date: Tue, 19 Sep 2023 16:39:00 -0400 Subject: [PATCH 2/4] Update collector/open-policy-agent/dashboards/overview/main.tf Co-authored-by: Nathan Slaughter <28688390+nslaughter@users.noreply.github.com> --- collector/open-policy-agent/dashboards/overview/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/open-policy-agent/dashboards/overview/main.tf b/collector/open-policy-agent/dashboards/overview/main.tf index af81b2c..a51fd7f 100644 --- a/collector/open-policy-agent/dashboards/overview/main.tf +++ b/collector/open-policy-agent/dashboards/overview/main.tf @@ -18,7 +18,7 @@ output "dashboard_url" { description = "OpenTelemetry Collector Open Policy Agent Dashboard URL" } -resource "lightstep_dashboard" "otel_collector_opa_dashboard" { +resource "lightstep_dashboard" "collector_openpolicyagent_overview" { project_name = var.lightstep_project dashboard_name = "Open Policy Agent" dashboard_description = "Monitoring dashboard for Open Policy Agent (OPA), allowing real-time tracking of resource utilization." From 5bd08fb4074bd41693e14f039a0d4acf880f708e Mon Sep 17 00:00:00 2001 From: Nathan Slaughter <28688390+nslaughter@users.noreply.github.com> Date: Tue, 19 Sep 2023 15:42:22 -0500 Subject: [PATCH 3/4] rename opa directory --- .../dashboards/overview/main.tf | 0 .../examples/compose/Dockerfile | 0 .../examples/compose/README.md | 0 .../examples/compose/bundle.tar.gz | Bin .../examples/compose/collector.yaml | 0 .../examples/compose/docker-compose.yaml | 0 .../examples/compose/echo_server.py | 0 .../examples/compose/policy/example-hr.rego | 0 .../examples/compose/policy/example-jwt.rego | 0 .../examples/compose/policy/example.rego | 0 .../examples/compose/requirements.txt | 0 .../metrics.csv | 0 12 files changed, 0 insertions(+), 0 deletions(-) rename collector/{open-policy-agent => openpolicyagent}/dashboards/overview/main.tf (100%) rename collector/{open-policy-agent => openpolicyagent}/examples/compose/Dockerfile (100%) rename collector/{open-policy-agent => openpolicyagent}/examples/compose/README.md (100%) rename collector/{open-policy-agent => openpolicyagent}/examples/compose/bundle.tar.gz (100%) rename collector/{open-policy-agent => openpolicyagent}/examples/compose/collector.yaml (100%) rename collector/{open-policy-agent => openpolicyagent}/examples/compose/docker-compose.yaml (100%) rename collector/{open-policy-agent => openpolicyagent}/examples/compose/echo_server.py (100%) rename collector/{open-policy-agent => openpolicyagent}/examples/compose/policy/example-hr.rego (100%) rename collector/{open-policy-agent => openpolicyagent}/examples/compose/policy/example-jwt.rego (100%) rename collector/{open-policy-agent => openpolicyagent}/examples/compose/policy/example.rego (100%) rename collector/{open-policy-agent => openpolicyagent}/examples/compose/requirements.txt (100%) rename collector/{open-policy-agent => openpolicyagent}/metrics.csv (100%) diff --git a/collector/open-policy-agent/dashboards/overview/main.tf b/collector/openpolicyagent/dashboards/overview/main.tf similarity index 100% rename from collector/open-policy-agent/dashboards/overview/main.tf rename to collector/openpolicyagent/dashboards/overview/main.tf diff --git a/collector/open-policy-agent/examples/compose/Dockerfile b/collector/openpolicyagent/examples/compose/Dockerfile similarity index 100% rename from collector/open-policy-agent/examples/compose/Dockerfile rename to collector/openpolicyagent/examples/compose/Dockerfile diff --git a/collector/open-policy-agent/examples/compose/README.md b/collector/openpolicyagent/examples/compose/README.md similarity index 100% rename from collector/open-policy-agent/examples/compose/README.md rename to collector/openpolicyagent/examples/compose/README.md diff --git a/collector/open-policy-agent/examples/compose/bundle.tar.gz b/collector/openpolicyagent/examples/compose/bundle.tar.gz similarity index 100% rename from collector/open-policy-agent/examples/compose/bundle.tar.gz rename to collector/openpolicyagent/examples/compose/bundle.tar.gz diff --git a/collector/open-policy-agent/examples/compose/collector.yaml b/collector/openpolicyagent/examples/compose/collector.yaml similarity index 100% rename from collector/open-policy-agent/examples/compose/collector.yaml rename to collector/openpolicyagent/examples/compose/collector.yaml diff --git a/collector/open-policy-agent/examples/compose/docker-compose.yaml b/collector/openpolicyagent/examples/compose/docker-compose.yaml similarity index 100% rename from collector/open-policy-agent/examples/compose/docker-compose.yaml rename to collector/openpolicyagent/examples/compose/docker-compose.yaml diff --git a/collector/open-policy-agent/examples/compose/echo_server.py b/collector/openpolicyagent/examples/compose/echo_server.py similarity index 100% rename from collector/open-policy-agent/examples/compose/echo_server.py rename to collector/openpolicyagent/examples/compose/echo_server.py diff --git a/collector/open-policy-agent/examples/compose/policy/example-hr.rego b/collector/openpolicyagent/examples/compose/policy/example-hr.rego similarity index 100% rename from collector/open-policy-agent/examples/compose/policy/example-hr.rego rename to collector/openpolicyagent/examples/compose/policy/example-hr.rego diff --git a/collector/open-policy-agent/examples/compose/policy/example-jwt.rego b/collector/openpolicyagent/examples/compose/policy/example-jwt.rego similarity index 100% rename from collector/open-policy-agent/examples/compose/policy/example-jwt.rego rename to collector/openpolicyagent/examples/compose/policy/example-jwt.rego diff --git a/collector/open-policy-agent/examples/compose/policy/example.rego b/collector/openpolicyagent/examples/compose/policy/example.rego similarity index 100% rename from collector/open-policy-agent/examples/compose/policy/example.rego rename to collector/openpolicyagent/examples/compose/policy/example.rego diff --git a/collector/open-policy-agent/examples/compose/requirements.txt b/collector/openpolicyagent/examples/compose/requirements.txt similarity index 100% rename from collector/open-policy-agent/examples/compose/requirements.txt rename to collector/openpolicyagent/examples/compose/requirements.txt diff --git a/collector/open-policy-agent/metrics.csv b/collector/openpolicyagent/metrics.csv similarity index 100% rename from collector/open-policy-agent/metrics.csv rename to collector/openpolicyagent/metrics.csv From 8c6525e66bddcf64cce08337a5908e79007f9aa1 Mon Sep 17 00:00:00 2001 From: Nathan Slaughter <28688390+nslaughter@users.noreply.github.com> Date: Tue, 19 Sep 2023 15:50:19 -0500 Subject: [PATCH 4/4] bump opa ver --- collector/openpolicyagent/examples/compose/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/openpolicyagent/examples/compose/docker-compose.yaml b/collector/openpolicyagent/examples/compose/docker-compose.yaml index beb4ecb..99fa14d 100644 --- a/collector/openpolicyagent/examples/compose/docker-compose.yaml +++ b/collector/openpolicyagent/examples/compose/docker-compose.yaml @@ -1,6 +1,6 @@ services: opa: - image: openpolicyagent/opa:0.40.0-rootless + image: openpolicyagent/opa:0.56.0-rootless ports: - "8181:8181" command: