From be9e2e87c78d6bde6f8766862e440bfe25a462bb Mon Sep 17 00:00:00 2001 From: Gabriel McGoldrick Date: Tue, 5 Nov 2024 17:05:27 +0000 Subject: [PATCH] OBSDOCS-1161 add content for validating that the monitoring stack is working --- ...ct-for-cluster-observability-operator.adoc | 61 ++++++++++++++ ...ck-for-cluster-observability-operator.adoc | 81 +++++++++++++++++++ ...ability-operator-to-monitor-a-service.adoc | 4 + 3 files changed, 146 insertions(+) create mode 100644 modules/monitoring-validating-a-monitoringstack-for-cluster-observability-operator.adoc diff --git a/modules/monitoring-creating-a-monitoringstack-object-for-cluster-observability-operator.adoc b/modules/monitoring-creating-a-monitoringstack-object-for-cluster-observability-operator.adoc index 703d00388835..fc1d37ab08b6 100644 --- a/modules/monitoring-creating-a-monitoringstack-object-for-cluster-observability-operator.adoc +++ b/modules/monitoring-creating-a-monitoringstack-object-for-cluster-observability-operator.adoc @@ -59,3 +59,64 @@ $ oc -n ns1-coo get monitoringstack NAME AGE example-coo-monitoring-stack 81m ---- + +. Run the following comand to retrieve information about the active targets from Prometheus and filter the output to list only targets labeled with `app=prometheus-coo-example-app`. This verifies which targets are discovered and actively monitored by Prometheus with this specific label. ++ +[source,terminal] +---- +$ oc -n oc -n ns1-coo exec -c prometheus prometheus-example-coo-monitoring-stack-0 -- curl -s 'http://localhost:9090/api/v1/targets' | jq '.data.activeTargets[].discoveredLabels | select(.__meta_kubernetes_endpoints_label_app=="prometheus-coo-example-app")' +---- ++ +.Example output +[source,json] +---- +{ + "__address__": "10.129.2.25:8080", + "__meta_kubernetes_endpoint_address_target_kind": "Pod", + "__meta_kubernetes_endpoint_address_target_name": "prometheus-coo-example-app-5d8cd498c7-9j2gj", + "__meta_kubernetes_endpoint_node_name": "ci-ln-8tt8vxb-72292-6cxjr-worker-a-wdfnz", + "__meta_kubernetes_endpoint_port_name": "web", + "__meta_kubernetes_endpoint_port_protocol": "TCP", + "__meta_kubernetes_endpoint_ready": "true", + "__meta_kubernetes_endpoints_annotation_endpoints_kubernetes_io_last_change_trigger_time": "2024-11-05T11:24:09Z", + "__meta_kubernetes_endpoints_annotationpresent_endpoints_kubernetes_io_last_change_trigger_time": "true", + "__meta_kubernetes_endpoints_label_app": "prometheus-coo-example-app", + "__meta_kubernetes_endpoints_labelpresent_app": "true", + "__meta_kubernetes_endpoints_name": "prometheus-coo-example-app", + "__meta_kubernetes_namespace": "ns1-coo", + "__meta_kubernetes_pod_annotation_k8s_ovn_org_pod_networks": "{\"default\":{\"ip_addresses\":[\"10.129.2.25/23\"],\"mac_address\":\"0a:58:0a:81:02:19\",\"gateway_ips\":[\"10.129.2.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.129.2.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.129.2.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.129.2.1\"}],\"ip_address\":\"10.129.2.25/23\",\"gateway_ip\":\"10.129.2.1\",\"role\":\"primary\"}}", + "__meta_kubernetes_pod_annotation_k8s_v1_cni_cncf_io_network_status": "[{\n \"name\": \"ovn-kubernetes\",\n \"interface\": \"eth0\",\n \"ips\": [\n \"10.129.2.25\"\n ],\n \"mac\": \"0a:58:0a:81:02:19\",\n \"default\": true,\n \"dns\": {}\n}]", + "__meta_kubernetes_pod_annotation_openshift_io_scc": "restricted-v2", + "__meta_kubernetes_pod_annotation_seccomp_security_alpha_kubernetes_io_pod": "runtime/default", + "__meta_kubernetes_pod_annotationpresent_k8s_ovn_org_pod_networks": "true", + "__meta_kubernetes_pod_annotationpresent_k8s_v1_cni_cncf_io_network_status": "true", + "__meta_kubernetes_pod_annotationpresent_openshift_io_scc": "true", + "__meta_kubernetes_pod_annotationpresent_seccomp_security_alpha_kubernetes_io_pod": "true", + "__meta_kubernetes_pod_controller_kind": "ReplicaSet", + "__meta_kubernetes_pod_controller_name": "prometheus-coo-example-app-5d8cd498c7", + "__meta_kubernetes_pod_host_ip": "10.0.128.2", + "__meta_kubernetes_pod_ip": "10.129.2.25", + "__meta_kubernetes_pod_label_app": "prometheus-coo-example-app", + "__meta_kubernetes_pod_label_pod_template_hash": "5d8cd498c7", + "__meta_kubernetes_pod_labelpresent_app": "true", + "__meta_kubernetes_pod_labelpresent_pod_template_hash": "true", + "__meta_kubernetes_pod_name": "prometheus-coo-example-app-5d8cd498c7-9j2gj", + "__meta_kubernetes_pod_node_name": "ci-ln-8tt8vxb-72292-6cxjr-worker-a-wdfnz", + "__meta_kubernetes_pod_phase": "Running", + "__meta_kubernetes_pod_ready": "true", + "__meta_kubernetes_pod_uid": "054c11b6-9a76-4827-a860-47f3a4596871", + "__meta_kubernetes_service_label_app": "prometheus-coo-example-app", + "__meta_kubernetes_service_labelpresent_app": "true", + "__meta_kubernetes_service_name": "prometheus-coo-example-app", + "__metrics_path__": "/metrics", + "__scheme__": "http", + "__scrape_interval__": "30s", + "__scrape_timeout__": "10s", + "job": "serviceMonitor/ns1-coo/prometheus-coo-example-monitor/0" +} +---- ++ +[NOTE] +==== +The above example uses link:https://jqlang.github.io/jq/[`jq` command-line JSON processor] to format the output for convenience. +==== diff --git a/modules/monitoring-validating-a-monitoringstack-for-cluster-observability-operator.adoc b/modules/monitoring-validating-a-monitoringstack-for-cluster-observability-operator.adoc new file mode 100644 index 000000000000..74b8760eb39b --- /dev/null +++ b/modules/monitoring-validating-a-monitoringstack-for-cluster-observability-operator.adoc @@ -0,0 +1,81 @@ +// Module included in the following assemblies: +// +// * observability/cluster-observability-operator/configuring-the-cluster-observability-operator-to-monitor-a-service.adoc + +:_mod-docs-content-type: PROCEDURE +[id="monitoring-validating-a-monitoringstack-for-cluster-observability-operator_{context}"] += Validating the monitoring stack + +To validate that the monitoring stack is working correctly, access the example service and then view the gathered metrics. + +.Prerequisites + +* You have access to the cluster as a user with the `cluster-admin` cluster role or as a user with administrative permissions for the namespace. +* You have installed the {coo-full}. +* You have deployed the `prometheus-coo-example-app` sample service in the `ns1-coo` namespace. +* You have created a `ServiceMonitor` object named `prometheus-coo-example-monitor` in the `ns1-coo` namespace. +* You have created a `MonitoringStack` object named `example-coo-monitoring-stack` in the `ns1-coo` namespace. + +.Procedure + +. Create a route to expose the example `prometheus-coo-example-app` service. From your terminal, run the command: ++ +[source,terminal] +---- +$ oc expose svc prometheus-coo-example-app +---- +. Access the route from your browser, or command line, to generate metrics. + +. Execute a query on the Prometheus pod to return the total HTTP requests metric: ++ +[source,terminal] +---- +$ oc -n ns1-coo exec -c prometheus prometheus-example-coo-monitoring-stack-0 -- curl -s 'http://localhost:9090/api/v1/query?query=http_requests_total' +---- ++ +.Example output (formatted using `jq` for convenience) +[source,json] +---- +{ + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": { + "__name__": "http_requests_total", + "code": "200", + "endpoint": "web", + "instance": "10.129.2.25:8080", + "job": "prometheus-coo-example-app", + "method": "get", + "namespace": "ns1-coo", + "pod": "prometheus-coo-example-app-5d8cd498c7-9j2gj", + "service": "prometheus-coo-example-app" + }, + "value": [ + 1730807483.632, + "3" + ] + }, + { + "metric": { + "__name__": "http_requests_total", + "code": "404", + "endpoint": "web", + "instance": "10.129.2.25:8080", + "job": "prometheus-coo-example-app", + "method": "get", + "namespace": "ns1-coo", + "pod": "prometheus-coo-example-app-5d8cd498c7-9j2gj", + "service": "prometheus-coo-example-app" + }, + "value": [ + 1730807483.632, + "0" + ] + } + ] + } +} +---- diff --git a/observability/cluster_observability_operator/configuring-the-cluster-observability-operator-to-monitor-a-service.adoc b/observability/cluster_observability_operator/configuring-the-cluster-observability-operator-to-monitor-a-service.adoc index ad91404bee2f..a1683f75824b 100644 --- a/observability/cluster_observability_operator/configuring-the-cluster-observability-operator-to-monitor-a-service.adoc +++ b/observability/cluster_observability_operator/configuring-the-cluster-observability-operator-to-monitor-a-service.adoc @@ -25,3 +25,7 @@ include::modules/monitoring-specifying-how-a-service-is-monitored-by-cluster-obs // Create a MonitoringStack object to discover the service monitor include::modules/monitoring-creating-a-monitoringstack-object-for-cluster-observability-operator.adoc[leveloffset=+1] + +// Validate a MonitoringStack +include::modules/monitoring-validating-a-monitoringstack-for-cluster-observability-operator.adoc[leveloffset=+1] +