[enterprise-4.12] OBSDOCS-1161 add content for validating that the monitoring stack is working

modules/monitoring-creating-a-monitoringstack-object-for-cluster-observability-operator.adoc

@@ -59,3 +59,64 @@ $ oc -n ns1-coo get monitoringstack
 NAME                         AGE
 example-coo-monitoring-stack   81m
 ----
+
+. Run the following comand to retrieve information about the active targets from Prometheus and filter the output to list only targets labeled with `app=prometheus-coo-example-app`. This verifies which targets are discovered and actively monitored by Prometheus with this specific label.
++
+[source,terminal]
+----
+$ oc -n oc -n ns1-coo exec -c prometheus prometheus-example-coo-monitoring-stack-0 -- curl -s 'http://localhost:9090/api/v1/targets' | jq '.data.activeTargets[].discoveredLabels | select(.__meta_kubernetes_endpoints_label_app=="prometheus-coo-example-app")'
+----
++
+.Example output
+[source,json]
+----
+{
+  "__address__": "10.129.2.25:8080",
+  "__meta_kubernetes_endpoint_address_target_kind": "Pod",
+  "__meta_kubernetes_endpoint_address_target_name": "prometheus-coo-example-app-5d8cd498c7-9j2gj",
+  "__meta_kubernetes_endpoint_node_name": "ci-ln-8tt8vxb-72292-6cxjr-worker-a-wdfnz",
+  "__meta_kubernetes_endpoint_port_name": "web",
+  "__meta_kubernetes_endpoint_port_protocol": "TCP",
+  "__meta_kubernetes_endpoint_ready": "true",
+  "__meta_kubernetes_endpoints_annotation_endpoints_kubernetes_io_last_change_trigger_time": "2024-11-05T11:24:09Z",
+  "__meta_kubernetes_endpoints_annotationpresent_endpoints_kubernetes_io_last_change_trigger_time": "true",
+  "__meta_kubernetes_endpoints_label_app": "prometheus-coo-example-app",
+  "__meta_kubernetes_endpoints_labelpresent_app": "true",
+  "__meta_kubernetes_endpoints_name": "prometheus-coo-example-app",
+  "__meta_kubernetes_namespace": "ns1-coo",
+  "__meta_kubernetes_pod_annotation_k8s_ovn_org_pod_networks": "{\"default\":{\"ip_addresses\":[\"10.129.2.25/23\"],\"mac_address\":\"0a:58:0a:81:02:19\",\"gateway_ips\":[\"10.129.2.1\"],\"routes\":[{\"dest\":\"10.128.0.0/14\",\"nextHop\":\"10.129.2.1\"},{\"dest\":\"172.30.0.0/16\",\"nextHop\":\"10.129.2.1\"},{\"dest\":\"100.64.0.0/16\",\"nextHop\":\"10.129.2.1\"}],\"ip_address\":\"10.129.2.25/23\",\"gateway_ip\":\"10.129.2.1\",\"role\":\"primary\"}}",
+  "__meta_kubernetes_pod_annotation_k8s_v1_cni_cncf_io_network_status": "[{\n    \"name\": \"ovn-kubernetes\",\n    \"interface\": \"eth0\",\n    \"ips\": [\n        \"10.129.2.25\"\n    ],\n    \"mac\": \"0a:58:0a:81:02:19\",\n    \"default\": true,\n    \"dns\": {}\n}]",
+  "__meta_kubernetes_pod_annotation_openshift_io_scc": "restricted-v2",
+  "__meta_kubernetes_pod_annotation_seccomp_security_alpha_kubernetes_io_pod": "runtime/default",
+  "__meta_kubernetes_pod_annotationpresent_k8s_ovn_org_pod_networks": "true",
+  "__meta_kubernetes_pod_annotationpresent_k8s_v1_cni_cncf_io_network_status": "true",
+  "__meta_kubernetes_pod_annotationpresent_openshift_io_scc": "true",
+  "__meta_kubernetes_pod_annotationpresent_seccomp_security_alpha_kubernetes_io_pod": "true",
+  "__meta_kubernetes_pod_controller_kind": "ReplicaSet",
+  "__meta_kubernetes_pod_controller_name": "prometheus-coo-example-app-5d8cd498c7",
+  "__meta_kubernetes_pod_host_ip": "10.0.128.2",
+  "__meta_kubernetes_pod_ip": "10.129.2.25",
+  "__meta_kubernetes_pod_label_app": "prometheus-coo-example-app",
+  "__meta_kubernetes_pod_label_pod_template_hash": "5d8cd498c7",
+  "__meta_kubernetes_pod_labelpresent_app": "true",
+  "__meta_kubernetes_pod_labelpresent_pod_template_hash": "true",
+  "__meta_kubernetes_pod_name": "prometheus-coo-example-app-5d8cd498c7-9j2gj",
+  "__meta_kubernetes_pod_node_name": "ci-ln-8tt8vxb-72292-6cxjr-worker-a-wdfnz",
+  "__meta_kubernetes_pod_phase": "Running",
+  "__meta_kubernetes_pod_ready": "true",
+  "__meta_kubernetes_pod_uid": "054c11b6-9a76-4827-a860-47f3a4596871",
+  "__meta_kubernetes_service_label_app": "prometheus-coo-example-app",
+  "__meta_kubernetes_service_labelpresent_app": "true",
+  "__meta_kubernetes_service_name": "prometheus-coo-example-app",
+  "__metrics_path__": "/metrics",
+  "__scheme__": "http",
+  "__scrape_interval__": "30s",
+  "__scrape_timeout__": "10s",
+  "job": "serviceMonitor/ns1-coo/prometheus-coo-example-monitor/0"
+}
+----
++
+[NOTE]
+====
+The above example uses link:https://jqlang.github.io/jq/[`jq` command-line JSON processor] to format the output for convenience.
+====

modules/monitoring-validating-a-monitoringstack-for-cluster-observability-operator.adoc

@@ -0,0 +1,81 @@
+// Module included in the following assemblies:
+//
+// * observability/cluster-observability-operator/configuring-the-cluster-observability-operator-to-monitor-a-service.adoc
+
+:_mod-docs-content-type: PROCEDURE
+[id="monitoring-validating-a-monitoringstack-for-cluster-observability-operator_{context}"]
+= Validating the monitoring stack
+
+To validate that the monitoring stack is working correctly, access the example service and then view the gathered metrics.
+
+.Prerequisites
+
+* You have access to the cluster as a user with the `cluster-admin` cluster role or as a user with administrative permissions for the namespace.
+* You have installed the {coo-full}.
+* You have deployed the `prometheus-coo-example-app` sample service in the `ns1-coo` namespace.
+* You have created a `ServiceMonitor` object named `prometheus-coo-example-monitor` in the `ns1-coo` namespace.
+* You have created a `MonitoringStack` object named `example-coo-monitoring-stack` in the `ns1-coo` namespace.
+
+.Procedure
+
+. Create a route to expose the example `prometheus-coo-example-app` service. From your terminal, run the command:
++
+[source,terminal]
+----
+$ oc expose svc prometheus-coo-example-app
+----
+. Access the route from your browser, or command line, to generate metrics.
+
+. Execute a query on the Prometheus pod to return the total HTTP requests metric:
++
+[source,terminal]
+----
+$ oc -n ns1-coo exec -c prometheus prometheus-example-coo-monitoring-stack-0 -- curl -s 'http://localhost:9090/api/v1/query?query=http_requests_total'
+----
++
+.Example output (formatted using `jq` for convenience)
+[source,json]
+----
+{
+  "status": "success",
+  "data": {
+    "resultType": "vector",
+    "result": [
+      {
+        "metric": {
+          "__name__": "http_requests_total",
+          "code": "200",
+          "endpoint": "web",
+          "instance": "10.129.2.25:8080",
+          "job": "prometheus-coo-example-app",
+          "method": "get",
+          "namespace": "ns1-coo",
+          "pod": "prometheus-coo-example-app-5d8cd498c7-9j2gj",
+          "service": "prometheus-coo-example-app"
+        },
+        "value": [
+          1730807483.632,
+          "3"
+        ]
+      },
+      {
+        "metric": {
+          "__name__": "http_requests_total",
+          "code": "404",
+          "endpoint": "web",
+          "instance": "10.129.2.25:8080",
+          "job": "prometheus-coo-example-app",
+          "method": "get",
+          "namespace": "ns1-coo",
+          "pod": "prometheus-coo-example-app-5d8cd498c7-9j2gj",
+          "service": "prometheus-coo-example-app"
+        },
+        "value": [
+          1730807483.632,
+          "0"
+        ]
+      }
+    ]
+  }
+}
+----

observability/cluster_observability_operator/configuring-the-cluster-observability-operator-to-monitor-a-service.adoc

@@ -25,3 +25,7 @@ include::modules/monitoring-specifying-how-a-service-is-monitored-by-cluster-obs
 
 // Create a MonitoringStack object to discover the service monitor
 include::modules/monitoring-creating-a-monitoringstack-object-for-cluster-observability-operator.adoc[leveloffset=+1]
+
+// Validate a MonitoringStack 
+include::modules/monitoring-validating-a-monitoringstack-for-cluster-observability-operator.adoc[leveloffset=+1]
+