From 0ad291f2f638287d916ef71e95c1c52a3bde67c7 Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Fri, 15 Sep 2023 14:04:41 +0200 Subject: [PATCH] Document troubleshooting OTEL Signed-off-by: Pavol Loffay --- _topic_maps/_topic_map.yml | 2 + .../distr-tracing-otel-troubleshooting.adoc | 13 ++++++ ...ng-otel-troubleshoot-logging-exporter.adoc | 30 ++++++++++++++ .../distr-tracing-otel-troubleshoot-logs.adoc | 24 +++++++++++ ...str-tracing-otel-troubleshoot-metrics.adoc | 40 +++++++++++++++++++ 5 files changed, 109 insertions(+) create mode 100644 distr_tracing/distr_tracing_otel/distr-tracing-otel-troubleshooting.adoc create mode 100644 modules/distr-tracing-otel-troubleshoot-logging-exporter.adoc create mode 100644 modules/distr-tracing-otel-troubleshoot-logs.adoc create mode 100644 modules/distr-tracing-otel-troubleshoot-metrics.adoc diff --git a/_topic_maps/_topic_map.yml b/_topic_maps/_topic_map.yml index 8c27d5ebcc0f..ea4029138679 100644 --- a/_topic_maps/_topic_map.yml +++ b/_topic_maps/_topic_map.yml @@ -3537,6 +3537,8 @@ Topics: File: distr-tracing-otel-configuring - Name: Use File: distr-tracing-otel-using + - Name: Troubleshooting + File: distr-tracing-otel-troubleshooting - Name: Migration File: distr-tracing-otel-migrating - Name: Removal diff --git a/distr_tracing/distr_tracing_otel/distr-tracing-otel-troubleshooting.adoc b/distr_tracing/distr_tracing_otel/distr-tracing-otel-troubleshooting.adoc new file mode 100644 index 000000000000..f5e70801179b --- /dev/null +++ b/distr_tracing/distr_tracing_otel/distr-tracing-otel-troubleshooting.adoc @@ -0,0 +1,13 @@ +:_content-type: ASSEMBLY +[id="distr-tracing-otel-troubleshoot"] += Troubleshooting the {OTELShortName} +include::_attributes/common-attributes.adoc[] +:context: distr-tracing-otel-troubleshoot + +toc::[] + +The OpenTelemetry Collector offers multiple ways to measure its health as well as investigate data ingestion issues. + +include::modules/distr-tracing-otel-troubleshoot-logs.adoc[leveloffset=+1] +include::modules/distr-tracing-otel-troubleshoot-metrics.adoc[leveloffset=+1] +include::modules/distr-tracing-otel-troubleshoot-logging-exporter.adoc[leveloffset=+1] diff --git a/modules/distr-tracing-otel-troubleshoot-logging-exporter.adoc b/modules/distr-tracing-otel-troubleshoot-logging-exporter.adoc new file mode 100644 index 000000000000..81842709e995 --- /dev/null +++ b/modules/distr-tracing-otel-troubleshoot-logging-exporter.adoc @@ -0,0 +1,30 @@ +// Module included in the following assemblies: +// +// * /distr_tracing/distr_tracing_otel/distr-tracing-otel-troubleshooting.adoc +:_content-type: PROCEDURE +[id="distr-tracing-otel-troubleshoot-logging-exporter_{context}"] += Logging exporter + +You can configure the logging exporter to export the collected data to the standard output. + +.Procedure + +. Configure the OpenTelemetry Collector custom resource as follows: ++ +[source,yaml] +---- + config: | + exporters: + logging: + verbosity: detailed + service: + pipelines: + traces: + exporters: [logging] + metrics: + exporters: [logging] + logs: + exporters: [logging] +---- + +. Use the `oc logs` command or the OpenShift console to export the logs to the standard output. diff --git a/modules/distr-tracing-otel-troubleshoot-logs.adoc b/modules/distr-tracing-otel-troubleshoot-logs.adoc new file mode 100644 index 000000000000..2ad1fab9ca33 --- /dev/null +++ b/modules/distr-tracing-otel-troubleshoot-logs.adoc @@ -0,0 +1,24 @@ +// Module included in the following assemblies: +// +// * /distr_tracing/distr_tracing_otel/distr-tracing-otel-troubleshooting.adoc +:_content-type: PROCEDURE +[id="distr-tracing-otel-troubleshoot-logs_{context}"] += Getting the OpenTelemetry Collector logs + +You can get the logs for the OpenTelemetry Collector as follows. + +.Procedure + +. Set the relevant log level in the OpenTelemetry Collector custom resource (CR): ++ +[source,yaml] +---- + config: | + service: + telemetry: + logs: + level: debug <1> +---- +<1> Collector's log level. Select one of the following values: `info`, `warn`, `error`, or `debug`. Defaults to `info`. + +. Use the `oc logs` command or the OpenShift console to retrieve the logs. diff --git a/modules/distr-tracing-otel-troubleshoot-metrics.adoc b/modules/distr-tracing-otel-troubleshoot-metrics.adoc new file mode 100644 index 000000000000..880b62c8e9de --- /dev/null +++ b/modules/distr-tracing-otel-troubleshoot-metrics.adoc @@ -0,0 +1,40 @@ +// Module included in the following assemblies: +// +// * /distr_tracing/distr_tracing_otel/distr-tracing-otel-troubleshooting.adoc +:_content-type: PROCEDURE +[id="distr-tracing-otel-troubleshoot-metrics_{context}"] += Exposing the metrics + +The OpenTelemetry Collector exposes the metrics about the data volumes it has processed. The following metrics are for spans, although similar metrics are exposed for metrics and logs signals: + +`otelcol_receiver_accepted_spans`:: The number of spans successfully pushed into the pipeline. +`otelcol_receiver_refused_spans`:: The number of spans that could not be pushed into the pipeline. +`otelcol_exporter_sent_spans`:: The number of spans successfully sent to destination. +`otelcol_exporter_enqueue_failed_spans`:: The number of spans failed to be added to the sending queue. + +The operator creates a `-collector-monitoring` telemetry service that you can use to scrape the metrics endpoint. + +.Procedure + +. Enable the telemetry service by adding the following lines in the OpenTelemetry Collector custom resource: + ++ +[source,yaml] +---- + config: | + service: + telemetry: + metrics: + address: ":8888" <1> +---- +<1> The address on which internal collector metrics are exposed. Defaults to `:8888`. +// TODO Operator 0.82.0 has spec.observability.metrics.enableMetrics config that creates ServiceMonitors for users + +. Retrieve the metrics by running the following command, which uses the port forwarding collector pod: ++ +[source,terminal] +---- +$ oc port-forward +---- + +. Access the metrics endpoint at `+http://localhost:8888/metrics+`.