From 7dd2675ff950cf72017e561467f27cad765adf5b Mon Sep 17 00:00:00 2001 From: Alekh Nema Date: Mon, 15 Jul 2024 21:48:11 +0530 Subject: [PATCH 1/3] Changes for cassandra opentelemetry doc after with metric collection and dashbaords --- .../opentelemetry/cassandra-opentelemetry.md | 78 ++++++++++++++++++- 1 file changed, 74 insertions(+), 4 deletions(-) diff --git a/docs/integrations/databases/opentelemetry/cassandra-opentelemetry.md b/docs/integrations/databases/opentelemetry/cassandra-opentelemetry.md index bad7a0d63e..081000fcfb 100644 --- a/docs/integrations/databases/opentelemetry/cassandra-opentelemetry.md +++ b/docs/integrations/databases/opentelemetry/cassandra-opentelemetry.md @@ -11,9 +11,9 @@ import TabItem from '@theme/TabItem'; Thumbnail icon Thumbnail icon -The [Cassandra](https://cassandra.apache.org/_/cassandra-basics.html) - OpenTelemetry app is a log based app that helps you monitor the availability, performance, health, and resource utilization of your Cassandra clusters. Preconfigured dashboards provide insight into resource utilization, cache/Gossip/Memtable statistics and Error and warnings. Cassandra logs are sent to Sumo Logic through OpenTelemetry [filelog receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/filelogreceiver). +The [Cassandra](https://cassandra.apache.org/_/cassandra-basics.html) - OpenTelemetry app is a log and metric based app that helps you monitor the availability, performance, health, and resource utilization of your Cassandra clusters. Preconfigured dashboards provide insight into resource utilization, cache/Gossip/Memtable statistics and Error and warnings along with the request served and latency, storage and compaction related metrics. Cassandra logs are sent to Sumo Logic through OpenTelemetry [filelog receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/filelogreceiver) and cassandra metrics are sent to Sumo Logic using [JMX opentelemetry receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/jmxreceiver) with the `target_system` set as [`cassandra`](https://github.com/open-telemetry/opentelemetry-java-contrib/blob/main/jmx-metrics/docs/target-systems/cassandra.md) -The app supports Logs from the open-source version of Cassandra. The App is tested on the 3.11.10 version of Cassandra. +The app supports Logs from the open-source version of Cassandra. The App is tested on the 4.0.0 version of Cassandra. Schematics @@ -21,13 +21,27 @@ The app supports Logs from the open-source version of Cassandra. The App is test Following are the [Fields](/docs/manage/fields/) which will be created as part of Cassandra App install if not already present: +- `deployment.environment`. User configured. Through this Cassandra cluster is identified by the environment where it resides. For example: dev, prod or qa. - `db.cluster.name`. User configured. Enter a name to identify this Cassandra cluster. This cluster name will be shown in the Sumo Logic dashboards. - `db.system`. Has fixed value of **cassandra**. -- `deployment.environment`. User configured. Through this Cassandra cluster is identified by the environment where it resides. For example: dev, prod or qa. +- `db.node.name`. Holds the value of the Fully Qualified Domain Name (FQDN) of the machine from which the OpenTelemetry collector is collecting logs and metrics. - `sumo.datasource`. Has fixed value of **cassandra**. ## Prerequisites +#### For metric collection + +JMX receiver collects Cassandra metrics from Cassandra server as part of the OpenTelemetry Collector (OTC). + + 1. Follow the instructions in [JMX - OpenTelemetry's prerequisites section](/docs/integrations/app-development/opentelemetry/jmx-opentelemetry/#prerequisites) to download the [JMX Metric Gatherer](https://github.com/open-telemetry/opentelemetry-java-contrib/blob/main/jmx-metrics/README.md). This gatherer is used by the [JMX Receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/jmxreceiver#details). + + 2. Set the JMX port by setting it as part of `JAVA_OPTS` for Tomcat startup. Usually it is set in the `/etc/systemd/system/cassandra.service` or `C:\Program Files\apache-tomcat\bin\tomcat.bat` file. + + ```json + JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=11099 -Dcom.sun.management.jmxremote.authenticate=true -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.password.file=${CASSANDRA_CONF_DIR}/jmx.password -Dcom.sun.management.jmxremote.access.file=${CASSANDRA_CONF_DIR}/jmx.access" + ``` + +#### For log collection Cassandra has three main logs: system.log, debug.log, and gc.log which hold general logging messages, debugging logging messages, and java garbage collection logs respectively. These logs by default live in `${CASSANDRA_HOME}/logs`, but most Linux distributions relocate logs to `/var/log/cassandra`. Operators can tune this location as well as what levels are logged using the provided logback.xml file. For more details on Cassandra logs, see[ this](https://cassandra.apache.org/doc/latest/troubleshooting/reading_logs.html) link. @@ -56,6 +70,7 @@ In this step, you will configure the yaml required for Cassandra Collection. Pat Below are the inputs required: - The path to system.log is required here. This file is typically located in `/var/log/cassandra`. If you're using a customized path, check the respective conf file for this information. +- For metrics, you're required to provide the JMX endpoint (default is `localhost:7199`) along with the `collection_interval` (default is 1 minute). You can add any custom fields which you want to be tagged with the data ingested in Sumo. Click on the **Download YAML File** button to get the yaml file. @@ -122,7 +137,41 @@ import LogsOutro from '../../../reuse/apps/opentelemetry/send-logs-outro.md'; INFO [ScheduledTasks:1] 2023-01-08 09:18:47,347 StatusLogger.java:101 - system.schema_aggregates ``` -## Sample queries  +## Sample metric message +{ + "queryId": "A", + "_source": "cassandra-otel", + "_metricId": "dasIFS6RvCVxEcsljWGpnQ", + "_sourceName": "Http Input", + "host.id": "i-071ade75d03650657", + "host.group": "cassandraHostGrp", + "telemetry.sdk.version": "1.38.0", + "os.type": "linux", + "sumo.datasource": "cassandra", + "db.system": "cassandra", + "db.node.name": "ip-54-163-607-30.ec2.internal", + "_sourceCategory": "Labs/cassandra-otel", + "deployment.environment": "dev", + "_contentType": "Carbon2", + "host.name": "ip-54-163-607-30.ec2.internal", + "metric": "cassandra.client.request.write.latency.50p", + "_collectorId": "000000000FB3B451", + "_sourceId": "000000005A60F624", + "unit": "us", + "db.cluster.name": "cassandra_cluster", + "telemetry.sdk.language": "java", + "telemetry.sdk.name": "opentelemetry", + "service.name": "unknown_service:java", + "_collector": "Labs - cassandra-otel", + "max": 229, + "min": 0, + "avg": 131.25, + "sum": 525, + "latest": 67, + "count": 4 +} + +## Sample Log queries  Following is the query from Cassandra App's overview Dashboard's Nodes Up Panel: @@ -136,6 +185,13 @@ Following is the query from Cassandra App's overview Dashboard's Nodes Up Panel: | sort by _timeslice asc ``` +## Sample Metrics queries  +Following is the query from Cassandra App's overview Dashboard's Number of Requests Panel: + +```sql +sumo.datasource=cassandra deployment.environment=* db.cluster.name=* db.node.name=* metric=cassandra.client.request.count | sum +``` + ## Viewing Cassandra Dashboards ### Overview @@ -202,3 +258,17 @@ Use this dashboard to: - Identify resource utilization. This can help you to determine whether resources are over-allocated or under-allocated. Resource Usage + +### Compaction + +The Cassandra - Compactions dashboard provides a insight into the completed and pending compaction tasks. +Compaction + +### Requests +The Cassandra - Requests dashboard gives insight into the number of request served, number of error request and their distribution by status and operation. Also you can monitor the read and write latency of the cluster instance using this dashboard. +Requests + +### Storage +The Cassandra - Storage dashboard provides insight into the current value of total hints of your cassandra cluster along with storage managed by the cluster. + +Storage \ No newline at end of file From 482753505b0e9288fbffb5f4bb36d92b656df4de Mon Sep 17 00:00:00 2001 From: John Pipkin Date: Wed, 17 Jul 2024 17:21:03 -0500 Subject: [PATCH 2/3] cassandraOtelMetrics --- .../databases/opentelemetry/cassandra-opentelemetry.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/integrations/databases/opentelemetry/cassandra-opentelemetry.md b/docs/integrations/databases/opentelemetry/cassandra-opentelemetry.md index 081000fcfb..c822d3cf8b 100644 --- a/docs/integrations/databases/opentelemetry/cassandra-opentelemetry.md +++ b/docs/integrations/databases/opentelemetry/cassandra-opentelemetry.md @@ -138,6 +138,8 @@ import LogsOutro from '../../../reuse/apps/opentelemetry/send-logs-outro.md'; ``` ## Sample metric message + +```json { "queryId": "A", "_source": "cassandra-otel", @@ -170,6 +172,7 @@ import LogsOutro from '../../../reuse/apps/opentelemetry/send-logs-outro.md'; "latest": 67, "count": 4 } +``` ## Sample Log queries  From 94a84089653c0d390c8a1a611d48302caaedfba2 Mon Sep 17 00:00:00 2001 From: Alekh Nema <91047769+sumoanema@users.noreply.github.com> Date: Wed, 24 Jul 2024 10:32:20 +0530 Subject: [PATCH 3/3] Apply suggestions from code review Co-authored-by: Jagadisha V <129049263+JV0812@users.noreply.github.com> --- .../opentelemetry/cassandra-opentelemetry.md | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/integrations/databases/opentelemetry/cassandra-opentelemetry.md b/docs/integrations/databases/opentelemetry/cassandra-opentelemetry.md index c822d3cf8b..2ed8d204b9 100644 --- a/docs/integrations/databases/opentelemetry/cassandra-opentelemetry.md +++ b/docs/integrations/databases/opentelemetry/cassandra-opentelemetry.md @@ -11,7 +11,9 @@ import TabItem from '@theme/TabItem'; Thumbnail icon Thumbnail icon -The [Cassandra](https://cassandra.apache.org/_/cassandra-basics.html) - OpenTelemetry app is a log and metric based app that helps you monitor the availability, performance, health, and resource utilization of your Cassandra clusters. Preconfigured dashboards provide insight into resource utilization, cache/Gossip/Memtable statistics and Error and warnings along with the request served and latency, storage and compaction related metrics. Cassandra logs are sent to Sumo Logic through OpenTelemetry [filelog receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/filelogreceiver) and cassandra metrics are sent to Sumo Logic using [JMX opentelemetry receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/jmxreceiver) with the `target_system` set as [`cassandra`](https://github.com/open-telemetry/opentelemetry-java-contrib/blob/main/jmx-metrics/docs/target-systems/cassandra.md) +The [Cassandra](https://cassandra.apache.org/_/cassandra-basics.html) app is a log and metrics-based app that helps you monitor the availability, performance, health, and resource utilization of your Cassandra clusters. The pre-configured dashboards provide insight into resource utilization, cache/Gossip/Memtable statistics, error and warnings, request served and latency, storage, and compaction. + +Cassandra logs are sent to Sumo Logic through OpenTelemetry [filelog receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/filelogreceiver) and cassandra metrics are sent to Sumo Logic using [JMX opentelemetry receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/jmxreceiver) with the `target_system` set as [`cassandra`](https://github.com/open-telemetry/opentelemetry-java-contrib/blob/main/jmx-metrics/docs/target-systems/cassandra.md). The app supports Logs from the open-source version of Cassandra. The App is tested on the 4.0.0 version of Cassandra. @@ -21,7 +23,7 @@ The app supports Logs from the open-source version of Cassandra. The App is test Following are the [Fields](/docs/manage/fields/) which will be created as part of Cassandra App install if not already present: -- `deployment.environment`. User configured. Through this Cassandra cluster is identified by the environment where it resides. For example: dev, prod or qa. +- `deployment.environment`. User configured. Through this Cassandra cluster is identified by the environment where it resides. For example: dev, prod, or qa. - `db.cluster.name`. User configured. Enter a name to identify this Cassandra cluster. This cluster name will be shown in the Sumo Logic dashboards. - `db.system`. Has fixed value of **cassandra**. - `db.node.name`. Holds the value of the Fully Qualified Domain Name (FQDN) of the machine from which the OpenTelemetry collector is collecting logs and metrics. @@ -35,7 +37,7 @@ JMX receiver collects Cassandra metrics from Cassandra server as part of the Ope 1. Follow the instructions in [JMX - OpenTelemetry's prerequisites section](/docs/integrations/app-development/opentelemetry/jmx-opentelemetry/#prerequisites) to download the [JMX Metric Gatherer](https://github.com/open-telemetry/opentelemetry-java-contrib/blob/main/jmx-metrics/README.md). This gatherer is used by the [JMX Receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/jmxreceiver#details). - 2. Set the JMX port by setting it as part of `JAVA_OPTS` for Tomcat startup. Usually it is set in the `/etc/systemd/system/cassandra.service` or `C:\Program Files\apache-tomcat\bin\tomcat.bat` file. + 2. Set the JMX port as part of `JAVA_OPTS` for Tomcat startup. Usually, it is set in the `/etc/systemd/system/cassandra.service` or `C:\Program Files\apache-tomcat\bin\tomcat.bat` file. ```json JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=11099 -Dcom.sun.management.jmxremote.authenticate=true -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.password.file=${CASSANDRA_CONF_DIR}/jmx.password -Dcom.sun.management.jmxremote.access.file=${CASSANDRA_CONF_DIR}/jmx.access" @@ -137,7 +139,7 @@ import LogsOutro from '../../../reuse/apps/opentelemetry/send-logs-outro.md'; INFO [ScheduledTasks:1] 2023-01-08 09:18:47,347 StatusLogger.java:101 - system.schema_aggregates ``` -## Sample metric message +## Sample metrics message ```json { @@ -174,7 +176,7 @@ import LogsOutro from '../../../reuse/apps/opentelemetry/send-logs-outro.md'; } ``` -## Sample Log queries  +## Sample log queries  Following is the query from Cassandra App's overview Dashboard's Nodes Up Panel: @@ -188,14 +190,14 @@ Following is the query from Cassandra App's overview Dashboard's Nodes Up Panel: | sort by _timeslice asc ``` -## Sample Metrics queries  +## Sample metrics query Following is the query from Cassandra App's overview Dashboard's Number of Requests Panel: ```sql sumo.datasource=cassandra deployment.environment=* db.cluster.name=* db.node.name=* metric=cassandra.client.request.count | sum ``` -## Viewing Cassandra Dashboards +## Viewing Cassandra dashboards ### Overview @@ -264,14 +266,16 @@ Use this dashboard to: ### Compaction -The Cassandra - Compactions dashboard provides a insight into the completed and pending compaction tasks. +The **Cassandra - Compactions** dashboard provides insight into the completed and pending compaction tasks. Compaction ### Requests -The Cassandra - Requests dashboard gives insight into the number of request served, number of error request and their distribution by status and operation. Also you can monitor the read and write latency of the cluster instance using this dashboard. + +The **Cassandra - Requests** dashboard provides insight into the number of request served, number of error request, and their distribution by status and operation. Also you can monitor the read and write latency of the cluster instance using this dashboard. Requests ### Storage -The Cassandra - Storage dashboard provides insight into the current value of total hints of your cassandra cluster along with storage managed by the cluster. + +The **Cassandra - Storage** dashboard provides insight into the current value of total hints of your Cassandra cluster along with storage managed by the cluster. Storage \ No newline at end of file