From db0287aa94ffb7badea785ab29af9757fa987144 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferri-Benedetti Date: Thu, 20 Nov 2025 17:20:19 +0100 Subject: [PATCH] Add elasticapm connector docs (#11214) * Add elasticapm connector docs * Fix inaccurate bit * Tech edits * Tech edit part 2 * Update docs/reference/edot-collector/components/elasticapmconnector.md Co-authored-by: Aleksandra Spilkowska <96738481+alexandra5000@users.noreply.github.com> --------- Co-authored-by: Aleksandra Spilkowska <96738481+alexandra5000@users.noreply.github.com> (cherry picked from commit b81e3fe06df8e33aa50d10577a33338d15ce6482) --- .../edot-collector/component-docs.yml | 8 +- docs/reference/edot-collector/components.md | 2 +- .../components/elasticapmconnector.md | 146 ++++++++++++++++++ .../components/elasticapmprocessor.md | 4 +- .../config/default-config-standalone.md | 4 +- docs/reference/edot-collector/toc.yml | 1 + 6 files changed, 157 insertions(+), 8 deletions(-) create mode 100644 docs/reference/edot-collector/components/elasticapmconnector.md diff --git a/docs/reference/edot-collector/component-docs.yml b/docs/reference/edot-collector/component-docs.yml index b1345e7b455..bd009740a6a 100644 --- a/docs/reference/edot-collector/component-docs.yml +++ b/docs/reference/edot-collector/component-docs.yml @@ -34,7 +34,9 @@ components: doc_path: "/reference/edot-collector/components/elasticapmprocessor.md" title: "Elastic APM processor" + # Connectors + elasticapmconnector: + doc_path: "/reference/edot-collector/components/elasticapmconnector.md" + title: "Elastic APM connector" + # Add more components as documentation is created - # elasticapmconnector: - # doc_path: "/reference/edot-collector/components/elasticapmconnector" - # title: "Elastic APM connector" diff --git a/docs/reference/edot-collector/components.md b/docs/reference/edot-collector/components.md index d71ae3bf462..541d0407fce 100644 --- a/docs/reference/edot-collector/components.md +++ b/docs/reference/edot-collector/components.md @@ -83,7 +83,7 @@ These components are included in EDOT Collector version 9.2.0. | [tailsamplingprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/tailsamplingprocessor) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.137.0 | | [transformprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/transformprocessor) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Core] | v0.137.0 | |***Connectors***|||| -| [elasticapmconnector](https://github.com/elastic/opentelemetry-collector-components/tree/main/connector/elasticapmconnector) | [Elastic Repo](https://github.com/elastic/opentelemetry-collector-components) | [Core] | v0.8.0 | +| [elasticapmconnector](/reference/edot-collector/components/elasticapmconnector.md) | [Elastic Repo](https://github.com/elastic/opentelemetry-collector-components) | [Core] | v0.8.0 | | [forwardconnector](https://github.com/open-telemetry/opentelemetry-collector/tree/main/connector/forwardconnector) | [OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector) | [Extended] | v0.137.0 | | [profilingmetricsconnector](https://github.com/elastic/opentelemetry-collector-components/tree/main/connector/profilingmetricsconnector) | [Elastic Repo](https://github.com/elastic/opentelemetry-collector-components) | [Extended] | v0.1.0 | | [routingconnector](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/connector/routingconnector) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Core] | v0.137.0 | diff --git a/docs/reference/edot-collector/components/elasticapmconnector.md b/docs/reference/edot-collector/components/elasticapmconnector.md new file mode 100644 index 00000000000..3b7f0ee0f75 --- /dev/null +++ b/docs/reference/edot-collector/components/elasticapmconnector.md @@ -0,0 +1,146 @@ +--- +navigation_title: Elastic {{product.apm}} connector +description: The Elastic {{product.apm}} connector is an OpenTelemetry Collector component that generates pre-aggregated APM metrics from trace data. +applies_to: + stack: + serverless: + observability: + product: + edot_collector: +products: + - id: elastic-agent + - id: observability + - id: edot-collector +--- + +# Elastic {{product.apm}} connector + +The Elastic {{product.apm}} connector generates pre-aggregated metrics from OpenTelemetry trace data. These metrics enable key {{product.apm}} features like service maps, transaction histograms, and service-level indicators with fast query performance. Instead of calculating metrics on the fly from potentially millions of transactions, the {{product.apm}} UIs can query pre-computed metric documents for quick data visualization. + +The connector works together with the [Elastic {{product.apm}} processor](elasticapmprocessor.md), which enriches OpenTelemetry traces with Elastic-specific attributes to ensure optimal compatibility with Elastic {{product.apm}} UIs. + +## Default usage in EDOT + +The `elasticapm` connector is included by default in EDOT Collector deployments that ingest trace data directly into {{es}}. It's not needed when using the [{{motlp}}](opentelemetry://reference/motlp.md), as the metric aggregation happens server-side. + +### Standalone deployments + +In standalone deployments, the Elastic {{product.apm}} connector is used in both agent and gateway modes: + +* **Agent mode**: The connector is part of the default [application and traces collection pipeline](../config/default-config-standalone.md#application-and-traces-collection-pipeline). It receives trace data from the pipeline, generates metrics, and forwards them to {{es}}. + +* **Gateway mode**: The connector is part of the [Gateway mode pipeline](../config/default-config-standalone.md#gateway-mode), where it generates metrics from traces received from other collectors running in agent mode before ingesting them into {{es}}. + + +### Kubernetes deployment + +In Kubernetes, the Elastic {{product.apm}} connector runs in the [Gateway collectors pipeline](../config/default-config-k8s.md#gateway-collectors-pipeline) when using direct ingestion to {{es}}. The Gateway receives traces from DaemonSet collectors, generates {{product.apm}} metrics, and writes both metrics and traces to {{es}}. + +For more details about the Kubernetes configuration, refer to [Default configuration (Kubernetes)](../config/default-config-k8s.md). + +## Example configuration + +The Elastic {{product.apm}} connector typically requires minimal configuration. Usually, an empty configuration block is sufficient: + +```yaml +connectors: + elasticapm: {} +``` + +When combined with the `elasticapm` processor in a complete pipeline: + +```yaml +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +connectors: + elasticapm: {} + +processors: + batch: + send_batch_size: 1000 + timeout: 1s + elasticapm: {} + +exporters: + elasticsearch/otel: + endpoints: + - ${ELASTIC_ENDPOINT} + api_key: ${ELASTIC_API_KEY} + mapping: + mode: otel + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch, elasticapm] + exporters: [elasticapm, elasticsearch/otel] + + metrics/aggregated-otel-metrics: + receivers: [elasticapm] + processors: [] + exporters: [elasticsearch/otel] +``` + +In this configuration, the `elasticapm` connector appears as both an exporter in the traces pipeline to generate {{product.apm}} metrics and as a receiver in the metrics pipeline to forward those metrics to {{es}}. + +## Generated metrics + +The Elastic {{product.apm}} connector generates the following types of aggregated metrics from trace data: + +| Metric Type | Description | +|------------|-------------| +| Transaction metrics | Aggregated statistics about service transactions, including throughput, latency distributions, and success rates. These metrics power service overview pages and transaction group views. | +| Service destination metrics | Metrics that track dependencies between services, showing how services communicate with databases, message queues, and other external systems. These metrics are used to build service maps. | +| Span metrics | Detailed metrics about individual span operations, including database queries, HTTP calls, and other operations. These provide granular performance insights for specific operations. | +| Service summary metrics | High-level service health metrics including error rates, throughput, and overall latency. These metrics enable quick health checks and alerting. | + +All metrics are generated with appropriate aggregation periods and follow Elastic's metric schema for seamless integration with {{product.apm}} UIs. + +## Aggregation intervals + +The connector aggregates metrics over multiple time intervals to provide different granularities for analysis. By default, metrics are aggregated at three intervals: 1 minute, 10 minutes, and 60 minutes. + +## Best practices + +Follow these recommendations when using the Elastic {{product.apm}} connector: + +* **Always pair with the elasticapm processor**: The connector and processor work together to provide the full Elastic {{product.apm}} experience. The processor enriches traces while the connector generates {{product.apm}} metrics. Include both in your pipeline configuration for complete functionality. + +* **Place the connector as an exporter in the traces pipeline**: Configure the Elastic {{product.apm}} connector as an exporter in your traces pipeline, alongside your final data destination. This ensures the connector receives processed trace data and can generate accurate metrics. + +* **Create a separate metrics pipeline for connector output**: Set up a dedicated metrics pipeline with the connector as the receiver. This isolates metric handling and makes it easier to apply metric-specific processing if needed. + +* **Use only for direct {{es}} ingestion**: If you're using the {{motlp}}, you don't need the Elastic {{product.apm}} connector, because the endpoint handles metric aggregation automatically. Using both can cause conflicts or duplicate metrics. + +* **Keep the connector updated**: The Elastic {{product.apm}} connector evolves with new Elastic {{product.apm}} features. Keep your EDOT Collector version current to benefit from the latest enhancements and compatibility improvements. + +* **Monitor connector performance**: The connector aggregates metrics in memory before flushing. For high-throughput environments, monitor memory usage and adjust the aggregation interval or deployment resources as needed. + +## Limitations + +Be aware of these constraints and behaviors when using the Elastic {{product.apm}} connector: + +* **Required for Elastic {{product.apm}} metrics**: Without the Elastic {{product.apm}} connector, you'll only have raw trace data in {{es}}. Service maps, transaction histograms, and other metric-driven {{product.apm}} features might require the pre-aggregated metrics that this connector generates. + +* **Not available in contrib OTel Collector**: The Elastic {{product.apm}} connector is an Elastic-specific component not included in the standard OpenTelemetry Collector or Collector Contrib distributions. To use it, you must either use EDOT Collector or [build a custom collector](../custom-collector.md) that includes Elastic's components. + +* **Memory usage scales with cardinality**: The connector maintains in-memory aggregations for unique combinations of service names, transaction names, and other dimensions. High-cardinality data (many unique values) increases memory requirements. Monitor memory usage in high-cardinality environments. + +* **Minimal configuration options**: Unlike some connectors, the Elastic {{product.apm}} connector operates with mostly fixed behavior and offers few configuration parameters. While this simplifies setup, it also means you have limited ability to customize the aggregation logic. + + +## Resources + +* [Contrib component: elasticapmconnector](https://github.com/elastic/opentelemetry-collector-components/tree/main/connector/elasticapmconnector) +* [Elastic {{product.apm}} processor](elasticapmprocessor.md) +* [Default configuration (Standalone)](../config/default-config-standalone.md#application-and-traces-collection-pipeline) +* [Default configuration (Kubernetes)](../config/default-config-k8s.md) +* [Build a custom collector with Elastic components](../custom-collector.md) + diff --git a/docs/reference/edot-collector/components/elasticapmprocessor.md b/docs/reference/edot-collector/components/elasticapmprocessor.md index 676833965f4..1b5ef259db0 100644 --- a/docs/reference/edot-collector/components/elasticapmprocessor.md +++ b/docs/reference/edot-collector/components/elasticapmprocessor.md @@ -17,7 +17,7 @@ products: The Elastic {{product.apm}} processor enriches OpenTelemetry trace data with Elastic-specific attributes and metadata, ensuring optimal compatibility with Elastic {{product.apm}} UIs and functionality. It bridges the gap between OpenTelemetry's trace format and Elastic's expectations, enabling features like service maps, transaction groups, and enhanced trace visualization. -The processor works together with the [Elastic {{product.apm}} connector](https://github.com/elastic/opentelemetry-collector-components/tree/main/connector/elasticapmconnector), which generates pre-aggregated {{product.apm}} metrics from trace data. +The processor works together with the [Elastic {{product.apm}} connector](elasticapmconnector.md), which generates pre-aggregated {{product.apm}} metrics from trace data. ## Default usage in EDOT @@ -138,7 +138,7 @@ Be aware of these constraints and behaviors when using the Elastic APM processor ## Resources * [Contrib component: elasticapmprocessor](https://github.com/elastic/opentelemetry-collector-components/tree/main/processor/elasticapmprocessor) -* [Elastic {{product.apm}} connector](https://github.com/elastic/opentelemetry-collector-components/tree/main/connector/elasticapmconnector) +* [Elastic {{product.apm}} connector](elasticapmconnector.md) * [Default configuration (Standalone)](../config/default-config-standalone.md#application-and-traces-collection-pipeline) * [Default configuration (Kubernetes)](../config/default-config-k8s.md) * [Build a custom collector with Elastic components](../custom-collector.md) diff --git a/docs/reference/edot-collector/config/default-config-standalone.md b/docs/reference/edot-collector/config/default-config-standalone.md index e72e4340202..48f17160421 100644 --- a/docs/reference/edot-collector/config/default-config-standalone.md +++ b/docs/reference/edot-collector/config/default-config-standalone.md @@ -83,7 +83,7 @@ Data is exported directly to {{es}} using the [`elasticsearch`] exporter in `OTe The application pipeline in the EDOT Collector receives data from OTel SDKs through the [`OTLP`] receiver. While logs and metrics are exported verbatim into {{es}}, traces require two additional components. -{applies_to}`edot_collector: ga 9.2` The [`elasticapm` processor](../components/elasticapmprocessor.md) enriches trace data with additional attributes that improve the user experience in the {{product.observability}} UIs. In addition, the [`elasticapm` connector](https://github.com/elastic/opentelemetry-collector-components/tree/main/connector/elasticapmconnector) generates pre-aggregated APM metrics from tracing data. +{applies_to}`edot_collector: ga 9.2` The [`elasticapm` processor](../components/elasticapmprocessor.md) enriches trace data with additional attributes that improve the user experience in the {{product.observability}} UIs. In addition, the [`elasticapm` connector](../components/elasticapmconnector.md) generates pre-aggregated APM metrics from tracing data. Application-related OTel data is ingested into {{es}} in OTel-native format using the [`elasticsearch`] exporter. @@ -372,7 +372,7 @@ The server expects incoming HTTP requests to include an API key with sufficient [`elasticinframetrics`]: https://github.com/elastic/opentelemetry-collector-components/tree/main/processor/elasticinframetricsprocessor [`elastictrace`]: https://github.com/elastic/opentelemetry-collector-components/tree/main/processor/elastictraceprocessor [`elasticapm` processor]: https://github.com/elastic/opentelemetry-collector-components/tree/main/processor/elasticapmprocessor -[`elasticapm` connector]: https://github.com/elastic/opentelemetry-collector-components/tree/main/connector/elasticapmconnector +[`elasticapm` connector]: ../components/elasticapmconnector.md [`resource`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourceprocessor [`resourcedetection`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor [`OTLP`]: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver/otlpreceiver diff --git a/docs/reference/edot-collector/toc.yml b/docs/reference/edot-collector/toc.yml index f8cd10618a0..afc9e0ffa79 100644 --- a/docs/reference/edot-collector/toc.yml +++ b/docs/reference/edot-collector/toc.yml @@ -18,6 +18,7 @@ toc: - file: components/elasticapmintakereceiver.md - file: components/k8sclusterreceiver.md - file: components/elasticapmprocessor.md + - file: components/elasticapmconnector.md - file: components/filelogreceiver.md - file: components/k8sobjectsreceiver.md - file: components/hostmetricsreceiver.md