From e001e9e742b5f6c649b619f8a10901f3134061aa Mon Sep 17 00:00:00 2001 From: Raphael Philipe Mendes da Silva Date: Fri, 7 Apr 2023 12:31:38 -0700 Subject: [PATCH] Chore: Add documentation for confmap providers and Kafka receiver and exporter (#521) * Chore: Add documentation for confmap providers Signed-off-by: Raphael Silva * Add documentation for Kakfa receiver/exporter Signed-off-by: Raphael Silva * Fix typos and improve documentation Signed-off-by: Raphael Silva * Add example of usage for the kafka components * Fix s3 urls --------- Signed-off-by: Raphael Silva Co-authored-by: bryan-aguilar <46550959+bryan-aguilar@users.noreply.github.com> --- src/config/sideBarData.js | 16 +- src/docs/components/confmap-providers.mdx | 315 ++++++++++++++++++ .../components/kafka-receiver-exporter.mdx | 148 ++++++++ 3 files changed, 472 insertions(+), 7 deletions(-) create mode 100644 src/docs/components/confmap-providers.mdx create mode 100644 src/docs/components/kafka-receiver-exporter.mdx diff --git a/src/config/sideBarData.js b/src/config/sideBarData.js index d61036051..659164625 100644 --- a/src/config/sideBarData.js +++ b/src/config/sideBarData.js @@ -98,18 +98,20 @@ export const sideBarData = [ { node: { id: nextId(), - label: "Components", + label: "ADOT Collector Components", items: [ + {label: "CloudWatch EMF Exporter", link: "/docs/getting-started/cloudwatch-metrics#cloudwatch-emf-exporter-awsemf"}, + {label: "Confmap Providers", link: "/docs/components/confmap-providers"}, + {label: "ECS Container Metrics Receiver", link: "/docs/components/ecs-metrics-receiver"}, + {label: "File Exporter", link: "/docs/components/misc-exporters#file-exporter"}, + {label: "Jaeger Receiver", link: "/docs/components/jaeger-zipkin-receiver#jaeger-receiver"}, + {label: "Kafka Receiver/Expoter", link: "/docs/components/kafka-receiver-exporter"}, + {label: "Logging Exporter", link: "/docs/components/misc-exporters#logging-exporter"}, {label: "OTLP Exporters", link: "/docs/components/otlp-exporter"}, {label: "Prometheus Exporters", link: "/docs/components/prometheus-exporters"}, - {label: "CloudWatch EMF Exporter", link: "/docs/getting-started/cloudwatch-metrics#cloudwatch-emf-exporter-awsemf"}, + {label: "StatsD Receiver", link: "/docs/components/statsd-receiver"}, {label: "X-Ray Exporter", link: "/docs/getting-started/x-ray#configuring-the-aws-x-ray-exporter"}, - {label: "Logging Exporter", link: "/docs/components/misc-exporters#logging-exporter"}, - {label: "File Exporter", link: "/docs/components/misc-exporters#file-exporter"}, - {label: "ECS Container Metrics Receiver", link: "/docs/components/ecs-metrics-receiver"}, {label: "X-Ray Receiver", link: "/docs/components/x-ray-receiver"}, - {label: "StatsD Receiver", link: "/docs/components/statsd-receiver"}, - {label: "Jaeger Receiver", link: "/docs/components/jaeger-zipkin-receiver#jaeger-receiver"}, {label: "Zipkin Receiver", link: "/docs/components/jaeger-zipkin-receiver#zipkin-receiver"}, ], link: "/docs/null" diff --git a/src/docs/components/confmap-providers.mdx b/src/docs/components/confmap-providers.mdx new file mode 100644 index 000000000..6b4c504a5 --- /dev/null +++ b/src/docs/components/confmap-providers.mdx @@ -0,0 +1,315 @@ +--- +title: 'Confmap providers - Get configuration from different sources' +description: | + Confmap represents the raw configuration for the OpenTelemetry Collector. A Confmap provider + is a Collector component that provides configuration retrieved from a URI to configure the collector. + This section will describe all the confmap providers supported by the ADOT collector. +path: '/docs/components/confmap-providers' +--- + +import SectionSeparator from "components/MdxSectionSeparator/sectionSeparator.jsx" + +The confmap providers are a type of OpenTelemetry collector component that is responsible by fetching configuration +from a URI. This configuration is be used to configure the Collector and all the other components. The URI for the confmap +providers is passed through the `-c/--config` command line parameter of the collector. The URI has the following format: +`:`. + +More technical details about this component can be found [here](https://github.com/open-telemetry/opentelemetry-collector/blob/main/confmap/README.md). + + + +## Confmap providers supported by the ADOT collector + +The ADOT collector support the following types of confmap providers: file, env, yaml, http, https and s3. + + +### File provider + +* scheme: `file`. Scheme is optional in the file provider and can be omitted when passing this as parameter. +* description: Retrieves configuration from the local file system +* examples: + * `/path/to/configuration.yaml` + * `file:/path/to/configuration.yaml` + * `c:\path\to\configuration.yaml` + * `file:c:\path\to\configuration.yaml` + +### Env provider + +* scheme: `env`. Scheme is optional. +* description: Retrieves configuration from an environment variable. +* examples: + * `env:ENVIRONMENT_FILE_NAME` + +### YAML provider + +* scheme: `yaml` +* description: Retrieves configuration directly from the command line, in yaml format. +* examples: + * `yaml:processors::batch::timeout: 2s` + +### HTTP provider + +* scheme: `http` +* description: Retrieves configuration from an http server. +* examples: + * `http://server/path/to/config.yaml` + +### HTTPS provider + +* scheme: `https` +* description: Retrieves configuration from an https server. It uses the certificate pool in the operating system to validate the hostname. +* examples: + * `https://server/path/to/config.yaml` + +### S3 provider + +* scheme: `s3` +* description: Retrieves configuration from AWS s3. This component uses the [default credentials chain](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials) to authenticate in s3 in order to fetch the configuration. +* examples: + * `https://.s3..amazonaws.com/path/to/config.yaml` + + + +## Passing command line parameters to the collector + +When installing the ADOT collector in EC2 from a RPM or MSI, you can pass the configuration parameter to a control +script that is responsible by managing the Operating system service that run the ADOT collector. Control scripts support +the following confmap providers: file, http, https and s3. + +### Linux + + +On Linux, to use any of the supported confmap providers, you can use the `-c` parameter of the the `aws-otel-collector-ctl` control script. + +``` +sudo /opt/aws/aws-otel-collector/bin/aws-otel-collector-ctl -c -a start" +``` + +Here are some examples: + +```bash +# File provider +sudo /opt/aws/aws-otel-collector/bin/aws-otel-collector-ctl -c /path/to/file.yaml -a start +# HTTP provider +sudo /opt/aws/aws-otel-collector/bin/aws-otel-collector-ctl -c "http://server/configuration.yaml" -a start +# HTTPS provider +sudo /opt/aws/aws-otel-collector/bin/aws-otel-collector-ctl -c "https://server/configuration.yaml" -a start +# S3 provider +sudo /opt/aws/aws-otel-collector/bin/aws-otel-collector-ctl -c "s3://bucket-example.s3.us-west-2.amazonaws.com/object.yaml" -a start +``` + +### Windows + +On Windows you can use the `-ConfigLocation` command line parameter of the `aws-otel-collector-ctl.ps1` control script to setup the ADOT collector service. + +Assuming you are in a Powershell session: +``` +. C:\\Program Files\\Amazon\\AwsOtelCollector\\aws-otel-collector-ctl.ps1' -ConfigLocation "" -Action start +``` + +Here are some examples: + +``` +# File provider +. 'C:\Program Files\Amazon\AwsOtelCollector\aws-otel-collector-ctl.ps1' -ConfigLocation 'C:\path\to\file.yaml' -a start +# HTTP provider +. 'C:\Program Files\Amazon\AwsOtelCollector\aws-otel-collector-ctl.ps1' -ConfigLocation 'http://server/configuration.yaml' -a start +# HTTPS provider +. 'C:\Program Files\Amazon\AwsOtelCollector\aws-otel-collector-ctl.ps1' -ConfigLocation 'https://server/configuration.yaml' -a start +# S3 provider +. 'C:\Program Files\Amazon\AwsOtelCollector\aws-otel-collector-ctl.ps1' -ConfigLocation 's3://bucket-example.s3.us-west-2.amazonaws.com/object.yaml' -a start +``` + +### Container environments + +In container environments, you can override the Docker [CMD instruction](https://docs.docker.com/engine/reference/builder/#cmd) to use the configuration from the URI that you want. This can be done +because all the parameters passed in the CMD instruction are passed to the ADOT collector since the [entrypoint for the ADOT collector image](https://github.com/aws-observability/aws-otel-collector/blob/main/cmd/awscollector/Dockerfile#L79) is the +Collector executable itself. + + +In ECS you can use the `command` property of the [environment container definition](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#container_definition_environment) to specify the +parameters that will be passed to the collector. + +Example: + +```json + { + "name": "aoc-collector", + "image": "public.ecr.aws/aws-observability/aws-otel-collector:latest", + "command": ["--config", ""], + "environment": [], + "environmentFiles": [], + "dependsOn": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/ecs-cwagent-sidecar-collector", + "awslogs-region": "us-west-2", + "awslogs-stream-prefix": "ecs", + "awslogs-create-group": "True" + } + } + } +``` + +Similarly, In Kubernetes you can use the `args` property of container definition to specify the command line parameters +that should be passed to the ADOT collector. + +Example: +```yaml +# create namespace +apiVersion: v1 +kind: Namespace +metadata: + name: aws-otel-eks + labels: + name: aws-otel-eks +--- +# create deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: aws-otel-eks-sidecar + namespace: aws-otel-eks + labels: + name: aws-otel-eks-sidecar +spec: + replicas: 1 + selector: + matchLabels: + name: aws-otel-eks-sidecar + template: + metadata: + labels: + name: aws-otel-eks-sidecar + spec: + containers: + - name: aws-otel-collector + image: "public.ecr.aws/aws-observability/aws-otel-collector:latest" + env: + - name: AWS_REGION + value: "us-west-2" + imagePullPolicy: Always + args: ["--config", ""] + resources: + limits: + cpu: 256m + memory: 512Mi + requests: + cpu: 32m + memory: 24Mi +``` + +## Embedding URIs in the configuration + +It is possible to embed configuration URIs in a configuration of the OpenTelemetry collector. These URIs will be expanded and replaced by the +content of the URI that they point to. To use this feature, you need to add the placeholders in the collector configuration with the +the following format: `${uri}`. You can provided URIs with any of the supported schemes. + + +The following pieces will demonstrate how this feature works. Supposed you have the following configuration that is passed to the +collector through the command line parameter. + +```yaml +extensions: + health_check: + +receivers: +${s3://example-bucket.s3.us-west-2.amazonaws.com/receivers.yaml} + +processors: +${s3://example-bucket.s3.us-west-2.amazonaws.com/processors.yaml} + +exporters: +${s3://example-bucket.s3.us-west-2.amazonaws.com/exporters.yaml} + + +service: + pipelines: + traces: + receivers: [otlp,awsxray] + processors: [batch/traces] + exporters: [awsxray] + metrics: + receivers: [otlp] + processors: [batch/metrics] + exporters: [awsemf] + + extensions: [health_check] +``` +In the example presented above, we are embeeding three different URIs. These embedded URIs will be expanded when the collector load the configuration. + +The following is the content of each URI: + +`s3://example-bucket.s3.us-west-2.amazonaws.com/receivers.yaml` + +``` + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + awsxray: + endpoint: 0.0.0.0:2000 + transport: udp +``` + +`s3://example-bucket.s3.us-west-2.amazonaws.com/processors.yaml` + +``` + batch/traces: + timeout: 1s + send_batch_size: 50 + batch/metrics: + timeout: 60s +``` + +`s3://example-bucket.s3.us-west-2.amazonaws.com/exporters.yaml` + +``` + awsxray: + awsemf: +``` + +The final configuration would look like: + +``` +extensions: + health_check: + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + awsxray: + endpoint: 0.0.0.0:2000 + transport: udp + +processors: + batch/traces: + timeout: 1s + send_batch_size: 50 + batch/metrics: + timeout: 60s + +exporters: + awsxray: + awsemf: + +service: + pipelines: + traces: + receivers: [otlp,awsxray] + processors: [batch/traces] + exporters: [awsxray] + metrics: + receivers: [otlp] + processors: [batch/metrics] + exporters: [awsemf] + + extensions: [health_check] + ``` diff --git a/src/docs/components/kafka-receiver-exporter.mdx b/src/docs/components/kafka-receiver-exporter.mdx new file mode 100644 index 000000000..cc84a9adc --- /dev/null +++ b/src/docs/components/kafka-receiver-exporter.mdx @@ -0,0 +1,148 @@ +--- +title: 'Kafka receiver and exporter' +description: | + There are two components that allows you to use Kafka as a transport mechanism for telemetry signals: + The Kafka receiver allows the collector to receive telemetry signals from Kafka while the Kafka exporter allows you to send them to Kafka. +path: '/docs/components/kafka-receiver-exporter' +--- + +import SectionSeparator from "components/MdxSectionSeparator/sectionSeparator.jsx" + + +The Kafka receiver and exporter allows you to send and receive telemetry signals (currently traces and metrics) from a Kafka cluster. The exporter will +produce telemetry signals and will submit it to a configured topic. A Kafka receiver can consume from this topic and further send +data to the telemetry pipeline configured in this collector instance. This component is flexible enough that you can have +multiple consumer groups for the same topic. + + + +## Upstream Kafka Exporter/Receiver documentation + +Please find bellow the documentation for each of these components: + +* [Kafka Exporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/kafkaexporter/README.md) +* [Kafka Receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/kafkareceiver/README.md) + +Notes: + +* These components don't create Kafka topics on your behalf. They need to be pre-created or you need to set the +cluster configuration to [auto create topics](https://kafka.apache.org/documentation/#brokerconfigs_auto.create.topics.enable). +* It is a good practice to set the `protocol_version` property to match the version of Kafka used in your cluster. +* The kafka exporter uses a synchronous producer that blocks and does not batch messages, therefore it should be used with batch and queued retry processors for higher throughput and resiliency. + +The ADOT collector is tested with Kafka Versions 2.8.1 and 3.2.0. + +## Integrating with Amazon Managed Streaming for Apache Kafka (MSK) + +The Kafka receiver and exporters can be integrated with [Amazon Managed Streaming for Apache Kafka (MSK)](https://aws.amazon.com/msk/). + +You can create a MSK cluster following [this guide](https://docs.aws.amazon.com/msk/latest/developerguide/getting-started.html). + * Select the authentication method accordingly to the options supported by the receivers/exporters. + * Enable TLS. + +After the cluster is provisioned, you can get the list of brokers by selecting the newly created cluster in the console and then +clicking in the `View client information` button. + +With the list of brokers, you can configure the collector accordingly. + +For a Kafka receiver the configuration would look like: + +``` + kafka/receiver: + auth: + tls: + insecure: false + protocol_version: 3.2.0 + topic: some-topic + brokers: + - b-2.testcluster.abc123.c13.kafka.us-west-2.amazonaws.com:9094 + - b-1.testcluster.abc123.c13.kafka.us-west-2.amazonaws.com:9094 + - b-3.testcluster.abc123.c13.kafka.us-west-2.amazonaws.com:9094 +``` + +For a Kafka exporter, the configuration would look like: +``` + kafka/exporter: + auth: + tls: + insecure: false + protocol_version: 3.2.0 + topic: some-topic + brokers: + - b-2.testcluster.abc123.c13.kafka.us-west-2.amazonaws.com:9094 + - b-1.testcluster.abc123.c13.kafka.us-west-2.amazonaws.com:9094 + - b-3.testcluster.abc123.c13.kafka.us-west-2.amazonaws.com:9094 +``` + +With such configuration you decouple producers (collector with exporter) and consumers (collector with receiver). This allow you to scale out consumers in case they perform some kind of processing on the telemetry signals as well as change the destination backend in the consumers without downtime as telemetry signals will still be recorded into the kafka topics while consumers are having their configuration changed. + +### Example + +In this section we are going to describe the simplest possible example to integrate the kafka receiver with the kafka exporter. We will be using two collector instances: A and B. Those instances are running in two different ec2 hosts. + +We are going to send telemetry data to collector A, which will submit this data to a Kafka topic. The collector B will be consuming from this same topic and it will receive this telemetry. + + +Configuration for instance A. + +``` +receivers: + statsd: + endpoint: 0.0.0.0:4567 + aggregation_interval: 5s + +exporters: + kafka/exporter: + protocol_version: "${extra_data.msk.kafka_version}" + auth: + tls: + insecure: false + topic: adot-collector-test + brokers: + - b-3.aocmskcluster281.test.c13.kafka.us-west-2.amazonaws.com:9094 + - b-1.aocmskcluster281.test.c13.kafka.us-west-2.amazonaws.com:9094 + - b-2.aocmskcluster281.test.c13.kafka.us-west-2.amazonaws.com:9094 + +service: + pipelines: + metrics: + receivers: [statsd] + exporters: [kafka/exporter] +``` + +Configuration for instance B. + +``` +receivers: + kafka/receiver: + topic: adot-collector-test + protocol_version: "2.8.1" + auth: + tls: + insecure: false + brokers: + - b-3.aocmskcluster281.test.c13.kafka.us-west-2.amazonaws.com:9094 + - b-1.aocmskcluster281.test.c13.kafka.us-west-2.amazonaws.com:9094 + - b-2.aocmskcluster281.test.c13.kafka.us-west-2.amazonaws.com:9094 + +exporters: + logging: + loglevel: debug + +service: + pipelines: + metrics: + receivers: [kafka/receiver] + exporters: [logging] +``` + +In the ec2 instance of the collector A we execute the following command to send telemetry to the statsd receiver: + +``` +echo "kafka.test.example:1|c" | nc -w 1 -u localhost 4567 +``` + +The following output is obtained in the collector B. +``` +{"level":"info","timestamp":"2023-04-05T03:56:23.531Z","message":"ResourceMetrics #0\nResource SchemaURL: \nScopeMetrics #0\nScopeMetrics SchemaURL: \nInstrumentationScope \nMetric #0\nDescriptor:\n -> Name: kafka.test.example\n -> Description: \n -> Unit: \n -> DataType: Sum\n -> IsMonotonic: false\n -> AggregationTemporality: Delta\nNumberDataPoints #0\nStartTimestamp: 2023-04-05 03:56:18.402491427 +0000 UTC\nTimestamp: 2023-04-05 03:56:23.401572412 +0000 UTC\nValue: 1\n","kind":"exporter","data_type":"metrics","name":"logging"} +```