Enable jmx receiver (#4067)

* Add jmx

* add jvm metrics

ci/Dockerfile

@@ -9,6 +9,11 @@ USER root
 RUN clean-apt install make \
             ca-certificates 
 
+RUN apt-get update && \
+apt-get -y install default-jre-headless
+
+ADD https://github.com/open-telemetry/opentelemetry-java-contrib/releases/download/v1.27.0/opentelemetry-jmx-metrics.jar /opt/opentelemetry-jmx-metrics.jar
+
 COPY --from=build /src/bin/otelcontribcol_* /otelcol-contrib
 USER 1001
 ENTRYPOINT ["/otelcol-contrib"]

ci/scripts/ci-deploy-demo.sh

@@ -12,6 +12,7 @@ set -x
 install_collector() {
   # Set the namespace and release name
   release_name="opentelemetry-collector"
+  release_name_deployment="opentelemetry-collector-deployment"
   namespace="otel"
 
   # if repo already exists, helm 3+ will skip
@@ -22,6 +23,11 @@ install_collector() {
     -f ./ci/values.yaml \
     --set-string image.tag="otelcolcontrib-v$CI_COMMIT_SHORT_SHA"
 
+  # --install collector that fetches jmx metrics. The jmx receiver cannot be used in the daemonset deployment
+  # as this would lead to duplicate metrics.
+  helm --debug upgrade "${release_name_deployment}" -n "${namespace}" open-telemetry/opentelemetry-collector --install \
+    -f ./ci/values-jmx.yaml \
+    --set-string image.tag="otelcolcontrib-v$CI_COMMIT_SHORT_SHA"
 }
 
 ###########################################################################################################

ci/values-jmx.yaml

@@ -0,0 +1,60 @@
+mode: deployment
+image:
+  repository: 172597598159.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib
+  tag: ""
+securityContext:
+  runAsUser: 0
+  runAsGroup: 0
+extraEnvs:
+  - name: HOST_IP
+    valueFrom:
+      fieldRef:
+        fieldPath: status.hostIP
+  - name: OTEL_COLLECTOR_NAME
+    value: $(HOST_IP)
+  - name: OTEL_EXPORTER_OTLP_ENDPOINT
+    value: http://$(OTEL_COLLECTOR_NAME):4317
+  - name: OTEL_RESOURCE_ATTRIBUTES
+    value: kafka_source=jmxreceiver
+config:
+  receivers:
+    jmx:
+      jar_path: /opt/opentelemetry-jmx-metrics.jar
+      endpoint: "opentelemetry-demo-kafka:1099"
+      target_system: kafka
+      log_level: trace
+    jmx/jvm:
+      jar_path: /opt/opentelemetry-jmx-metrics.jar
+      endpoint: "opentelemetry-demo-kafka:1099"
+      target_system: jvm
+      log_level: trace
+    jmx/consumer:
+      jar_path: /opt/opentelemetry-jmx-metrics.jar
+      endpoint: "opentelemetry-demo-frauddetectionservice:1098"
+      target_system: kafka-consumer
+      log_level: trace
+    jmx/producer:
+      jar_path: /opt/opentelemetry-jmx-metrics.jar
+      endpoint: "opentelemetry-demo-orderproducer:1097"
+      target_system: kafka-producer
+      log_level: trace
+    zookeeper:
+      endpoint: "opentelemetry-demo-zookeeper:2181"
+      collection_interval: 20s
+      initial_delay: 1s
+  exporters:
+    otlp:
+      endpoint: ${env:OTEL_EXPORTER_OTLP_ENDPOINT}
+      tls:
+        insecure: true
+  processors:
+    batch:
+      send_batch_max_size: 1000
+      send_batch_size: 100
+      timeout: 10s
+  service:
+    pipelines:
+      metrics:
+        receivers: [jmx, jmx/jvm, jmx/consumer, jmx/producer, zookeeper]
+        processors: [batch]
+        exporters: [otlp]

ci/values.yaml

@@ -20,7 +20,7 @@ securityContext:
 resources:
   limits:
     cpu: 256m
-    memory: 1Gi
+    memory: 2Gi
 presets:
   logsCollection:
     enabled: true
@@ -109,10 +109,13 @@ config:
         http:
           endpoint: 0.0.0.0:4318
   exporters:
+    logging:
+      verbosity: detailed
     datadog:
       host_metadata:
         tags: ['env:otel']
       metrics:
+        resource_attributes_as_tags: true
         histograms:
           mode: counters
           send_count_sum_metrics: true
@@ -123,6 +126,33 @@ config:
       api:
         key: "$DD_API_KEY"
   processors:
+    attributes:
+      actions:
+      - key: log.file.path
+        pattern: \/var\/log\/pods\/otel_opentelemetry-demo-(?P<service_name>.*?)-.*
+        action: extract
+      - key: log.file.path
+        pattern: \/var\/log\/pods\/otel_opentelemetry-demo-(?P<source>.*?)-.*
+        action: extract
+      - key: service
+        from_attribute: service_name
+        action: upsert
+    attributes/kafkasrc:
+      include:
+        match_type: regexp
+        attributes:
+          - {key: "source", value: 'kafka|frauddetectionservice|orderproducer'}
+      actions:
+      # this makes sure the source for logs that come from kafka|frauddetectionservice|orderproducer is
+      # set to kafka. The OOTB box kafka dashboard filters logs on source:kafka. This will still not work
+      # as it will be a log attribute and can only be searched by @source:kafka. We would need to support
+      # attributes_as_tags or find another solution for this to work.
+      - key: source
+        action: update
+        value: "kafka"
+    memory_limiter:
+      check_interval: 1s
+      limit_mib: 500
     resourcedetection:
       # ensures host.name and other important resource tags
       # get picked up
@@ -201,5 +231,5 @@ config:
         processors: [resourcedetection, k8sattributes, batch]
         exporters: [datadog]
       logs:
-        processors: [resourcedetection, k8sattributes, batch]
+        processors: [memory_limiter, resourcedetection, k8sattributes, attributes, attributes/kafkasrc, batch]
         exporters: [datadog]