Sumologic exporter and metric pipelines into OT Collector & agent (#1647

)



Co-authored-by: Patryk Małek <pmalek@sumologic.com>

deploy/helm/sumologic/conf/traces/traces.otelcol.conf.yaml

@@ -20,5 +20,7 @@
 {{- $yamlFile := replace "processors.source.exclude_container_regex.replace" $excludeContainerRegex $yamlFile }}
 {{- $yamlFile := replace "processors.source.exclude_host_regex.replace" $excludeHostRegex $yamlFile }}
 {{- $yamlFile := replace "processors.resource.cluster.replace" $clusterName $yamlFile }}
+{{- $yamlFile := replace "exporters.sumologic.source_name.replace" $sourceName $yamlFile }}
+{{- $yamlFile := replace "exporters.sumologic.source_category.replace" $sourceCategory $yamlFile }}
 
 {{- tpl ($yamlFile | replace ": '{{" ": {{" | replace "}}'" "}}") . | nindent 2 }}

deploy/helm/sumologic/templates/_helpers.tpl

@@ -965,15 +965,30 @@ Example:
 {{- define "kubernetes.sources.envs" -}}
 {{- $ctx := .Context -}}
 {{- $type := .Type -}}
-{{- range $key, $source := (index .Context.sumologic.collector.sources $type) }}
-        - name: {{ template "terraform.sources.endpoint" (include "terraform.sources.name" (dict "Name" $key "Type" $type)) }}
-          valueFrom:
-            secretKeyRef:
-              name: sumologic
-              key: {{ template "terraform.sources.config-map-variable" (dict "Type" $type "Context" $ctx "Name" $key) }}
+{{- range $name, $source := (index .Context.sumologic.collector.sources $type) }}
+{{ include "kubernetes.sources.env" (dict "Context" $ctx "Type" $type  "Name" $name ) }}
 {{- end }}
 {{- end -}}
 
+{{/*
+Generate fluentd envs for given source type:
+
+Example:
+
+{{ include "kubernetes.sources.env" (dict "Context" .Values "Type" "metrics" "Name" $name ) }}
+*/}}
+{{- define "kubernetes.sources.env" -}}
+{{- $ctx := .Context -}}
+{{- $type := .Type -}}
+{{- $name := .Name -}}
+- name: {{ template "terraform.sources.endpoint" (include "terraform.sources.name" (dict "Name" $name "Type" $type)) }}
+  valueFrom:
+    secretKeyRef:
+      name: sumologic
+      key: {{ template "terraform.sources.config-map-variable" (dict "Type" $type "Context" $ctx "Name" $name) }}
+{{- end -}}
+
+
 {{/*
 Generate a space separated list of quoted values:
 

deploy/helm/sumologic/templates/events/fluentd/statefulset.yaml

@@ -98,7 +98,7 @@ spec:
           periodSeconds: 5
         env:
 {{- $ctx := .Values -}}
-{{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "events")}}
+{{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "events") | nindent 8 }}
 {{- if .Values.fluentd.events.extraEnvVars }}
 {{ toYaml .Values.fluentd.events.extraEnvVars | nindent 8 }}
 {{- end }}

deploy/helm/sumologic/templates/logs/fluentd/statefulset.yaml

@@ -145,9 +145,9 @@ spec:
 {{- end }}
         env:
 {{- $ctx := .Values -}}
-{{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "logs")}}
+{{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "logs") | nindent 8 }}
 {{- if .Values.sumologic.traces.enabled }}
-{{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "traces")}}
+{{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "traces") | nindent 8 }}
 {{- end }}
         {{- if .Values.fluentd.logs.extraEnvVars }}
 {{ toYaml .Values.fluentd.logs.extraEnvVars | nindent 8 }}

deploy/helm/sumologic/templates/metrics/fluentd/statefulset.yaml

@@ -140,7 +140,7 @@ spec:
 {{- end }}
         env:
 {{- $ctx := .Values -}}
-{{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "metrics")}}
+{{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "metrics") | nindent 8 }}
         {{- if .Values.fluentd.metrics.extraEnvVars }}
 {{ toYaml .Values.fluentd.metrics.extraEnvVars | nindent 8 }}
         {{- end }}

deploy/helm/sumologic/templates/traces/otel-agent/daemonset.yaml

@@ -62,7 +62,8 @@ spec:
         - name: GOGC
           value: "80"
 {{- $ctx := .Values -}}
-{{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "traces") }}
+{{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "traces") | nindent 8 }}
+{{- include "kubernetes.sources.env" (dict "Context" $ctx "Type" "metrics" "Name" "default" ) | nindent 8 }}
 {{- if .Values.otelagent.daemonset.extraEnvVars }}
 {{- toYaml .Values.otelagent.daemonset.extraEnvVars | nindent 8 }}
 {{- end }}

deploy/helm/sumologic/templates/traces/otelcol/deployment.yaml

@@ -65,7 +65,8 @@ spec:
         - name: GOGC
           value: "80"
 {{- $ctx := .Values -}}
-{{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "traces") }}
+{{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "traces") | nindent 8 }}
+{{- include "kubernetes.sources.env" (dict "Context" $ctx "Type" "metrics" "Name" "default" ) | nindent 8 }}
 {{- if .Values.otelcol.deployment.extraEnvVars }}
 {{- toYaml .Values.otelcol.deployment.extraEnvVars | nindent 8 }}
 {{- end }}

deploy/helm/sumologic/values.yaml

@@ -3064,6 +3064,10 @@ otelagent:
           receivers: [jaeger, opencensus, otlp, zipkin]
           processors: [memory_limiter, k8s_tagger, batch]
           exporters: [otlp]
+        metrics:
+          receivers: [otlp]
+          processors: [memory_limiter, k8s_tagger, batch]
+          exporters: [otlp]
 
 ## Configure otelcol
 otelcol:
@@ -3221,6 +3225,13 @@ otelcol:
         send_batch_max_size: 512
         ## Time duration after which a batch will be sent regardless of size
         timeout: 5s
+
+      ## Add System metadata - needed for Sumologic exporter source_host
+      resourcedetection:
+        detectors: [system]
+        timeout: 10s
+        override: false
+
     extensions:
       health_check: {}
       memory_ballast:
@@ -3238,6 +3249,47 @@ otelcol:
       otlphttp:
         traces_endpoint: ${SUMO_ENDPOINT_DEFAULT_TRACES_SOURCE}
         compression: gzip
+      sumologic:
+        endpoint: ${SUMO_ENDPOINT_DEFAULT_METRICS_SOURCE}
+        ## Compression encoding format, either empty string (""), gzip or deflate (default gzip).
+        ## Empty string means no compression
+        compress_encoding: gzip
+        ## Max HTTP request body size in bytes before compression (if applied). By default 1_048_576 (1MB) is used.
+        max_request_body_size: 1_048_576  # 1MB
+        ## Format to use when sending logs to Sumo. (default json) (possible values: json, text)
+        log_format: text
+        ## Format of the metrics to be sent (default is prometheus) (possible values: carbon2, prometheus)
+        ## carbon2 and graphite are going to be supported soon.
+        metric_format: prometheus
+        ## Desired source category. Useful if you want to override the source category configured for the source.
+        source_category: "exporters.sumologic.source_category.replace"
+        ## Desired source name. Useful if you want to override the source name configured for the source.
+        source_name: "exporters.sumologic.source_name.replace"
+        ## Desired host name. Useful if you want to override the source host configured for the source.
+        source_host: "%{host.name}"
+        ## List of regexes for attributes which should be send as metadata
+        metadata_attributes:
+          - k8s.*
+          - host.name
+        ## Timeout for every attempt to send data to Sumo Logic backend. Maximum connection timeout is 55s.
+        timeout: 5s
+        retry_on_failure:
+          enabled: true
+          ## Time to wait after the first failure before retrying
+          initial_interval: 5s
+          ## Upper bound on backoff
+          max_interval: 30s
+          ## Maximum amount of time spent trying to send a batch
+          max_elapsed_time: 120s
+        sending_queue:
+          enabled: false
+          ## Number of consumers that dequeue batches
+          num_consumers: 10
+          ## Maximum number of batches kept in memory before data
+          ## User should calculate this as num_seconds * requests_per_second where:
+          ## num_seconds is the number of seconds to buffer in case of a backend outage
+          ## requests_per_second is the average number of requests per seconds.
+          queue_size: 5000
     service:
       extensions: [health_check, memory_ballast]
       pipelines:
@@ -3248,6 +3300,10 @@ otelcol:
           exporters: [otlphttp]
           ## and uncomment next one
           # exporters: [zipkin]
+        metrics:
+          receivers: [otlp]
+          processors: [memory_limiter, k8s_tagger, resourcedetection, resource, batch]
+          exporters: [sumologic]
 
 metadata:
   ## Configure image for Opentelemetry Collector (for logs and metrics)

tests/helm/tracing/static/collection-monitoring-false.output.yaml

@@ -15,6 +15,28 @@ data:
       otlphttp:
         compression: gzip
         traces_endpoint: ${SUMO_ENDPOINT_DEFAULT_TRACES_SOURCE}
+      sumologic:
+        compress_encoding: gzip
+        endpoint: ${SUMO_ENDPOINT_DEFAULT_METRICS_SOURCE}
+        log_format: text
+        max_request_body_size: 1048576
+        metadata_attributes:
+        - k8s.*
+        - host.name
+        metric_format: prometheus
+        retry_on_failure:
+          enabled: true
+          initial_interval: 5s
+          max_elapsed_time: 120s
+          max_interval: 30s
+        sending_queue:
+          enabled: false
+          num_consumers: 10
+          queue_size: 5000
+        source_category: 
+        source_host: '%{host.name}'
+        source_name: 
+        timeout: 5s
       zipkin:
         endpoint: ${SUMO_ENDPOINT_DEFAULT_TRACES_SOURCE}
     extensions:
@@ -63,6 +85,11 @@ data:
         - action: upsert
           key: k8s.cluster.name
           value: kubernetes
+      resourcedetection:
+        detectors:
+        - system
+        override: false
+        timeout: 10s
       source:
         annotation_prefix: k8s.pod.annotation.
         collector: "kubernetes"
@@ -106,6 +133,17 @@ data:
       - health_check
       - memory_ballast
       pipelines:
+        metrics:
+          exporters:
+          - sumologic
+          processors:
+          - memory_limiter
+          - k8s_tagger
+          - resourcedetection
+          - resource
+          - batch
+          receivers:
+          - otlp
         traces:
           exporters:
           - otlphttp

tests/helm/tracing/static/simple.output.yaml

@@ -15,6 +15,28 @@ data:
       otlphttp:
         compression: gzip
         traces_endpoint: ${SUMO_ENDPOINT_DEFAULT_TRACES_SOURCE}
+      sumologic:
+        compress_encoding: gzip
+        endpoint: ${SUMO_ENDPOINT_DEFAULT_METRICS_SOURCE}
+        log_format: text
+        max_request_body_size: 1048576
+        metadata_attributes:
+        - k8s.*
+        - host.name
+        metric_format: prometheus
+        retry_on_failure:
+          enabled: true
+          initial_interval: 5s
+          max_elapsed_time: 120s
+          max_interval: 30s
+        sending_queue:
+          enabled: false
+          num_consumers: 10
+          queue_size: 5000
+        source_category: 
+        source_host: '%{host.name}'
+        source_name: 
+        timeout: 5s
       zipkin:
         endpoint: ${SUMO_ENDPOINT_DEFAULT_TRACES_SOURCE}
     extensions:
@@ -63,6 +85,11 @@ data:
         - action: upsert
           key: k8s.cluster.name
           value: kubernetes
+      resourcedetection:
+        detectors:
+        - system
+        override: false
+        timeout: 10s
       source:
         annotation_prefix: k8s.pod.annotation.
         collector: "kubernetes"
@@ -106,6 +133,17 @@ data:
       - health_check
       - memory_ballast
       pipelines:
+        metrics:
+          exporters:
+          - sumologic
+          processors:
+          - memory_limiter
+          - k8s_tagger
+          - resourcedetection
+          - resource
+          - batch
+          receivers:
+          - otlp
         traces:
           exporters:
           - otlphttp