feat(sumologic-mock): add full support to events (#3545)

* feat(sumologic-mock): add full support to events Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * chore: changelog Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> --------- Signed-off-by: Dominik Rosiek <drosiek@sumologic.com>
SumoLogic · Feb 14, 2024 · 8ea6b68 · 8ea6b68
1 parent b1c5c45
commit 8ea6b68
Show file tree

Hide file tree

Showing 13 changed files with 152 additions and 5 deletions.
diff --git a/.changelog/3545.changed.txt b/.changelog/3545.changed.txt
@@ -0,0 +1 @@
+feat(sumologic-mock): add full support to events
diff --git a/deploy/helm/sumologic/README.md b/deploy/helm/sumologic/README.md
@@ -490,6 +490,9 @@ The following table lists the configurable parameters of the Sumo Logic chart an
 | `debug.metrics.metadata.forwardToSumologicMock`                                       | Set to `true` to forward metrics processed by metadata statefulset also to Sumo Logic Mock.                                                                                                                                                                                                                | `false`                                                                                                                                                                              |
 | `debug.metrics.metadata.print`                                                        | Set to `true` to print logs processed by metric metadata pods on standard output.                                                                                                                                                                                                                          | `false`                                                                                                                                                                              |
 | `debug.metrics.metadata.stopLogsIngestion`                                            | Set to `true` to prevent sending metric metadata logs to Sumo Logic. It prevents increasing ingestion and costs in case of huge amount of metric metadata logs.                                                                                                                                            | `false`                                                                                                                                                                              |
+| `debug.events.print`                                                                  | Set to `true` to print events processed by events collector on standard output.                                                                                                                                                                                                                            | `false`                                                                                                                                                                              |
+| `debug.events.stopLogsIngestion`                                                      | Set to `true` to prevent sending event collector logs to Sumo Logic. It prevents increasing ingestion and costs in case of huge amount of event collector logs.                                                                                                                                            | `false`                                                                                                                                                                              |
+| `debug.events.forwardToSumologicMock`                                                 | Set to `true` to forward events also to Sumo Logic Mock.                                                                                                                                                                                                                                                   | `false`                                                                                                                                                                              |
 | `debug.sumologicMock.deployment.affinity`                                             | Affinity for the Sumo Logic Mock deployment.                                                                                                                                                                                                                                                               | `{}`                                                                                                                                                                                 |
 | `debug.sumologicMock.deployment.extraArgs`                                            | Extra Arguments for the Sumo Logic Mock deployment. See [list of supported arguments][sumologic_mock_arguments]                                                                                                                                                                                            | `[]`                                                                                                                                                                                 |
 | `debug.sumologicMock.deployment.nodeSelector`                                         | Node selector the Sumo Logic Mock deployment.                                                                                                                                                                                                                                                              | `{}`                                                                                                                                                                                 |

diff --git a/deploy/helm/sumologic/conf/events/otelcol/config.yaml b/deploy/helm/sumologic/conf/events/otelcol/config.yaml
@@ -10,6 +10,21 @@ exporters:
       storage: file_storage
 {{- end }}
 
+{{- if eq .Values.debug.events.print true }}
+  debug:
+    verbosity: detailed
+{{- end }}
+
+{{- if eq (include "sumologic-mock.forward-events" .) "true" }}
+  sumologic/sumologic-mock:
+    client: {{ include "sumologic.sumo_client" . }}
+    endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
+    log_format: {{ include "sumologic.events.exporter.format" . }}
+    clear_logs_timestamp: false # this only affects the otlp format
+    sending_queue:
+      enabled: true
+{{- end }}
+
 extensions:
 {{- if .Values.sumologic.events.persistence.enabled }}
   file_storage:
@@ -64,6 +79,12 @@ service:
     logs/events:
       exporters:
         - sumologic
+{{- if eq .Values.debug.events.print true }}
+        - debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-events" .) "true" }}
+        - sumologic/sumologic-mock
+{{- end }}
       processors:
         - memory_limiter
         - resource/add_cluster

diff --git a/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml
@@ -79,6 +79,9 @@ receivers:
       {{ if eq .Values.debug.instrumentation.tracesSampler.stopLogsIngestion true }}
       {{ include "tracesSampler.collector.files.list" . }}
       {{ end }}
+      {{ if eq .Values.debug.events.stopLogsIngestion true }}
+      {{ include "events.collector.files.list" . }}
+      {{ end }}
 
 {{ if lt (int (include "kubernetes.minor" .)) 24 }}
     ## sets fingerprint_size to 17kb in order to match the longest possible docker line (which by default is 16kb)

diff --git a/deploy/helm/sumologic/conf/logs/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/otelcol/config.yaml
@@ -21,7 +21,7 @@ exporters:
 
 {{- if eq (include "sumologic-mock.forward-logs-metadata" .) "true" }}
   sumologic/sumologic-mock-containers:
-    endpoint: http://{{ template "sumologic-mock.hostname" . }}:{{ template "sumologic-mock.port" . }}/receiver
+    endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
     json_logs:
       add_timestamp: false
     log_format: {{ include "logs.otelcol.container.exporter.format" . }}
@@ -52,7 +52,7 @@ exporters:
 
 {{- if eq (include "sumologic-mock.forward-logs-metadata" .) "true" }}
   sumologic/sumologic-mock-systemd:
-    endpoint: http://{{ template "sumologic-mock.hostname" . }}:{{ template "sumologic-mock.port" . }}/receiver
+    endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
     json_logs:
       add_timestamp: false
     log_format: json
@@ -84,7 +84,7 @@ exporters:
 {{- if eq (include "sumologic-mock.forward-logs-metadata" .) "true" }}
   sumologic/sumologic-mock:
     client: {{ include "sumologic.sumo_client" . }}
-    endpoint: http://{{ template "sumologic-mock.hostname" . }}:{{ template "sumologic-mock.port" . }}/receiver
+    endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
     log_format: otlp
     clear_logs_timestamp: false
     sending_queue:

diff --git a/deploy/helm/sumologic/conf/metrics/otelcol/exporters.yaml b/deploy/helm/sumologic/conf/metrics/otelcol/exporters.yaml
@@ -7,7 +7,7 @@ debug:
 sumologic/sumologic-mock-default:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: {{ include "metrics.otelcol.exporter.format" . }}
-  endpoint: http://{{ template "sumologic-mock.hostname" . }}:{{ template "sumologic-mock.port" . }}/receiver
+  endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
   ## Sumo doesn't yet natively support OTLP Histograms
   decompose_otlp_histograms: true
   ## Configuration for sending queue
@@ -28,7 +28,7 @@ sumologic/sumologic-mock-default:
 sumologic/sumologic-mock-http:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
-  endpoint: http://{{ template "sumologic-mock.hostname" . }}:{{ template "sumologic-mock.port" . }}/receiver
+  endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}

diff --git a/deploy/helm/sumologic/templates/_helpers/_events.tpl b/deploy/helm/sumologic/templates/_helpers/_events.tpl
@@ -115,3 +115,7 @@ otlp
 {{- fail "`sumologic.events.sourceType` can only be `http` or `otlp`" -}}
 {{- end -}}
 {{- end -}}
+
+{{- define "events.collector.files.list" -}}
+- /var/log/pods/{{ template "sumologic.namespace" . }}_{{ template "sumologic.metadata.name.events" . }}*/*/*.log
+{{- end -}}
diff --git a/deploy/helm/sumologic/templates/_helpers/_sumologic_mock.tpl b/deploy/helm/sumologic/templates/_helpers/_sumologic_mock.tpl
@@ -55,6 +55,12 @@ true
 {{- end -}}
 {{- end -}}
 
+{{- define "sumologic-mock.forward-events"}}
+{{- if and (eq .Values.debug.sumologicMock.enabled true) (eq .Values.debug.events.forwardToSumologicMock true) -}}
+true
+{{- end -}}
+{{- end -}}
+
 {{- define "sumologic-mock.forward-metrics-metadata"}}
 {{- if and (eq .Values.debug.sumologicMock.enabled true) (eq .Values.debug.metrics.metadata.forwardToSumologicMock true) -}}
 true
@@ -83,3 +89,7 @@ true
 {{- define "sumologic.labels.sumologic-mock" -}}
 sumologic.com/app: sumologic-mock
 {{- end -}}
+
+{{- define "sumologic-mock.receiver-endpoint" -}}
+http://{{ template "sumologic-mock.hostname" . }}:{{ template "sumologic-mock.port" . }}/receiver
+{{- end -}}
diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -2389,3 +2389,8 @@ debug:
     tracesSampler:
       print: false
       stopLogsIngestion: false
+
+  events:
+    print: false
+    stopLogsIngestion: false
+    forwardToSumologicMock: false
diff --git a/tests/helm/testdata/goldenfile/events_otc/sumologic-mock.input.yaml b/tests/helm/testdata/goldenfile/events_otc/sumologic-mock.input.yaml
@@ -0,0 +1,6 @@
+debug:
+  sumologicMock:
+    enabled: true
+  events:
+    print: true
+    forwardToSumologicMock: true
diff --git a/tests/helm/testdata/goldenfile/events_otc/sumologic-mock.output.yaml b/tests/helm/testdata/goldenfile/events_otc/sumologic-mock.output.yaml
@@ -0,0 +1,91 @@
+---
+# Source: sumologic/templates/events/otelcol/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: RELEASE-NAME-sumologic-otelcol-events
+  namespace: sumologic
+  labels:
+    app: RELEASE-NAME-sumologic-otelcol-events
+    chart: "sumologic-%CURRENT_CHART_VERSION%"
+    release: "RELEASE-NAME"
+    heritage: "Helm"
+data:
+  config.yaml: |
+    exporters:
+      debug:
+        verbosity: detailed
+      sumologic:
+        clear_logs_timestamp: false
+        client: k8s_%CURRENT_CHART_VERSION%
+        endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_EVENTS_SOURCE}
+        log_format: otlp
+        sending_queue:
+          enabled: true
+          storage: file_storage
+      sumologic/sumologic-mock:
+        clear_logs_timestamp: false
+        client: k8s_%CURRENT_CHART_VERSION%
+        endpoint: http://RELEASE-NAME-sumologic-mock.sumologic:3000/receiver
+        log_format: otlp
+        sending_queue:
+          enabled: true
+    extensions:
+      file_storage:
+        directory: /var/lib/storage/events
+        timeout: 10s
+      health_check: {}
+      pprof: {}
+    processors:
+      batch:
+        send_batch_max_size: 2048
+        send_batch_size: 1024
+        timeout: 1s
+      memory_limiter:
+        check_interval: 1s
+        limit_percentage: 70
+        spike_limit_percentage: 20
+      resource/add_cluster:
+        attributes:
+        - action: upsert
+          key: cluster
+          value: kubernetes
+      source:
+        collector: kubernetes
+        source_category: kubernetes/events
+        source_category_prefix: ""
+        source_category_replace_dash: /
+        source_name: events
+      sumologic:
+        add_cloud_namespace: false
+      transform/add_timestamp:
+        log_statements:
+        - context: log
+          statements:
+          - set(time, Now()) where time_unix_nano == 0
+          - set(attributes["timestamp"], Int(time_unix_nano / 1000000))
+    receivers:
+      raw_k8s_events: {}
+    service:
+      extensions:
+      - health_check
+      - file_storage
+      - pprof
+      pipelines:
+        logs/events:
+          exporters:
+          - sumologic
+          - debug
+          - sumologic/sumologic-mock
+          processors:
+          - memory_limiter
+          - resource/add_cluster
+          - source
+          - sumologic
+          - transform/add_timestamp
+          - batch
+          receivers:
+          - raw_k8s_events
+      telemetry:
+        logs:
+          level: info
diff --git a/tests/helm/testdata/goldenfile/logs_otc/debug.input.yaml b/tests/helm/testdata/goldenfile/logs_otc/debug.input.yaml
@@ -22,3 +22,5 @@ debug:
       stopLogsIngestion: true
     tracesSampler:
       stopLogsIngestion: true
+  events:
+    stopLogsIngestion: true
diff --git a/tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml b/tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml
@@ -66,6 +66,7 @@ data:
         - /var/log/pods/sumologic_RELEASE-NAME-sumologic-otelcol-instrumentation*/*/*.log
         - /var/log/pods/sumologic_RELEASE-NAME-sumologic-traces-gateway*/*/*.log
         - /var/log/pods/sumologic_RELEASE-NAME-sumologic-traces-sampler*/*/*.log
+        - /var/log/pods/sumologic_RELEASE-NAME-sumologic-otelcol-events*/*/*.log
         include:
         - /var/log/pods/*/*/*.log
         include_file_name: false