feat(logs): add ability to use OTLP source

SumoLogic · May 9, 2023 · ddda1e0 · ddda1e0
1 parent 59211e6
commit ddda1e0
Show file tree

Hide file tree

Showing 15 changed files with 149 additions and 6 deletions.
diff --git a/.changelog/3040.added.txt b/.changelog/3040.added.txt
@@ -0,0 +1 @@
+feat(logs): add ability to use OTLP source
diff --git a/deploy/helm/sumologic/conf/logs/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/otelcol/config.yaml
@@ -18,6 +18,7 @@ exporters:
     endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
     json_logs:
       add_timestamp: false
+    endpoint: {{ include "logs.otelcol.exporter.endpoint" . }}
     log_format: {{ include "logs.otelcol.container.exporter.format" . }}
     sending_queue:
       enabled: true
@@ -34,7 +35,8 @@ exporters:
     endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
     json_logs:
       add_timestamp: false
-    log_format: json
+    endpoint: {{ include "logs.otelcol.exporter.endpoint" . }}
+    log_format: {{ include "logs.otelcol.systemd.exporter.format" . }}
     sending_queue:
       enabled: true
       num_consumers: 10

diff --git a/deploy/helm/sumologic/templates/_helpers/_logs.tpl b/deploy/helm/sumologic/templates/_helpers/_logs.tpl
@@ -106,13 +106,51 @@ Return the log format for the Sumologic exporter for container logs.
 */}}
 {{- define "logs.otelcol.container.exporter.format" -}}
 {{- $jsonFormats := list "json" "fields" "json_merge" -}}
+{{- if eq .Values.sumologic.logs.sourceType "otlp" -}}
+{{- "otlp" -}}
+{{- else if eq .Values.sumologic.logs.sourceType "http" -}}
 {{- if has .Values.sumologic.logs.container.format $jsonFormats -}}
 {{- "json" -}}
 {{- else if eq .Values.sumologic.logs.container.format "text" -}}
 {{- "text" -}}
 {{- else -}}
 {{- fail "`sumologic.logs.container.format` can only be `json`, `text`, `json_merge` or `fields`" -}}
 {{- end -}}
+{{- else -}}
+{{- fail "`sumologic.logs.sourceType` can only be `http` or `otlp`" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the log format for the Sumologic exporter for systemd and kubelet logs.
+
+'{{ include "logs.otelcol.systemd.exporter.format" . }}'
+*/}}
+{{- define "logs.otelcol.systemd.exporter.format" -}}
+{{- if eq .Values.sumologic.logs.sourceType "http" -}}
+{{- "json" -}}
+{{- else if eq .Values.sumologic.logs.sourceType "otlp" -}}
+{{- "otlp" -}}
+{{- else -}}
+{{- fail "`sumologic.logs.sourceType` can only be `http` or `otlp`" -}}
+{{- end -}}
+{{- end -}}
+
+
+
+{{/*
+Return the endpoint for the Sumologic exporter for logs.
+
+'{{ include "logs.otelcol.exporter.endpoint" . }}'
+*/}}
+{{- define "logs.otelcol.exporter.endpoint" -}}
+{{- if eq .Values.sumologic.logs.sourceType "http" -}}
+{{- "${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}" -}}
+{{- else if eq .Values.sumologic.logs.sourceType "otlp" -}}
+{{- "${SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE}" -}}
+{{- else -}}
+{{- fail "`sumologic.logs.sourceType` can only be `http` or `otlp`" -}}
+{{- end -}}
 {{- end -}}
 
 {{- define "sumologic.labels.app.logs" -}}

diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -188,7 +188,11 @@ sumologic:
         default:
           name: logs
           config-name: endpoint-logs
-
+        default-otlp:
+          name: logs-otlp
+          config-name: endpoint-logs-otlp
+          properties:
+            content_type: Otlp
           ## Properties can be used to extend default settings, such as processing rules, fields etc
           # properties:
           #  filters:
@@ -397,6 +401,8 @@ sumologic:
       - service
       - statefulset
 
+    sourceType: http
+
   ### Metrics configuration
   ## Set the enabled flag to false for disabling metrics ingestion altogether.
   metrics:

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc_statefulset/basic.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc_statefulset/basic.output.yaml
@@ -91,11 +91,11 @@ spec:
             - name: pprof
               containerPort: 1777
               protocol: TCP
-            - name: otlphttp2
-              containerPort: 4319
+            - containerPort: 4319
+              name: otlphttp2
               protocol: TCP
-            - name: otlphttp3
-              containerPort: 4320
+            - containerPort: 4320
+              name: otlphttp3
               protocol: TCP
           livenessProbe:
             httpGet:
@@ -136,6 +136,12 @@ spec:
                 secretKeyRef:
                   name: sumologic
                   key: endpoint-logs
+            - name: SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE
+              valueFrom:
+                secretKeyRef:
+                  name: sumologic
+                  key: endpoint-logs-otlp
+
             - name: NO_PROXY
               value: kubernetes.default.svc
             - name: VALUE_FROM_SECRET

diff --git a/tests/helm/testdata/goldenfile/terraform/all_fields.output.yaml b/tests/helm/testdata/goldenfile/terraform/all_fields.output.yaml
@@ -344,6 +344,7 @@ data:
     locals {
       default_events_source                       = "events"
       default_logs_source                         = "logs"
+      default_otlp_logs_source                    = "logs-otlp"
       apiserver_metrics_source                    = "apiserver-metrics"
       control_plane_metrics_source                = "control-plane-metrics"
       controller_metrics_source                   = "kube-controller-manager-metrics"
@@ -475,6 +476,12 @@ data:
         collector_id = sumologic_collector.collector.id
     }
 
+    resource "sumologic_http_source" "default_otlp_logs_source" {
+        name         = local.default_otlp_logs_source
+        collector_id = sumologic_collector.collector.id
+        content_type = "Otlp"
+    }
+
     resource "sumologic_http_source" "apiserver_metrics_source" {
         name         = local.apiserver_metrics_source
         collector_id = sumologic_collector.collector.id
@@ -575,6 +582,7 @@ data:
       data = {
         endpoint-events                           = sumologic_http_source.default_events_source.url
         endpoint-logs                             = sumologic_http_source.default_logs_source.url
+        endpoint-logs-otlp                        = sumologic_http_source.default_otlp_logs_source.url
         endpoint-metrics-apiserver                = sumologic_http_source.apiserver_metrics_source.url
         endpoint-control_plane_metrics_source     = sumologic_http_source.control_plane_metrics_source.url
         endpoint-metrics-kube-controller-manager  = sumologic_http_source.controller_metrics_source.url
@@ -718,6 +726,7 @@ data:
     true  # prevent to render empty if; then
     terraform import sumologic_http_source.default_events_source "${COLLECTOR_NAME}/events"
     terraform import sumologic_http_source.default_logs_source "${COLLECTOR_NAME}/logs"
+    terraform import sumologic_http_source.default_otlp_logs_source "${COLLECTOR_NAME}/logs-otlp"
     terraform import sumologic_http_source.apiserver_metrics_source "${COLLECTOR_NAME}/apiserver-metrics"
     terraform import sumologic_http_source.control_plane_metrics_source "${COLLECTOR_NAME}/control-plane-metrics"
     terraform import sumologic_http_source.controller_metrics_source "${COLLECTOR_NAME}/kube-controller-manager-metrics"

diff --git a/tests/helm/testdata/goldenfile/terraform/collector_fields.output.yaml b/tests/helm/testdata/goldenfile/terraform/collector_fields.output.yaml
@@ -344,6 +344,7 @@ data:
     locals {
       default_events_source                       = "events"
       default_logs_source                         = "logs"
+      default_otlp_logs_source                    = "logs-otlp"
       apiserver_metrics_source                    = "apiserver-metrics"
       control_plane_metrics_source                = "control-plane-metrics"
       controller_metrics_source                   = "kube-controller-manager-metrics"
@@ -476,6 +477,12 @@ data:
         collector_id = sumologic_collector.collector.id
     }
 
+    resource "sumologic_http_source" "default_otlp_logs_source" {
+        name         = local.default_otlp_logs_source
+        collector_id = sumologic_collector.collector.id
+        content_type = "Otlp"
+    }
+
     resource "sumologic_http_source" "apiserver_metrics_source" {
         name         = local.apiserver_metrics_source
         collector_id = sumologic_collector.collector.id
@@ -531,6 +538,7 @@ data:
       data = {
         endpoint-events                           = sumologic_http_source.default_events_source.url
         endpoint-logs                             = sumologic_http_source.default_logs_source.url
+        endpoint-logs-otlp                        = sumologic_http_source.default_otlp_logs_source.url
         endpoint-metrics-apiserver                = sumologic_http_source.apiserver_metrics_source.url
         endpoint-control_plane_metrics_source     = sumologic_http_source.control_plane_metrics_source.url
         endpoint-metrics-kube-controller-manager  = sumologic_http_source.controller_metrics_source.url
@@ -673,6 +681,7 @@ data:
     true  # prevent to render empty if; then
     terraform import sumologic_http_source.default_events_source "${COLLECTOR_NAME}/events"
     terraform import sumologic_http_source.default_logs_source "${COLLECTOR_NAME}/logs"
+    terraform import sumologic_http_source.default_otlp_logs_source "${COLLECTOR_NAME}/logs-otlp"
     terraform import sumologic_http_source.apiserver_metrics_source "${COLLECTOR_NAME}/apiserver-metrics"
     terraform import sumologic_http_source.control_plane_metrics_source "${COLLECTOR_NAME}/control-plane-metrics"
     terraform import sumologic_http_source.controller_metrics_source "${COLLECTOR_NAME}/kube-controller-manager-metrics"

diff --git a/tests/helm/testdata/goldenfile/terraform/default.output.yaml b/tests/helm/testdata/goldenfile/terraform/default.output.yaml
@@ -344,6 +344,7 @@ data:
     locals {
       default_events_source                       = "events"
       default_logs_source                         = "logs"
+      default_otlp_logs_source                    = "logs-otlp"
       apiserver_metrics_source                    = "apiserver-metrics"
       control_plane_metrics_source                = "control-plane-metrics"
       controller_metrics_source                   = "kube-controller-manager-metrics"
@@ -474,6 +475,12 @@ data:
         collector_id = sumologic_collector.collector.id
     }
 
+    resource "sumologic_http_source" "default_otlp_logs_source" {
+        name         = local.default_otlp_logs_source
+        collector_id = sumologic_collector.collector.id
+        content_type = "Otlp"
+    }
+
     resource "sumologic_http_source" "apiserver_metrics_source" {
         name         = local.apiserver_metrics_source
         collector_id = sumologic_collector.collector.id
@@ -529,6 +536,7 @@ data:
       data = {
         endpoint-events                           = sumologic_http_source.default_events_source.url
         endpoint-logs                             = sumologic_http_source.default_logs_source.url
+        endpoint-logs-otlp                        = sumologic_http_source.default_otlp_logs_source.url
         endpoint-metrics-apiserver                = sumologic_http_source.apiserver_metrics_source.url
         endpoint-control_plane_metrics_source     = sumologic_http_source.control_plane_metrics_source.url
         endpoint-metrics-kube-controller-manager  = sumologic_http_source.controller_metrics_source.url
@@ -671,6 +679,7 @@ data:
     true  # prevent to render empty if; then
     terraform import sumologic_http_source.default_events_source "${COLLECTOR_NAME}/events"
     terraform import sumologic_http_source.default_logs_source "${COLLECTOR_NAME}/logs"
+    terraform import sumologic_http_source.default_otlp_logs_source "${COLLECTOR_NAME}/logs-otlp"
     terraform import sumologic_http_source.apiserver_metrics_source "${COLLECTOR_NAME}/apiserver-metrics"
     terraform import sumologic_http_source.control_plane_metrics_source "${COLLECTOR_NAME}/control-plane-metrics"
     terraform import sumologic_http_source.controller_metrics_source "${COLLECTOR_NAME}/kube-controller-manager-metrics"

diff --git a/tests/helm/testdata/goldenfile/terraform/disable_default_metrics.output.yaml b/tests/helm/testdata/goldenfile/terraform/disable_default_metrics.output.yaml
@@ -344,6 +344,7 @@ data:
     locals {
       default_events_source                       = "events"
       default_logs_source                         = "logs"
+      default_otlp_logs_source                    = "logs-otlp"
       apiserver_metrics_source                    = "apiserver-metrics"
       control_plane_metrics_source                = "control-plane-metrics"
       controller_metrics_source                   = "kube-controller-manager-metrics"
@@ -473,6 +474,12 @@ data:
         collector_id = sumologic_collector.collector.id
     }
 
+    resource "sumologic_http_source" "default_otlp_logs_source" {
+        name         = local.default_otlp_logs_source
+        collector_id = sumologic_collector.collector.id
+        content_type = "Otlp"
+    }
+
     resource "sumologic_http_source" "apiserver_metrics_source" {
         name         = local.apiserver_metrics_source
         collector_id = sumologic_collector.collector.id
@@ -523,6 +530,7 @@ data:
       data = {
         endpoint-events                           = sumologic_http_source.default_events_source.url
         endpoint-logs                             = sumologic_http_source.default_logs_source.url
+        endpoint-logs-otlp                        = sumologic_http_source.default_otlp_logs_source.url
         endpoint-metrics-apiserver                = sumologic_http_source.apiserver_metrics_source.url
         endpoint-control_plane_metrics_source     = sumologic_http_source.control_plane_metrics_source.url
         endpoint-metrics-kube-controller-manager  = sumologic_http_source.controller_metrics_source.url
@@ -664,6 +672,7 @@ data:
     true  # prevent to render empty if; then
     terraform import sumologic_http_source.default_events_source "${COLLECTOR_NAME}/events"
     terraform import sumologic_http_source.default_logs_source "${COLLECTOR_NAME}/logs"
+    terraform import sumologic_http_source.default_otlp_logs_source "${COLLECTOR_NAME}/logs-otlp"
     terraform import sumologic_http_source.apiserver_metrics_source "${COLLECTOR_NAME}/apiserver-metrics"
     terraform import sumologic_http_source.control_plane_metrics_source "${COLLECTOR_NAME}/control-plane-metrics"
     terraform import sumologic_http_source.controller_metrics_source "${COLLECTOR_NAME}/kube-controller-manager-metrics"

diff --git a/tests/helm/testdata/goldenfile/terraform/disabled_dashboards.output.yaml b/tests/helm/testdata/goldenfile/terraform/disabled_dashboards.output.yaml
@@ -344,6 +344,7 @@ data:
     locals {
       default_events_source                       = "events"
       default_logs_source                         = "logs"
+      default_otlp_logs_source                    = "logs-otlp"
       apiserver_metrics_source                    = "apiserver-metrics"
       control_plane_metrics_source                = "control-plane-metrics"
       controller_metrics_source                   = "kube-controller-manager-metrics"
@@ -474,6 +475,12 @@ data:
         collector_id = sumologic_collector.collector.id
     }
 
+    resource "sumologic_http_source" "default_otlp_logs_source" {
+        name         = local.default_otlp_logs_source
+        collector_id = sumologic_collector.collector.id
+        content_type = "Otlp"
+    }
+
     resource "sumologic_http_source" "apiserver_metrics_source" {
         name         = local.apiserver_metrics_source
         collector_id = sumologic_collector.collector.id
@@ -529,6 +536,7 @@ data:
       data = {
         endpoint-events                           = sumologic_http_source.default_events_source.url
         endpoint-logs                             = sumologic_http_source.default_logs_source.url
+        endpoint-logs-otlp                        = sumologic_http_source.default_otlp_logs_source.url
         endpoint-metrics-apiserver                = sumologic_http_source.apiserver_metrics_source.url
         endpoint-control_plane_metrics_source     = sumologic_http_source.control_plane_metrics_source.url
         endpoint-metrics-kube-controller-manager  = sumologic_http_source.controller_metrics_source.url
@@ -671,6 +679,7 @@ data:
     true  # prevent to render empty if; then
     terraform import sumologic_http_source.default_events_source "${COLLECTOR_NAME}/events"
     terraform import sumologic_http_source.default_logs_source "${COLLECTOR_NAME}/logs"
+    terraform import sumologic_http_source.default_otlp_logs_source "${COLLECTOR_NAME}/logs-otlp"
     terraform import sumologic_http_source.apiserver_metrics_source "${COLLECTOR_NAME}/apiserver-metrics"
     terraform import sumologic_http_source.control_plane_metrics_source "${COLLECTOR_NAME}/control-plane-metrics"
     terraform import sumologic_http_source.controller_metrics_source "${COLLECTOR_NAME}/kube-controller-manager-metrics"

diff --git a/tests/helm/testdata/goldenfile/terraform/disabled_monitors.output.yaml b/tests/helm/testdata/goldenfile/terraform/disabled_monitors.output.yaml
@@ -344,6 +344,7 @@ data:
     locals {
       default_events_source                       = "events"
       default_logs_source                         = "logs"
+      default_otlp_logs_source                    = "logs-otlp"
       apiserver_metrics_source                    = "apiserver-metrics"
       control_plane_metrics_source                = "control-plane-metrics"
       controller_metrics_source                   = "kube-controller-manager-metrics"
@@ -474,6 +475,12 @@ data:
         collector_id = sumologic_collector.collector.id
     }
 
+    resource "sumologic_http_source" "default_otlp_logs_source" {
+        name         = local.default_otlp_logs_source
+        collector_id = sumologic_collector.collector.id
+        content_type = "Otlp"
+    }
+
     resource "sumologic_http_source" "apiserver_metrics_source" {
         name         = local.apiserver_metrics_source
         collector_id = sumologic_collector.collector.id
@@ -529,6 +536,7 @@ data:
       data = {
         endpoint-events                           = sumologic_http_source.default_events_source.url
         endpoint-logs                             = sumologic_http_source.default_logs_source.url
+        endpoint-logs-otlp                        = sumologic_http_source.default_otlp_logs_source.url
         endpoint-metrics-apiserver                = sumologic_http_source.apiserver_metrics_source.url
         endpoint-control_plane_metrics_source     = sumologic_http_source.control_plane_metrics_source.url
         endpoint-metrics-kube-controller-manager  = sumologic_http_source.controller_metrics_source.url
@@ -671,6 +679,7 @@ data:
     true  # prevent to render empty if; then
     terraform import sumologic_http_source.default_events_source "${COLLECTOR_NAME}/events"
     terraform import sumologic_http_source.default_logs_source "${COLLECTOR_NAME}/logs"
+    terraform import sumologic_http_source.default_otlp_logs_source "${COLLECTOR_NAME}/logs-otlp"
     terraform import sumologic_http_source.apiserver_metrics_source "${COLLECTOR_NAME}/apiserver-metrics"
     terraform import sumologic_http_source.control_plane_metrics_source "${COLLECTOR_NAME}/control-plane-metrics"
     terraform import sumologic_http_source.controller_metrics_source "${COLLECTOR_NAME}/kube-controller-manager-metrics"

diff --git a/tests/helm/testdata/goldenfile/terraform/monitors_with_email_notifications.output.yaml b/tests/helm/testdata/goldenfile/terraform/monitors_with_email_notifications.output.yaml
@@ -344,6 +344,7 @@ data:
     locals {
       default_events_source                       = "events"
       default_logs_source                         = "logs"
+      default_otlp_logs_source                    = "logs-otlp"
       apiserver_metrics_source                    = "apiserver-metrics"
       control_plane_metrics_source                = "control-plane-metrics"
       controller_metrics_source                   = "kube-controller-manager-metrics"
@@ -480,6 +481,12 @@ data:
         collector_id = sumologic_collector.collector.id
     }
 
+    resource "sumologic_http_source" "default_otlp_logs_source" {
+        name         = local.default_otlp_logs_source
+        collector_id = sumologic_collector.collector.id
+        content_type = "Otlp"
+    }
+
     resource "sumologic_http_source" "apiserver_metrics_source" {
         name         = local.apiserver_metrics_source
         collector_id = sumologic_collector.collector.id
@@ -535,6 +542,7 @@ data:
       data = {
         endpoint-events                           = sumologic_http_source.default_events_source.url
         endpoint-logs                             = sumologic_http_source.default_logs_source.url
+        endpoint-logs-otlp                        = sumologic_http_source.default_otlp_logs_source.url
         endpoint-metrics-apiserver                = sumologic_http_source.apiserver_metrics_source.url
         endpoint-control_plane_metrics_source     = sumologic_http_source.control_plane_metrics_source.url
         endpoint-metrics-kube-controller-manager  = sumologic_http_source.controller_metrics_source.url
@@ -677,6 +685,7 @@ data:
     true  # prevent to render empty if; then
     terraform import sumologic_http_source.default_events_source "${COLLECTOR_NAME}/events"
     terraform import sumologic_http_source.default_logs_source "${COLLECTOR_NAME}/logs"
+    terraform import sumologic_http_source.default_otlp_logs_source "${COLLECTOR_NAME}/logs-otlp"
     terraform import sumologic_http_source.apiserver_metrics_source "${COLLECTOR_NAME}/apiserver-metrics"
     terraform import sumologic_http_source.control_plane_metrics_source "${COLLECTOR_NAME}/control-plane-metrics"
     terraform import sumologic_http_source.controller_metrics_source "${COLLECTOR_NAME}/kube-controller-manager-metrics"