feat(logs): add ability to use OTLP source

.changelog/3040.added.txt

@@ -0,0 +1 @@
+feat(logs): add ability to use OTLP source

deploy/helm/sumologic/README.md

@@ -90,6 +90,7 @@ The following table lists the configurable parameters of the Sumo Logic chart an
 | `sumologic.logs.defaultFluentd.excludeUnitRegex`                                    | Configuration option only for Fluentd, a regular expression for unit. Matching unit will be excluded from Sumo. The logs will still be sent to FluentD. .                                                                                                                                                                                                            | `""`                                                                                                                                                                                 |
 | `sumologic.logs.fields`                                                             | Fields to be created at Sumo Logic to ensure logs are tagged with relevant metadata. [Sumo Logic help](https://help.sumologic.com/docs/manage/fields/#manage-fields)                                                                                                                                                                                                 | `["cluster", "container", "daemonset", "deployment", "host", "namespace", "node", "pod", "service", "statefulset"]`                                                                  |
 | `sumologic.logs.metadata.provider`                                                  | Defines logs metadata enrichment provider - `otelcol` or `fluentd`. `otelcol` is the default and is recommended. `fluentd` is deprecated.                                                                                                                                                                                                                            | `otelcol`                                                                                                                                                                            |
+| `sumologic.logs.sourceType`                                                         | The type of the Sumo Logic source being used for logs ingestion. Can be `http` for [HTTP Source](https://help.sumologic.com/docs/send-data/hosted-collectors/http-source/logs-metrics/) or `otlp` for [OTLP/HTTP Source](https://help.sumologic.com/docs/send-data/hosted-collectors/http-source/otlp/).                                                             | `http`                                                                                                                                                                               |
 | `sumologic.metrics.enabled`                                                         | Set the enabled flag to false for disabling metrics ingestion altogether.                                                                                                                                                                                                                                                                                            | `true`                                                                                                                                                                               |
 | `sumologic.metrics.metadata.provider`                                               | Defines metrics metadata enrichment provider - `otelcol` or `fluentd`. `otelcol` is the default and is recommended. `fluentd` is deprecated.                                                                                                                                                                                                                         | `otelcol`                                                                                                                                                                            |
 | `sumologic.metrics.otelcol.extraProcessors`                                         | Extra processors configuration for metrics pipeline. See [/docs/collecting-application-metrics.md#metrics-modifications](/docs/collecting-application-metrics.md#metrics-modifications) for more information.                                                                                                                                                        | `[]`                                                                                                                                                                                 |

deploy/helm/sumologic/conf/logs/otelcol/config.yaml

@@ -1,16 +1,5 @@
-extensions:
-  health_check: {}
-{{ if .Values.metadata.persistence.enabled }}
-  ## Configuration for File Storage extension
-  file_storage:
-    directory: /var/lib/storage/otc
-    timeout: 10s
-    compaction:
-      on_rebound: true
-      directory: /tmp
-{{ end }}
-  pprof: {}
 exporters:
+{{- if eq .Values.sumologic.logs.sourceType "http" }}
 {{ if .Values.sumologic.logs.container.enabled }}
   sumologic/containers:
     endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
@@ -24,7 +13,7 @@ exporters:
 {{- if .Values.metadata.persistence.enabled }}
       storage: file_storage
 {{- end }}
-{{ end }}
+{{- end }}
 
 {{ if .Values.sumologic.logs.systemd.enabled }}
   sumologic/systemd:
@@ -39,18 +28,33 @@ exporters:
 {{- if .Values.metadata.persistence.enabled }}
       storage: file_storage
 {{- end }}
-{{ end }}
+{{- end }}
+
+{{ else }}
+  sumologic:
+    endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE}
+    log_format: otlp
+    clear_logs_timestamp: false
+    sending_queue:
+      enabled: true
+      num_consumers: 10
+      queue_size: 10_000
+{{- if .Values.metadata.persistence.enabled }}
+      storage: file_storage
+{{- end }}
+{{- end }}
 
 extensions:
+  health_check: {}
 {{ if .Values.metadata.persistence.enabled }}
+  ## Configuration for File Storage extension
   file_storage:
-    compaction:
-      directory: /tmp
-      on_rebound: true
     directory: /var/lib/storage/otc
     timeout: 10s
+    compaction:
+      on_rebound: true
+      directory: /tmp
 {{ end }}
-  health_check: {}
   pprof: {}
 
 processors:
@@ -460,7 +464,7 @@ service:
 {{ if .Values.sumologic.logs.container.enabled }}
     logs/fluent/containers:
       exporters:
-        - sumologic/containers
+{{ include "logs.otelcol.container.exporters" . | indent 8 }}
       processors:
         - memory_limiter
         - filter/include_fluent_tag_containers
@@ -495,7 +499,7 @@ service:
 {{ if .Values.sumologic.logs.systemd.enabled }}
     logs/fluent/kubelet:
       exporters:
-        - sumologic/systemd
+{{ include "logs.otelcol.kubelet.exporters" . | indent 8 }}
       processors:
         - memory_limiter
         - filter/include_fluent_tag_host
@@ -516,7 +520,7 @@ service:
 
     logs/fluent/systemd:
       exporters:
-        - sumologic/systemd
+{{ include "logs.otelcol.systemd.exporters" . | indent 8 }}
       processors:
         - memory_limiter
         - filter/include_fluent_tag_host
@@ -546,7 +550,7 @@ service:
     ## - attributes/fluent_containers functionality is being replaced by otellogs operators
     logs/otlp/containers:
       exporters:
-        - sumologic/containers
+{{ include "logs.otelcol.container.exporters" . | indent 8 }}
       processors:
         - memory_limiter
         - filter/include_containers
@@ -582,7 +586,7 @@ service:
     ## - added transform/remove_attributes processor
     logs/otlp/kubelet:
       exporters:
-        - sumologic/systemd
+{{ include "logs.otelcol.kubelet.exporters" . | indent 8 }}
       processors:
         - memory_limiter
         - filter/include_fluent_tag_host
@@ -608,7 +612,7 @@ service:
     ## - added transform/remove_attributes processor
     logs/otlp/systemd:
       exporters:
-        - sumologic/systemd
+{{ include "logs.otelcol.systemd.exporters" . | indent 8 }}
       processors:
         - memory_limiter
         - filter/include_fluent_tag_host

deploy/helm/sumologic/templates/_helpers/_logs.tpl

@@ -115,6 +115,51 @@ Return the log format for the Sumologic exporter for container logs.
 {{- end -}}
 {{- end -}}
 
+{{/*
+Return the exporters for container log pipeline.
+
+'{{ include "logs.otelcol.container.exporters" . }}'
+*/}}
+{{- define "logs.otelcol.container.exporters" -}}
+{{- if eq .Values.sumologic.logs.sourceType "http" -}}
+- sumologic/containers
+{{- else if eq .Values.sumologic.logs.sourceType "otlp" }}
+- sumologic
+{{- else -}}
+{{- fail "`sumologic.logs.sourceType` can only be `http` or `otlp`" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the exporters for systemd log pipeline.
+
+'{{ include "logs.otelcol.systemd.exporters" . }}'
+*/}}
+{{- define "logs.otelcol.systemd.exporters" -}}
+{{- if eq .Values.sumologic.logs.sourceType "http" -}}
+- sumologic/systemd
+{{- else if eq .Values.sumologic.logs.sourceType "otlp" }}
+- sumologic
+{{- else -}}
+{{- fail "`sumologic.logs.sourceType` can only be `http` or `otlp`" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the exporters for kubelet log pipeline.
+
+'{{ include "logs.otelcol.kubelet.exporters" . }}'
+*/}}
+{{- define "logs.otelcol.kubelet.exporters" -}}
+{{- if eq .Values.sumologic.logs.sourceType "http" }}
+- sumologic/systemd
+{{- else if eq .Values.sumologic.logs.sourceType "otlp" }}
+- sumologic
+{{- else }}
+{{- fail "`sumologic.logs.sourceType` can only be `http` or `otlp`" -}}
+{{- end -}}
+{{- end -}}
+
 {{- define "sumologic.labels.app.logs" -}}
 {{- if eq .Values.sumologic.logs.metadata.provider "fluentd" -}}
 {{ template "sumologic.labels.app.fluentd" . }}-logs

deploy/helm/sumologic/values.yaml

@@ -201,6 +201,11 @@ sumologic:
           #   - name: "Test Exclude Debug"
           #     filter_type: "Exclude"
           #     regexp: ".*DEBUG.*"
+        default-otlp:
+          name: logs-otlp
+          config-name: endpoint-logs-otlp
+          properties:
+            content_type: Otlp
       events:
         default:
           name: events
@@ -417,6 +422,8 @@ sumologic:
       - service
       - statefulset
 
+    sourceType: http
+
   ### Metrics configuration
   ## Set the enabled flag to false for disabling metrics ingestion altogether.
   metrics:

docs/otlp-source.md

@@ -0,0 +1,41 @@
+# Using the OTLP source
+
+Historically, agents used by this Chart sent logs and metrics data to a [generic HTTP Source][http_source] in Sumo Logic. Ever since the
+[version 3][v3] release, the Chart predominantly uses the [OpenTelemetry Collector][otel], and consequently also the [OTLP protocol][otlp].
+The data needed to be converted to the formats the generic HTTP source supports before being sent.
+
+Recently, we've added support for directly sending data using the OTLP protocol to Sumo Logic. This is achieved using the [OTLP
+source][otlp_source]. This document explains how to use this new source with the Helm Chart.
+
+**Note:** This Chart currently only supports OTLP Source for the following signal types: logs
+
+## Enabling the OTLP source
+
+Add the following to your configuration:
+
+```yaml
+sumologic:
+  logs:
+    sourceType: otlp
+```
+
+**Note:** The source is automatically created during Chart installation. This setting simply makes the Chart start sending data to it.
+
+## Benefits
+
+Sending data directly via OTLP is more efficient, as we skip the conversion step. OTLP is also a binary-encoded format, which improves the
+efficiency further.
+
+### Logs
+
+As a structured log format, OTLP frees us from the need to parse metadata out of the log body on the Sumo side. This makes the following
+features work without additional manual configuration:
+
+- multiline parsing for the `text` log format
+- correct timestamps for the `text` log format
+
+[http_source]: https://help.sumologic.com/docs/send-data/hosted-collectors/http-source/logs-metrics/
+[otlp_source]: https://help.sumologic.com/docs/send-data/hosted-collectors/http-source/otlp/
+[v3]: https://github.com/SumoLogic/sumologic-kubernetes-collection/releases/tag/v3.0.0
+[otel]: ./opentelemetry-collector.md
+[otlp]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md

tests/helm/logs_test.go

@@ -261,6 +261,37 @@ sumologic:
 	}
 }
 
+func TestMetadataLogOtlpSource(t *testing.T) {
+	t.Parallel()
+	templatePath := "templates/logs/otelcol/configmap.yaml"
+	valuesYaml := `
+sumologic:
+  logs:
+    sourceType: otlp
+`
+
+	var otelConfig struct {
+		Exporters  map[string]map[string]interface{}
+		Processors map[string]interface{}
+		Service    struct {
+			Pipelines map[string]struct {
+				Receivers  []string
+				Processors []string
+				Exporters  []string
+			}
+		}
+	}
+
+	otelConfigYaml := GetOtelConfigYaml(t, valuesYaml, templatePath)
+	err := yaml.Unmarshal([]byte(otelConfigYaml), &otelConfig)
+	require.NoError(t, err)
+	require.ElementsMatch(t, []string{"sumologic"}, keys(otelConfig.Exporters))
+	require.Equal(t, "otlp", otelConfig.Exporters["sumologic"]["log_format"])
+	require.ElementsMatch(t, []string{"sumologic"}, otelConfig.Service.Pipelines["logs/otlp/containers"].Exporters)
+	require.ElementsMatch(t, []string{"sumologic"}, otelConfig.Service.Pipelines["logs/otlp/systemd"].Exporters)
+	require.ElementsMatch(t, []string{"sumologic"}, otelConfig.Service.Pipelines["logs/otlp/kubelet"].Exporters)
+}
+
 func TestCollectorOtelConfigMerge(t *testing.T) {
 	t.Parallel()
 	templatePath := "templates/logs/collector/otelcol/configmap.yaml"

tests/helm/testdata/goldenfile/metadata_logs_otc_statefulset/basic.output.yaml

@@ -117,6 +117,11 @@ spec:
                 secretKeyRef:
                   name: sumologic
                   key: endpoint-logs
+            - name: SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE
+              valueFrom:
+                secretKeyRef:
+                  name: sumologic
+                  key: endpoint-logs-otlp
 
             - name: NO_PROXY
               value: kubernetes.default.svc

tests/helm/testdata/goldenfile/metadata_logs_otc_statefulset/custom.output.yaml

@@ -140,6 +140,11 @@ spec:
                 secretKeyRef:
                   name: sumologic
                   key: endpoint-logs
+            - name: SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE
+              valueFrom:
+                secretKeyRef:
+                  name: sumologic
+                  key: endpoint-logs-otlp
 
             - name: NO_PROXY
               value: kubernetes.default.svc

tests/helm/testdata/goldenfile/terraform/all_fields.output.yaml

@@ -345,6 +345,7 @@ data:
     locals {
       default_events_source                       = "events"
       default_logs_source                         = "logs"
+      default_otlp_logs_source                    = "logs-otlp"
       apiserver_metrics_source                    = "apiserver-metrics"
       control_plane_metrics_source                = "control-plane-metrics"
       controller_metrics_source                   = "kube-controller-manager-metrics"
@@ -484,6 +485,12 @@ data:
         }
     }
 
+    resource "sumologic_http_source" "default_otlp_logs_source" {
+        name         = local.default_otlp_logs_source
+        collector_id = sumologic_collector.collector.id
+        content_type = "Otlp"
+    }
+
     resource "sumologic_http_source" "apiserver_metrics_source" {
         name         = local.apiserver_metrics_source
         collector_id = sumologic_collector.collector.id
@@ -584,6 +591,7 @@ data:
       data = {
         endpoint-events                           = sumologic_http_source.default_events_source.url
         endpoint-logs                             = sumologic_http_source.default_logs_source.url
+        endpoint-logs-otlp                        = sumologic_http_source.default_otlp_logs_source.url
         endpoint-metrics-apiserver                = sumologic_http_source.apiserver_metrics_source.url
         endpoint-control_plane_metrics_source     = sumologic_http_source.control_plane_metrics_source.url
         endpoint-metrics-kube-controller-manager  = sumologic_http_source.controller_metrics_source.url
@@ -727,6 +735,7 @@ data:
     true  # prevent to render empty if; then
     terraform import sumologic_http_source.default_events_source "${COLLECTOR_NAME}/events"
     terraform import sumologic_http_source.default_logs_source "${COLLECTOR_NAME}/logs"
+    terraform import sumologic_http_source.default_otlp_logs_source "${COLLECTOR_NAME}/logs-otlp"
     terraform import sumologic_http_source.apiserver_metrics_source "${COLLECTOR_NAME}/apiserver-metrics"
     terraform import sumologic_http_source.control_plane_metrics_source "${COLLECTOR_NAME}/control-plane-metrics"
     terraform import sumologic_http_source.controller_metrics_source "${COLLECTOR_NAME}/kube-controller-manager-metrics"

tests/helm/testdata/goldenfile/terraform/collector_fields.output.yaml

@@ -345,6 +345,7 @@ data:
     locals {
       default_events_source                       = "events"
       default_logs_source                         = "logs"
+      default_otlp_logs_source                    = "logs-otlp"
       apiserver_metrics_source                    = "apiserver-metrics"
       control_plane_metrics_source                = "control-plane-metrics"
       controller_metrics_source                   = "kube-controller-manager-metrics"
@@ -485,6 +486,12 @@ data:
         }
     }
 
+    resource "sumologic_http_source" "default_otlp_logs_source" {
+        name         = local.default_otlp_logs_source
+        collector_id = sumologic_collector.collector.id
+        content_type = "Otlp"
+    }
+
     resource "sumologic_http_source" "apiserver_metrics_source" {
         name         = local.apiserver_metrics_source
         collector_id = sumologic_collector.collector.id
@@ -540,6 +547,7 @@ data:
       data = {
         endpoint-events                           = sumologic_http_source.default_events_source.url
         endpoint-logs                             = sumologic_http_source.default_logs_source.url
+        endpoint-logs-otlp                        = sumologic_http_source.default_otlp_logs_source.url
         endpoint-metrics-apiserver                = sumologic_http_source.apiserver_metrics_source.url
         endpoint-control_plane_metrics_source     = sumologic_http_source.control_plane_metrics_source.url
         endpoint-metrics-kube-controller-manager  = sumologic_http_source.controller_metrics_source.url
@@ -682,6 +690,7 @@ data:
     true  # prevent to render empty if; then
     terraform import sumologic_http_source.default_events_source "${COLLECTOR_NAME}/events"
     terraform import sumologic_http_source.default_logs_source "${COLLECTOR_NAME}/logs"
+    terraform import sumologic_http_source.default_otlp_logs_source "${COLLECTOR_NAME}/logs-otlp"
     terraform import sumologic_http_source.apiserver_metrics_source "${COLLECTOR_NAME}/apiserver-metrics"
     terraform import sumologic_http_source.control_plane_metrics_source "${COLLECTOR_NAME}/control-plane-metrics"
     terraform import sumologic_http_source.controller_metrics_source "${COLLECTOR_NAME}/kube-controller-manager-metrics"