[exporter/datasetexporter] Various improvements and fixes for the Dat…

…aSet Log Exporter plugin (#23672)

# Related issue
#20660

## Remove OtelExporter - Log - prefix from all the DataSet event message
fields

This prefix was added in the past for testing purposes, but it's not
desired.

## Correctly handle LogRecord severity

The code has been updated to correctly map ``LogRecord.SeverityNumber``
and ``LogRecord.SeverityText`` field value to the DataSet event log
event severity (https://app.scalyr.com/help/parsing-logs#specialAttrs,
https://github.com/scalyr/logstash-output-scalyr/blob/master/lib/logstash/outputs/scalyr.rb#L70)

Redundant `severity.number` and `severity.text` attribute has also been
removed from DataSet event.

## Remove redundant flags and flags.is_sampled attribute

I removed redundant `flags` and `flags.is_sampled` attribute from the
DataSet events. DataSet doesn't do anything with those fields and they
are also redundant - if event is ingested into DataSet and displayed in
the UI this means it has already been sampled so `flags.is_sampled`
tells us no new information.

---------

Co-authored-by: tomas.zdara <tomas.zdara@sentinelone.com>

.chloggen/dataset-exporter-various-improvements-and-fixes.yaml

@@ -0,0 +1,20 @@
+# Use this changelog template to create an entry for release notes.
+# If your change doesn't affect end users, such as a test fix or a tooling change,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: exporter/datasetexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Correctly map LogRecord severity to DataSet severity, remove redundant DataSet event message field prefix (OtelExporter - Log -) and remove redundant DataSet event fields (flags, flags.is_sampled)."
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [20660, 23672]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

exporter/datasetexporter/logs_exporter.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/scalyr/dataset-go/pkg/api/add_events"
@@ -19,6 +20,21 @@ import (
 
 var now = time.Now
 
+// If a LogRecord doesn't contain severity or we can't map it to a valid DataSet severity, we use
+// this value (3 - INFO) instead
+const defaultDataSetSeverityLevel int = dataSetLogLevelInfo
+
+// Constants for valid DataSet log levels (aka Event.sev int field value)
+const (
+	dataSetLogLevelFinest = 0
+	dataSetLogLevelTrace  = 1
+	dataSetLogLevelDebug  = 2
+	dataSetLogLevelInfo   = 3
+	dataSetLogLevelWarn   = 4
+	dataSetLogLevelError  = 5
+	dataSetLogLevelFatal  = 6
+)
+
 func createLogsExporter(ctx context.Context, set exporter.CreateSettings, config component.Config) (exporter.Logs, error) {
 	cfg := castConfig(config)
 	e, err := newDatasetExporter("logs", cfg, set)
@@ -65,6 +81,90 @@ func buildBody(attrs map[string]interface{}, value pcommon.Value) string {
 	return message
 }
 
+// Function maps OTel severity on the LogRecord to DataSet severity level (number)
+func mapOtelSeverityToDataSetSeverity(log plog.LogRecord) int {
+	// This function maps OTel severity level to DataSet severity levels
+	//
+	// Valid OTel levels - https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber
+	// and valid DataSet ones - https://github.com/scalyr/logstash-output-scalyr/blob/master/lib/logstash/outputs/scalyr.rb#L70
+	sevNum := log.SeverityNumber()
+	sevText := log.SeverityText()
+
+	dataSetSeverity := defaultDataSetSeverityLevel
+
+	if sevNum > 0 {
+		dataSetSeverity = mapLogRecordSevNumToDataSetSeverity(sevNum)
+	} else if sevText != "" {
+		// Per docs, SeverityNumber is optional so if it's not present we fall back to SeverityText
+		// https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitytext
+		dataSetSeverity = mapLogRecordSeverityTextToDataSetSeverity(sevText)
+	}
+
+	return dataSetSeverity
+}
+
+func mapLogRecordSevNumToDataSetSeverity(sevNum plog.SeverityNumber) int {
+	// Maps LogRecord.SeverityNumber field value to DataSet severity value.
+	dataSetSeverity := defaultDataSetSeverityLevel
+
+	if sevNum <= 0 {
+		return dataSetSeverity
+	}
+
+	// See https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber
+	// for OTEL mappings
+	switch sevNum {
+	case 1, 2, 3, 4:
+		// TRACE
+		dataSetSeverity = dataSetLogLevelTrace
+	case 5, 6, 7, 8:
+		// DEBUG
+		dataSetSeverity = dataSetLogLevelDebug
+	case 9, 10, 11, 12:
+		// INFO
+		dataSetSeverity = dataSetLogLevelInfo
+	case 13, 14, 15, 16:
+		// WARN
+		dataSetSeverity = dataSetLogLevelWarn
+	case 17, 18, 19, 20:
+		// ERROR
+		dataSetSeverity = dataSetLogLevelError
+	case 21, 22, 23, 24:
+		// FATAL / CRITICAL / EMERGENCY
+		dataSetSeverity = dataSetLogLevelFatal
+	}
+
+	return dataSetSeverity
+}
+
+func mapLogRecordSeverityTextToDataSetSeverity(sevText string) int {
+	// Maps LogRecord.SeverityText field value to DataSet severity value.
+	dataSetSeverity := defaultDataSetSeverityLevel
+
+	if sevText == "" {
+		return dataSetSeverity
+	}
+
+	switch strings.ToLower(sevText) {
+	case "fine", "finest":
+		dataSetSeverity = dataSetLogLevelFinest
+	case "trace":
+		dataSetSeverity = dataSetLogLevelTrace
+	case "debug":
+		dataSetSeverity = dataSetLogLevelDebug
+	case "info", "information":
+		dataSetSeverity = dataSetLogLevelInfo
+	case "warn", "warning":
+		dataSetSeverity = dataSetLogLevelWarn
+	case "error":
+		dataSetSeverity = dataSetLogLevelError
+	case "fatal", "critical", "emergency":
+		dataSetSeverity = dataSetLogLevelFatal
+	}
+
+	return dataSetSeverity
+}
+
 func buildEventFromLog(
 	log plog.LogRecord,
 	resource pcommon.Resource,
@@ -75,30 +175,22 @@ func buildEventFromLog(
 	event := add_events.Event{}
 
 	observedTs := log.ObservedTimestamp().AsTime()
-	if sevNum := log.SeverityNumber(); sevNum > 0 {
-		attrs["severity.number"] = sevNum
-		event.Sev = int(sevNum)
-	}
+
+	event.Sev = mapOtelSeverityToDataSetSeverity(log)
 
 	if timestamp := log.Timestamp().AsTime(); !timestamp.Equal(time.Unix(0, 0)) {
 		event.Ts = strconv.FormatInt(timestamp.UnixNano(), 10)
 	}
 
 	if body := log.Body().AsString(); body != "" {
-		attrs["message"] = fmt.Sprintf(
-			"OtelExporter - Log - %s",
-			buildBody(attrs, log.Body()),
-		)
+		attrs["message"] = buildBody(attrs, log.Body())
 	}
 	if dropped := log.DroppedAttributesCount(); dropped > 0 {
 		attrs["dropped_attributes_count"] = dropped
 	}
 	if !observedTs.Equal(time.Unix(0, 0)) {
 		attrs["observed.timestamp"] = observedTs.String()
 	}
-	if sevText := log.SeverityText(); sevText != "" {
-		attrs["severity.text"] = sevText
-	}
 	if span := log.SpanID().String(); span != "" {
 		attrs["span_id"] = span
 	}
@@ -111,9 +203,6 @@ func buildEventFromLog(
 	if event.Ts == "" {
 		// ObservedTimestamp should always be set, but in case if it's not, we fall back to
 		// current time
-		// TODO: We should probably also do a rate limited log message here since this
-		// could indicate an issue with the current setup in case most events don't contain
-		// a timestamp.
 		if !observedTs.Equal(time.Unix(0, 0)) {
 			event.Ts = strconv.FormatInt(observedTs.UnixNano(), 10)
 		} else {
@@ -122,8 +211,6 @@ func buildEventFromLog(
 	}
 
 	updateWithPrefixedValues(attrs, "attributes.", ".", log.Attributes().AsRaw(), 0)
-	attrs["flags"] = log.Flags()
-	attrs["flag.is_sampled"] = log.Flags().IsSampled()
 
 	if settings.ExportResourceInfo {
 		updateWithPrefixedValues(attrs, "resource.attributes.", ".", resource.Attributes().AsRaw(), 0)