Skip to content

Commit

Permalink
add batcher config to splunkhec exporter (#32563)
Browse files Browse the repository at this point in the history
**Description:** 
Add a new feature in splunk hec exporter
- utilitize batching framework
open-telemetry/opentelemetry-collector#9738 in
splunk hec exporter
- adds batcher config in splunk hec exporter config and append it as
exporter option if enabled

**Link to tracking Issue:**
Resolves
#32545

**Testing:**
Added unit test for the config

**Documentation:**
Updated the README file
  • Loading branch information
splunkericl committed May 4, 2024
1 parent effd258 commit df40aae
Show file tree
Hide file tree
Showing 6 changed files with 63 additions and 3 deletions.
27 changes: 27 additions & 0 deletions .chloggen/exporter-splunkhec-addbatcherconfig.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Use this changelog template to create an entry for release notes.

# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: enhancement

# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
component: exporter/splunkhec

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: add experimental exporter batcher config

# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
issues: [32545]

# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext:

# If your change doesn't affect end users or the exported elements of any package,
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
# Optional: The change log or logs in which this entry should be included.
# e.g. '[user]' or '[user, api]'
# Include 'user' if the change is relevant to end users.
# Include 'api' if there is a change to a library API.
# Default: '[user]'
change_logs: [user]
1 change: 1 addition & 0 deletions exporter/splunkhecexporter/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ The following configuration options can also be configured:
- `telemetry/enabled` (default: false): Specifies whether to enable telemetry inside splunk hec exporter.
- `telemetry/override_metrics_names` (default: empty map): Specifies the metrics name to overrides in splunk hec exporter.
- `telemetry/extra_attributes` (default: empty map): Specifies the extra metrics attributes in splunk hec exporter.
- `batcher`(Experimental, disabled by default): Specifies batching configuration on the exporter. Information about the configuration can be found [here](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md)

In addition, this exporter offers queued retry which is enabled by default.
Information about queued retry configuration parameters can be found
Expand Down
5 changes: 5 additions & 0 deletions exporter/splunkhecexporter/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"go.opentelemetry.io/collector/config/confighttp"
"go.opentelemetry.io/collector/config/configopaque"
"go.opentelemetry.io/collector/config/configretry"
"go.opentelemetry.io/collector/exporter/exporterbatcher"
"go.opentelemetry.io/collector/exporter/exporterhelper"

"github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk"
Expand Down Expand Up @@ -68,6 +69,10 @@ type Config struct {
exporterhelper.QueueSettings `mapstructure:"sending_queue"`
configretry.BackOffConfig `mapstructure:"retry_on_failure"`

// Experimental: This configuration is at the early stage of development and may change without backward compatibility
// until https://github.com/open-telemetry/opentelemetry-collector/issues/8122 is resolved.
BatcherConfig exporterbatcher.Config `mapstructure:"batcher"`

// LogDataEnabled can be used to disable sending logs by the exporter.
LogDataEnabled bool `mapstructure:"log_data_enabled"`

Expand Down
11 changes: 11 additions & 0 deletions exporter/splunkhecexporter/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"go.opentelemetry.io/collector/config/configretry"
"go.opentelemetry.io/collector/config/configtls"
"go.opentelemetry.io/collector/confmap/confmaptest"
"go.opentelemetry.io/collector/exporter/exporterbatcher"
"go.opentelemetry.io/collector/exporter/exporterhelper"

"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter/internal/metadata"
Expand Down Expand Up @@ -90,6 +91,16 @@ func TestLoadConfig(t *testing.T) {
NumConsumers: 2,
QueueSize: 10,
},
BatcherConfig: exporterbatcher.Config{
Enabled: true,
FlushTimeout: time.Second,
MinSizeConfig: exporterbatcher.MinSizeConfig{
MinSizeItems: 1,
},
MaxSizeConfig: exporterbatcher.MaxSizeConfig{
MaxSizeItems: 10,
},
},
HecToOtelAttrs: splunk.HecToOtelAttrs{
Source: "mysource",
SourceType: "mysourcetype",
Expand Down
17 changes: 14 additions & 3 deletions exporter/splunkhecexporter/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"go.opentelemetry.io/collector/config/configretry"
"go.opentelemetry.io/collector/consumer"
"go.opentelemetry.io/collector/exporter"
"go.opentelemetry.io/collector/exporter/exporterbatcher"
"go.opentelemetry.io/collector/exporter/exporterhelper"
conventions "go.opentelemetry.io/collector/semconv/v1.6.1"

Expand Down Expand Up @@ -58,6 +59,9 @@ func NewFactory() exporter.Factory {
}

func createDefaultConfig() component.Config {
batcherCfg := exporterbatcher.NewDefaultConfig()
batcherCfg.Enabled = false

defaultMaxConns := defaultMaxIdleCons
defaultIdleConnTimeout := defaultIdleConnTimeout
return &Config{
Expand All @@ -74,6 +78,7 @@ func createDefaultConfig() component.Config {
SplunkAppName: defaultSplunkAppName,
BackOffConfig: configretry.NewDefaultBackOffConfig(),
QueueSettings: exporterhelper.NewDefaultQueueSettings(),
BatcherConfig: batcherCfg,
DisableCompression: false,
MaxContentLengthLogs: defaultContentLengthLogsLimit,
MaxContentLengthMetrics: defaultContentLengthMetricsLimit,
Expand Down Expand Up @@ -119,7 +124,9 @@ func createTracesExporter(
exporterhelper.WithRetry(cfg.BackOffConfig),
exporterhelper.WithQueue(cfg.QueueSettings),
exporterhelper.WithStart(c.start),
exporterhelper.WithShutdown(c.stop))
exporterhelper.WithShutdown(c.stop),
exporterhelper.WithBatcher(cfg.BatcherConfig),
)

if err != nil {
return nil, err
Expand Down Expand Up @@ -152,7 +159,9 @@ func createMetricsExporter(
exporterhelper.WithRetry(cfg.BackOffConfig),
exporterhelper.WithQueue(cfg.QueueSettings),
exporterhelper.WithStart(c.start),
exporterhelper.WithShutdown(c.stop))
exporterhelper.WithShutdown(c.stop),
exporterhelper.WithBatcher(cfg.BatcherConfig),
)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -184,7 +193,9 @@ func createLogsExporter(
exporterhelper.WithRetry(cfg.BackOffConfig),
exporterhelper.WithQueue(cfg.QueueSettings),
exporterhelper.WithStart(c.start),
exporterhelper.WithShutdown(c.stop))
exporterhelper.WithShutdown(c.stop),
exporterhelper.WithBatcher(cfg.BatcherConfig),
)

if err != nil {
return nil, err
Expand Down
5 changes: 5 additions & 0 deletions exporter/splunkhecexporter/testdata/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ splunk_hec/allsettings:
initial_interval: 10s
max_interval: 60s
max_elapsed_time: 10m
batcher:
enabled: true
flush_timeout: 1s
min_size_items: 1
max_size_items: 10
splunk_app_name: "OpenTelemetry-Collector Splunk Exporter"
splunk_app_version: "v0.0.1"
hec_metadata_to_otel_attrs:
Expand Down

0 comments on commit df40aae

Please sign in to comment.