From 0de7e888d8a8bd56270b8fe86fa16f3cb80933ad Mon Sep 17 00:00:00 2001 From: Martin Majlis <122797378+martin-majlis-s1@users.noreply.github.com> Date: Mon, 24 Jun 2024 13:31:06 +0200 Subject: [PATCH] [exporter/datasetexporter] Upgrade to dataset-go v0.19.0 (#33675) **Description:** This PR is upgrading dataset-go to version v0.19.0 - https://github.com/scalyr/dataset-go/releases/tag/v0.19.0 This PR is also fixing: * #33498 - config validation is not called during conversion, therefore I had to remove tests from logs and trace exporter. These tests are now moved to the factory. * #32533 - fixing failing integration tests, that were never updated and therefore the configuration contained invalid values **Link to tracking Issue:** #33498, #32533, #33675 **Testing:** Unit tests and integration tests. **Documentation:** fixes #32533 --------- Signed-off-by: Alex Boten <223565+codeboten@users.noreply.github.com> Co-authored-by: Curtis Robert Co-authored-by: Dmitrii Anoshin Co-authored-by: Ishan Shanware Co-authored-by: Evan Bradley <11745660+evan-bradley@users.noreply.github.com> Co-authored-by: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Co-authored-by: Alex Boten <223565+codeboten@users.noreply.github.com> Co-authored-by: Paulo Janotti Co-authored-by: Joshua MacDonald Co-authored-by: Rong Hu --- .../datasetexporter-update-to-0.19.0.yaml | 30 +++++++++++ cmd/otelcontribcol/go.mod | 2 +- cmd/otelcontribcol/go.sum | 4 +- exporter/datasetexporter/config.go | 54 +++++++++---------- exporter/datasetexporter/datasetexporter.go | 8 +-- exporter/datasetexporter/factory_test.go | 23 +++++++- exporter/datasetexporter/go.mod | 2 +- exporter/datasetexporter/go.sum | 4 +- .../logs_exporter_stress_test.go | 11 ++-- .../datasetexporter/logs_exporter_test.go | 24 --------- .../datasetexporter/traces_exporter_test.go | 22 -------- 11 files changed, 91 insertions(+), 93 deletions(-) create mode 100644 .chloggen/datasetexporter-update-to-0.19.0.yaml diff --git a/.chloggen/datasetexporter-update-to-0.19.0.yaml b/.chloggen/datasetexporter-update-to-0.19.0.yaml new file mode 100644 index 0000000000000..afa3e58e520f8 --- /dev/null +++ b/.chloggen/datasetexporter-update-to-0.19.0.yaml @@ -0,0 +1,30 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: 'bug_fix' + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: datasetexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Upgrade dataset-go to v0.19.0 and fix found issues + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [33498, 32533, 33675] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + Upgrade `dataset-go` library from v0.18.0 to v0.19.0. + Enable skipped integration test and adjust the test so it is passing again. + Do not validate configuration, let the framework run the validation. + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/cmd/otelcontribcol/go.mod b/cmd/otelcontribcol/go.mod index e5466d98499ef..61d7906ef996d 100644 --- a/cmd/otelcontribcol/go.mod +++ b/cmd/otelcontribcol/go.mod @@ -691,7 +691,7 @@ require ( github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/samber/lo v1.38.1 // indirect github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25 // indirect - github.com/scalyr/dataset-go v0.18.0 // indirect + github.com/scalyr/dataset-go v0.19.0 // indirect github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect github.com/segmentio/asm v1.2.0 // indirect diff --git a/cmd/otelcontribcol/go.sum b/cmd/otelcontribcol/go.sum index 0528765b43a6b..b3d72d209457a 100644 --- a/cmd/otelcontribcol/go.sum +++ b/cmd/otelcontribcol/go.sum @@ -2105,8 +2105,8 @@ github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25 h1:/8rfZAdFfafRXOgz+ZpMZZWZ5pYggCY9t7e/BvjaBHM= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= -github.com/scalyr/dataset-go v0.18.0 h1:CTv7kk/FGdiicTWo3h1brFusHD1yjhVGINFamP8uukw= -github.com/scalyr/dataset-go v0.18.0/go.mod h1:4x0JK5X0UdhZ2TEO3kHu9pTELDRc3WsrBBwQfkOPZKc= +github.com/scalyr/dataset-go v0.19.0 h1:rlpvUnyKM9PD+b+sXqJVwktkQ+vkecKSVjSs+Z3IioI= +github.com/scalyr/dataset-go v0.19.0/go.mod h1:k/A+KhdEyffuTGb1n+jSWg8J5ikV9iYVKejVGjGXXoA= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds= diff --git a/exporter/datasetexporter/config.go b/exporter/datasetexporter/config.go index 22734ba4719af..3f6fe531708fd 100644 --- a/exporter/datasetexporter/config.go +++ b/exporter/datasetexporter/config.go @@ -202,39 +202,33 @@ func (c *Config) String() string { return s } -func (c *Config) convert() (*ExporterConfig, error) { - err := c.Validate() - if err != nil { - return nil, fmt.Errorf("config is not valid: %w", err) - } - +func (c *Config) convert() *ExporterConfig { return &ExporterConfig{ - datasetConfig: &datasetConfig.DataSetConfig{ - Endpoint: c.DatasetURL, - Tokens: datasetConfig.DataSetTokens{WriteLog: string(c.APIKey)}, - BufferSettings: buffer_config.DataSetBufferSettings{ - MaxLifetime: c.BufferSettings.MaxLifetime, - PurgeOlderThan: c.BufferSettings.PurgeOlderThan, - MaxSize: buffer.LimitBufferSize, - GroupBy: c.BufferSettings.GroupBy, - RetryInitialInterval: c.BufferSettings.RetryInitialInterval, - RetryMaxInterval: c.BufferSettings.RetryMaxInterval, - RetryMaxElapsedTime: c.BufferSettings.RetryMaxElapsedTime, - RetryMultiplier: backoff.DefaultMultiplier, - RetryRandomizationFactor: backoff.DefaultRandomizationFactor, - RetryShutdownTimeout: c.BufferSettings.RetryShutdownTimeout, - }, - ServerHostSettings: server_host_config.DataSetServerHostSettings{ - UseHostName: c.ServerHostSettings.UseHostName, - ServerHost: c.ServerHostSettings.ServerHost, - }, - Debug: c.Debug, + datasetConfig: &datasetConfig.DataSetConfig{ + Endpoint: c.DatasetURL, + Tokens: datasetConfig.DataSetTokens{WriteLog: string(c.APIKey)}, + BufferSettings: buffer_config.DataSetBufferSettings{ + MaxLifetime: c.BufferSettings.MaxLifetime, + PurgeOlderThan: c.BufferSettings.PurgeOlderThan, + MaxSize: buffer.LimitBufferSize, + GroupBy: c.BufferSettings.GroupBy, + RetryInitialInterval: c.BufferSettings.RetryInitialInterval, + RetryMaxInterval: c.BufferSettings.RetryMaxInterval, + RetryMaxElapsedTime: c.BufferSettings.RetryMaxElapsedTime, + RetryMultiplier: backoff.DefaultMultiplier, + RetryRandomizationFactor: backoff.DefaultRandomizationFactor, + RetryShutdownTimeout: c.BufferSettings.RetryShutdownTimeout, }, - tracesSettings: c.TracesSettings, - logsSettings: c.LogsSettings, - serverHostSettings: c.ServerHostSettings, + ServerHostSettings: server_host_config.DataSetServerHostSettings{ + UseHostName: c.ServerHostSettings.UseHostName, + ServerHost: c.ServerHostSettings.ServerHost, + }, + Debug: c.Debug, }, - nil + tracesSettings: c.TracesSettings, + logsSettings: c.LogsSettings, + serverHostSettings: c.ServerHostSettings, + } } type ExporterConfig struct { diff --git a/exporter/datasetexporter/datasetexporter.go b/exporter/datasetexporter/datasetexporter.go index 4166094bd5ae6..8f50db3a2a20c 100644 --- a/exporter/datasetexporter/datasetexporter.go +++ b/exporter/datasetexporter/datasetexporter.go @@ -36,13 +36,7 @@ func newDatasetExporter(entity string, config *Config, set exporter.Settings) (* zap.String("id.string", set.ID.String()), zap.String("id.name", set.ID.Name()), ) - exporterCfg, err := config.convert() - if err != nil { - return nil, fmt.Errorf( - "cannot convert config: %s; %w", - config.String(), err, - ) - } + exporterCfg := config.convert() userAgent := fmt.Sprintf( "%s;%s;%s", "OtelCollector", diff --git a/exporter/datasetexporter/factory_test.go b/exporter/datasetexporter/factory_test.go index 0599415d13292..a2693366d3d17 100644 --- a/exporter/datasetexporter/factory_test.go +++ b/exporter/datasetexporter/factory_test.go @@ -153,6 +153,17 @@ func TestLoadConfig(t *testing.T) { } } +func TestValidateConfigs(t *testing.T) { + tests := createExporterTests() + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + err := component.ValidateConfig(tt.config) + assert.Equal(t, tt.expectedError, err) + }) + } +} + type CreateTest struct { name string config component.Config @@ -169,7 +180,17 @@ func createExporterTests() []CreateTest { { name: "broken", config: &Config{}, - expectedError: fmt.Errorf("cannot get DataSetExporter: cannot convert config: DatasetURL: ; APIKey: [REDACTED] (0); Debug: false; BufferSettings: {MaxLifetime:0s PurgeOlderThan:0s GroupBy:[] RetryInitialInterval:0s RetryMaxInterval:0s RetryMaxElapsedTime:0s RetryShutdownTimeout:0s}; LogsSettings: {ExportResourceInfo:false ExportResourcePrefix: ExportScopeInfo:false ExportScopePrefix: DecomposeComplexMessageField:false DecomposedComplexMessagePrefix: exportSettings:{ExportSeparator: ExportDistinguishingSuffix:}}; TracesSettings: {exportSettings:{ExportSeparator: ExportDistinguishingSuffix:}}; ServerHostSettings: {UseHostName:false ServerHost:}; BackOffConfig: {Enabled:false InitialInterval:0s RandomizationFactor:0 Multiplier:0 MaxInterval:0s MaxElapsedTime:0s}; QueueSettings: {Enabled:false NumConsumers:0 QueueSize:0 StorageID:}; TimeoutSettings: {Timeout:0s}; config is not valid: api_key is required"), + expectedError: fmt.Errorf("api_key is required"), + }, + { + name: "missing-url", + config: &Config{APIKey: "AAA"}, + expectedError: fmt.Errorf("dataset_url is required"), + }, + { + name: "missing-key", + config: &Config{DatasetURL: "bbb"}, + expectedError: fmt.Errorf("api_key is required"), }, { name: "valid", diff --git a/exporter/datasetexporter/go.mod b/exporter/datasetexporter/go.mod index 729ded3bed3cd..9f7aa5e028d30 100644 --- a/exporter/datasetexporter/go.mod +++ b/exporter/datasetexporter/go.mod @@ -6,7 +6,7 @@ require ( github.com/google/uuid v1.6.0 // github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage v0.103.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.103.0 - github.com/scalyr/dataset-go v0.18.0 + github.com/scalyr/dataset-go v0.19.0 github.com/stretchr/testify v1.9.0 go.opentelemetry.io/collector/component v0.103.0 go.opentelemetry.io/collector/confmap v0.103.0 diff --git a/exporter/datasetexporter/go.sum b/exporter/datasetexporter/go.sum index 6845dd9c9f4b9..4a4f3738ca0ea 100644 --- a/exporter/datasetexporter/go.sum +++ b/exporter/datasetexporter/go.sum @@ -60,8 +60,8 @@ github.com/prometheus/procfs v0.15.0 h1:A82kmvXJq2jTu5YUhSGNlYoxh85zLnKgPz4bMZgI github.com/prometheus/procfs v0.15.0/go.mod h1:Y0RJ/Y5g5wJpkTisOtqwDSo4HwhGmLB4VQSw2sQJLHk= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/scalyr/dataset-go v0.18.0 h1:CTv7kk/FGdiicTWo3h1brFusHD1yjhVGINFamP8uukw= -github.com/scalyr/dataset-go v0.18.0/go.mod h1:4x0JK5X0UdhZ2TEO3kHu9pTELDRc3WsrBBwQfkOPZKc= +github.com/scalyr/dataset-go v0.19.0 h1:rlpvUnyKM9PD+b+sXqJVwktkQ+vkecKSVjSs+Z3IioI= +github.com/scalyr/dataset-go v0.19.0/go.mod h1:k/A+KhdEyffuTGb1n+jSWg8J5ikV9iYVKejVGjGXXoA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= diff --git a/exporter/datasetexporter/logs_exporter_stress_test.go b/exporter/datasetexporter/logs_exporter_stress_test.go index 5c3ece7c57a1c..3ef6222fd8f12 100644 --- a/exporter/datasetexporter/logs_exporter_stress_test.go +++ b/exporter/datasetexporter/logs_exporter_stress_test.go @@ -28,8 +28,6 @@ import ( ) func TestConsumeLogsManyLogsShouldSucceed(t *testing.T) { - t.Skip("TODO: Skipping due to https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/32533") - const maxDelay = 200 * time.Millisecond createSettings := exportertest.NewNopSettings() @@ -52,7 +50,7 @@ func TestConsumeLogsManyLogsShouldSucceed(t *testing.T) { for _, ev := range cer.Events { processedEvents.Add(1) - key, found := ev.Attrs["body.str"] + key, found := ev.Attrs["key"] assert.True(t, found) mutex.Lock() sKey := key.(string) @@ -82,11 +80,18 @@ func TestConsumeLogsManyLogsShouldSucceed(t *testing.T) { BufferSettings: BufferSettings{ MaxLifetime: maxDelay, GroupBy: []string{"attributes.container_id"}, + RetryInitialInterval: maxDelay, + RetryMaxInterval: 10 * maxDelay, + RetryMaxElapsedTime: 50 * maxDelay, RetryShutdownTimeout: time.Minute, + PurgeOlderThan: 100 * maxDelay, }, BackOffConfig: configretry.NewDefaultBackOffConfig(), QueueSettings: exporterhelper.NewDefaultQueueSettings(), TimeoutSettings: exporterhelper.NewDefaultTimeoutSettings(), + ServerHostSettings: ServerHostSettings{ + UseHostName: true, + }, } logs, err := createLogsExporter(context.Background(), createSettings, config) diff --git a/exporter/datasetexporter/logs_exporter_test.go b/exporter/datasetexporter/logs_exporter_test.go index 36bfda2dcafcb..7eb223da94fd5 100644 --- a/exporter/datasetexporter/logs_exporter_test.go +++ b/exporter/datasetexporter/logs_exporter_test.go @@ -31,30 +31,6 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/testdata" ) -func TestCreateLogsExporter(t *testing.T) { - ctx := context.Background() - createSettings := exportertest.NewNopSettings() - tests := createExporterTests() - - for _, tt := range tests { - t.Run(tt.name, func(*testing.T) { - logs, err := createLogsExporter(ctx, createSettings, tt.config) - - if err == nil { - assert.Nil(t, tt.expectedError, tt.name) - assert.NotNil(t, logs, tt.name) - } else { - if tt.expectedError == nil { - assert.Nil(t, err, tt.name) - } else { - assert.Equal(t, tt.expectedError.Error(), err.Error(), tt.name) - assert.Nil(t, logs, tt.name) - } - } - }) - } -} - func TestBuildBody(t *testing.T) { slice := pcommon.NewValueSlice() err := slice.FromRaw([]any{1, 2, 3}) diff --git a/exporter/datasetexporter/traces_exporter_test.go b/exporter/datasetexporter/traces_exporter_test.go index fdd9bf91fec88..f74c8f63c0b50 100644 --- a/exporter/datasetexporter/traces_exporter_test.go +++ b/exporter/datasetexporter/traces_exporter_test.go @@ -4,7 +4,6 @@ package datasetexporter import ( - "context" "fmt" "strings" "testing" @@ -12,33 +11,12 @@ import ( "github.com/scalyr/dataset-go/pkg/api/add_events" "github.com/stretchr/testify/assert" - "go.opentelemetry.io/collector/exporter/exportertest" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/ptrace" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/testdata" ) -func TestCreateTracesExporter(t *testing.T) { - ctx := context.Background() - createSettings := exportertest.NewNopSettings() - tests := createExporterTests() - - for _, tt := range tests { - t.Run(tt.name, func(*testing.T) { - logs, err := createTracesExporter(ctx, createSettings, tt.config) - - if err == nil { - assert.Nil(t, tt.expectedError) - assert.NotNil(t, logs) - } else { - assert.Equal(t, tt.expectedError.Error(), err.Error()) - assert.Nil(t, logs) - } - }) - } -} - func generateTEvent1Raw() *add_events.Event { return &add_events.Event{ Thread: "TT",