From 46b660aa56986b25ce888494863cfe1e26bfa37b Mon Sep 17 00:00:00 2001 From: Raj Nishtala Date: Tue, 16 Apr 2024 10:08:41 -0400 Subject: [PATCH] feat(otrm): Support validating filter processors in the source templates --- .changelog/1548.added.txt | 1 + pkg/extension/opampextension/components.go | 2 + pkg/extension/opampextension/go.mod | 5 + pkg/extension/opampextension/go.sum | 14 +++ .../opampextension/opamp_agent_test.go | 40 +++++++ .../opamp.d/opamp-filter-processor.yaml | 103 ++++++++++++++++++ 6 files changed, 165 insertions(+) create mode 100644 .changelog/1548.added.txt create mode 100644 pkg/extension/opampextension/testdata/opamp.d/opamp-filter-processor.yaml diff --git a/.changelog/1548.added.txt b/.changelog/1548.added.txt new file mode 100644 index 0000000000..6340fa1a40 --- /dev/null +++ b/.changelog/1548.added.txt @@ -0,0 +1 @@ +feat(otrm): Support validating filter processors in the source templates \ No newline at end of file diff --git a/pkg/extension/opampextension/components.go b/pkg/extension/opampextension/components.go index 336f3d2e55..956450b45d 100644 --- a/pkg/extension/opampextension/components.go +++ b/pkg/extension/opampextension/components.go @@ -20,6 +20,7 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension" "github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension" "github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/apachereceiver" @@ -80,6 +81,7 @@ func Components() ( memorylimiterprocessor.NewFactory(), resourcedetectionprocessor.NewFactory(), resourceprocessor.NewFactory(), + filterprocessor.NewFactory(), ) errs = multierr.Append(errs, err) diff --git a/pkg/extension/opampextension/go.mod b/pkg/extension/opampextension/go.mod index ada5ce73dd..128fff4722 100644 --- a/pkg/extension/opampextension/go.mod +++ b/pkg/extension/opampextension/go.mod @@ -17,6 +17,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.98.0 github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.98.0 github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage v0.98.0 + github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.98.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.98.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.98.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/apachereceiver v0.98.0 @@ -59,6 +60,7 @@ require ( github.com/IBM/sarama v1.43.1 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/Showmax/go-fqdn v1.0.0 // indirect + github.com/alecthomas/participle/v2 v2.1.1 // indirect github.com/apache/thrift v0.20.0 // indirect github.com/armon/go-metrics v0.4.1 // indirect github.com/aws/aws-sdk-go v1.51.17 // indirect @@ -108,6 +110,7 @@ require ( github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/serf v0.10.1 // indirect + github.com/iancoleman/strcase v0.3.0 // indirect github.com/imdario/mergo v0.3.11 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/influxdata/go-syslog/v3 v3.0.1-0.20230911200830-875f5bc594a4 // indirect @@ -150,6 +153,8 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/internal/kafka v0.98.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/metadataproviders v0.98.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchpersignal v0.98.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.98.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.98.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.98.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure v0.98.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.98.0 // indirect diff --git a/pkg/extension/opampextension/go.sum b/pkg/extension/opampextension/go.sum index 94e8a73d5a..ec2e31de05 100644 --- a/pkg/extension/opampextension/go.sum +++ b/pkg/extension/opampextension/go.sum @@ -62,6 +62,12 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Showmax/go-fqdn v1.0.0 h1:0rG5IbmVliNT5O19Mfuvna9LL7zlHyRfsSvBPZmF9tM= github.com/Showmax/go-fqdn v1.0.0/go.mod h1:SfrFBzmDCtCGrnHhoDjuvFnKsWjEQX/Q9ARZvOrJAko= +github.com/alecthomas/assert/v2 v2.3.0 h1:mAsH2wmvjsuvyBvAmCtm7zFsBlb8mIHx5ySLVdDZXL0= +github.com/alecthomas/assert/v2 v2.3.0/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= +github.com/alecthomas/participle/v2 v2.1.1 h1:hrjKESvSqGHzRb4yW1ciisFJ4p3MGYih6icjJvbsmV8= +github.com/alecthomas/participle/v2 v2.1.1/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c= +github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk= +github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -335,7 +341,11 @@ github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= +github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= @@ -523,6 +533,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchpersignal v0. github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchpersignal v0.98.0/go.mod h1:90/iHUjO/6DGVa161GDsdbntj1r06DRS8ZQBvw+PNOY= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.98.0 h1:Y4UbYtpt+AsCVj4W8HKxLSlKNrMLhxypmA34GkwjJGE= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.98.0/go.mod h1:a1eqRzzePACd9YamtiY0+42P4dn1pB0Ozc0B3P6JGMs= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.98.0 h1:vgVVJgTRQzgkIf8iODqKiS5VoMUyUPHRQjAUMOeLJt8= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.98.0/go.mod h1:5RtSFx/r557j1/Sy8+MO+N0ulfEzDwSNwEKo7bdHvYQ= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.98.0 h1:lZGvpoHCYoEFBDLhnKGGAgGJyX2l2jGZUgC1kTz/esI= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.98.0/go.mod h1:fOHCXaWnVcKJPY9Jxeoyl7tSuuhZ/PPN92ib8eyjSLQ= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.98.0 h1:F1Vnzkhg0Trncmvw65Mq1wBKPgX2wI3aGE90J7CXhuU= @@ -537,6 +549,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/zipkin github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/zipkin v0.98.0/go.mod h1:CgCTyjyZCHfm+nci5W1gqL55imJQXjuETC5fzg54Pus= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/winperfcounters v0.98.0 h1:Jq5ya0oIU9gx0XTf8rvnqPwD6CPqq/smsg1N9Mhb6Vg= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/winperfcounters v0.98.0/go.mod h1:7gAWx+978lf6DlSyOm6fR7E8a4szffwbiAauM7ArPw8= +github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.98.0 h1:+o1Jm65Yp8Z/EXHwimEfWQlRrnYNpB0ITAH9TFlY96c= +github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.98.0/go.mod h1:dHnrsXrapzrkj/G5zIYdavY/5d+n1J9qeYXKrepLYf4= github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.98.0 h1:DBCsfHZJt6zEL6aBcZZhHSnKjcLcBbI1c9IAPK1/aLg= github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.98.0/go.mod h1:khpK0DS7FJzQmRo2alCBFK4pa2bV9NPLQTLqfx43REI= github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.98.0 h1:+xPdlUjZiMTRrZK059U8zPP/IlhYoDt8jT+WpdT+9WE= diff --git a/pkg/extension/opampextension/opamp_agent_test.go b/pkg/extension/opampextension/opamp_agent_test.go index 42180957e7..dc2a715c67 100644 --- a/pkg/extension/opampextension/opamp_agent_test.go +++ b/pkg/extension/opampextension/opamp_agent_test.go @@ -404,6 +404,46 @@ func TestApplyRemoteConfigMissingProcessor(t *testing.T) { assert.Equal(t, len(o.effectiveConfig), 0) } +func TestApplyFilterProcessorConfig(t *testing.T) { + d, err := os.MkdirTemp("", "opamp.d") + assert.NoError(t, err) + defer os.RemoveAll(d) + + cfg := createDefaultConfig().(*Config) + cfg.RemoteConfigurationDirectory = d + set := extensiontest.NewNopCreateSettings() + o, err := newOpampAgent(cfg, set.Logger, set.BuildInfo, set.Resource) + assert.NoError(t, err) + + assert.Equal(t, len(o.effectiveConfig), 0) + + path := filepath.Join("testdata", "opamp.d", "opamp-filter-processor.yaml") + rb, err := os.ReadFile(path) + assert.NoError(t, err) + + rc := &protobufs.AgentRemoteConfig{ + Config: &protobufs.AgentConfigMap{ + ConfigMap: map[string]*protobufs.AgentConfigFile{ + "default": { + Body: rb, + }, + }, + }, + ConfigHash: []byte("b2b1e3e7f45d564db1c0b621bbf67008"), + } + + changed, err := o.applyRemoteConfig(rc) + assert.NoError(t, err) + assert.True(t, changed) + assert.NotEqual(t, len(o.effectiveConfig), 0) + + cfg.AcceptsRemoteConfiguration = false + changed, err = o.applyRemoteConfig(rc) + assert.False(t, changed) + assert.Error(t, err) + assert.Equal(t, "OpAMP agent does not accept remote configuration", err.Error()) +} + func TestShutdown(t *testing.T) { cfg := createDefaultConfig().(*Config) cfg.ClientConfig.Auth = nil diff --git a/pkg/extension/opampextension/testdata/opamp.d/opamp-filter-processor.yaml b/pkg/extension/opampextension/testdata/opamp.d/opamp-filter-processor.yaml new file mode 100644 index 0000000000..2000aa9ec3 --- /dev/null +++ b/pkg/extension/opampextension/testdata/opamp.d/opamp-filter-processor.yaml @@ -0,0 +1,103 @@ + receivers: + hostmetrics/09c98ea4-6686-4ab9-b3cd-f04070906441: + collection_interval: 5m + scrapers: + network: + windowseventlog/Application/09c98ea4-6686-4ab9-b3cd-f04070906441: + channel: Application + operators: + - type: add + field: resource["_sourceName"] + value: Application + windowseventlog/Security/09c98ea4-6686-4ab9-b3cd-f04070906441: + channel: Security + operators: + - type: add + field: resource["_sourceName"] + value: Security + windowseventlog/System/09c98ea4-6686-4ab9-b3cd-f04070906441: + channel: System + operators: + - type: add + field: resource["_sourceName"] + value: System + windowseventlog/Microsoft-Windows-PowerShell/Operational/09c98ea4-6686-4ab9-b3cd-f04070906441: + channel: Microsoft-Windows-PowerShell/Operational + operators: + - type: add + field: resource["_sourceName"] + value: Microsoft-Windows-PowerShell/Operational + windowseventlog/Amazon EC2Launch/09c98ea4-6686-4ab9-b3cd-f04070906441: + channel: Amazon EC2Launch + operators: + - type: add + field: resource["_sourceName"] + value: Amazon EC2Launch + processors: + memory_limiter: + check_interval: 5s + limit_percentage: 75 + spike_limit_percentage: 20 + batch: + send_batch_size: 1_024 + timeout: 1s + resourcedetection/system: + detectors: ["system"] + system: + hostname_sources: ["dns","os"] + resource/metrics/09c98ea4-6686-4ab9-b3cd-f04070906441: + attributes: + - key: _sourceName + value: windows + action: insert + resource/09c98ea4-6686-4ab9-b3cd-f04070906441: + attributes: + - key: _source + value: sample_win_st + action: insert + - key: _contentType + value: OpenTelemetry + action: insert + - key: sumo.datasource + value: windows + action: insert + filter/logs/09c98ea4-6686-4ab9-b3cd-f04070906441: + logs: + log_record: + - not(IsMatch(body, ".*\"id\":40962.*") or IsMatch(body, ".*\"id\":8197.*")) + - IsMatch(body, ".*\"id\":8197.*") + exporters: + sumologic: + sending_queue: + enabled: true + storage: file_storage + retry_on_failure: + max_elapsed_time: 0 + service: + pipelines: + metrics/windows/09c98ea4-6686-4ab9-b3cd-f04070906441: + receivers: + - hostmetrics/09c98ea4-6686-4ab9-b3cd-f04070906441 + processors: + - memory_limiter + - resourcedetection/system + - batch + - resource/metrics/09c98ea4-6686-4ab9-b3cd-f04070906441 + - resource/09c98ea4-6686-4ab9-b3cd-f04070906441 + exporters: + - sumologic + logs/windows/09c98ea4-6686-4ab9-b3cd-f04070906441: + receivers: + - windowseventlog/Application/09c98ea4-6686-4ab9-b3cd-f04070906441 + - windowseventlog/Security/09c98ea4-6686-4ab9-b3cd-f04070906441 + - windowseventlog/System/09c98ea4-6686-4ab9-b3cd-f04070906441 + - windowseventlog/Microsoft-Windows-PowerShell/Operational/09c98ea4-6686-4ab9-b3cd-f04070906441 + - windowseventlog/Amazon EC2Launch/09c98ea4-6686-4ab9-b3cd-f04070906441 + processors: + - memory_limiter + - resourcedetection/system + - batch + - resource/09c98ea4-6686-4ab9-b3cd-f04070906441 + - filter/logs/09c98ea4-6686-4ab9-b3cd-f04070906441 + exporters: + - sumologic