Skip to content

Commit

Permalink
feat(otrm): Support validating filter processors in the source templates
Browse files Browse the repository at this point in the history
  • Loading branch information
rnishtala-sumo committed Apr 16, 2024
1 parent e791f9d commit 46b660a
Show file tree
Hide file tree
Showing 6 changed files with 165 additions and 0 deletions.
1 change: 1 addition & 0 deletions .changelog/1548.added.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
feat(otrm): Support validating filter processors in the source templates
2 changes: 2 additions & 0 deletions pkg/extension/opampextension/components.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension"
"github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension"
"github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage"
"github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor"
"github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor"
"github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor"
"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/apachereceiver"
Expand Down Expand Up @@ -80,6 +81,7 @@ func Components() (
memorylimiterprocessor.NewFactory(),
resourcedetectionprocessor.NewFactory(),
resourceprocessor.NewFactory(),
filterprocessor.NewFactory(),
)
errs = multierr.Append(errs, err)

Expand Down
5 changes: 5 additions & 0 deletions pkg/extension/opampextension/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ require (
github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.98.0
github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.98.0
github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage v0.98.0
github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.98.0
github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.98.0
github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.98.0
github.com/open-telemetry/opentelemetry-collector-contrib/receiver/apachereceiver v0.98.0
Expand Down Expand Up @@ -59,6 +60,7 @@ require (
github.com/IBM/sarama v1.43.1 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/Showmax/go-fqdn v1.0.0 // indirect
github.com/alecthomas/participle/v2 v2.1.1 // indirect
github.com/apache/thrift v0.20.0 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/aws/aws-sdk-go v1.51.17 // indirect
Expand Down Expand Up @@ -108,6 +110,7 @@ require (
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/hashicorp/serf v0.10.1 // indirect
github.com/iancoleman/strcase v0.3.0 // indirect
github.com/imdario/mergo v0.3.11 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/influxdata/go-syslog/v3 v3.0.1-0.20230911200830-875f5bc594a4 // indirect
Expand Down Expand Up @@ -150,6 +153,8 @@ require (
github.com/open-telemetry/opentelemetry-collector-contrib/internal/kafka v0.98.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/internal/metadataproviders v0.98.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchpersignal v0.98.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.98.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.98.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.98.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure v0.98.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.98.0 // indirect
Expand Down
14 changes: 14 additions & 0 deletions pkg/extension/opampextension/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,12 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/Showmax/go-fqdn v1.0.0 h1:0rG5IbmVliNT5O19Mfuvna9LL7zlHyRfsSvBPZmF9tM=
github.com/Showmax/go-fqdn v1.0.0/go.mod h1:SfrFBzmDCtCGrnHhoDjuvFnKsWjEQX/Q9ARZvOrJAko=
github.com/alecthomas/assert/v2 v2.3.0 h1:mAsH2wmvjsuvyBvAmCtm7zFsBlb8mIHx5ySLVdDZXL0=
github.com/alecthomas/assert/v2 v2.3.0/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ=
github.com/alecthomas/participle/v2 v2.1.1 h1:hrjKESvSqGHzRb4yW1ciisFJ4p3MGYih6icjJvbsmV8=
github.com/alecthomas/participle/v2 v2.1.1/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c=
github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
Expand Down Expand Up @@ -335,7 +341,11 @@ github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR
github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0=
github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI=
github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
Expand Down Expand Up @@ -523,6 +533,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchpersignal v0.
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchpersignal v0.98.0/go.mod h1:90/iHUjO/6DGVa161GDsdbntj1r06DRS8ZQBvw+PNOY=
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.98.0 h1:Y4UbYtpt+AsCVj4W8HKxLSlKNrMLhxypmA34GkwjJGE=
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.98.0/go.mod h1:a1eqRzzePACd9YamtiY0+42P4dn1pB0Ozc0B3P6JGMs=
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.98.0 h1:vgVVJgTRQzgkIf8iODqKiS5VoMUyUPHRQjAUMOeLJt8=
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.98.0/go.mod h1:5RtSFx/r557j1/Sy8+MO+N0ulfEzDwSNwEKo7bdHvYQ=
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.98.0 h1:lZGvpoHCYoEFBDLhnKGGAgGJyX2l2jGZUgC1kTz/esI=
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.98.0/go.mod h1:fOHCXaWnVcKJPY9Jxeoyl7tSuuhZ/PPN92ib8eyjSLQ=
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.98.0 h1:F1Vnzkhg0Trncmvw65Mq1wBKPgX2wI3aGE90J7CXhuU=
Expand All @@ -537,6 +549,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/zipkin
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/zipkin v0.98.0/go.mod h1:CgCTyjyZCHfm+nci5W1gqL55imJQXjuETC5fzg54Pus=
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/winperfcounters v0.98.0 h1:Jq5ya0oIU9gx0XTf8rvnqPwD6CPqq/smsg1N9Mhb6Vg=
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/winperfcounters v0.98.0/go.mod h1:7gAWx+978lf6DlSyOm6fR7E8a4szffwbiAauM7ArPw8=
github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.98.0 h1:+o1Jm65Yp8Z/EXHwimEfWQlRrnYNpB0ITAH9TFlY96c=
github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.98.0/go.mod h1:dHnrsXrapzrkj/G5zIYdavY/5d+n1J9qeYXKrepLYf4=
github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.98.0 h1:DBCsfHZJt6zEL6aBcZZhHSnKjcLcBbI1c9IAPK1/aLg=
github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.98.0/go.mod h1:khpK0DS7FJzQmRo2alCBFK4pa2bV9NPLQTLqfx43REI=
github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.98.0 h1:+xPdlUjZiMTRrZK059U8zPP/IlhYoDt8jT+WpdT+9WE=
Expand Down
40 changes: 40 additions & 0 deletions pkg/extension/opampextension/opamp_agent_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,46 @@ func TestApplyRemoteConfigMissingProcessor(t *testing.T) {
assert.Equal(t, len(o.effectiveConfig), 0)
}

func TestApplyFilterProcessorConfig(t *testing.T) {
d, err := os.MkdirTemp("", "opamp.d")
assert.NoError(t, err)
defer os.RemoveAll(d)

cfg := createDefaultConfig().(*Config)
cfg.RemoteConfigurationDirectory = d
set := extensiontest.NewNopCreateSettings()
o, err := newOpampAgent(cfg, set.Logger, set.BuildInfo, set.Resource)
assert.NoError(t, err)

assert.Equal(t, len(o.effectiveConfig), 0)

path := filepath.Join("testdata", "opamp.d", "opamp-filter-processor.yaml")
rb, err := os.ReadFile(path)
assert.NoError(t, err)

rc := &protobufs.AgentRemoteConfig{
Config: &protobufs.AgentConfigMap{
ConfigMap: map[string]*protobufs.AgentConfigFile{
"default": {
Body: rb,
},
},
},
ConfigHash: []byte("b2b1e3e7f45d564db1c0b621bbf67008"),
}

changed, err := o.applyRemoteConfig(rc)
assert.NoError(t, err)
assert.True(t, changed)
assert.NotEqual(t, len(o.effectiveConfig), 0)

cfg.AcceptsRemoteConfiguration = false
changed, err = o.applyRemoteConfig(rc)
assert.False(t, changed)
assert.Error(t, err)
assert.Equal(t, "OpAMP agent does not accept remote configuration", err.Error())
}

func TestShutdown(t *testing.T) {
cfg := createDefaultConfig().(*Config)
cfg.ClientConfig.Auth = nil
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
receivers:
hostmetrics/09c98ea4-6686-4ab9-b3cd-f04070906441:
collection_interval: 5m
scrapers:
network:
windowseventlog/Application/09c98ea4-6686-4ab9-b3cd-f04070906441:
channel: Application
operators:
- type: add
field: resource["_sourceName"]
value: Application
windowseventlog/Security/09c98ea4-6686-4ab9-b3cd-f04070906441:
channel: Security
operators:
- type: add
field: resource["_sourceName"]
value: Security
windowseventlog/System/09c98ea4-6686-4ab9-b3cd-f04070906441:
channel: System
operators:
- type: add
field: resource["_sourceName"]
value: System
windowseventlog/Microsoft-Windows-PowerShell/Operational/09c98ea4-6686-4ab9-b3cd-f04070906441:
channel: Microsoft-Windows-PowerShell/Operational
operators:
- type: add
field: resource["_sourceName"]
value: Microsoft-Windows-PowerShell/Operational
windowseventlog/Amazon EC2Launch/09c98ea4-6686-4ab9-b3cd-f04070906441:
channel: Amazon EC2Launch
operators:
- type: add
field: resource["_sourceName"]
value: Amazon EC2Launch
processors:
memory_limiter:
check_interval: 5s
limit_percentage: 75
spike_limit_percentage: 20
batch:
send_batch_size: 1_024
timeout: 1s
resourcedetection/system:
detectors: ["system"]
system:
hostname_sources: ["dns","os"]
resource/metrics/09c98ea4-6686-4ab9-b3cd-f04070906441:
attributes:
- key: _sourceName
value: windows
action: insert
resource/09c98ea4-6686-4ab9-b3cd-f04070906441:
attributes:
- key: _source
value: sample_win_st
action: insert
- key: _contentType
value: OpenTelemetry
action: insert
- key: sumo.datasource
value: windows
action: insert
filter/logs/09c98ea4-6686-4ab9-b3cd-f04070906441:
logs:
log_record:
- not(IsMatch(body, ".*\"id\":40962.*") or IsMatch(body, ".*\"id\":8197.*"))
- IsMatch(body, ".*\"id\":8197.*")
exporters:
sumologic:
sending_queue:
enabled: true
storage: file_storage
retry_on_failure:
max_elapsed_time: 0
service:
pipelines:
metrics/windows/09c98ea4-6686-4ab9-b3cd-f04070906441:
receivers:
- hostmetrics/09c98ea4-6686-4ab9-b3cd-f04070906441
processors:
- memory_limiter
- resourcedetection/system
- batch
- resource/metrics/09c98ea4-6686-4ab9-b3cd-f04070906441
- resource/09c98ea4-6686-4ab9-b3cd-f04070906441
exporters:
- sumologic
logs/windows/09c98ea4-6686-4ab9-b3cd-f04070906441:
receivers:
- windowseventlog/Application/09c98ea4-6686-4ab9-b3cd-f04070906441
- windowseventlog/Security/09c98ea4-6686-4ab9-b3cd-f04070906441
- windowseventlog/System/09c98ea4-6686-4ab9-b3cd-f04070906441
- windowseventlog/Microsoft-Windows-PowerShell/Operational/09c98ea4-6686-4ab9-b3cd-f04070906441
- windowseventlog/Amazon EC2Launch/09c98ea4-6686-4ab9-b3cd-f04070906441
processors:
- memory_limiter
- resourcedetection/system
- batch
- resource/09c98ea4-6686-4ab9-b3cd-f04070906441
- filter/logs/09c98ea4-6686-4ab9-b3cd-f04070906441
exporters:
- sumologic

0 comments on commit 46b660a

Please sign in to comment.