Skip to content

Commit

Permalink
NR-279356: Add new multiline parser config param to logging config (#…
Browse files Browse the repository at this point in the history 
…1881)
  • Loading branch information
@SivaKumarP127
SivaKumarP127 authored Jun 14, 2024
1 parent 38edb1b commit 4cacb5c
Show file tree
Hide file tree
Showing 3 changed files with 218 additions and 22 deletions.
47 changes: 25 additions & 22 deletions pkg/integrations/v4/logs/cfg.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,18 +89,19 @@ type YAML struct {

// LogCfg logging integration config from customer defined YAML.
type LogCfg struct {
Name string `yaml:"name"`
File string `yaml:"file"` // ...
MaxLineKb int `yaml:"max_line_kb"` // Setup the max value of the buffer while reading lines.
Systemd string `yaml:"systemd"` // ...
Pattern string `yaml:"pattern"`
Attributes map[string]string `yaml:"attributes"`
Syslog *LogSyslogCfg `yaml:"syslog"`
Tcp *LogTcpCfg `yaml:"tcp"`
Fluentbit *LogExternalFBCfg `yaml:"fluentbit"`
Winlog *LogWinlogCfg `yaml:"winlog"`
Winevtlog *LogWinevtlogCfg `yaml:"winevtlog"`
targetFilesCnt int
Name string `yaml:"name"`
File string `yaml:"file"` // ...
MaxLineKb int `yaml:"max_line_kb"` // Setup the max value of the buffer while reading lines.
Systemd string `yaml:"systemd"` // ...
Pattern string `yaml:"pattern"`
Attributes map[string]string `yaml:"attributes"`
Syslog *LogSyslogCfg `yaml:"syslog"`
Tcp *LogTcpCfg `yaml:"tcp"`

Check failure on line 99 in pkg/integrations/v4/logs/cfg.go

View workflow job for this annotation

GitHub Actions / linter-linux / Run Linter 

var-naming: struct field Tcp should be TCP (revive)

Check failure on line 99 in pkg/integrations/v4/logs/cfg.go

View workflow job for this annotation

GitHub Actions / linter-macos / Lint tests 

var-naming: struct field Tcp should be TCP (revive)

Check failure on line 99 in pkg/integrations/v4/logs/cfg.go

View workflow job for this annotation

GitHub Actions / linter-windows / Lint tests 

var-naming: struct field Tcp should be TCP (revive)
Fluentbit *LogExternalFBCfg `yaml:"fluentbit"`
Winlog *LogWinlogCfg `yaml:"winlog"`
Winevtlog *LogWinevtlogCfg `yaml:"winevtlog"`
MultilineParser string `yaml:"multilineParser"`
targetFilesCnt int
}

// LogSyslogCfg logging integration config from customer defined YAML, specific for the Syslog input plugin
Expand Down Expand Up @@ -187,6 +188,7 @@ type FBCfgInput struct {
BufferMaxSize string // plugin: tail
MemBufferLimit string // plugin: tail
PathKey string // plugin: tail
MultilineParser string // plugin: tail
SkipLongLines string // always on
Systemd_Filter string // plugin: systemd
Channels string // plugin: winlog
Expand Down Expand Up @@ -389,7 +391,7 @@ func parseConfigBlock(l LogCfg, logsHomeDir string, fbOSConfig FBOSConfig) (inpu

// Single file
func parseFileInput(l LogCfg, dbPath string) (input FBCfgInput, filters []FBCfgFilter) {
input = newFileInput(l.File, dbPath, l.Name, getBufferMaxSize(l))
input = newFileInput(l.File, dbPath, l.Name, getBufferMaxSize(l), l.MultilineParser)
filters = append(filters, newRecordModifierFilterForInput(l.Name, fbInputTypeTail, l.Attributes))
filters = parsePattern(l, fbGrepFieldForTail, filters)
return input, filters
Expand Down Expand Up @@ -544,16 +546,17 @@ func newFBExternalConfig(l LogExternalFBCfg) FBCfgExternal {
}
}

func newFileInput(filePath string, dbPath string, tag string, bufSize int) FBCfgInput {
func newFileInput(filePath string, dbPath string, tag string, bufSize int, multilineParser string) FBCfgInput {
return FBCfgInput{
Name: fbInputTypeTail,
PathKey: "filePath",
Path: filePath,
DB: dbPath,
Tag: tag,
BufferMaxSize: fmt.Sprintf("%dk", bufSize),
MemBufferLimit: fmt.Sprintf("%dk", memBufferLimit),
SkipLongLines: "On",
Name: fbInputTypeTail,
PathKey: "filePath",
Path: filePath,
DB: dbPath,
Tag: tag,
BufferMaxSize: fmt.Sprintf("%dk", bufSize),
MemBufferLimit: fmt.Sprintf("%dk", memBufferLimit),
MultilineParser: multilineParser,
SkipLongLines: "On",
}
}

Expand Down
3 changes: 3 additions & 0 deletions pkg/integrations/v4/logs/cfg_template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ var fbConfigFormat = `{{- range .Inputs }}
{{- if .SkipLongLines }}
Skip_Long_Lines {{ .SkipLongLines }}
{{- end }}
{{- if .MultilineParser }}
Multiline.Parser {{ .MultilineParser }}
{{- end }}
{{- if .PathKey }}
Path_Key {{ .PathKey }}
{{- end }}
Expand Down
190 changes: 190 additions & 0 deletions pkg/integrations/v4/logs/cfg_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1929,3 +1929,193 @@ func TestNewFbConfForTargetFileCount(t *testing.T) {
})
}
}

func TestMultilineParserFBCfgFormat(t *testing.T) {
expected := `
[INPUT]
Name tail
Path /path/to/folder/*
Buffer_Max_Size 32k
Skip_Long_Lines On
Multiline.Parser go
Path_Key filePath
Tag some-folder
DB fb.db
[FILTER]
Name grep
Match some-folder
Regex log foo
[FILTER]
Name record_modifier
Match some-folder
Record "fb.input" "tail"
Record "key1" "value1"
Record "key2" "value2"
Record "key3 with space" "value3 with space"
[FILTER]
Name record_modifier
Match *
Record "entity.guid.INFRA" "testGUID"
Record "fb.source" "nri-agent"
[OUTPUT]
Name newrelic
Match *
licenseKey ${NR_LICENSE_KEY_ENV_VAR}
validateProxyCerts false
@INCLUDE /path/to/fb/config
`

fbCfg := FBCfg{
Inputs: []FBCfgInput{
{
Name: "tail",
Tag: "some-folder",
DB: "fb.db",
Path: "/path/to/folder/*",
BufferMaxSize: "32k",
SkipLongLines: "On",
MultilineParser: "go",
PathKey: "filePath",
},
},
Filters: []FBCfgFilter{
{
Name: "grep",
Match: "some-folder",
Regex: "log foo",
},
{
Name: "record_modifier",
Match: "some-folder",
Records: map[string]string{
"fb.input": "tail",
"key1": "value1",
"key2": "value2",
"key3 with space": "value3 with space",
},
},
{
Name: "record_modifier",
Match: "*",
Records: map[string]string{
"entity.guid.INFRA": "testGUID",
"fb.source": "nri-agent",
},
},
},
Output: FBCfgOutput{
Name: "newrelic",
Match: "*",
LicenseKey: "licenseKey",
},
ExternalCfg: FBCfgExternal{
CfgFilePath: "/path/to/fb/config",
ParsersFilePath: "/path/to/fb/parsers",
},
}

result, extCfg, err := fbCfg.Format()
assert.Empty(t, err)
assert.Equal(t, "/path/to/fb/parsers", extCfg.ParsersFilePath)
assert.Equal(t, expected, result)
}

func TestMlParserFBCfgWithMultipleParsers(t *testing.T) {
expected := `
[INPUT]
Name tail
Path /path/to/folder/*
Buffer_Max_Size 32k
Skip_Long_Lines On
Multiline.Parser go, java, python
Path_Key filePath
Tag some-folder
DB fb.db
[FILTER]
Name grep
Match some-folder
Regex log foo
[FILTER]
Name record_modifier
Match some-folder
Record "fb.input" "tail"
Record "key1" "value1"
Record "key2" "value2"
Record "key3 with space" "value3 with space"
[FILTER]
Name record_modifier
Match *
Record "entity.guid.INFRA" "testGUID"
Record "fb.source" "nri-agent"
[OUTPUT]
Name newrelic
Match *
licenseKey ${NR_LICENSE_KEY_ENV_VAR}
validateProxyCerts false
@INCLUDE /path/to/fb/config
`

fbCfg := FBCfg{
Inputs: []FBCfgInput{
{
Name: "tail",
Tag: "some-folder",
DB: "fb.db",
Path: "/path/to/folder/*",
BufferMaxSize: "32k",
SkipLongLines: "On",
MultilineParser: "go, java, python",
PathKey: "filePath",
},
},
Filters: []FBCfgFilter{
{
Name: "grep",
Match: "some-folder",
Regex: "log foo",
},
{
Name: "record_modifier",
Match: "some-folder",
Records: map[string]string{
"fb.input": "tail",
"key1": "value1",
"key2": "value2",
"key3 with space": "value3 with space",
},
},
{
Name: "record_modifier",
Match: "*",
Records: map[string]string{
"entity.guid.INFRA": "testGUID",
"fb.source": "nri-agent",
},
},
},
Output: FBCfgOutput{
Name: "newrelic",
Match: "*",
LicenseKey: "licenseKey",
},
ExternalCfg: FBCfgExternal{
CfgFilePath: "/path/to/fb/config",
ParsersFilePath: "/path/to/fb/parsers",
},
}

result, extCfg, err := fbCfg.Format()
assert.Empty(t, err)
assert.Equal(t, "/path/to/fb/parsers", extCfg.ParsersFilePath)
assert.Equal(t, expected, result)
}

0 comments on commit 4cacb5c

Please sign in to comment.