Skip to content

Commit

Permalink
logparser input plugin
Browse files Browse the repository at this point in the history
closes #102
closes #328
  • Loading branch information
sparrc committed Jun 7, 2016
1 parent ad88a94 commit 1bf3b27
Show file tree
Hide file tree
Showing 15 changed files with 747 additions and 123 deletions.
1 change: 1 addition & 0 deletions Godeps
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ github.com/shirou/gopsutil 586bb697f3ec9f8ec08ffefe18f521a64534037c
github.com/soniah/gosnmp b1b4f885b12c5dcbd021c5cee1c904110de6db7d
github.com/streadway/amqp b4f3ceab0337f013208d31348b578d83c0064744
github.com/stretchr/testify 1f4a1643a57e798696635ea4c126e9127adb7d3c
github.com/vjeantet/grok 83bfdfdfd1a8146795b28e547a8e3c8b28a466c2
github.com/wvanbergen/kafka 46f9a1cf3f670edec492029fadded9c2d9e18866
github.com/wvanbergen/kazoo-go 0f768712ae6f76454f987c3356177e138df258f8
github.com/zensqlmonitor/go-mssqldb ffe5510c6fa5e15e6d983210ab501c815b56b363
Expand Down
79 changes: 79 additions & 0 deletions filter/filter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package filter

import (
"strings"

"github.com/gobwas/glob"
)

type Filter interface {
Match(string) bool
}

// CompileFilter takes a list of string filters and returns a Filter interface
// for matching a given string against the filter list. The filter list
// supports glob matching too, ie:
//
// f, _ := CompileFilter([]string{"cpu", "mem", "net*"})
// f.Match("cpu") // true
// f.Match("network") // true
// f.Match("memory") // false
//
func CompileFilter(filters []string) (Filter, error) {
// return if there is nothing to compile
if len(filters) == 0 {
return nil, nil
}

// check if we can compile a non-glob filter
noGlob := true
for _, filter := range filters {
if hasMeta(filter) {
noGlob = false
break
}
}

switch {
case noGlob:
// return non-globbing filter if not needed.
return compileFilterNoGlob(filters), nil
case len(filters) == 1:
return glob.Compile(filters[0])
default:
return glob.Compile("{" + strings.Join(filters, ",") + "}")
}
}

// hasMeta reports whether path contains any magic glob characters.
func hasMeta(s string) bool {
return strings.IndexAny(s, "*?[") >= 0
}

type filter struct {
m map[string]struct{}
}

func (f *filter) Match(s string) bool {
_, ok := f.m[s]
return ok
}

type filtersingle struct {
s string
}

func (f *filtersingle) Match(s string) bool {
return f.s == s
}

func compileFilterNoGlob(filters []string) Filter {
if len(filters) == 1 {
return &filtersingle{s: filters[0]}
}
out := filter{m: make(map[string]struct{})}
for _, filter := range filters {
out.m[filter] = struct{}{}
}
return &out
}
96 changes: 96 additions & 0 deletions filter/filter_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package filter

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestCompileFilter(t *testing.T) {
f, err := CompileFilter([]string{})
assert.NoError(t, err)
assert.Nil(t, f)

f, err = CompileFilter([]string{"cpu"})
assert.NoError(t, err)
assert.True(t, f.Match("cpu"))
assert.False(t, f.Match("cpu0"))
assert.False(t, f.Match("mem"))

f, err = CompileFilter([]string{"cpu*"})
assert.NoError(t, err)
assert.True(t, f.Match("cpu"))
assert.True(t, f.Match("cpu0"))
assert.False(t, f.Match("mem"))

f, err = CompileFilter([]string{"cpu", "mem"})
assert.NoError(t, err)
assert.True(t, f.Match("cpu"))
assert.False(t, f.Match("cpu0"))
assert.True(t, f.Match("mem"))

f, err = CompileFilter([]string{"cpu", "mem", "net*"})
assert.NoError(t, err)
assert.True(t, f.Match("cpu"))
assert.False(t, f.Match("cpu0"))
assert.True(t, f.Match("mem"))
assert.True(t, f.Match("network"))
}

var benchbool bool

func BenchmarkFilterSingleNoGlobFalse(b *testing.B) {
f, _ := CompileFilter([]string{"cpu"})
var tmp bool
for n := 0; n < b.N; n++ {
tmp = f.Match("network")
}
benchbool = tmp
}

func BenchmarkFilterSingleNoGlobTrue(b *testing.B) {
f, _ := CompileFilter([]string{"cpu"})
var tmp bool
for n := 0; n < b.N; n++ {
tmp = f.Match("cpu")
}
benchbool = tmp
}

func BenchmarkFilter(b *testing.B) {
f, _ := CompileFilter([]string{"cpu", "mem", "net*"})
var tmp bool
for n := 0; n < b.N; n++ {
tmp = f.Match("network")
}
benchbool = tmp
}

func BenchmarkFilterNoGlob(b *testing.B) {
f, _ := CompileFilter([]string{"cpu", "mem", "net"})
var tmp bool
for n := 0; n < b.N; n++ {
tmp = f.Match("net")
}
benchbool = tmp
}

func BenchmarkFilter2(b *testing.B) {
f, _ := CompileFilter([]string{"aa", "bb", "c", "ad", "ar", "at", "aq",
"aw", "az", "axxx", "ab", "cpu", "mem", "net*"})
var tmp bool
for n := 0; n < b.N; n++ {
tmp = f.Match("network")
}
benchbool = tmp
}

func BenchmarkFilter2NoGlob(b *testing.B) {
f, _ := CompileFilter([]string{"aa", "bb", "c", "ad", "ar", "at", "aq",
"aw", "az", "axxx", "ab", "cpu", "mem", "net"})
var tmp bool
for n := 0; n < b.N; n++ {
tmp = f.Match("net")
}
benchbool = tmp
}
23 changes: 0 additions & 23 deletions internal/internal.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ import (
"strings"
"time"
"unicode"

"github.com/gobwas/glob"
)

const alphanum string = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
Expand Down Expand Up @@ -209,27 +207,6 @@ func WaitTimeout(c *exec.Cmd, timeout time.Duration) error {
}
}

// CompileFilter takes a list of glob "filters", ie:
// ["MAIN.*", "CPU.*", "NET"]
// and compiles them into a glob object. This glob object can
// then be used to match keys to the filter.
func CompileFilter(filters []string) (glob.Glob, error) {
var out glob.Glob

// return if there is nothing to compile
if len(filters) == 0 {
return out, nil
}

var err error
if len(filters) == 1 {
out, err = glob.Compile(filters[0])
} else {
out, err = glob.Compile("{" + strings.Join(filters, ",") + "}")
}
return out, err
}

// RandomSleep will sleep for a random amount of time up to max.
// If the shutdown channel is closed, it will return before it has finished
// sleeping.
Expand Down
31 changes: 0 additions & 31 deletions internal/internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,37 +107,6 @@ func TestRunError(t *testing.T) {
assert.Error(t, err)
}

func TestCompileFilter(t *testing.T) {
f, err := CompileFilter([]string{})
assert.NoError(t, err)
assert.Nil(t, f)

f, err = CompileFilter([]string{"cpu"})
assert.NoError(t, err)
assert.True(t, f.Match("cpu"))
assert.False(t, f.Match("cpu0"))
assert.False(t, f.Match("mem"))

f, err = CompileFilter([]string{"cpu*"})
assert.NoError(t, err)
assert.True(t, f.Match("cpu"))
assert.True(t, f.Match("cpu0"))
assert.False(t, f.Match("mem"))

f, err = CompileFilter([]string{"cpu", "mem"})
assert.NoError(t, err)
assert.True(t, f.Match("cpu"))
assert.False(t, f.Match("cpu0"))
assert.True(t, f.Match("mem"))

f, err = CompileFilter([]string{"cpu", "mem", "net*"})
assert.NoError(t, err)
assert.True(t, f.Match("cpu"))
assert.False(t, f.Match("cpu0"))
assert.True(t, f.Match("mem"))
assert.True(t, f.Match("network"))
}

func TestRandomSleep(t *testing.T) {
// test that zero max returns immediately
s := time.Now()
Expand Down
36 changes: 17 additions & 19 deletions internal/models/filter.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,80 +3,78 @@ package internal_models
import (
"fmt"

"github.com/gobwas/glob"

"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/filter"
)

// TagFilter is the name of a tag, and the values on which to filter
type TagFilter struct {
Name string
Filter []string
filter glob.Glob
filter filter.Filter
}

// Filter containing drop/pass and tagdrop/tagpass rules
type Filter struct {
NameDrop []string
nameDrop glob.Glob
nameDrop filter.Filter
NamePass []string
namePass glob.Glob
namePass filter.Filter

FieldDrop []string
fieldDrop glob.Glob
fieldDrop filter.Filter
FieldPass []string
fieldPass glob.Glob
fieldPass filter.Filter

TagDrop []TagFilter
TagPass []TagFilter

TagExclude []string
tagExclude glob.Glob
tagExclude filter.Filter
TagInclude []string
tagInclude glob.Glob
tagInclude filter.Filter

IsActive bool
}

// Compile all Filter lists into glob.Glob objects.
// Compile all Filter lists into filter.Filter objects.
func (f *Filter) CompileFilter() error {
var err error
f.nameDrop, err = internal.CompileFilter(f.NameDrop)
f.nameDrop, err = filter.CompileFilter(f.NameDrop)
if err != nil {
return fmt.Errorf("Error compiling 'namedrop', %s", err)
}
f.namePass, err = internal.CompileFilter(f.NamePass)
f.namePass, err = filter.CompileFilter(f.NamePass)
if err != nil {
return fmt.Errorf("Error compiling 'namepass', %s", err)
}

f.fieldDrop, err = internal.CompileFilter(f.FieldDrop)
f.fieldDrop, err = filter.CompileFilter(f.FieldDrop)
if err != nil {
return fmt.Errorf("Error compiling 'fielddrop', %s", err)
}
f.fieldPass, err = internal.CompileFilter(f.FieldPass)
f.fieldPass, err = filter.CompileFilter(f.FieldPass)
if err != nil {
return fmt.Errorf("Error compiling 'fieldpass', %s", err)
}

f.tagExclude, err = internal.CompileFilter(f.TagExclude)
f.tagExclude, err = filter.CompileFilter(f.TagExclude)
if err != nil {
return fmt.Errorf("Error compiling 'tagexclude', %s", err)
}
f.tagInclude, err = internal.CompileFilter(f.TagInclude)
f.tagInclude, err = filter.CompileFilter(f.TagInclude)
if err != nil {
return fmt.Errorf("Error compiling 'taginclude', %s", err)
}

for i, _ := range f.TagDrop {
f.TagDrop[i].filter, err = internal.CompileFilter(f.TagDrop[i].Filter)
f.TagDrop[i].filter, err = filter.CompileFilter(f.TagDrop[i].Filter)
if err != nil {
return fmt.Errorf("Error compiling 'tagdrop', %s", err)
}
}
for i, _ := range f.TagPass {
f.TagPass[i].filter, err = internal.CompileFilter(f.TagPass[i].Filter)
f.TagPass[i].filter, err = filter.CompileFilter(f.TagPass[i].Filter)
if err != nil {
return fmt.Errorf("Error compiling 'tagpass', %s", err)
}
Expand Down

0 comments on commit 1bf3b27

Please sign in to comment.