From 9548a08c6f39a79ba837dbbaa1b86bcefba6ce07 Mon Sep 17 00:00:00 2001 From: Jerome Froelich Date: Thu, 29 Jun 2017 17:57:55 -0400 Subject: [PATCH] [filters] Various updates to the filters package to support current filters in collector --- filters/filter.go | 103 +++++++++++++++-- filters/filter_test.go | 256 +++++++++++++++++++++++++++++++++-------- 2 files changed, 304 insertions(+), 55 deletions(-) diff --git a/filters/filter.go b/filters/filter.go index 75a5118..72d1ede 100644 --- a/filters/filter.go +++ b/filters/filter.go @@ -28,7 +28,10 @@ import ( var ( errInvalidFilterPattern = errors.New("invalid filter pattern defined") - allowAllFilter Filter = allowFilter{} + matchAll Filter = matchAllFilter{} + matchNone Filter = matchNoneFilter{} + timestamp Filter = timestampFilter{} + uuid Filter = uuidFilter{} singleAnyCharFilterForwards chainFilter = &singleAnyCharFilter{backwards: false} singleAnyCharFilterBackwards chainFilter = &singleAnyCharFilter{backwards: true} ) @@ -112,7 +115,7 @@ func newWildcardFilter(pattern []byte) (Filter, error) { if len(pattern) == 1 { // Whole thing is wildcard. - return newAllowFilter(), nil + return NewMatchAllFilter(), nil } if wIdx == len(pattern)-1 { @@ -217,13 +220,6 @@ func newRangeFilter(pattern []byte, backwards bool, seg chainSegment) (Filter, e return newMultiChainFilter(filters, seg, backwards), nil } -// allowFilter is a filter that allows all. -type allowFilter struct{} - -func newAllowFilter() Filter { return allowAllFilter } -func (f allowFilter) String() string { return "All" } -func (f allowFilter) Matches(val []byte) bool { return true } - // equalityFilter is a filter that matches exact values. type equalityFilter struct { pattern []byte @@ -279,6 +275,95 @@ func (f *negationFilter) Matches(val []byte) bool { return !f.filter.Matches(val) } +// matchAllFilter is a filter that matches all input. +type matchAllFilter struct{} + +// NewMatchAllFilter returns a filter that matches all input. +func NewMatchAllFilter() Filter { return matchAll } +func (f matchAllFilter) String() string { return "All" } +func (f matchAllFilter) Matches(val []byte) bool { return true } + +// matchNoneFilter is a filter that does not match any input. +type matchNoneFilter struct{} + +// NewMatchNoneFilter returns a filter that does not match any input. +func NewMatchNoneFilter() Filter { return matchNone } +func (f matchNoneFilter) String() string { return "None" } +func (f matchNoneFilter) Matches(val []byte) bool { return false } + +// timestampFilter is a filter that matches timestamps. +type timestampFilter struct{} + +// NewTimestampFilter returns a filter that matches any inputs that +// contain timestamps. +func NewTimestampFilter() Filter { return timestamp } +func (f timestampFilter) String() string { return "Timestamp" } +func (f timestampFilter) Matches(val []byte) bool { + if len(val) < 10 { + return false + } + + var count int + for i := 1; i-count <= len(val)-9; i++ { + // Only look for timestamps between 1400000000 (2014-05-12) and + // 2000000000 (2033-05-18). + if count == 0 { + if val[i-1] == '1' && '4' <= val[i] && val[i] <= '9' { + count = 2 + } + continue + } + + if '0' <= val[i] && val[i] <= '9' { + count++ + } else { + count = 0 + } + + if count == 10 { + return true + } + } + + return false +} + +// uuidFilter is a filter that matches UUID's. +type uuidFilter struct{} + +// NewUUIDFilter returns a filter that matches any inputs that contain UUID's, +// which are defined as 32 consecutive hexidecimal characters, ignoring hyphens. +func NewUUIDFilter() Filter { return uuid } +func (f uuidFilter) String() string { return "UUID" } +func (f uuidFilter) Matches(val []byte) bool { + var ( + count int + end = len(val) + ) + + for i := 0; i-count <= end-32; i++ { + // Ignore hyphens. + if val[i] == '-' { + continue + } + + if !isHex(val[i]) { + count = 0 + continue + } + + if count++; count >= 32 { + return true + } + } + return false +} + +// isHex returns a bool indicating whether a byte is a hexidecimal character. +func isHex(b byte) bool { + return ('a' <= b && b <= 'f') || ('0' <= b && b <= '9') || ('A' <= b && b <= 'F') +} + // multiFilter chains multiple filters together with a logicalOp. type multiFilter struct { filters []Filter diff --git a/filters/filter_test.go b/filters/filter_test.go index f5edbf9..0a154ad 100644 --- a/filters/filter_test.go +++ b/filters/filter_test.go @@ -29,9 +29,16 @@ import ( func TestFilters(t *testing.T) { filters := genAndValidateFilters(t, []testPattern{ - testPattern{pattern: "f[A-z]?*", expectedStr: "StartsWith(Equals(\"f\") then Range(\"A-z\") then AnyChar)"}, - testPattern{pattern: "*ba[a-z]", expectedStr: "EndsWith(Equals(\"ba\") then Range(\"a-z\"))"}, - testPattern{pattern: "fo*?ba[!0-9][0-9]{8,9}", expectedStr: "StartsWith(Equals(\"fo\")) && EndsWith(AnyChar then Equals(\"ba\") then Not(Range(\"0-9\")) then Range(\"0-9\") then Range(\"8,9\"))"}, + testPattern{ + pattern: "f[A-z]?*", + expectedStr: "StartsWith(Equals(\"f\") then Range(\"A-z\") then AnyChar)"}, + testPattern{ + pattern: "*ba[a-z]", + expectedStr: "EndsWith(Equals(\"ba\") then Range(\"a-z\"))"}, + testPattern{ + pattern: "fo*?ba[!0-9][0-9]{8,9}", + expectedStr: "StartsWith(Equals(\"fo\")) && EndsWith(AnyChar then Equals(\"ba\") then " + + "Not(Range(\"0-9\")) then Range(\"0-9\") then Range(\"8,9\"))"}, }) inputs := []testInput{ @@ -44,9 +51,12 @@ func TestFilters(t *testing.T) { } for _, input := range inputs { - for i, expectedMatch := range input.matches { - require.Equal(t, expectedMatch, filters[i].Matches(input.val), - fmt.Sprintf("input: %s, pattern: %s", input.val, filters[i].String())) + for i, expected := range input.matches { + actual := filters[i].Matches(input.val) + require.Equal( + t, expected, actual, + fmt.Sprintf("input: '%s', pattern: '%s'", input.val, filters[i].String()), + ) } } } @@ -57,28 +67,55 @@ func TestEqualityFilter(t *testing.T) { {val: "fo", match: false}, {val: "foob", match: false}, } + f := newEqualityFilter([]byte("foo")) for _, input := range inputs { - require.Equal(t, input.match, f.Matches([]byte(input.val))) + actual := f.Matches([]byte(input.val)) + require.Equal( + t, input.match, actual, + fmt.Sprintf("input: '%s', pattern: '%s'", input.val, f.String()), + ) } } func TestEmptyFilter(t *testing.T) { + inputs := []mockFilterData{ + {val: "", match: true}, + {val: " ", match: false}, + {val: "foo", match: false}, + } + f, err := NewFilter(nil) require.NoError(t, err) - require.True(t, f.Matches([]byte(""))) - require.False(t, f.Matches([]byte(" "))) - require.False(t, f.Matches([]byte("foo"))) + for _, input := range inputs { + actual := f.Matches([]byte(input.val)) + require.Equal( + t, input.match, actual, + fmt.Sprintf("input: '%s', pattern: '%s'", input.val, f.String()), + ) + } } func TestWildcardFilters(t *testing.T) { filters := genAndValidateFilters(t, []testPattern{ - testPattern{pattern: "foo", expectedStr: "Equals(\"foo\")"}, - testPattern{pattern: "*bar", expectedStr: "EndsWith(Equals(\"bar\"))"}, - testPattern{pattern: "baz*", expectedStr: "StartsWith(Equals(\"baz\"))"}, - testPattern{pattern: "*cat*", expectedStr: "Contains(\"cat\")"}, - testPattern{pattern: "foo*bar", expectedStr: "StartsWith(Equals(\"foo\")) && EndsWith(Equals(\"bar\"))"}, - testPattern{pattern: "*", expectedStr: "All"}, + testPattern{ + pattern: "foo", + expectedStr: "Equals(\"foo\")"}, + testPattern{ + pattern: "*bar", + expectedStr: "EndsWith(Equals(\"bar\"))"}, + testPattern{ + pattern: "baz*", + expectedStr: "StartsWith(Equals(\"baz\"))"}, + testPattern{ + pattern: "*cat*", + expectedStr: "Contains(\"cat\")"}, + testPattern{ + pattern: "foo*bar", + expectedStr: "StartsWith(Equals(\"foo\")) && EndsWith(Equals(\"bar\"))"}, + testPattern{ + pattern: "*", + expectedStr: "All"}, }) inputs := []testInput{ @@ -95,24 +132,47 @@ func TestWildcardFilters(t *testing.T) { } for _, input := range inputs { - for i, expectedMatch := range input.matches { - require.Equal(t, expectedMatch, filters[i].Matches(input.val), - fmt.Sprintf("input: %s, pattern: %s", input.val, filters[i].String())) + for i, expected := range input.matches { + actual := filters[i].Matches(input.val) + require.Equal( + t, expected, actual, + fmt.Sprintf("input: '%s', pattern: '%s'", input.val, filters[i].String()), + ) } } } func TestRangeFilters(t *testing.T) { filters := genAndValidateFilters(t, []testPattern{ - testPattern{pattern: "fo[a-zA-Z0-9]", expectedStr: "Equals(\"fo\") then Range(\"a-z || A-Z || 0-9\")"}, - testPattern{pattern: "f[o-o]?", expectedStr: "Equals(\"f\") then Range(\"o-o\") then AnyChar"}, - testPattern{pattern: "???", expectedStr: "AnyChar then AnyChar then AnyChar"}, - testPattern{pattern: "ba?", expectedStr: "Equals(\"ba\") then AnyChar"}, - testPattern{pattern: "[!cC]ar", expectedStr: "Not(Range(\"cC\")) then Equals(\"ar\")"}, - testPattern{pattern: "ba?[0-9][!a-z]9", expectedStr: "Equals(\"ba\") then AnyChar then Range(\"0-9\") then Not(Range(\"a-z\")) then Equals(\"9\")"}, - testPattern{pattern: "{ba,fo,car}*", expectedStr: "StartsWith(Range(\"ba,fo,car\"))"}, - testPattern{pattern: "ba{r,t}*[!a-zA-Z]", expectedStr: "StartsWith(Equals(\"ba\") then Range(\"r,t\")) && EndsWith(Not(Range(\"a-z || A-Z\")))"}, - testPattern{pattern: "*{9}", expectedStr: "EndsWith(Range(\"9\"))"}, + testPattern{ + pattern: "fo[a-zA-Z0-9]", + expectedStr: "Equals(\"fo\") then Range(\"a-z || A-Z || 0-9\")"}, + testPattern{ + pattern: "f[o-o]?", + expectedStr: "Equals(\"f\") then Range(\"o-o\") then AnyChar"}, + testPattern{ + pattern: "???", + expectedStr: "AnyChar then AnyChar then AnyChar"}, + testPattern{ + pattern: "ba?", + expectedStr: "Equals(\"ba\") then AnyChar"}, + testPattern{ + pattern: "[!cC]ar", + expectedStr: "Not(Range(\"cC\")) then Equals(\"ar\")"}, + testPattern{ + pattern: "ba?[0-9][!a-z]9", + expectedStr: "Equals(\"ba\") then AnyChar then Range(\"0-9\") " + + "then Not(Range(\"a-z\")) then Equals(\"9\")"}, + testPattern{ + pattern: "{ba,fo,car}*", + expectedStr: "StartsWith(Range(\"ba,fo,car\"))"}, + testPattern{ + pattern: "ba{r,t}*[!a-zA-Z]", + expectedStr: "StartsWith(Equals(\"ba\") then Range(\"r,t\")) && " + + "EndsWith(Not(Range(\"a-z || A-Z\")))"}, + testPattern{ + pattern: "*{9}", + expectedStr: "EndsWith(Range(\"9\"))"}, }) inputs := []testInput{ @@ -130,9 +190,11 @@ func TestRangeFilters(t *testing.T) { } for _, input := range inputs { - for i, expectedMatch := range input.matches { - require.Equal(t, expectedMatch, filters[i].Matches(input.val), - fmt.Sprintf("input: %s, pattern: %s", input.val, filters[i].String())) + for i, expected := range input.matches { + actual := filters[i].Matches(input.val) + require.Equal( + t, expected, actual, + fmt.Sprintf("input: '%s', pattern: '%s'", input.val, filters[i].String())) } } } @@ -156,21 +218,36 @@ func TestMultiFilter(t *testing.T) { } for _, input := range inputs { - for i, expectedMatch := range input.matches { - require.Equal(t, expectedMatch, filters[i].Matches(input.val), - fmt.Sprintf("input: %s, pattern: %s", input.val, filters[i].String())) + for i, expected := range input.matches { + actual := filters[i].Matches(input.val) + require.Equal( + t, expected, actual, + fmt.Sprintf("input: '%s', pattern: '%s'", input.val, filters[i].String()), + ) } } } func TestNegationFilter(t *testing.T) { filters := genAndValidateFilters(t, []testPattern{ - testPattern{pattern: "!foo", expectedStr: "Not(Equals(\"foo\"))"}, - testPattern{pattern: "!*bar", expectedStr: "Not(EndsWith(Equals(\"bar\")))"}, - testPattern{pattern: "!baz*", expectedStr: "Not(StartsWith(Equals(\"baz\")))"}, - testPattern{pattern: "!*cat*", expectedStr: "Not(Contains(\"cat\"))"}, - testPattern{pattern: "!foo*bar", expectedStr: "Not(StartsWith(Equals(\"foo\")) && EndsWith(Equals(\"bar\")))"}, - testPattern{pattern: "foo!", expectedStr: "Equals(\"foo!\")"}, + testPattern{ + pattern: "!foo", + expectedStr: "Not(Equals(\"foo\"))"}, + testPattern{ + pattern: "!*bar", + expectedStr: "Not(EndsWith(Equals(\"bar\")))"}, + testPattern{ + pattern: "!baz*", + expectedStr: "Not(StartsWith(Equals(\"baz\")))"}, + testPattern{ + pattern: "!*cat*", + expectedStr: "Not(Contains(\"cat\"))"}, + testPattern{ + pattern: "!foo*bar", + expectedStr: "Not(StartsWith(Equals(\"foo\")) && EndsWith(Equals(\"bar\")))"}, + testPattern{ + pattern: "foo!", + expectedStr: "Equals(\"foo!\")"}, }) inputs := []testInput{ @@ -186,13 +263,96 @@ func TestNegationFilter(t *testing.T) { } for _, input := range inputs { - for i, expectedMatch := range input.matches { - require.Equal(t, expectedMatch, filters[i].Matches(input.val), - fmt.Sprintf("input: %s, pattern: %s", input.val, filters[i].String())) + for i, expected := range input.matches { + actual := filters[i].Matches(input.val) + require.Equal( + t, expected, actual, + fmt.Sprintf("input: '%s', pattern: '%s'", input.val, filters[i].String()), + ) } } } +func TestMatchAllFilter(t *testing.T) { + inputs := []mockFilterData{ + {val: "", match: true}, + {val: "foo", match: true}, + } + + f := NewMatchAllFilter() + for _, input := range inputs { + actual := f.Matches([]byte(input.val)) + require.Equal( + t, input.match, actual, + fmt.Sprintf("input: '%s', pattern: '%s'", input.val, f.String()), + ) + } +} + +func TestMatchNoneFilter(t *testing.T) { + inputs := []mockFilterData{ + {val: "", match: false}, + {val: "foo", match: false}, + } + + f := NewMatchNoneFilter() + for _, input := range inputs { + actual := f.Matches([]byte(input.val)) + require.Equal( + t, input.match, actual, + fmt.Sprintf("input: '%s', pattern: '%s'", input.val, f.String()), + ) + } +} + +func TestTimestampFilter(t *testing.T) { + inputs := []mockFilterData{ + {val: "", match: false}, + {val: "foo", match: false}, + {val: "1399999999", match: false}, + {val: "2000000000", match: false}, + {val: "1400000000", match: true}, + {val: "1999999999", match: true}, + {val: "1498768589", match: true}, + {val: "foo.1498768589", match: true}, + {val: "1498768589.foo", match: true}, + {val: "foo.1498768589.bar", match: true}, + } + + f := NewTimestampFilter() + for _, input := range inputs { + actual := f.Matches([]byte(input.val)) + require.Equal( + t, input.match, actual, + fmt.Sprintf("input: '%s', pattern: '%s'", input.val, f.String()), + ) + } +} + +func TestUUIDFilter(t *testing.T) { + inputs := []mockFilterData{ + {val: "", match: false}, + {val: "foo", match: false}, + {val: "c52efb39805149b98bee15346037f727", match: true}, + {val: "foo.c52efb39805149b98bee15346037f727", match: true}, + {val: "c52efb39805149b98bee15346037f727.foo", match: true}, + {val: "foo.c52efb39805149b98bee15346037f727.bar", match: true}, + {val: "c52efb39-8051-49b9-8bee-15346037f727", match: true}, + {val: "foo.c52efb39-8051-49b9-8bee-15346037f727", match: true}, + {val: "c52efb39-8051-49b9-8bee-15346037f727.foo", match: true}, + {val: "foo.c52efb39-8051-49b9-8bee-15346037f727.bar", match: true}, + } + + f := NewUUIDFilter() + for _, input := range inputs { + actual := f.Matches([]byte(input.val)) + require.Equal( + t, input.match, actual, + fmt.Sprintf("input: '%s', pattern: '%s'", input.val, f.String()), + ) + } +} + func TestBadPatterns(t *testing.T) { patterns := []string{ "!", // negation of nothing is everything, so user should use *. @@ -218,7 +378,7 @@ func TestBadPatterns(t *testing.T) { for _, pattern := range patterns { _, err := NewFilter([]byte(pattern)) - require.Error(t, err, fmt.Sprintf("pattern: %s", pattern)) + require.Error(t, err, fmt.Sprintf("pattern: '%s'", pattern)) } } @@ -252,7 +412,9 @@ func TestMultiCharSequenceFilter(t *testing.T) { validateLookup(t, f, "123book", true, "123") } -func validateLookup(t *testing.T, f chainFilter, val string, expectedMatch bool, expectedRemainder string) { +func validateLookup( + t *testing.T, f chainFilter, val string, expectedMatch bool, expectedRemainder string, +) { remainder, match := f.matches([]byte(val)) require.Equal(t, expectedMatch, match) require.Equal(t, expectedRemainder, string(remainder)) @@ -277,7 +439,9 @@ func genAndValidateFilters(t *testing.T, patterns []testPattern) []Filter { filters := make([]Filter, len(patterns)) for i, pattern := range patterns { filters[i], err = NewFilter([]byte(pattern.pattern)) - require.NoError(t, err, fmt.Sprintf("No error expected, but got: %v for pattern: %s", err, pattern.pattern)) + require.NoError( + t, err, fmt.Sprintf("No error expected, but got: %v for pattern: '%s'", err, pattern.pattern), + ) require.Equal(t, pattern.expectedStr, filters[i].String()) }