diff --git a/libbeat/common/mapval/core.go b/libbeat/common/mapval/core.go index 07631851b1d..79e769efd9b 100644 --- a/libbeat/common/mapval/core.go +++ b/libbeat/common/mapval/core.go @@ -18,6 +18,7 @@ package mapval import ( + "reflect" "sort" "strings" @@ -39,32 +40,41 @@ func Optional(id IsDef) IsDef { // Map is the type used to define schema definitions for Schema. type Map map[string]interface{} +// Slice is a convenience []interface{} used to declare schema defs. +type Slice []interface{} + // Validator is the result of Schema and is run against the map you'd like to test. -type Validator func(common.MapStr) *Results +type Validator func(common.MapStr) (*Results, error) // Compose combines multiple SchemaValidators into a single one. func Compose(validators ...Validator) Validator { - return func(actual common.MapStr) *Results { + return func(actual common.MapStr) (r *Results, err error) { results := make([]*Results, len(validators)) for idx, validator := range validators { - results[idx] = validator(actual) + results[idx], err = validator(actual) + if err != nil { + return nil, err + } } combined := NewResults() for _, r := range results { - r.EachResult(func(path string, vr ValueResult) bool { + r.EachResult(func(path Path, vr ValueResult) bool { combined.record(path, vr) return true }) } - return combined + return combined, err } } // Strict is used when you want any unspecified keys that are encountered to be considered errors. func Strict(laxValidator Validator) Validator { - return func(actual common.MapStr) *Results { - results := laxValidator(actual) + return func(actual common.MapStr) (*Results, error) { + results, err := laxValidator(actual) + if err != nil { + return results, err + } // The inner workings of this are a little weird // We use a hash of dotted paths to track the results @@ -83,61 +93,69 @@ func Strict(laxValidator Validator) Validator { } sort.Strings(validatedPaths) - walk(actual, func(woi walkObserverInfo) { - _, validatedExactly := results.Fields[woi.dottedPath] + err = walk(actual, func(woi walkObserverInfo) { + _, validatedExactly := results.Fields[woi.path.String()] if validatedExactly { return // This key was tested, passes strict test } // Search returns the point just before an actual match (since we ruled out an exact match with the cheaper // hash check above. We have to validate the actual match with a prefix check as well - matchIdx := sort.SearchStrings(validatedPaths, woi.dottedPath) - if matchIdx < len(validatedPaths) && strings.HasPrefix(validatedPaths[matchIdx], woi.dottedPath) { + matchIdx := sort.SearchStrings(validatedPaths, woi.path.String()) + if matchIdx < len(validatedPaths) && strings.HasPrefix(validatedPaths[matchIdx], woi.path.String()) { return } - results.record(woi.dottedPath, StrictFailureVR) + results.merge(StrictFailureResult(woi.path)) }) - return results + return results, err } } // Schema takes a Map and returns an executable Validator function. func Schema(expected Map) Validator { - return func(actual common.MapStr) *Results { + return func(actual common.MapStr) (*Results, error) { return walkValidate(expected, actual) } } -func walkValidate(expected Map, actual common.MapStr) (results *Results) { +func walkValidate(expected Map, actual common.MapStr) (results *Results, err error) { results = NewResults() - walk( + err = walk( common.MapStr(expected), func(expInfo walkObserverInfo) { - - actualKeyExists, _ := actual.HasKey(expInfo.dottedPath) - actualV, _ := actual.GetValue(expInfo.dottedPath) + actualKeyExists, actualV := expInfo.path.GetFrom(actual) // If this is a definition use it, if not, check exact equality isDef, isIsDef := expInfo.value.(IsDef) if !isIsDef { - // We don't check maps for equality, we check their properties - // individual via our own traversal, so bail early - if _, isMS := actualV.(common.MapStr); isMS { - return - } - isDef = IsEqual(expInfo.value) + if !interfaceIsCollection(expInfo.value) { + isDef = IsEqual(expInfo.value) + } else if interfaceIsCollection(actualV) { + // We don't check collections for equality, we check their properties + // individual via our own traversal, so bail early unless the collection + // is empty. The one exception + if reflect.ValueOf(actualV).Len() > 0 { + return + } + + isDef = IsEqual(expInfo.value) + } } if !isDef.optional || isDef.optional && actualKeyExists { - results.record( - expInfo.dottedPath, - isDef.check(actualV, actualKeyExists), - ) + var checkRes *Results + checkRes, err = isDef.check(expInfo.path, actualV, actualKeyExists) + results.merge(checkRes) } }) - return results + return results, err +} + +func interfaceIsCollection(o interface{}) bool { + kind := reflect.ValueOf(o).Kind() + return kind == reflect.Map || kind == reflect.Slice } diff --git a/libbeat/common/mapval/core_test.go b/libbeat/common/mapval/core_test.go index 941545c6d5b..9e4480d8e18 100644 --- a/libbeat/common/mapval/core_test.go +++ b/libbeat/common/mapval/core_test.go @@ -23,9 +23,18 @@ import ( "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/elastic/beats/libbeat/common" ) +func assertValidator(t *testing.T, validator Validator, input common.MapStr) { + res, err := validator(input) + require.NoError(t, err) + + assertResults(t, res) +} + // assertResults validates the schema passed successfully. func assertResults(t *testing.T, r *Results) *Results { for _, err := range r.Errors() { @@ -40,10 +49,11 @@ func TestFlat(t *testing.T) { "baz": 1, } - results := Schema(Map{ + results, err := Schema(Map{ "foo": "bar", "baz": IsIntGt(0), })(m) + require.NoError(t, err) assertResults(t, results) } @@ -53,9 +63,10 @@ func TestBadFlat(t *testing.T) { fakeT := new(testing.T) - results := Schema(Map{ + results, err := Schema(Map{ "notafield": IsDuration, })(m) + require.NoError(t, err) assertResults(fakeT, results) @@ -63,7 +74,7 @@ func TestBadFlat(t *testing.T) { result := results.Fields["notafield"][0] assert.False(t, result.Valid) - assert.Equal(t, result.Message, KeyMissingVR.Message) + assert.Equal(t, result, KeyMissingVR) } func TestNested(t *testing.T) { @@ -74,13 +85,15 @@ func TestNested(t *testing.T) { }, } - results := Schema(Map{ + results, err := Schema(Map{ "foo": Map{ "bar": "baz", }, "foo.dur": IsDuration, })(m) + require.NoError(t, err) + assertResults(t, results) assert.Len(t, results.Fields, 2, "One result per matcher") @@ -97,20 +110,23 @@ func TestComposition(t *testing.T) { composed := Compose(fooValidator, bazValidator) // Test that the validators work individually - assertResults(t, fooValidator(m)) - assertResults(t, bazValidator(m)) + assertValidator(t, fooValidator, m) + assertValidator(t, bazValidator, m) // Test that the composition of them works - assertResults(t, composed(m)) + assertValidator(t, composed, m) - assert.Len(t, composed(m).Fields, 2) + composedRes, _ := composed(m) + assert.Len(t, composedRes.Fields, 2) badValidator := Schema(Map{"notakey": "blah"}) badComposed := Compose(badValidator, composed) fakeT := new(testing.T) - assertResults(fakeT, badComposed(m)) - assert.Len(t, badComposed(m).Fields, 3) + assertValidator(fakeT, badComposed, m) + badComposedRes, _ := badComposed(m) + + assert.Len(t, badComposedRes.Fields, 3) assert.True(t, fakeT.Failed()) } @@ -135,16 +151,18 @@ func TestStrictFunc(t *testing.T) { }, }) - assertResults(t, validValidator(m)) + assertValidator(t, validValidator, m) partialValidator := Schema(Map{ "foo": "bar", }) // Should pass, since this is not a strict check - assertResults(t, partialValidator(m)) + assertValidator(t, partialValidator, m) + + res, err := Strict(partialValidator)(m) + require.NoError(t, err) - res := Strict(partialValidator)(m) assert.Equal(t, []ValueResult{StrictFailureVR}, res.DetailedErrors().Fields["baz"]) assert.Equal(t, []ValueResult{StrictFailureVR}, res.DetailedErrors().Fields["nest.very.deep"]) assert.Nil(t, res.DetailedErrors().Fields["bar"]) @@ -160,7 +178,7 @@ func TestOptional(t *testing.T) { "non": Optional(IsEqual("foo")), }) - assertResults(t, validator(m)) + assertValidator(t, validator, m) } func TestExistence(t *testing.T) { @@ -173,7 +191,7 @@ func TestExistence(t *testing.T) { "non": KeyMissing, }) - assertResults(t, validator(m)) + assertValidator(t, validator, m) } func TestComplex(t *testing.T) { @@ -188,9 +206,10 @@ func TestComplex(t *testing.T) { }, "slice": []string{"pizza", "pasta", "and more"}, "empty": nil, + "arr": []common.MapStr{{"foo": "bar"}, {"foo": "baz"}}, } - res := Schema(Map{ + validator := Schema(Map{ "foo": "bar", "hash": Map{ "baz": 1, @@ -202,7 +221,164 @@ func TestComplex(t *testing.T) { "slice": []string{"pizza", "pasta", "and more"}, "empty": KeyPresent, "doesNotExist": KeyMissing, - })(m) + "arr": IsArrayOf(Schema(Map{"foo": IsStringContaining("a")})), + }) - assertResults(t, res) + assertValidator(t, validator, m) +} + +func TestLiteralArray(t *testing.T) { + m := common.MapStr{ + "a": []interface{}{ + []interface{}{1, 2, 3}, + []interface{}{"foo", "bar"}, + "hello", + }, + } + + validator := Schema(Map{ + "a": []interface{}{ + []interface{}{1, 2, 3}, + []interface{}{"foo", "bar"}, + "hello", + }, + }) + + goodRes, err := validator(m) + require.NoError(t, err) + + assertResults(t, goodRes) + // We evaluate multidimensional slice as a single field for now + // This is kind of easier, but maybe we should do our own traversal later. + assert.Len(t, goodRes.Fields, 6) +} + +func TestStringSlice(t *testing.T) { + m := common.MapStr{ + "a": []string{"a", "b"}, + } + + validator := Schema(Map{ + "a": []string{"a", "b"}, + }) + + goodRes, err := validator(m) + require.NoError(t, err) + + assertResults(t, goodRes) + // We evaluate multidimensional slices as a single field for now + // This is kind of easier, but maybe we should do our own traversal later. + assert.Len(t, goodRes.Fields, 2) +} + +func TestEmptySlice(t *testing.T) { + // In the case of an empty Slice, the validator will compare slice type + // In this case we're treating the slice as a value and doing a literal comparison + // Users should use an IsDef testing for an empty slice (that can use reflection) + // if they need something else. + m := common.MapStr{ + "a": []interface{}{}, + "b": []string{}, + } + + validator := Schema(Map{ + "a": []interface{}{}, + "b": []string{}, + }) + + goodRes, err := validator(m) + require.NoError(t, err) + + assertResults(t, goodRes) + assert.Len(t, goodRes.Fields, 2) +} + +func TestLiteralMdSlice(t *testing.T) { + m := common.MapStr{ + "a": [][]int{ + {1, 2, 3}, + {4, 5, 6}, + }, + } + + validator := Schema(Map{ + "a": [][]int{ + {1, 2, 3}, + {4, 5, 6}, + }, + }) + + goodRes, err := validator(m) + require.NoError(t, err) + + assertResults(t, goodRes) + // We evaluate multidimensional slices as a single field for now + // This is kind of easier, but maybe we should do our own traversal later. + assert.Len(t, goodRes.Fields, 6) + + badValidator := Strict(Schema(Map{ + "a": [][]int{ + {1, 2, 3}, + }, + })) + + badRes, err := badValidator(m) + require.NoError(t, err) + + assert.False(t, badRes.Valid) + assert.Len(t, badRes.Fields, 7) + // The reason the len is 4 is that there is 1 extra slice + 4 values. + assert.Len(t, badRes.Errors(), 4) +} + +func TestSliceOfIsDefs(t *testing.T) { + m := common.MapStr{ + "a": []int{1, 2, 3}, + "b": []interface{}{"foo", "bar", 3}, + } + + goodV := Schema(Map{ + "a": []interface{}{IsIntGt(0), IsIntGt(1), 3}, + "b": []interface{}{IsStringContaining("o"), "bar", IsIntGt(2)}, + }) + + assertValidator(t, goodV, m) + + badV := Schema(Map{ + "a": []interface{}{IsIntGt(100), IsIntGt(1), 3}, + "b": []interface{}{IsStringContaining("X"), "bar", IsIntGt(2)}, + }) + badRes, err := badV(m) + require.NoError(t, err) + + assert.False(t, badRes.Valid) + assert.Len(t, badRes.Errors(), 2) +} + +func TestMatchArrayAsValue(t *testing.T) { + m := common.MapStr{ + "a": []int{1, 2, 3}, + "b": []interface{}{"foo", "bar", 3}, + } + + goodV := Schema(Map{ + "a": []int{1, 2, 3}, + "b": []interface{}{"foo", "bar", 3}, + }) + + assertValidator(t, goodV, m) + + badV := Schema(Map{ + "a": "robot", + "b": []interface{}{"foo", "bar", 3}, + }) + + badRes, err := badV(m) + require.NoError(t, err) + + assert.False(t, badRes.Valid) + assert.False(t, badRes.Fields["a"][0].Valid) + for _, f := range badRes.Fields["b"] { + assert.True(t, f.Valid) + } } diff --git a/libbeat/common/mapval/is_defs.go b/libbeat/common/mapval/is_defs.go index 59f009ffa44..dc8d7d7971e 100644 --- a/libbeat/common/mapval/is_defs.go +++ b/libbeat/common/mapval/is_defs.go @@ -19,10 +19,11 @@ package mapval import ( "fmt" + "reflect" "strings" "time" - "github.com/stretchr/testify/assert" + "github.com/elastic/beats/libbeat/common" ) // KeyPresent checks that the given key is in the map, even if it has a nil value. @@ -31,6 +32,28 @@ var KeyPresent = IsDef{name: "check key present"} // KeyMissing checks that the given key is not present defined. var KeyMissing = IsDef{name: "check key not present", checkKeyMissing: true} +// IsArrayOf validates that the array at the given key is an array of objects all validatable +// via the given Validator. +func IsArrayOf(validator Validator) IsDef { + return Is("array of maps", func(path Path, v interface{}) (*Results, error) { + vArr, isArr := v.([]common.MapStr) + if !isArr { + return SimpleResult(path, false, "Expected array at given path"), nil + } + + results := NewResults() + + var err error + for idx, curMap := range vArr { + var validatorRes *Results + validatorRes, err = validator(curMap) + results.mergeUnderPrefix(path.ExtendSlice(idx), validatorRes) + } + + return results, err + }) +} + // IsAny takes a variable number of IsDef's and combines them with a logical OR. If any single definition // matches the key will be marked as valid. func IsAny(of ...IsDef) IsDef { @@ -40,108 +63,103 @@ func IsAny(of ...IsDef) IsDef { } isName := fmt.Sprintf("either %#v", names) - return Is(isName, func(v interface{}) ValueResult { + return Is(isName, func(path Path, v interface{}) (*Results, error) { for _, def := range of { - vr := def.check(v, true) + var err error + vr, err := def.check(path, v, true) if vr.Valid { - return vr + return vr, err } } - return ValueResult{ + return SimpleResult( + path, false, fmt.Sprintf("Value was none of %#v, actual value was %#v", names, v), - } + ), nil }) } // IsStringContaining validates that the the actual value contains the specified substring. func IsStringContaining(needle string) IsDef { - return Is("is string containing", func(v interface{}) ValueResult { + return Is("is string containing", func(path Path, v interface{}) (*Results, error) { strV, ok := v.(string) if !ok { - return ValueResult{ + return SimpleResult( + path, false, fmt.Sprintf("Unable to convert '%v' to string", v), - } + ), nil } if !strings.Contains(strV, needle) { - return ValueResult{ + return SimpleResult( + path, false, fmt.Sprintf("String '%s' did not contain substring '%s'", strV, needle), - } + ), nil } - return ValidVR + return ValidResult(path), nil }) } // IsDuration tests that the given value is a duration. -var IsDuration = Is("is a duration", func(v interface{}) ValueResult { +var IsDuration = Is("is a duration", func(path Path, v interface{}) (*Results, error) { if _, ok := v.(time.Duration); ok { - return ValidVR + return ValidResult(path), nil } - return ValueResult{ + return SimpleResult( + path, false, fmt.Sprintf("Expected a time.duration, got '%v' which is a %T", v, v), - } + ), nil }) // IsEqual tests that the given object is equal to the actual object. func IsEqual(to interface{}) IsDef { - return Is("equals", func(v interface{}) ValueResult { - if assert.ObjectsAreEqual(v, to) { - return ValidVR + return Is("equals", func(path Path, v interface{}) (*Results, error) { + if reflect.DeepEqual(v, to) { + return ValidResult(path), nil } - return ValueResult{ + return SimpleResult( + path, false, fmt.Sprintf("objects not equal: actual(%v) != expected(%v)", v, to), - } - }) -} - -// IsEqualToValue tests that the given value is equal to the actual value. -func IsEqualToValue(to interface{}) IsDef { - return Is("equals", func(v interface{}) ValueResult { - if assert.ObjectsAreEqualValues(v, to) { - return ValidVR - } - return ValueResult{ - false, - fmt.Sprintf("values not equal: actual(%v) != expected(%v)", v, to), - } + ), nil }) } // IsNil tests that a value is nil. -var IsNil = Is("is nil", func(v interface{}) ValueResult { +var IsNil = Is("is nil", func(path Path, v interface{}) (*Results, error) { if v == nil { - return ValidVR + return ValidResult(path), nil } - return ValueResult{ + return SimpleResult( + path, false, fmt.Sprintf("Value %v is not nil", v), - } + ), nil }) func intGtChecker(than int) ValueValidator { - return func(v interface{}) ValueResult { + return func(path Path, v interface{}) (*Results, error) { n, ok := v.(int) if !ok { msg := fmt.Sprintf("%v is a %T, but was expecting an int!", v, v) - return ValueResult{false, msg} + return SimpleResult(path, false, msg), nil } if n > than { - return ValidVR + return ValidResult(path), nil } - return ValueResult{ + return SimpleResult( + path, false, fmt.Sprintf("%v is not greater than %v", n, than), - } + ), nil } } diff --git a/libbeat/common/mapval/is_defs_test.go b/libbeat/common/mapval/is_defs_test.go index 2ee6dedea37..4e656ffeed0 100644 --- a/libbeat/common/mapval/is_defs_test.go +++ b/libbeat/common/mapval/is_defs_test.go @@ -23,21 +23,30 @@ import ( "time" "github.com/stretchr/testify/assert" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/common" ) -func assertIsDefValid(t *testing.T, id IsDef, value interface{}) { - res := id.check(value, true) +func assertIsDefValid(t *testing.T, id IsDef, value interface{}) *Results { + res, err := id.check(MustParsePath("p"), value, true) + require.NoError(t, err) + if !res.Valid { assert.Fail( t, "Expected Valid IsDef", - "Isdef %#v was not valid for value %#v with error: ", id, value, res.Message, + "Isdef %#v was not valid for value %#v with error: ", id, value, res.Errors(), ) } + return res } -func assertIsDefInvalid(t *testing.T, id IsDef, value interface{}) { - res := id.check(value, true) +func assertIsDefInvalid(t *testing.T, id IsDef, value interface{}) *Results { + res, err := id.check(MustParsePath("p"), value, true) + require.NoError(t, err) + if res.Valid { assert.Fail( t, @@ -47,6 +56,30 @@ func assertIsDefInvalid(t *testing.T, id IsDef, value interface{}) { value, ) } + return res +} + +func TestIsArrayOf(t *testing.T) { + validator := Schema(Map{"foo": "bar"}) + + id := IsArrayOf(validator) + + goodMap := common.MapStr{"foo": "bar"} + goodMapArr := []common.MapStr{goodMap, goodMap} + + goodRes := assertIsDefValid(t, id, goodMapArr) + goodFields := goodRes.Fields + assert.Len(t, goodFields, 2) + assert.Contains(t, goodFields, "p.[0].foo") + assert.Contains(t, goodFields, "p.[1].foo") + + badMap := common.MapStr{"foo": "bot"} + badMapArr := []common.MapStr{badMap} + + badRes := assertIsDefInvalid(t, id, badMapArr) + badFields := badRes.Fields + assert.Len(t, badFields, 1) + assert.Contains(t, badFields, "p.[0].foo") } func TestIsAny(t *testing.T) { diff --git a/libbeat/common/mapval/path.go b/libbeat/common/mapval/path.go new file mode 100644 index 00000000000..0b759e224d8 --- /dev/null +++ b/libbeat/common/mapval/path.go @@ -0,0 +1,172 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mapval + +import ( + "fmt" + "reflect" + "regexp" + "strconv" + "strings" + + "github.com/elastic/beats/libbeat/common" +) + +// PathComponentType indicates the type of PathComponent. +type PathComponentType int + +const ( + // PCMapKey is the Type for map keys. + PCMapKey = iota + // PCSliceIdx is the Type for slice indices. + PCSliceIdx +) + +// PathComponent structs represent one breadcrumb in a Path. +type PathComponent struct { + Type PathComponentType // One of PCMapKey or PCSliceIdx + Key string // Populated for maps + Index int // Populated for slices +} + +func (pc PathComponent) String() string { + if pc.Type == PCSliceIdx { + return fmt.Sprintf("[%d]", pc.Index) + } + return pc.Key +} + +// Path represents the path within a nested set of maps. +type Path []PathComponent + +// ExtendSlice is used to add a new PathComponent of the PCSliceIdx type. +func (p Path) ExtendSlice(index int) Path { + return p.extend( + PathComponent{PCSliceIdx, "", index}, + ) +} + +// ExtendMap adds a new PathComponent of the PCMapKey type. +func (p Path) ExtendMap(key string) Path { + return p.extend( + PathComponent{PCMapKey, key, -1}, + ) +} + +func (p Path) extend(pc PathComponent) Path { + out := make(Path, len(p)+1) + copy(out, p) + out[len(p)] = pc + return out +} + +// Concat combines two paths into a new path without modifying any existing paths. +func (p Path) Concat(other Path) Path { + out := make(Path, 0, len(p)+len(other)) + out = append(out, p...) + return append(out, other...) +} + +func (p Path) String() string { + out := make([]string, len(p)) + for idx, pc := range p { + out[idx] = pc.String() + } + return strings.Join(out, ".") +} + +// Last returns the last PathComponent in this Path. +func (p Path) Last() PathComponent { + return p[len(p)-1] +} + +// GetFrom takes a map and fetches the given path from it. +func (p Path) GetFrom(m common.MapStr) (exists bool, value interface{}) { + value = m + exists = true + for _, pc := range p { + switch reflect.TypeOf(value).Kind() { + case reflect.Map: + converted := interfaceToMapStr(value) + value, exists = converted[pc.Key] + case reflect.Slice: + converted := sliceToSliceOfInterfaces(value) + if pc.Index < len(converted) { + exists = true + value = converted[pc.Index] + } else { + exists = false + value = nil + } + default: + panic("Unexpected type") + } + + if exists == false { + return exists, nil + } + } + + return exists, value +} + +var arrMatcher = regexp.MustCompile("\\[(\\d+)\\]") + +// InvalidPathString is the error type returned from unparseable paths. +type InvalidPathString string + +func (ps InvalidPathString) Error() string { + return fmt.Sprintf("Invalid path Path: %#v", ps) +} + +// ParsePath parses a path of form key.[0].otherKey.[1] into a Path object. +func ParsePath(in string) (p Path, err error) { + keyParts := strings.Split(in, ".") + + p = make(Path, len(keyParts)) + for idx, part := range keyParts { + r := arrMatcher.FindStringSubmatch(part) + pc := PathComponent{} + if len(r) > 0 { + pc.Type = PCSliceIdx + // Cannot fail, validated by regexp already + pc.Index, err = strconv.Atoi(r[1]) + if err != nil { + return p, err + } + } else if len(part) > 0 { + pc.Type = PCMapKey + pc.Key = part + } else { + return p, InvalidPathString(in) + } + + p[idx] = pc + } + + return p, nil +} + +// MustParsePath is a convenience method for parsing paths that have been previously validated +func MustParsePath(in string) Path { + out, err := ParsePath(in) + if err != nil { + panic(err) + } + return out +} diff --git a/libbeat/common/mapval/reflect_tools.go b/libbeat/common/mapval/reflect_tools.go new file mode 100644 index 00000000000..c97c4b650dc --- /dev/null +++ b/libbeat/common/mapval/reflect_tools.go @@ -0,0 +1,62 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mapval + +import ( + "reflect" + + "github.com/elastic/beats/libbeat/common" +) + +func interfaceToMapStr(o interface{}) common.MapStr { + newMap := common.MapStr{} + rv := reflect.ValueOf(o) + + for _, key := range rv.MapKeys() { + mapV := rv.MapIndex(key) + keyStr := key.Interface().(string) + var value interface{} + + if !mapV.IsNil() { + value = mapV.Interface().(interface{}) + } + + newMap[keyStr] = value + } + return newMap +} + +func sliceToSliceOfInterfaces(o interface{}) []interface{} { + rv := reflect.ValueOf(o) + converted := make([]interface{}, rv.Len()) + for i := 0; i < rv.Len(); i++ { + var indexV = rv.Index(i) + var convertedValue interface{} + if indexV.Type().Kind() == reflect.Interface { + if !indexV.IsNil() { + convertedValue = indexV.Interface().(interface{}) + } else { + convertedValue = nil + } + } else { + convertedValue = indexV.Interface().(interface{}) + } + converted[i] = convertedValue + } + return converted +} diff --git a/libbeat/common/mapval/results.go b/libbeat/common/mapval/results.go index 14279c31f36..e6dd7e06841 100644 --- a/libbeat/common/mapval/results.go +++ b/libbeat/common/mapval/results.go @@ -35,11 +35,51 @@ func NewResults() *Results { } } -func (r *Results) record(path string, result ValueResult) { - if r.Fields[path] == nil { - r.Fields[path] = []ValueResult{result} +// SimpleResult provides a convenient and simple method for creating a *Results object for a single validation. +// It's a very common way for validators to return a *Results object, and is generally simpler than +// using SingleResult. +func SimpleResult(path Path, valid bool, msg string) *Results { + vr := ValueResult{valid, msg} + return SingleResult(path, vr) +} + +// SingleResult returns a *Results object with a single validated value at the given path +// using the provided ValueResult as its sole validation. +func SingleResult(path Path, result ValueResult) *Results { + r := NewResults() + r.record(path, result) + return r +} + +func (r *Results) merge(other *Results) { + for path, valueResults := range other.Fields { + for _, valueResult := range valueResults { + r.record(MustParsePath(path), valueResult) + } + } +} + +func (r *Results) mergeUnderPrefix(prefix Path, other *Results) { + if len(prefix) == 0 { + // If the prefix is empty, just use standard merge + // No need to add the dots + r.merge(other) + return + } + + for path, valueResults := range other.Fields { + for _, valueResult := range valueResults { + parsed := MustParsePath(path) + r.record(prefix.Concat(parsed), valueResult) + } + } +} + +func (r *Results) record(path Path, result ValueResult) { + if r.Fields[path.String()] == nil { + r.Fields[path.String()] = []ValueResult{result} } else { - r.Fields[path] = append(r.Fields[path], result) + r.Fields[path.String()] = append(r.Fields[path.String()], result) } if !result.Valid { @@ -50,10 +90,10 @@ func (r *Results) record(path string, result ValueResult) { // EachResult executes the given callback once per Value result. // The provided callback can return true to keep iterating, or false // to stop. -func (r Results) EachResult(f func(string, ValueResult) bool) { +func (r Results) EachResult(f func(Path, ValueResult) bool) { for path, pathResults := range r.Fields { for _, result := range pathResults { - if !f(path, result) { + if !f(MustParsePath(path), result) { return } } @@ -63,7 +103,7 @@ func (r Results) EachResult(f func(string, ValueResult) bool) { // DetailedErrors returns a new Results object consisting only of error data. func (r *Results) DetailedErrors() *Results { errors := NewResults() - r.EachResult(func(path string, vr ValueResult) bool { + r.EachResult(func(path Path, vr ValueResult) bool { if !vr.Valid { errors.record(path, vr) } @@ -75,7 +115,7 @@ func (r *Results) DetailedErrors() *Results { // ValueResultError is used to represent an error validating an individual value. type ValueResultError struct { - path string + path Path valueResult ValueResult } @@ -88,7 +128,7 @@ func (vre ValueResultError) Error() string { func (r Results) Errors() []error { errors := make([]error, 0) - r.EachResult(func(path string, vr ValueResult) bool { + r.EachResult(func(path Path, vr ValueResult) bool { if !vr.Valid { errors = append(errors, ValueResultError{path, vr}) } diff --git a/libbeat/common/mapval/results_test.go b/libbeat/common/mapval/results_test.go index 9649379a506..50cc1d254a6 100644 --- a/libbeat/common/mapval/results_test.go +++ b/libbeat/common/mapval/results_test.go @@ -32,8 +32,8 @@ func TestEmpty(t *testing.T) { func TestWithError(t *testing.T) { r := NewResults() - r.record("foo", KeyMissingVR) - r.record("bar", ValidVR) + r.record(MustParsePath("foo"), KeyMissingVR) + r.record(MustParsePath("bar"), ValidVR) assert.False(t, r.Valid) diff --git a/libbeat/common/mapval/value.go b/libbeat/common/mapval/value.go index 911cc638c2f..8eb6325bd7e 100644 --- a/libbeat/common/mapval/value.go +++ b/libbeat/common/mapval/value.go @@ -24,7 +24,7 @@ type ValueResult struct { } // A ValueValidator is used to validate a value in a Map. -type ValueValidator func(v interface{}) ValueResult +type ValueValidator func(path Path, v interface{}) (*Results, error) // An IsDef defines the type of check to do. // Generally only name and checker are set. optional and checkKeyMissing are @@ -36,28 +36,38 @@ type IsDef struct { checkKeyMissing bool } -func (id IsDef) check(v interface{}, keyExists bool) ValueResult { +func (id IsDef) check(path Path, v interface{}, keyExists bool) (*Results, error) { if id.checkKeyMissing { if !keyExists { - return ValidVR + return ValidResult(path), nil } - return ValueResult{false, "key should not exist!"} + return SimpleResult(path, false, "this key should not exist"), nil } if !id.optional && !keyExists { - return KeyMissingVR + return KeyMissingResult(path), nil } if id.checker != nil { - return id.checker(v) + return id.checker(path, v) } - return ValidVR + return ValidResult(path), nil +} + +// ValidResult is a convenience value for Valid results. +func ValidResult(path Path) *Results { + return SimpleResult(path, true, "is valid") } // ValidVR is a convenience value for Valid results. -var ValidVR = ValueResult{true, ""} +var ValidVR = ValueResult{true, "is valid"} + +// KeyMissingResult is emitted when a key was expected, but was not present. +func KeyMissingResult(path Path) *Results { + return SingleResult(path, KeyMissingVR) +} // KeyMissingVR is emitted when a key was expected, but was not present. var KeyMissingVR = ValueResult{ @@ -65,5 +75,13 @@ var KeyMissingVR = ValueResult{ "expected this key to be present", } +// StrictFailureResult is emitted when Strict() is used, and an unexpected field is found. +func StrictFailureResult(path Path) *Results { + return SingleResult(path, StrictFailureVR) +} + // StrictFailureVR is emitted when Strict() is used, and an unexpected field is found. -var StrictFailureVR = ValueResult{false, "unexpected field encountered during strict validation"} +var StrictFailureVR = ValueResult{ + false, + "unexpected field encountered during strict validation", +} diff --git a/libbeat/common/mapval/walk.go b/libbeat/common/mapval/walk.go index 94bd10665f4..4a0f46d840f 100644 --- a/libbeat/common/mapval/walk.go +++ b/libbeat/common/mapval/walk.go @@ -18,56 +18,63 @@ package mapval import ( - "strings" + "reflect" "github.com/elastic/beats/libbeat/common" ) type walkObserverInfo struct { - key string - value interface{} - currentMap common.MapStr - rootMap common.MapStr - path []string - dottedPath string + key PathComponent + value interface{} + rootMap common.MapStr + path Path } // walkObserver functions run once per object in the tree. type walkObserver func(info walkObserverInfo) // walk is a shorthand way to walk a tree. -func walk(m common.MapStr, wo walkObserver) { - walkFull(m, m, []string{}, wo) +func walk(m common.MapStr, wo walkObserver) error { + return walkFullMap(m, m, Path{}, wo) } -// walkFull walks the given MapStr tree. -// TODO: Handle slices/arrays. We intentionally don't handle list types now because we don't need it (yet) -// and it isn't clear in the context of validation what the right thing is to do there beyond letting the user -// perform a custom validation -func walkFull(m common.MapStr, root common.MapStr, path []string, wo walkObserver) { - for k, v := range m { - splitK := strings.Split(k, ".") - newPath := make([]string, len(path)+len(splitK)) - copy(newPath, path) - copy(newPath[len(path):], splitK) - - dottedPath := strings.Join(newPath, ".") +func walkFull(o interface{}, root common.MapStr, path Path, wo walkObserver) error { + wo(walkObserverInfo{path.Last(), o, root, path}) - wo(walkObserverInfo{k, v, m, root, newPath, dottedPath}) + switch reflect.TypeOf(o).Kind() { + case reflect.Map: + converted := interfaceToMapStr(o) + err := walkFullMap(converted, root, path, wo) + if err != nil { + return err + } + case reflect.Slice: + converted := sliceToSliceOfInterfaces(o) - // Walk nested maps - vIsMap := false - var mapV common.MapStr - if convertedMS, ok := v.(common.MapStr); ok { - mapV = convertedMS - vIsMap = true - } else if convertedM, ok := v.(Map); ok { - mapV = common.MapStr(convertedM) - vIsMap = true + for idx, v := range converted { + newPath := path.ExtendSlice(idx) + err := walkFull(v, root, newPath, wo) + if err != nil { + return err + } } + } - if vIsMap { - walkFull(mapV, root, newPath, wo) + return nil +} + +// walkFullMap walks the given MapStr tree. +func walkFullMap(m common.MapStr, root common.MapStr, path Path, wo walkObserver) error { + for k, v := range m { + //TODO: Handle this error + additionalPath, err := ParsePath(k) + if err != nil { + return err } + newPath := path.Concat(additionalPath) + + walkFull(v, root, newPath, wo) } + + return nil } diff --git a/libbeat/testing/mapvaltest/mapvaltest.go b/libbeat/testing/mapvaltest/mapvaltest.go index 3e28a591088..c1ed597669b 100644 --- a/libbeat/testing/mapvaltest/mapvaltest.go +++ b/libbeat/testing/mapvaltest/mapvaltest.go @@ -28,6 +28,8 @@ import ( "github.com/davecgh/go-spew/spew" + "github.com/stretchr/testify/require" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/mapval" ) @@ -35,7 +37,8 @@ import ( // Test takes the output from a Validator invocation and runs test assertions on the result. // If you are using this library for testing you will probably want to run Test(t, Schema(Map{...}), actual) as a pattern. func Test(t *testing.T, v mapval.Validator, m common.MapStr) *mapval.Results { - r := v(m) + r, err := v(m) + require.NoError(t, err) if !r.Valid { assert.Fail(