Enhance mapval to handle slices (elastic#7784)

* Enhance mapval to handle arrays

Introduces initial support for slices into mapval. There are lots of good refactor opportunities this opens up, but I want to draw the line somewhere close.

I came across a case where I needed array support in elastic#7545 which I'll be rebasing on this PR.

This required a lot of internals refactoring. The highlights are:

While the code is more complex, there are fewer special cases, and the control flow is less complicated
Making value validation less of a special case
Redesigning the walk function to work correctly with slices. This required using reflect extensively.
Validating a value involves returning a full mapval.Results map which gets merged into the main Validator results. This is a better internal design and allows for more flexibility. It does mean that custom IsDef validations have a slightly more complicated signature since they need to know the path of the thing they're validating, but that's a good thing in terms of power.
Introducing a Path type instead of using simple strings for paths.
Moved away from using the MapStr Get method to fetch subkeys. This was required to allow users to specify slice subscript.
Tests was added for comparing slices in various ways. As literals, with nested matchers, etc.

* Validate Map objects when compiling schema. Rename Schema->Compile.

We no longer need to return an error from actually executing a Validator as a result.

This also fixes a bug where we would expand keynames in input maps to validators.

heartbeat/hbtest/hbtestutil.go

@@ -61,7 +61,7 @@ func ServerPort(server *httptest.Server) (uint16, error) {
 // MonitorChecks creates a skima.Validator that represents the "monitor" field present
 // in all heartbeat events.
 func MonitorChecks(id string, host string, ip string, scheme string, status string) mapval.Validator {
-	return mapval.Schema(mapval.Map{
+	return mapval.MustCompile(mapval.Map{
 		"monitor": mapval.Map{
 			// TODO: This is only optional because, for some reason, TCP returns
 			// this value, but HTTP does not. We should fix this
@@ -78,14 +78,14 @@ func MonitorChecks(id string, host string, ip string, scheme string, status stri
 // TCPBaseChecks checks the minimum TCP response, which is only issued
 // without further fields when the endpoint does not respond.
 func TCPBaseChecks(port uint16) mapval.Validator {
-	return mapval.Schema(mapval.Map{"tcp.port": port})
+	return mapval.MustCompile(mapval.Map{"tcp.port": port})
 }
 
 // ErrorChecks checks the standard heartbeat error hierarchy, which should
 // consist of a message (or a mapval isdef that can match the message) and a type under the error key.
 // The message is checked only as a substring since exact string matches can be fragile due to platform differences.
 func ErrorChecks(msgSubstr string, errType string) mapval.Validator {
-	return mapval.Schema(mapval.Map{
+	return mapval.MustCompile(mapval.Map{
 		"error": mapval.Map{
 			"message": mapval.IsStringContaining(msgSubstr),
 			"type":    errType,
@@ -98,6 +98,6 @@ func ErrorChecks(msgSubstr string, errType string) mapval.Validator {
 func RespondingTCPChecks(port uint16) mapval.Validator {
 	return mapval.Compose(
 		TCPBaseChecks(port),
-		mapval.Schema(mapval.Map{"tcp.rtt.connect.us": mapval.IsDuration}),
+		mapval.MustCompile(mapval.Map{"tcp.rtt.connect.us": mapval.IsDuration}),
 	)
 }

heartbeat/monitors/active/http/http_test.go

@@ -62,15 +62,15 @@ func checkServer(t *testing.T, handlerFunc http.HandlerFunc) (*httptest.Server,
 // The minimum response is just the URL. Only to be used for unreachable server
 // tests.
 func httpBaseChecks(url string) mapval.Validator {
-	return mapval.Schema(mapval.Map{
+	return mapval.MustCompile(mapval.Map{
 		"http.url": url,
 	})
 }
 
 func respondingHTTPChecks(url string, statusCode int) mapval.Validator {
 	return mapval.Compose(
 		httpBaseChecks(url),
-		mapval.Schema(mapval.Map{
+		mapval.MustCompile(mapval.Map{
 			"http": mapval.Map{
 				"response.status_code":   statusCode,
 				"rtt.content.us":         mapval.IsDuration,

heartbeat/monitors/active/tcp/tcp_test.go

@@ -77,7 +77,7 @@ func TestUpEndpointJob(t *testing.T) {
 				"up",
 			),
 			hbtest.RespondingTCPChecks(port),
-			mapval.Schema(mapval.Map{
+			mapval.MustCompile(mapval.Map{
 				"resolve": mapval.Map{
 					"host":   "localhost",
 					"ip":     "127.0.0.1",

libbeat/common/mapval/compiled_schema.go

@@ -0,0 +1,44 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mapval
+
+import "github.com/elastic/beats/libbeat/common"
+
+type flatValidator struct {
+	path  Path
+	isDef IsDef
+}
+
+// CompiledSchema represents a compiled definition for driving a Validator.
+type CompiledSchema []flatValidator
+
+// Check executes the the checks within the CompiledSchema
+func (cs CompiledSchema) Check(actual common.MapStr) *Results {
+	results := NewResults()
+	for _, pv := range cs {
+		actualV, actualKeyExists := pv.path.GetFrom(actual)
+
+		if !pv.isDef.optional || pv.isDef.optional && actualKeyExists {
+			var checkRes *Results
+			checkRes = pv.isDef.check(pv.path, actualV, actualKeyExists)
+			results.merge(checkRes)
+		}
+	}
+
+	return results
+}

libbeat/common/mapval/core.go

@@ -18,6 +18,7 @@
 package mapval
 
 import (
+	"reflect"
 	"sort"
 	"strings"
 
@@ -36,10 +37,13 @@ func Optional(id IsDef) IsDef {
 	return id
 }
 
-// Map is the type used to define schema definitions for Schema.
+// Map is the type used to define schema definitions for Compile.
 type Map map[string]interface{}
 
-// Validator is the result of Schema and is run against the map you'd like to test.
+// Slice is a convenience []interface{} used to declare schema defs.
+type Slice []interface{}
+
+// Validator is the result of Compile and is run against the map you'd like to test.
 type Validator func(common.MapStr) *Results
 
 // Compose combines multiple SchemaValidators into a single one.
@@ -52,7 +56,7 @@ func Compose(validators ...Validator) Validator {
 
 		combined := NewResults()
 		for _, r := range results {
-			r.EachResult(func(path string, vr ValueResult) bool {
+			r.EachResult(func(path Path, vr ValueResult) bool {
 				combined.record(path, vr)
 				return true
 			})
@@ -83,61 +87,63 @@ func Strict(laxValidator Validator) Validator {
 		}
 		sort.Strings(validatedPaths)
 
-		walk(actual, func(woi walkObserverInfo) {
-			_, validatedExactly := results.Fields[woi.dottedPath]
+		walk(actual, false, func(woi walkObserverInfo) error {
+			_, validatedExactly := results.Fields[woi.path.String()]
 			if validatedExactly {
-				return // This key was tested, passes strict test
+				return nil // This key was tested, passes strict test
 			}
 
 			// Search returns the point just before an actual match (since we ruled out an exact match with the cheaper
 			// hash check above. We have to validate the actual match with a prefix check as well
-			matchIdx := sort.SearchStrings(validatedPaths, woi.dottedPath)
-			if matchIdx < len(validatedPaths) && strings.HasPrefix(validatedPaths[matchIdx], woi.dottedPath) {
-				return
+			matchIdx := sort.SearchStrings(validatedPaths, woi.path.String())
+			if matchIdx < len(validatedPaths) && strings.HasPrefix(validatedPaths[matchIdx], woi.path.String()) {
+				return nil
 			}
 
-			results.record(woi.dottedPath, StrictFailureVR)
+			results.merge(StrictFailureResult(woi.path))
+
+			return nil
 		})
 
 		return results
 	}
 }
 
-// Schema takes a Map and returns an executable Validator function.
-func Schema(expected Map) Validator {
-	return func(actual common.MapStr) *Results {
-		return walkValidate(expected, actual)
-	}
-}
-
-func walkValidate(expected Map, actual common.MapStr) (results *Results) {
-	results = NewResults()
-	walk(
-		common.MapStr(expected),
-		func(expInfo walkObserverInfo) {
-
-			actualKeyExists, _ := actual.HasKey(expInfo.dottedPath)
-			actualV, _ := actual.GetValue(expInfo.dottedPath)
-
-			// If this is a definition use it, if not, check exact equality
-			isDef, isIsDef := expInfo.value.(IsDef)
+// Compile takes the given map, validates the paths within it, and returns
+// a Validator that can check real data.
+func Compile(in Map) (validator Validator, err error) {
+	compiled := make(CompiledSchema, 0)
+	err = walk(common.MapStr(in), true, func(current walkObserverInfo) error {
+
+		// Determine whether we should test this value
+		// We want to test all values except collections that contain a value
+		// If a collection contains a value, we check those 'leaf' values instead
+		rv := reflect.ValueOf(current.value)
+		kind := rv.Kind()
+		isCollection := kind == reflect.Map || kind == reflect.Slice
+		isNonEmptyCollection := isCollection && rv.Len() > 0
+
+		if !isNonEmptyCollection {
+			isDef, isIsDef := current.value.(IsDef)
 			if !isIsDef {
-				// We don't check maps for equality, we check their properties
-				// individual via our own traversal, so bail early
-				if _, isMS := actualV.(common.MapStr); isMS {
-					return
-				}
-
-				isDef = IsEqual(expInfo.value)
+				isDef = IsEqual(current.value)
 			}
 
-			if !isDef.optional || isDef.optional && actualKeyExists {
-				results.record(
-					expInfo.dottedPath,
-					isDef.check(actualV, actualKeyExists),
-				)
-			}
-		})
+			compiled = append(compiled, flatValidator{current.path, isDef})
+		}
+		return nil
+	})
 
-	return results
+	return func(actual common.MapStr) *Results {
+		return compiled.Check(actual)
+	}, err
+}
+
+// MustCompile compiles the given map, panic-ing if that map is invalid.
+func MustCompile(in Map) Validator {
+	compiled, err := Compile(in)
+	if err != nil {
+		panic(err)
+	}
+	return compiled
 }