Merge 6af3cbd into 44eba8c

filter/metrics_parser_test.go

@@ -23,6 +23,7 @@ func TestParseMetric(t *testing.T) {
 			"Invalid.value 12g5 1234567890",
 			"No.value.two.spaces  1234567890",
 			"No.timestamp.space.in.the.end 12 ",
+			"No.value.no.timestamp",
 			"No.timestamp 12",
 			" 12 1234567890",
 			"Non-ascii.こんにちは 12 1234567890",
@@ -37,9 +38,11 @@ func TestParseMetric(t *testing.T) {
 			"Newline.in.the.end 12 1234567890\n",
 			"Newline.in.the.end 12 1234567890\r",
 			"Newline.in.the.end 12 1234567890\r\n",
-			";empty.name.but.with.label= 1 2",
-			"no.labels.but.delimiter.in.the.end; 1 2",
-			"empty.label.name;= 1 2",
+			";Empty.name.but.with.label= 1 2",
+			"No.labels.but.delimiter.in.the.end; 1 2",
+			"Empty.label.name;= 1 2",
+			"Only.label.name;name 1 2",
+			"Too.Many.=.In.Label;name=value= 1 2",
 		}
 
 		for _, invalidMetric := range invalidMetrics {

filter/pattern_index.go

@@ -0,0 +1,161 @@
+package filter
+
+import (
+	"fmt"
+	"path"
+	"strings"
+
+	"github.com/moira-alert/moira"
+
+	"github.com/vova616/xxhash"
+)
+
+var asteriskHash = xxhash.Checksum32([]byte("*"))
+
+// PatternNode contains pattern node
+type PatternNode struct {
+	Children   []*PatternNode
+	Part       string
+	Hash       uint32
+	Prefix     string
+	InnerParts []string
+}
+
+// PatternIndex helps to index patterns and allows to match them by metric
+type PatternIndex struct {
+	Root *PatternNode
+}
+
+// NewPatternIndex creates new PatternIndex using patterns
+func NewPatternIndex(patterns []string) *PatternIndex {
+	root := &PatternNode{}
+
+	for _, pattern := range patterns {
+		currentNode := root
+		parts := strings.Split(pattern, ".")
+		if hasEmptyParts(parts) {
+			continue
+		}
+		for _, part := range parts {
+			found := false
+			for _, child := range currentNode.Children {
+				if part == child.Part {
+					currentNode = child
+					found = true
+					break
+				}
+			}
+			if !found {
+				newNode := &PatternNode{Part: part}
+
+				if currentNode.Prefix == "" {
+					newNode.Prefix = part
+				} else {
+					newNode.Prefix = fmt.Sprintf("%s.%s", currentNode.Prefix, part)
+				}
+
+				if part == "*" || !strings.ContainsAny(part, "{*?") {
+					newNode.Hash = xxhash.Checksum32([]byte(part))
+				} else {
+					if strings.Contains(part, "{") && strings.Contains(part, "}") {
+						prefix, bigSuffix := split2(part, "{")
+						inner, suffix := split2(bigSuffix, "}")
+						innerParts := strings.Split(inner, ",")
+
+						newNode.InnerParts = make([]string, 0, len(innerParts))
+						for _, innerPart := range innerParts {
+							newNode.InnerParts = append(newNode.InnerParts, fmt.Sprintf("%s%s%s", prefix, innerPart, suffix))
+						}
+					} else {
+						newNode.InnerParts = []string{part}
+					}
+				}
+				currentNode.Children = append(currentNode.Children, newNode)
+				currentNode = newNode
+			}
+		}
+	}
+
+	return &PatternIndex{Root: root}
+}
+
+// MatchPatterns allows to match pattern by metric
+func (source *PatternIndex) MatchPatterns(metric string) []string {
+	currentLevel := []*PatternNode{source.Root}
+	var found, index int
+	for i, c := range metric {
+		if c == '.' {
+			part := metric[index:i]
+
+			if len(part) == 0 {
+				return []string{}
+			}
+
+			index = i + 1
+
+			currentLevel, found = findPart(part, currentLevel)
+			if found == 0 {
+				return []string{}
+			}
+		}
+	}
+
+	part := metric[index:]
+	currentLevel, found = findPart(part, currentLevel)
+	if found == 0 {
+		return []string{}
+	}
+
+	matched := make([]string, 0, found)
+	for _, node := range currentLevel {
+		if len(node.Children) == 0 {
+			matched = append(matched, node.Prefix)
+		}
+	}
+
+	return matched
+}
+
+func findPart(part string, currentLevel []*PatternNode) ([]*PatternNode, int) {
+	nextLevel := make([]*PatternNode, 0, 64)
+	hash := xxhash.Checksum32(moira.UnsafeStringToBytes(part))
+	for _, node := range currentLevel {
+		for _, child := range node.Children {
+			match := false
+
+			if child.Hash == asteriskHash || child.Hash == hash {
+				match = true
+			} else if len(child.InnerParts) > 0 {
+				for _, innerPart := range child.InnerParts {
+					innerMatch, _ := path.Match(innerPart, part)
+					if innerMatch {
+						match = true
+						break
+					}
+				}
+			}
+
+			if match {
+				nextLevel = append(nextLevel, child)
+			}
+		}
+	}
+	return nextLevel, len(nextLevel)
+}
+
+func split2(s, sep string) (string, string) {
+	splitResult := strings.SplitN(s, sep, 2)
+	if len(splitResult) < 2 {
+		return splitResult[0], ""
+	}
+	return splitResult[0], splitResult[1]
+}
+
+func hasEmptyParts(parts []string) bool {
+	for _, part := range parts {
+		if part == "" {
+			return true
+		}
+	}
+	return false
+}

filter/pattern_index_test.go

@@ -0,0 +1,57 @@
+package filter
+
+import "testing"
+
+import . "github.com/smartystreets/goconvey/convey"
+
+func TestPatternIndex(t *testing.T) {
+	Convey("Given patterns, should build index and match patterns", t, func() {
+		patterns := []string{
+			"Simple.matching.pattern",
+			"Star.single.*",
+			"Star.*.double.any*",
+			"Bracket.{one,two,three}.pattern",
+			"Bracket.pr{one,two,three}suf",
+			"Complex.matching.pattern",
+			"Complex.*.*",
+			"Complex.*.",
+			"Complex.*{one,two,three}suf*.pattern",
+			"Question.?at_begin",
+			"Question.at_the_end?",
+		}
+
+		index := NewPatternIndex(patterns)
+		testCases := []struct {
+			Metric          string
+			MatchedPatterns []string
+		}{
+			{"Simple.matching.pattern", []string{"Simple.matching.pattern"}},
+			{"Star.single.anything", []string{"Star.single.*"}},
+			{"Star.anything.double.anything", []string{"Star.*.double.any*"}},
+			{"Bracket.one.pattern", []string{"Bracket.{one,two,three}.pattern"}},
+			{"Bracket.two.pattern", []string{"Bracket.{one,two,three}.pattern"}},
+			{"Bracket.three.pattern", []string{"Bracket.{one,two,three}.pattern"}},
+			{"Bracket.pronesuf", []string{"Bracket.pr{one,two,three}suf"}},
+			{"Bracket.prtwosuf", []string{"Bracket.pr{one,two,three}suf"}},
+			{"Bracket.prthreesuf", []string{"Bracket.pr{one,two,three}suf"}},
+			{"Complex.matching.pattern", []string{"Complex.matching.pattern", "Complex.*.*"}},
+			{"Complex.anything.pattern", []string{"Complex.*.*"}},
+			{"Complex.prefixonesuffix.pattern", []string{"Complex.*.*", "Complex.*{one,two,three}suf*.pattern"}},
+			{"Complex.prefixtwofix.pattern", []string{"Complex.*.*"}},
+			{"Complex.anything.pattern", []string{"Complex.*.*"}},
+			{"Question.1at_begin", []string{"Question.?at_begin"}},
+			{"Question.at_the_end2", []string{"Question.at_the_end?"}},
+			{"Two.dots..together", []string{}},
+			{"Simple.notmatching.pattern", []string{}},
+			{"Star.nothing", []string{}},
+			{"Bracket.one.nothing", []string{}},
+			{"Bracket.nothing.pattern", []string{}},
+			{"Complex.prefixonesuffix", []string{}},
+		}
+
+		for _, testCase := range testCases {
+			matchedPatterns := index.MatchPatterns(testCase.Metric)
+			So(matchedPatterns, ShouldResemble, testCase.MatchedPatterns)
+		}
+	})
+}

filter/patterns/worker.go

@@ -31,7 +31,7 @@ func NewRefreshPatternWorker(database moira.Database, metrics *graphite.FilterMe
 
 // Start process to refresh pattern tree every second
 func (worker *RefreshPatternWorker) Start() error {
-	err := worker.patternStorage.RefreshTree()
+	err := worker.patternStorage.Refresh()
 	if err != nil {
 		worker.logger.Errorf("pattern refresh failed: %s", err.Error())
 		return err
@@ -46,7 +46,7 @@ func (worker *RefreshPatternWorker) Start() error {
 				return nil
 			case <-checkTicker.C:
 				timer := time.Now()
-				err := worker.patternStorage.RefreshTree()
+				err := worker.patternStorage.Refresh()
 				if err != nil {
 					worker.logger.Errorf("Pattern refresh failed: %s", err.Error())
 				}