From c066b5885efb57decafbee9fbbb532b4e2041529 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Tue, 2 Jan 2018 03:47:56 -0500 Subject: [PATCH] consider exception or log message for grouping key if nothing else is available (#435) closes #402 --- processor/error/event.go | 61 ++++++++++++++----- processor/error/event_test.go | 55 +++++++++++------ .../TestProcessErrorFull.approved.json | 6 +- .../TestProcessErrorNullValues.approved.json | 8 +-- processor/error/payload_test.go | 2 +- 5 files changed, 89 insertions(+), 43 deletions(-) diff --git a/processor/error/event.go b/processor/error/event.go index 8cec7a967a6..5aed08d20b3 100644 --- a/processor/error/event.go +++ b/processor/error/event.go @@ -4,6 +4,7 @@ import ( "crypto/md5" "encoding/hex" "fmt" + "hash" "io" "strconv" "time" @@ -132,40 +133,68 @@ func (e *Event) addGroupingKey() { e.add("grouping_key", e.calcGroupingKey()) } -func (e *Event) calcGroupingKey() string { - hash := md5.New() +type groupingKey struct { + hash hash.Hash + empty bool +} - add := func(s *string) bool { - if s != nil { - io.WriteString(hash, *s) - } - return s != nil +func newGroupingKey() *groupingKey { + return &groupingKey{ + hash: md5.New(), + empty: true, + } +} + +func (k *groupingKey) add(s *string) bool { + if s == nil { + return false } + io.WriteString(k.hash, *s) + k.empty = false + return true +} - addEither := func(s *string, s2 string) { - if ok := add(s); ok == false { - add(&s2) - } +func (k *groupingKey) addEither(s1 *string, s2 string) { + if ok := k.add(s1); !ok { + k.add(&s2) } +} + +func (k *groupingKey) String() string { + return hex.EncodeToString(k.hash.Sum(nil)) +} + +// calcGroupingKey computes a value for deduplicating errors - events with +// same grouping key can be collapsed together. +func (e *Event) calcGroupingKey() string { + k := newGroupingKey() var st m.Stacktrace if e.Exception != nil { - add(e.Exception.Type) + k.add(e.Exception.Type) st = e.Exception.Stacktrace } if e.Log != nil { - add(e.Log.ParamMessage) + k.add(e.Log.ParamMessage) if st == nil || len(st) == 0 { st = e.Log.Stacktrace } } for _, fr := range st { - addEither(fr.Module, fr.Filename) - addEither(fr.Function, string(fr.Lineno)) + k.addEither(fr.Module, fr.Filename) + k.addEither(fr.Function, string(fr.Lineno)) + } + + if k.empty { + if e.Exception != nil { + k.add(&e.Exception.Message) + } else if e.Log != nil { + k.add(&e.Log.Message) + } } - return hex.EncodeToString(hash.Sum(nil)) + return k.String() } func (e *Event) add(key string, val interface{}) { diff --git a/processor/error/event_test.go b/processor/error/event_test.go index e4baa3cec5b..e5ee3950f1d 100644 --- a/processor/error/event_test.go +++ b/processor/error/event_test.go @@ -4,6 +4,7 @@ import ( "crypto/md5" "encoding/hex" "fmt" + "io" "testing" "github.com/stretchr/testify/assert" @@ -82,9 +83,14 @@ func TestEventTransform(t *testing.T) { context := common.MapStr{"user": common.MapStr{"id": "888"}, "c1": "val"} - emptyOut := common.MapStr{ - "grouping_key": hex.EncodeToString(md5.New().Sum(nil)), - } + baseExceptionHash := md5.New() + io.WriteString(baseExceptionHash, baseException().Message) + // 706a38d554b47b8f82c6b542725c05dc + baseExceptionGroupingKey := hex.EncodeToString(baseExceptionHash.Sum(nil)) + + baseLogHash := md5.New() + io.WriteString(baseLogHash, baseLog().Message) + baseLogGroupingKey := hex.EncodeToString(baseLogHash.Sum(nil)) tests := []struct { Event Event @@ -92,49 +98,60 @@ func TestEventTransform(t *testing.T) { Msg string }{ { - Event: Event{}, - Output: emptyOut, - Msg: "Minimal Event, default stacktrace transformation fn", - }, - { - Event: Event{Exception: baseException().withCode("13")}, + Event: Event{}, Output: common.MapStr{ - "exception": common.MapStr{"code": "13", "message": "exception message"}, "grouping_key": hex.EncodeToString(md5.New().Sum(nil)), }, - Msg: "Minimal Event, default stacktrace transformation fn", + Msg: "Minimal Event", }, { Event: Event{Log: baseLog()}, Output: common.MapStr{ "log": common.MapStr{"message": "error log message"}, - "grouping_key": hex.EncodeToString(md5.New().Sum(nil)), + "grouping_key": baseLogGroupingKey, + }, + Msg: "Minimal Event wth log", + }, + { + Event: Event{Exception: baseException(), Log: baseLog()}, + Output: common.MapStr{ + "exception": common.MapStr{"message": "exception message"}, + "log": common.MapStr{"message": "error log message"}, + "grouping_key": baseExceptionGroupingKey, }, - Msg: "Minimal Event wth log, default stacktrace transformation fn", + Msg: "Minimal Event wth log and exception", + }, + { + Event: Event{Exception: baseException()}, + Output: common.MapStr{ + "exception": common.MapStr{"message": "exception message"}, + "grouping_key": baseExceptionGroupingKey, + }, + Msg: "Minimal Event with exception", }, { Event: Event{Exception: baseException().withCode("13")}, Output: common.MapStr{ "exception": common.MapStr{"message": "exception message", "code": "13"}, - "grouping_key": hex.EncodeToString(md5.New().Sum(nil)), + "grouping_key": baseExceptionGroupingKey, }, - Msg: "Minimal Event wth exception, string code, default stacktrace transformation fn", + Msg: "Minimal Event wth exception, string code", }, { Event: Event{Exception: baseException().withCode(13)}, Output: common.MapStr{ "exception": common.MapStr{"message": "exception message", "code": "13"}, - "grouping_key": hex.EncodeToString(md5.New().Sum(nil)), + "grouping_key": baseExceptionGroupingKey, }, - Msg: "Minimal Event wth exception, int code, default stacktrace transformation fn", + Msg: "Minimal Event wth exception, int code", }, { Event: Event{Exception: baseException().withCode(13.0)}, Output: common.MapStr{ "exception": common.MapStr{"message": "exception message", "code": "13"}, - "grouping_key": hex.EncodeToString(md5.New().Sum(nil)), + "grouping_key": baseExceptionGroupingKey, }, - Msg: "Minimal Event wth exception, float code, default stacktrace transformation fn", + Msg: "Minimal Event wth exception, float code", }, { Event: Event{ diff --git a/processor/error/package_tests/TestProcessErrorFull.approved.json b/processor/error/package_tests/TestProcessErrorFull.approved.json index 92f411cbff3..86473cf08dd 100644 --- a/processor/error/package_tests/TestProcessErrorFull.approved.json +++ b/processor/error/package_tests/TestProcessErrorFull.approved.json @@ -290,7 +290,7 @@ "code": "35", "message": "foo is not defined" }, - "grouping_key": "d41d8cd98f00b204e9800998ecf8427e", + "grouping_key": "f6b5a2877d9b00d5b32b44c9db039f11", "id": "9f0e9d68-c185-4d21-a6f4-4673ed561ec8" }, "processor": { @@ -338,7 +338,7 @@ "exception": { "message": "foo.bar is not a function" }, - "grouping_key": "d41d8cd98f00b204e9800998ecf8427e", + "grouping_key": "5be374e988ceb5382d62c7ab53764663", "id": "9f0e9d68-c185-4d21-a6f4-4673ed561ec8" }, "processor": { @@ -383,7 +383,7 @@ } }, "error": { - "grouping_key": "d41d8cd98f00b204e9800998ecf8427e", + "grouping_key": "d6b3f958dfea98dc9ed2b57d5f0c48bb", "id": "9f0e9d67-c185-4d21-a6f4-4673ed561ec8", "log": { "message": "Cannot read property 'baz' of undefined" diff --git a/processor/error/package_tests/TestProcessErrorNullValues.approved.json b/processor/error/package_tests/TestProcessErrorNullValues.approved.json index 80e1c0d9931..3134539bbef 100644 --- a/processor/error/package_tests/TestProcessErrorNullValues.approved.json +++ b/processor/error/package_tests/TestProcessErrorNullValues.approved.json @@ -19,7 +19,7 @@ "exception": { "message": "The username root is unknown" }, - "grouping_key": "d41d8cd98f00b204e9800998ecf8427e", + "grouping_key": "b221d3265304e99841ca9be334c00851", "log": { "message": "My service could not talk to the database named foobar" } @@ -79,7 +79,7 @@ "exception": { "message": "foo is not defined" }, - "grouping_key": "d41d8cd98f00b204e9800998ecf8427e" + "grouping_key": "f6b5a2877d9b00d5b32b44c9db039f11" }, "processor": { "event": "error", @@ -133,7 +133,7 @@ } }, "error": { - "grouping_key": "d41d8cd98f00b204e9800998ecf8427e", + "grouping_key": "5be374e988ceb5382d62c7ab53764663", "log": { "message": "foo.bar is not a function" } @@ -164,7 +164,7 @@ "user": null }, "error": { - "grouping_key": "d41d8cd98f00b204e9800998ecf8427e", + "grouping_key": "d6b3f958dfea98dc9ed2b57d5f0c48bb", "log": { "message": "Cannot read property 'baz' of undefined" } diff --git a/processor/error/payload_test.go b/processor/error/payload_test.go index 81f7903a37c..107a9256841 100644 --- a/processor/error/payload_test.go +++ b/processor/error/payload_test.go @@ -65,7 +65,7 @@ func TestPayloadTransform(t *testing.T) { }, }, "error": common.MapStr{ - "grouping_key": "d41d8cd98f00b204e9800998ecf8427e", + "grouping_key": "706a38d554b47b8f82c6b542725c05dc", "exception": common.MapStr{"message": "exception message"}, "log": common.MapStr{"message": "error log message"}, },