Skip to content

Commit

Permalink
feat: update error table with new columns (#4356)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ItsSudip
ItsSudip committed Feb 8, 2024
1 parent 26d4729 commit b8b5527
Show file tree
Hide file tree
Showing 4 changed files with 161 additions and 42 deletions.
50 changes: 35 additions & 15 deletions enterprise/reporting/error_reporting.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ var ErrorDetailReportsColumns = []string{
"count",
"status_code",
"event_type",
"event_name",
"sample_event",
"sample_response",
"error_code",
"error_message",
}
Expand Down Expand Up @@ -188,7 +191,6 @@ func (edr *ErrorDetailReporter) Report(ctx context.Context, metrics []*types.PUR
"sourceId": metric.ConnectionDetails.SourceID,
"destinationId": metric.ConnectionDetails.DestinationID,
}).Count(int(metric.StatusDetail.Count))

_, err = stmt.Exec(
workspaceID,
edr.namespace,
Expand All @@ -203,6 +205,9 @@ func (edr *ErrorDetailReporter) Report(ctx context.Context, metrics []*types.PUR
metric.StatusDetail.Count,
metric.StatusDetail.StatusCode,
metric.StatusDetail.EventType,
metric.StatusDetail.EventName,
string(metric.StatusDetail.SampleEvent),
metric.StatusDetail.SampleResponse,
errDets.ErrorCode,
errDets.ErrorMessage,
)
Expand Down Expand Up @@ -398,6 +403,9 @@ func (edr *ErrorDetailReporter) getReports(ctx context.Context, currentMs int64,
"count",
"status_code",
"event_type",
"event_name",
"sample_event",
"sample_response",
"error_code",
"error_message",
"dest_type",
Expand Down Expand Up @@ -427,6 +435,9 @@ func (edr *ErrorDetailReporter) getReports(ctx context.Context, currentMs int64,
"count",
"status_code",
"event_type",
"event_name"
"sample_event",
"sample_response",
"error_code",
"error_message",
"dest_type",
Expand All @@ -448,6 +459,9 @@ func (edr *ErrorDetailReporter) getReports(ctx context.Context, currentMs int64,
&dbEdMetric.Count,
&dbEdMetric.EDErrorDetails.StatusCode,
&dbEdMetric.EDErrorDetails.EventType,
&dbEdMetric.EDErrorDetails.EventName,
&dbEdMetric.EDErrorDetails.SampleEvent,
&dbEdMetric.EDErrorDetails.SampleResponse,
&dbEdMetric.EDErrorDetails.ErrorCode,
&dbEdMetric.EDErrorDetails.ErrorMessage,
&dbEdMetric.EDConnectionDetails.DestType,
Expand Down Expand Up @@ -505,16 +519,19 @@ func (edr *ErrorDetailReporter) aggregate(reports []*types.EDReportsDB) []*types
ReportedAt: firstReport.ReportedAt * 60 * 1000,
},
}
var errs []types.EDErrorDetails

reportsCountMap := lo.CountValuesBy(reports, func(rep *types.EDReportsDB) types.EDErrorDetails {
return types.EDErrorDetails{
messageMap := make(map[string]int)
reportsCountMap := make(map[types.EDErrorDetails]int64)
for index, rep := range reports {
messageMap[rep.EDErrorDetails.ErrorMessage] = index
errDet := types.EDErrorDetails{
StatusCode: rep.StatusCode,
ErrorCode: rep.ErrorCode,
ErrorMessage: rep.ErrorMessage,
EventType: rep.EventType,
EventName: rep.EventName,
}
})
reportsCountMap[errDet] += rep.Count
}

reportGrpKeys := lo.Keys(reportsCountMap)
sort.SliceStable(reportGrpKeys, func(i, j int) bool {
Expand All @@ -525,15 +542,18 @@ func (edr *ErrorDetailReporter) aggregate(reports []*types.EDReportsDB) []*types
irep.ErrorMessage < jrep.ErrorMessage ||
irep.EventType < jrep.EventType)
})
for _, rep := range reportGrpKeys {
errs = append(errs, types.EDErrorDetails{
StatusCode: rep.StatusCode,
ErrorCode: rep.ErrorCode,
ErrorMessage: rep.ErrorMessage,
EventType: rep.EventType,
Count: reportsCountMap[rep],
})
}
errs := lo.MapToSlice(reportsCountMap, func(rep types.EDErrorDetails, count int64) types.EDErrorDetails {
return types.EDErrorDetails{
StatusCode: rep.StatusCode,
ErrorCode: rep.ErrorCode,
ErrorMessage: rep.ErrorMessage,
EventType: rep.EventType,
EventName: rep.EventName,
SampleEvent: reports[messageMap[rep.ErrorMessage]].SampleEvent,
SampleResponse: reports[messageMap[rep.ErrorMessage]].SampleResponse,
Count: count,
}
})
edrSchema.Errors = errs
edrortingMetrics = append(edrortingMetrics, &edrSchema)
}
Expand Down
135 changes: 110 additions & 25 deletions enterprise/reporting/reporting_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,7 @@ func TestAggregationLogic(t *testing.T) {
ReportMetadata: types.ReportMetadata{
ReportedAt: 124335445,
},
Count: 5,
},
{
PU: "dest_transformer",
Expand All @@ -479,6 +480,7 @@ func TestAggregationLogic(t *testing.T) {
ReportMetadata: types.ReportMetadata{
ReportedAt: 124335445,
},
Count: 10,
},
{
PU: "dest_transformer",
Expand All @@ -503,6 +505,7 @@ func TestAggregationLogic(t *testing.T) {
ReportMetadata: types.ReportMetadata{
ReportedAt: 124335445,
},
Count: 10,
},
{
PU: "dest_transformer",
Expand All @@ -527,6 +530,7 @@ func TestAggregationLogic(t *testing.T) {
ReportMetadata: types.ReportMetadata{
ReportedAt: 124335446,
},
Count: 1,
},
// error occurred at router level(assume this is batching enabled)
{
Expand All @@ -544,14 +548,74 @@ func TestAggregationLogic(t *testing.T) {
DestType: "DES_1",
},
EDErrorDetails: types.EDErrorDetails{
StatusCode: 500,
ErrorCode: "",
ErrorMessage: "Cannot read type property of undefined", // some error during batching
EventType: "identify",
StatusCode: 500,
ErrorCode: "",
ErrorMessage: "Cannot read type property of undefined", // some error during batching
EventType: "track",
EventName: "Page View",
SampleResponse: "some response",
SampleEvent: "some sample event",
},
ReportMetadata: types.ReportMetadata{
ReportedAt: 124335446,
},
Count: 6,
},
{
PU: "router",
EDInstanceDetails: types.EDInstanceDetails{
WorkspaceID: "wsp1",
InstanceID: "instance-1",
Namespace: "nmspc",
},
EDConnectionDetails: types.EDConnectionDetails{
SourceID: "src-1",
SourceDefinitionId: "src-def-1",
DestinationDefinitionId: "des-def-1",
DestinationID: "des-1",
DestType: "DES_1",
},
EDErrorDetails: types.EDErrorDetails{
StatusCode: 500,
ErrorCode: "",
ErrorMessage: "Cannot read type property of undefined", // some error during batching
EventType: "track",
EventName: "Page View",
SampleResponse: "different response",
SampleEvent: "different sample event",
},
ReportMetadata: types.ReportMetadata{
ReportedAt: 124335446,
},
Count: 10,
},
{
PU: "router",
EDInstanceDetails: types.EDInstanceDetails{
WorkspaceID: "wsp1",
InstanceID: "instance-1",
Namespace: "nmspc",
},
EDConnectionDetails: types.EDConnectionDetails{
SourceID: "src-1",
SourceDefinitionId: "src-def-1",
DestinationDefinitionId: "des-def-1",
DestinationID: "des-1",
DestType: "DES_1",
},
EDErrorDetails: types.EDErrorDetails{
StatusCode: 401,
ErrorCode: "",
ErrorMessage: "Request had invalid authentication credentials Expected OAuth access token login cookie or other valid authentication credential See ",
EventType: "track",
EventName: "Page View",
SampleResponse: "some different response",
SampleEvent: "some different sample event",
},
ReportMetadata: types.ReportMetadata{
ReportedAt: 124335446,
},
Count: 10,
},
}
configSubscriber := newConfigSubscriber(logger.NOP)
Expand All @@ -578,18 +642,24 @@ func TestAggregationLogic(t *testing.T) {
},
Errors: []types.EDErrorDetails{
{
StatusCode: dbErrs[0].StatusCode,
ErrorCode: dbErrs[0].ErrorCode,
ErrorMessage: dbErrs[0].ErrorMessage,
EventType: dbErrs[0].EventType,
Count: 1,
StatusCode: dbErrs[0].StatusCode,
ErrorCode: dbErrs[0].ErrorCode,
ErrorMessage: dbErrs[0].ErrorMessage,
EventType: dbErrs[0].EventType,
EventName: dbErrs[0].EventName,
Count: 5,
SampleResponse: dbErrs[0].SampleResponse,
SampleEvent: dbErrs[0].SampleEvent,
},
{
StatusCode: dbErrs[1].StatusCode,
ErrorCode: dbErrs[1].ErrorCode,
ErrorMessage: dbErrs[1].ErrorMessage,
EventType: dbErrs[1].EventType,
Count: 2,
StatusCode: dbErrs[1].StatusCode,
ErrorCode: dbErrs[1].ErrorCode,
ErrorMessage: dbErrs[1].ErrorMessage,
EventType: dbErrs[1].EventType,
EventName: dbErrs[1].EventName,
Count: 20,
SampleResponse: dbErrs[1].SampleResponse,
SampleEvent: dbErrs[1].SampleEvent,
},
},
},
Expand All @@ -612,11 +682,14 @@ func TestAggregationLogic(t *testing.T) {
},
Errors: []types.EDErrorDetails{
{
StatusCode: dbErrs[3].StatusCode,
ErrorCode: dbErrs[3].ErrorCode,
ErrorMessage: dbErrs[3].ErrorMessage,
EventType: dbErrs[3].EventType,
Count: 1,
StatusCode: dbErrs[3].StatusCode,
ErrorCode: dbErrs[3].ErrorCode,
ErrorMessage: dbErrs[3].ErrorMessage,
EventType: dbErrs[3].EventType,
EventName: dbErrs[3].EventName,
Count: 1,
SampleResponse: dbErrs[3].SampleResponse,
SampleEvent: dbErrs[3].SampleEvent,
},
},
},
Expand All @@ -639,15 +712,27 @@ func TestAggregationLogic(t *testing.T) {
},
Errors: []types.EDErrorDetails{
{
StatusCode: dbErrs[4].StatusCode,
ErrorCode: dbErrs[4].ErrorCode,
ErrorMessage: dbErrs[4].ErrorMessage,
EventType: dbErrs[4].EventType,
Count: 1,
StatusCode: dbErrs[4].StatusCode,
ErrorCode: dbErrs[4].ErrorCode,
ErrorMessage: dbErrs[4].ErrorMessage,
EventType: dbErrs[4].EventType,
EventName: dbErrs[4].EventName,
Count: 16,
SampleResponse: dbErrs[5].SampleResponse,
SampleEvent: dbErrs[5].SampleEvent,
},
{
StatusCode: dbErrs[6].StatusCode,
ErrorCode: dbErrs[6].ErrorCode,
ErrorMessage: dbErrs[6].ErrorMessage,
EventType: dbErrs[6].EventType,
EventName: dbErrs[6].EventName,
Count: 10,
SampleResponse: dbErrs[6].SampleResponse,
SampleEvent: dbErrs[6].SampleEvent,
},
},
},
}

require.Equal(t, reportResults, reportingMetrics)
}
11 changes: 11 additions & 0 deletions sql/migrations/error_detail_reports/000002_alter_error_detail_reports.up.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
---
--- Error Detail Reports
---
ALTER TABLE
error_detail_reports
ADD
COLUMN IF NOT EXISTS event_name TEXT DEFAULT '',
ADD
COLUMN IF NOT EXISTS sample_event JSONB NOT NULL DEFAULT '{}'::jsonb,
ADD
COLUMN IF NOT EXISTS sample_response TEXT DEFAULT '';
7 changes: 5 additions & 2 deletions utils/types/reporting_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,11 @@ type EDErrorDetails struct {
ErrorCode string `json:"errorCode"`
ErrorMessage string `json:"errorMessage"`
// TODO: need to check with team if this makes sense ?
EventType string `json:"-"`
Count int `json:"count"`
EventType string `json:"eventType"`
EventName string `json:"eventName"`
Count int64 `json:"count"`
SampleResponse string `json:"sampleResponse"`
SampleEvent string `json:"sampleEvent"`
}

type EDReportsDB struct {
Expand Down

0 comments on commit b8b5527

Please sign in to comment.