Skip to content

Commit

Permalink
chore: reduce cardinality of warehouse metrics (#4364)
Browse files Browse the repository at this point in the history
* chore: reduce cardinality of warehouse metrics

* addressed comments
  • Loading branch information
mihir20 authored and atzoum committed Feb 12, 2024
1 parent 09b9f71 commit 2cd917c
Showing 1 changed file with 11 additions and 18 deletions.
29 changes: 11 additions & 18 deletions warehouse/router/upload_stats.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"slices"
"strings"

"github.com/rudderlabs/rudder-go-kit/config"
"github.com/rudderlabs/rudder-go-kit/stats"
"github.com/rudderlabs/rudder-server/utils/misc"
"github.com/rudderlabs/rudder-server/warehouse/logfield"
Expand Down Expand Up @@ -109,43 +108,37 @@ func (job *UploadJob) generateUploadAbortedMetrics() {
}

func (job *UploadJob) recordTableLoad(tableName string, numEvents int64) {
capturedTableName := strings.ToLower(tableName)
rudderAPISupportedEventTypes := []string{"tracks", "identifies", "pages", "screens", "aliases", "groups"}
if slices.Contains(rudderAPISupportedEventTypes, strings.ToLower(tableName)) {
// record total events synced (ignoring additional row synced to the event table for e.g.track call)
job.counterStat(`event_delivery`, warehouseutils.Tag{
Name: "tableName",
Value: strings.ToLower(tableName),
}).Count(int(numEvents))
if !slices.Contains(rudderAPISupportedEventTypes, capturedTableName) {
// making all other tableName as other, to reduce cardinality
capturedTableName = "others"
}

skipMetricTagForEachEventTable := config.GetBool("Warehouse.skipMetricTagForEachEventTable", false)
if skipMetricTagForEachEventTable {
standardTablesToRecordEventsMetric := []string{"tracks", "users", "identifies", "pages", "screens", "aliases", "groups", "rudder_discards"}
if !slices.Contains(standardTablesToRecordEventsMetric, strings.ToLower(tableName)) {
// club all event table metric tags under one tag to avoid too many tags
tableName = "others"
}
}
job.counterStat(`event_delivery`, warehouseutils.Tag{
Name: "tableName",
Value: capturedTableName,
}).Count(int(numEvents))

job.counterStat(`rows_synced`, warehouseutils.Tag{
Name: "tableName",
Value: strings.ToLower(tableName),
Value: capturedTableName,
}).Count(int(numEvents))

// Delay for the oldest event in the batch
firstEventAt, err := job.stagingFileRepo.FirstEventForUpload(job.ctx, job.upload)
if err != nil {
job.logger.Errorf("[WH]: Failed to generate delay metrics: %s, Err: %v", job.warehouse.Identifier, err)
return
}

if !job.upload.Retried {
conf := job.warehouse.Destination.Config
syncFrequency := "1440"
if conf[warehouseutils.SyncFrequency] != nil {
syncFrequency, _ = conf[warehouseutils.SyncFrequency].(string)
}
job.timerStat("event_delivery_time",
warehouseutils.Tag{Name: "tableName", Value: strings.ToLower(tableName)},
warehouseutils.Tag{Name: "tableName", Value: capturedTableName},
warehouseutils.Tag{Name: "syncFrequency", Value: syncFrequency},
).Since(firstEventAt)
}
Expand Down

0 comments on commit 2cd917c

Please sign in to comment.