Skip to content

Commit

Permalink
fix incorrect timestamp replacement
Browse files Browse the repository at this point in the history
  • Loading branch information
Vadman97 committed Apr 17, 2024
1 parent 72775a6 commit 8844ac6
Show file tree
Hide file tree
Showing 4 changed files with 13 additions and 9 deletions.
4 changes: 2 additions & 2 deletions backend/clickhouse/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func (client *Client) WriteErrorGroups(ctx context.Context, groups []*model.Erro
NewStruct(new(ClickhouseErrorGroup)).
InsertInto(ErrorGroupsTable, chGroups...).
BuildWithFlavor(sqlbuilder.ClickHouse)
sql, args = replaceTimestampInserts(sql, args, 10, map[int]bool{1: true, 2: true}, MicroSeconds)
sql, args = replaceTimestampInserts(sql, args, map[int]bool{1: true, 2: true}, MicroSeconds)
return client.conn.Exec(chCtx, sql, args...)
}

Expand Down Expand Up @@ -157,7 +157,7 @@ func (client *Client) WriteErrorObjects(ctx context.Context, objects []*model.Er
NewStruct(new(ClickhouseErrorObject)).
InsertInto(ErrorObjectsTable, chObjects...).
BuildWithFlavor(sqlbuilder.ClickHouse)
sql, args = replaceTimestampInserts(sql, args, 14, map[int]bool{1: true}, MicroSeconds)
sql, args = replaceTimestampInserts(sql, args, map[int]bool{1: true}, MicroSeconds)
return client.conn.Exec(chCtx, sql, args...)
}

Expand Down
12 changes: 8 additions & 4 deletions backend/clickhouse/insert.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"regexp"
"slices"
"strings"
)

type TimeUnit uint8
Expand All @@ -15,14 +16,17 @@ const (
NanoSeconds TimeUnit = 9
)

var pattern = regexp.MustCompile(`\?`)
var keysPattern = regexp.MustCompile(`^INSERT INTO \w+ \(([^)]+)\) VALUES`)
var argPattern = regexp.MustCompile(`\?`)

// replaceTimestampInserts updates direct timestamp inserts to accept int64 unix values
func replaceTimestampInserts(sql string, args []interface{}, numColumns int, columnsToReplace map[int]bool, scale TimeUnit) (string, []interface{}) {
func replaceTimestampInserts(sql string, args []interface{}, columnsToReplace map[int]bool, scale TimeUnit) (string, []interface{}) {
keysMatch := keysPattern.FindStringSubmatch(sql)
keys := strings.Split(keysMatch[1], ",")
var replaced, found int
sql = pattern.ReplaceAllStringFunc(sql, func(s string) string {
sql = argPattern.ReplaceAllStringFunc(sql, func(s string) string {
defer func() { found++ }()
idx := found % numColumns
idx := found % len(keys)
if !columnsToReplace[idx] {
return "?"
}
Expand Down
2 changes: 1 addition & 1 deletion backend/clickhouse/insert_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ func Benchmark_replaceTimestampInserts(b *testing.B) {
for i := 0; i < b.N; i++ {
replaceTimestampInserts("INSERT INTO error_groups (ProjectID, CreatedAt, UpdatedAt, ID, Event, Status, Type, ErrorTagID, ErrorTagTitle, ErrorTagDescription) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", []interface{}{
1, ts, ts + 1, 2, "event", "status", "type", 3, "title", "description",
}, 10, map[int]bool{0: true, 1: true, 2: true, 7: true, 9: true}, NanoSeconds)
}, map[int]bool{0: true, 1: true, 2: true, 7: true, 9: true}, NanoSeconds)
}
assert.Less(b, b.Elapsed()/time.Duration(b.N), time.Millisecond)
}
4 changes: 2 additions & 2 deletions backend/clickhouse/sessions.go
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ func (client *Client) WriteSessions(ctx context.Context, sessions []*model.Sessi
NewStruct(new(ClickhouseSession)).
InsertInto(SessionsTable, chSessions...).
BuildWithFlavor(sqlbuilder.ClickHouse)
sessionsSql, sessionsArgs = replaceTimestampInserts(sessionsSql, sessionsArgs, 34, map[int]bool{7: true, 8: true}, MicroSeconds)
sessionsSql, sessionsArgs = replaceTimestampInserts(sessionsSql, sessionsArgs, map[int]bool{7: true, 8: true}, MicroSeconds)
return client.conn.Exec(chCtx, sessionsSql, sessionsArgs...)
})
}
Expand All @@ -228,7 +228,7 @@ func (client *Client) WriteSessions(ctx context.Context, sessions []*model.Sessi
NewStruct(new(ClickhouseField)).
InsertInto(FieldsTable, chFields...).
BuildWithFlavor(sqlbuilder.ClickHouse)
fieldsSql, fieldsArgs = replaceTimestampInserts(fieldsSql, fieldsArgs, 6, map[int]bool{3: true}, MicroSeconds)
fieldsSql, fieldsArgs = replaceTimestampInserts(fieldsSql, fieldsArgs, map[int]bool{3: true}, MicroSeconds)
return client.conn.Exec(chCtx, fieldsSql, fieldsArgs...)
})
}
Expand Down

0 comments on commit 8844ac6

Please sign in to comment.