diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3f4d1eed..c537ff9e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -50,6 +50,34 @@ jobs: env: CGO_ENABLED: 1 + - name: Test + run: make test + env: + CGO_ENABLED: 1 + + - name: Test (with GMT+5) + run: | + go clean -testcache + TZ=Etc/GMT+5 make test + env: + CGO_ENABLED: 1 + +# Some tests are broen in GMT-5 +# --- FAIL: TestProm1UnpackFast (1.34s) +# prometheus_test.go:74: +# Error Trace: prometheus_test.go:74 +# Error: Not equal: +# expected: +# Test: TestProm1UnpackFast +# FAIL +# FAIL github.com/lomik/carbon-clickhouse/receiver 2.515s + # - name: Test (with GMT-5) + # run: | + # go clean -testcache + # TZ=Etc/GMT-5 make test + # env: + # CGO_ENABLED: 1 + - name: Set up Ruby uses: ruby/setup-ruby@v1 with: diff --git a/Dockerfile b/Dockerfile index a49785db..ddf091ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ WORKDIR /go/src/github.com/lomik/carbon-clickhouse COPY . . -RUN apk --no-cache add make git +RUN apk --no-cache add make git tzdata RUN make diff --git a/README.md b/README.md index 63d7d853..c1be17c9 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,11 @@ Usage of carbon-clickhouse: -version=false: Print version ``` +Date are broken by default (not always in UTC), but this used from start of project, and can produce some bugs. +Change to UTC requires points/index/tags tables rebuild (Date recalc to true UTC) or queries with wide Date range. +Set `data.utc-date = true` for this. +Without UTC date is required to run carbon-clickhouse and graphite-clickhouse in one timezone. + ```toml [common] # Prefix for store all internal carbon-clickhouse graphs. Supported macroses: {host} @@ -115,6 +120,9 @@ compression = "none" # For "lz4" 0 means use normal LZ4, >=1 use LZ4HC with this depth (the higher - the better compression, but slower) compression-level = 0 +# Date are broken by default (not always in UTC) +#utc-date = false + [upload.graphite] type = "points" table = "graphite" diff --git a/carbon/config.go b/carbon/config.go index d56156c4..f14a4451 100644 --- a/carbon/config.go +++ b/carbon/config.go @@ -8,6 +8,7 @@ import ( "time" "github.com/BurntSushi/toml" + rb "github.com/lomik/carbon-clickhouse/helper/RowBinary" "github.com/lomik/carbon-clickhouse/helper/config" "github.com/lomik/carbon-clickhouse/helper/tags" "github.com/lomik/carbon-clickhouse/uploader" @@ -93,6 +94,7 @@ type dataConfig struct { AutoInterval *config.ChunkAutoInterval `toml:"chunk-auto-interval"` CompAlgo *config.Compression `toml:"compression"` CompLevel int `toml:"compression-level"` + UTCDate bool `toml:"utc-date"` } // Config ... @@ -286,5 +288,9 @@ func ReadConfig(filename string) (*Config, error) { } } + if cfg.Data.UTCDate { + rb.SetUTCDate() + } + return cfg, nil } diff --git a/helper/RowBinary/date.go b/helper/RowBinary/date.go index c588a665..e8d01638 100644 --- a/helper/RowBinary/date.go +++ b/helper/RowBinary/date.go @@ -2,6 +2,18 @@ package RowBinary import "time" +var TimestampToDays func(timestamp uint32) uint16 + +// UTCTimestampToDays is always UTC, but mismatch SlowTimestampToDays and need points/index/tags table rebuild (with Date recalc) +func SetUTCDate() { + TimestampToDays = UTCTimestampToDays +} + +// PrecalcTimestampToDays is broken, not always UTC, like SlowTimestampToDays, but used from start of project +func SetDefaultDate() { + TimestampToDays = PrecalcTimestampToDays +} + var daysTimestampStart []int64 func init() { @@ -14,13 +26,11 @@ func init() { daysTimestampStart = append(daysTimestampStart, ts) t = t.Add(24 * time.Hour) } + SetDefaultDate() } -func TimestampToDays(timestamp uint32) uint16 { - if int64(timestamp) < daysTimestampStart[0] { - return 0 - } - +// PrecalcTimestampToDays is broken, not always UTC, like SlowTimestampToDays +func PrecalcTimestampToDays(timestamp uint32) uint16 { i := int(timestamp / 86400) ts := int64(timestamp) @@ -28,7 +38,6 @@ func TimestampToDays(timestamp uint32) uint16 { // fallback to slow method return SlowTimestampToDays(timestamp) } - FindLoop: for { if ts < daysTimestampStart[i] { @@ -43,7 +52,27 @@ FindLoop: } } +// SlowTimestampToDays is broken, not always UTC func SlowTimestampToDays(timestamp uint32) uint16 { t := time.Unix(int64(timestamp), 0) return uint16(time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC).Unix() / 86400) } + +// TimestampToDaysFormat is pair for SlowTimestampToDays and broken also for symmetric +func TimestampToDaysFormat(timestamp int64) string { + t := time.Unix(timestamp, 0) + return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC).Format("2006-01-02") +} + +// TimeToDaysFormat like TimestampDaysFormat, but for time.Time +func TimeToDaysFormat(t time.Time) string { + return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC).Format("2006-01-02") +} + +func UTCTimestampToDays(timestamp uint32) uint16 { + return uint16(timestamp / 86400) +} + +func UTCTimestampToDaysFormat(timestamp uint32) string { + return time.Unix(int64(timestamp), 0).UTC().Format("2006-01-02") +} diff --git a/helper/RowBinary/date_test.go b/helper/RowBinary/date_test.go index 2c875911..78952485 100644 --- a/helper/RowBinary/date_test.go +++ b/helper/RowBinary/date_test.go @@ -1,10 +1,111 @@ package RowBinary import ( + "os" + "strconv" "testing" "time" ) +var runBroken bool + +func init() { + if os.Getenv("RUN_BRKEN_TESTS") == "1" { + runBroken = true + } +} + +// SlowTimestampToDays is broken on some cases +// +// TZ=Etc/GMT-5 RUN_BRKEN_TESTS=1 make test +// --- FAIL: TestSlowTimestampToDays (0.00s) +// +// --- FAIL: TestSlowTimestampToDays/1668106870_2022-11-11T00:01:10+05:00,_UTC_2022-11-10T19:01:10Z_[0] (0.00s) +// date_test.go:62: TimestampDaysFormat() = 19307 (2022-11-11), want 19306 (2022-11-10) +// --- FAIL: TestSlowTimestampToDays/1668193200_2022-11-12T00:00:00+05:00,_UTC_2022-11-11T19:00:00Z_[1] (0.00s) +// date_test.go:62: TimestampDaysFormat() = 19308 (2022-11-12), want 19307 (2022-11-11) +// +// FAIL +// FAIL github.com/lomik/carbon-clickhouse/helper/RowBinary 3.328s +// +// $ TZ=Etc/GMT+5 RUN_BRKEN_TESTS=1 make test +// go test -race ./... +// --- FAIL: TestSlowTimestampToDays (0.00s) +// +// --- FAIL: TestSlowTimestampToDays/1668106870_2022-11-10T14:01:10-05:00,_UTC_2022-11-10T19:01:10Z_[0] (0.00s) +// date_test.go:62: TimestampDaysFormat() = 19306 (2022-11-09), want 19306 (2022-11-10) +// --- FAIL: TestSlowTimestampToDays/1668193200_2022-11-11T14:00:00-05:00,_UTC_2022-11-11T19:00:00Z_[1] (0.00s) +// date_test.go:62: TimestampDaysFormat() = 19307 (2022-11-10), want 19307 (2022-11-11) +// --- FAIL: TestSlowTimestampToDays/1668124800_2022-11-10T19:00:00-05:00,_UTC_2022-11-11T00:00:00Z_[2] (0.00s) +// date_test.go:62: TimestampDaysFormat() = 19306 (2022-11-09), want 19307 (2022-11-11) +// --- FAIL: TestSlowTimestampToDays/1668142799_2022-11-10T23:59:59-05:00,_UTC_2022-11-11T04:59:59Z_[3] (0.00s) +// date_test.go:62: TimestampDaysFormat() = 19306 (2022-11-09), want 19307 (2022-11-11) +// --- FAIL: TestSlowTimestampToDays/1650776160_2022-04-23T23:56:00-05:00,_UTC_2022-04-24T04:56:00Z_[4] (0.00s) +// date_test.go:62: TimestampDaysFormat() = 19105 (2022-04-22), want 19106 (2022-04-24) +// +// --- FAIL: TestTimestampToDays (0.00s) +func TestSlowTimestampToDays(t *testing.T) { + if !runBroken { + t.Log("skip broken test, set RUN_BRKEN_TESTS=1 for run") + return + } + tests := []struct { + ts uint32 + want uint16 + wantStr string + }{ + { + ts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC + // select toDate(1650776160,'UTC') + // 2022-11-10 + want: 19306, + wantStr: "2022-11-10", + }, + { + ts: 1668193200, // 2022-11-12 00:00:00 +05:00 ; 2022-11-11 19:00:00 UTC + // SELECT Date(19307) + // 2022-11-11 + want: 19307, + wantStr: "2022-11-11", + }, + { + ts: 1668124800, // 2022-11-11 00:00:00 UTC + want: 19307, + wantStr: "2022-11-11", + }, + { + ts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC + want: 19307, + wantStr: "2022-11-11", + }, + { + ts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7) + // 2022-04-24 4:56:00 + // select toDate(1650776160,'UTC') + // 2022-04-24 + // select toDate(1650776160,'Etc/GMT+7') + // 2022-04-23 + want: 19106, + wantStr: "2022-04-24", + }, + } + for i, tt := range tests { + t.Run(strconv.FormatInt(int64(tt.ts), 10)+" "+time.Unix(int64(tt.ts), 0).Format(time.RFC3339)+", UTC "+time.Unix(int64(tt.ts), 0).UTC().Format(time.RFC3339)+" ["+strconv.Itoa(i)+"]", func(t *testing.T) { + got := SlowTimestampToDays(tt.ts) + // gotStr := dayStart.Add(time.Duration(int64(got) * 24 * 3600 * 1e9)).Format("2006-01-02") + gotStr := time.Unix(int64(got)*24*3600, 0).Format("2006-01-02") + if gotStr != tt.wantStr || got != tt.want { + t.Errorf("TimestampDaysFormat() = %v (%s), want %v (%s)", got, gotStr, tt.want, tt.wantStr) + } + convStr := UTCTimestampToDaysFormat(uint32(tt.want) * 24 * 3600) + if convStr != tt.wantStr { + t.Errorf("conversion got %s, want %s", convStr, tt.wantStr) + } + }) + } +} + +// TimestampToDays is broken on some cases like SlowTimestampToDays func TestTimestampToDays(t *testing.T) { ts := uint32(0) end := uint32(time.Now().Unix()) + 473040000 // +15 years @@ -12,12 +113,19 @@ func TestTimestampToDays(t *testing.T) { d1 := SlowTimestampToDays(ts) d2 := TimestampToDays(ts) if d1 != d2 { - t.FailNow() + t.Fatalf("SlowTimestampToDays(%d)=%d, TimestampToDays(%d)=%d", ts, d1, ts, d2) + } + ts1 := time.Unix(int64(d1)*86400, 0) + ds1 := time.Date(ts1.Year(), ts1.Month(), ts1.Day(), 0, 0, 0, 0, time.UTC).Format("2006-01-02") + ds2 := TimestampToDaysFormat(int64(d1) * 86400) + if ds1 != ds2 { + t.Fatalf("SlowTimestampToDays(%d)=%d, format error %s %s", ts, d1, ds1, ds2) } ts += 780 // step 13 minutes } } + func BenchmarkTimestampToDays(b *testing.B) { timestamp := uint32(time.Now().Unix()) x := SlowTimestampToDays(timestamp) @@ -39,3 +147,58 @@ func BenchmarkSlowTimestampToDays(b *testing.B) { } } } + +func TestUTCTimestampToDays(t *testing.T) { + tests := []struct { + ts uint32 + want uint16 + wantStr string + }{ + { + ts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC + // select toDate(1650776160,'UTC') + // 2022-11-10 + want: 19306, + wantStr: "2022-11-10", + }, + { + ts: 1668193200, // 2022-11-12 00:00:00 +05:00 ; 2022-11-11 19:00:00 UTC + // SELECT Date(19307) + // 2022-11-11 + // SELECT toDate(1668193200, 'UTC') + // 2022-11-11 + want: 19307, + wantStr: "2022-11-11", + }, + { + ts: 1668124800, // 2022-11-11 00:00:00 UTC + want: 19307, + wantStr: "2022-11-11", + }, + { + ts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC + want: 19307, + wantStr: "2022-11-11", + }, + { + ts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7) + // 2022-04-24 4:56:00 + // select toDate(1650776160,'UTC') + // 2022-04-24 + // select toDate(1650776160,'Etc/GMT+7') + // 2022-04-23 + want: 19106, + wantStr: "2022-04-24", + }, + } + for i, tt := range tests { + t.Run(strconv.FormatInt(int64(tt.ts), 10)+" "+time.Unix(int64(tt.ts), 0).Format(time.RFC3339)+", UTC "+time.Unix(int64(tt.ts), 0).UTC().Format(time.RFC3339)+" ["+strconv.Itoa(i)+"]", func(t *testing.T) { + got := UTCTimestampToDays(tt.ts) + // gotStr := dayStart.Add(time.Duration(int64(got) * 24 * 3600 * 1e9)).Format("2006-01-02") + gotStr := UTCTimestampToDaysFormat(uint32(got) * 86400) + if gotStr != tt.wantStr || got != tt.want { + t.Errorf("TimestampDaysFormat() = %v (%s), want %v (%s)", got, gotStr, tt.want, tt.wantStr) + } + }) + } +}