From 88f2005ba626291b827971c3e96f923a7bc04323 Mon Sep 17 00:00:00 2001 From: msaf1980 Date: Fri, 11 Nov 2022 01:14:38 +0500 Subject: [PATCH] Allow to use Date in UTC (or wide from/until for migrate) --- .github/workflows/tests.yml | 28 + autocomplete/autocomplete.go | 16 +- autocomplete/autocomplete_test.go | 23 +- cmd/e2e-test/carbon-clickhouse.go | 17 +- cmd/e2e-test/checks.go | 54 +- cmd/e2e-test/clickhouse.go | 56 ++ cmd/e2e-test/e2etesting.go | 60 ++- cmd/e2e-test/graphite-clickhouse.go | 7 + config/config.go | 13 + doc/config.md | 2 + finder/date_reverse.go | 13 +- finder/date_reverse_test.go | 49 ++ finder/index.go | 33 +- finder/index_test.go | 62 +++ finder/tagged.go | 17 +- finder/tagged_test.go | 55 ++ helper/date/date.go | 103 ++++ helper/date/date_test.go | 254 +++++++++ helper/datetime/datetime.go | 25 +- helper/datetime/datetime_test.go | 1 + issues/daytime/carbon-clickhouse.conf.tpl | 45 ++ ...graphite-clickhouse-internal-aggr.conf.tpl | 35 ++ issues/daytime/graphite-clickhouse.conf.tpl | 35 ++ issues/daytime/test.toml | 490 ++++++++++++++++++ pkg/where/where.go | 5 +- render/data/query_test.go | 19 +- tests/agg_oneblock/test.toml | 3 + tests/one_table/test.toml | 158 ++++++ 28 files changed, 1591 insertions(+), 87 deletions(-) create mode 100644 finder/date_reverse_test.go create mode 100644 helper/date/date.go create mode 100644 helper/date/date_test.go create mode 100644 issues/daytime/carbon-clickhouse.conf.tpl create mode 100644 issues/daytime/graphite-clickhouse-internal-aggr.conf.tpl create mode 100644 issues/daytime/graphite-clickhouse.conf.tpl create mode 100644 issues/daytime/test.toml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c8afc1166..231bbf4e5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,11 +54,39 @@ jobs: env: CGO_ENABLED: 1 + - name: Test (with GMT-5) + run: | + go clean -testcache + TZ=Etc/GMT-5 make test + env: + CGO_ENABLED: 1 + + - name: Test (with GMT+5) + run: | + go clean -testcache + TZ=Etc/GMT+5 make test + env: + CGO_ENABLED: 1 + - name: Integration tests run: | make e2e-test ./e2e-test -config tests + # TODO (msaf1980): find a way to set TZ in carbon-clickhouse docker (or run locally) + # run with clickhouse.date-format = "both" + # - name: Integration tests (with Etc/GMT-5) + # run: | + # make e2e-test + # sudo timedatectl set-timezone Etc/GMT-5 + # ./e2e-test -config issues/daytime + + # - name: Integration tests (with Etc/GMT+5) + # run: | + # make e2e-test + # sudo timedatectl set-timezone Etc/GMT+5 + # ./e2e-test -config issues/daytime + - name: Set up Ruby uses: ruby/setup-ruby@v1 with: diff --git a/autocomplete/autocomplete.go b/autocomplete/autocomplete.go index 9e0d80a76..9a71c9a95 100644 --- a/autocomplete/autocomplete.go +++ b/autocomplete/autocomplete.go @@ -13,6 +13,7 @@ import ( "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/finder" "github.com/lomik/graphite-clickhouse/helper/clickhouse" + "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/helper/utils" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/scope" @@ -43,6 +44,12 @@ func NewValues(config *config.Config) *Handler { return h } +func dateString(autocompleteDays int, tm time.Time) (string, string) { + fromDate := date.FromTimeToDaysFormat(tm.AddDate(0, 0, -autocompleteDays)) + untilDate := date.UntilTimeToDaysFormat(tm) + return fromDate, untilDate +} + func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { logger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("autocomplete") r = r.WithContext(scope.WithLogger(r.Context(), logger)) @@ -172,9 +179,8 @@ func (h *Handler) ServeTags(w http.ResponseWriter, r *http.Request) { } } - now := time.Now() - fromDate := now.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Format("2006-01-02") - untilDate := now.Format("2006-01-02") + // TODO (msaf1980) fix for disable daily + fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, start) var key string @@ -358,8 +364,8 @@ func (h *Handler) ServeValues(w http.ResponseWriter, r *http.Request) { } } - fromDate := start.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Format("2006-01-02") - untilDate := start.Format("2006-01-02") + // TODO (msaf1980) fix for disable daily + fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, start) var key string diff --git a/autocomplete/autocomplete_test.go b/autocomplete/autocomplete_test.go index 0a9e0cfde..6c868b324 100644 --- a/autocomplete/autocomplete_test.go +++ b/autocomplete/autocomplete_test.go @@ -10,8 +10,8 @@ import ( "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/tests/clickhouse" + chtest "github.com/lomik/graphite-clickhouse/helper/tests/clickhouse" "github.com/lomik/graphite-clickhouse/metrics" - "github.com/stretchr/testify/assert" ) @@ -52,7 +52,7 @@ func testResponce(t *testing.T, step int, h *Handler, tt *testStruct, wantCached func TestHandler_ServeValues(t *testing.T) { metrics.DisableMetrics() - srv := clickhouse.NewTestServer() + srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() @@ -60,11 +60,10 @@ func TestHandler_ServeValues(t *testing.T) { h := NewTags(cfg) - from := "1636432127" - until := "1636442929" - - fromDate := time.Now().AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Format("2006-01-02") - untilDate := time.Now().Format("2006-01-02") + now := time.Now() + until := strconv.FormatInt(now.Unix(), 10) + from := strconv.FormatInt(now.Add(-time.Minute).Unix(), 10) + fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) srv.AddResponce( "SELECT substr(arrayJoin(Tags), 6) AS value FROM graphite_tagged WHERE (((Tag1='environment=production') AND (arrayExists((x) -> x='project=web', Tags))) AND (arrayJoin(Tags) LIKE 'host=%')) AND "+ @@ -98,6 +97,7 @@ func TestHandler_ServeValues(t *testing.T) { } func TestTagsAutocomplete_ServeValuesCached(t *testing.T) { + metrics.DisableMetrics() srv := clickhouse.NewTestServer() defer srv.Close() @@ -118,11 +118,10 @@ func TestTagsAutocomplete_ServeValuesCached(t *testing.T) { h := NewTags(cfg) - from := "1636432127" - until := "1636442929" - - fromDate := time.Now().AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Format("2006-01-02") - untilDate := time.Now().Format("2006-01-02") + now := time.Now() + until := strconv.FormatInt(now.Unix(), 10) + from := strconv.FormatInt(now.Add(-time.Minute).Unix(), 10) + fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) srv.AddResponce( "SELECT substr(arrayJoin(Tags), 6) AS value FROM graphite_tagged WHERE (((Tag1='environment=production') AND (arrayExists((x) -> x='project=web', Tags))) AND (arrayJoin(Tags) LIKE 'host=%')) AND "+ diff --git a/cmd/e2e-test/carbon-clickhouse.go b/cmd/e2e-test/carbon-clickhouse.go index 3876d0a07..54a460aa6 100644 --- a/cmd/e2e-test/carbon-clickhouse.go +++ b/cmd/e2e-test/carbon-clickhouse.go @@ -2,11 +2,13 @@ package main import ( "errors" + "fmt" "io/ioutil" "os" "os/exec" "path" "path/filepath" + "strings" "text/template" ) @@ -79,22 +81,29 @@ func (c *CarbonClickhouse) Start(testDir, clickhouseURL, clickhouseContainer str return "", err } - // tz, _ := localTZLocationName() + tz := os.Getenv("TZ") cchStart := []string{"run", "-d", "--name", c.container, "-p", c.address + ":2003", "-v", c.storeDir + ":/etc/carbon-clickhouse", + // TZ, need to be same as graphite-clickhouse for prevent bugs, ike issue #184 + "-v", "/etc/timezone:/etc/timezone:ro", + "-v", "/etc/localtime:/etc/localtime:ro", + "-e", "TZ=" + tz, "--link", clickhouseContainer, } - if c.TZ != "" { - cchStart = append(cchStart, "-e", "TZ="+c.TZ) - } cchStart = append(cchStart, c.DockerImage+":"+c.Version) cmd := exec.Command(DockerBinary, cchStart...) out, err := cmd.CombinedOutput() + if err == nil { + dateLocal, _ := exec.Command("date").Output() + dateLocalStr := strings.TrimRight(string(dateLocal), "\n") + _, dateOnCCH := containerExec(c.container, []string{"date"}) + fmt.Printf("date local %s, on carbon-clickhouse %s\n", dateLocalStr, dateOnCCH) + } return string(out), err } diff --git a/cmd/e2e-test/checks.go b/cmd/e2e-test/checks.go index 7dddf1b4f..51e76ab64 100644 --- a/cmd/e2e-test/checks.go +++ b/cmd/e2e-test/checks.go @@ -63,18 +63,32 @@ func compareFindMatch(errors *[]string, name, url string, actual, expected []cli } } -func verifyMetricsFind(address string, check *MetricsFindCheck) []string { +func verifyMetricsFind(ch *Clickhouse, gch *GraphiteClickhouse, check *MetricsFindCheck) []string { var errors []string httpClient := http.Client{ Timeout: check.Timeout, } + address := gch.URL() for _, format := range check.Formats { name := "" if url, result, respHeader, err := client.MetricsFind(&httpClient, address, format, check.Query, check.from, check.until); err == nil { + id := requestId(respHeader) if check.ErrorRegexp != "" { - errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", requestId(respHeader), url, check.ErrorRegexp)) + errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", id, url, check.ErrorRegexp)) } compareFindMatch(&errors, name, url, result, check.Result, check.InCache, check.CacheTTL, respHeader) + if len(result) == 0 && len(check.Result) > 0 { + gch.Grep(id) + if len(check.DumpIfEmpty) > 0 { + for _, q := range check.DumpIfEmpty { + if out, err := ch.Query(q); err == nil { + fmt.Fprintf(os.Stderr, "%s\n%s", q, out) + } else { + fmt.Fprintf(os.Stderr, "%s: %s\n", err.Error(), q) + } + } + } + } if check.CacheTTL > 0 && check.ErrorRegexp == "" { // second query must be find-cached @@ -124,11 +138,12 @@ func compareTags(errors *[]string, name, url string, actual, expected []string, } } -func verifyTags(address string, check *TagsCheck) []string { +func verifyTags(ch *Clickhouse, gch *GraphiteClickhouse, check *TagsCheck) []string { var errors []string httpClient := http.Client{ Timeout: check.Timeout, } + address := gch.URL() for _, format := range check.Formats { var ( result []string @@ -145,10 +160,23 @@ func verifyTags(address string, check *TagsCheck) []string { } if err == nil { + id := requestId(respHeader) if check.ErrorRegexp != "" { - errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", requestId(respHeader), url, check.ErrorRegexp)) + errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", id, url, check.ErrorRegexp)) } compareTags(&errors, name, url, result, check.Result, check.InCache, check.CacheTTL, respHeader) + if len(result) == 0 && len(check.Result) > 0 { + gch.Grep(id) + if len(check.DumpIfEmpty) > 0 { + for _, q := range check.DumpIfEmpty { + if out, err := ch.Query(q); err == nil { + fmt.Fprintf(os.Stderr, "%s\n%s", q, out) + } else { + fmt.Fprintf(os.Stderr, "%s: %s\n", err.Error(), q) + } + } + } + } if check.CacheTTL > 0 && check.ErrorRegexp == "" { // second query must be find-cached @@ -243,20 +271,34 @@ func compareRender(errors *[]string, name, url string, actual, expected []client } } -func verifyRender(address string, check *RenderCheck, defaultPreision time.Duration) []string { +func verifyRender(ch *Clickhouse, gch *GraphiteClickhouse, check *RenderCheck, defaultPreision time.Duration) []string { var errors []string httpClient := http.Client{ Timeout: check.Timeout, } + address := gch.URL() from := datetime.TimestampTruncate(check.from, defaultPreision) until := datetime.TimestampTruncate(check.until, defaultPreision) for _, format := range check.Formats { if url, result, respHeader, err := client.Render(&httpClient, address, format, check.Targets, from, until); err == nil { + id := requestId(respHeader) name := "" if check.ErrorRegexp != "" { - errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", requestId(respHeader), url, check.ErrorRegexp)) + errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", id, url, check.ErrorRegexp)) } compareRender(&errors, name, url, result, check.result, check.InCache, respHeader, check.CacheTTL) + if len(result) == 0 && len(check.result) > 0 { + gch.Grep(id) + if len(check.DumpIfEmpty) > 0 { + for _, q := range check.DumpIfEmpty { + if out, err := ch.Query(q); err == nil { + fmt.Fprintf(os.Stderr, "%s\n%s", q, out) + } else { + fmt.Fprintf(os.Stderr, "%s: %s\n", err.Error(), q) + } + } + } + } if check.CacheTTL > 0 && check.ErrorRegexp == "" { // second query must be find-cached diff --git a/cmd/e2e-test/clickhouse.go b/cmd/e2e-test/clickhouse.go index feeff1183..b89dbbf65 100644 --- a/cmd/e2e-test/clickhouse.go +++ b/cmd/e2e-test/clickhouse.go @@ -1,8 +1,17 @@ package main import ( + "bytes" "errors" + "fmt" + "io/ioutil" + "net/http" + "os" "os/exec" + "strings" + "time" + + "github.com/msaf1980/go-stringutils" ) var ClickhouseContainerName = "clickhouse-server-gch-test" @@ -107,3 +116,50 @@ func (c *Clickhouse) Container() string { func (c *Clickhouse) Exec(sql string) (bool, string) { return containerExec(c.container, []string{"sh", "-c", "clickhouse-client -q '" + sql + "'"}) } + +func (c *Clickhouse) Query(sql string) (string, error) { + reader := strings.NewReader(sql) + request, err := http.NewRequest("POST", c.URL(), reader) + if err != nil { + return "", err + } + + httpClient := http.Client{ + Timeout: time.Minute, + } + resp, err := httpClient.Do(request) + if err != nil { + return "", err + } + msg, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + if resp.StatusCode != http.StatusOK { + return "", errors.New(resp.Status + ": " + string(bytes.TrimRight(msg, "\n"))) + } + return string(msg), nil +} + +func (c *Clickhouse) IsUp() bool { + if len(c.container) == 0 { + return false + } + req, err := http.DefaultClient.Get(c.url) + if err != nil { + return false + } + return req.StatusCode == http.StatusOK +} + +func (c *Clickhouse) Logs() { + if len(c.container) == 0 { + return + } + + chArgs := []string{"logs", c.container} + + cmd := exec.Command(DockerBinary, chArgs...) + out, _ := cmd.CombinedOutput() + fmt.Fprintln(os.Stderr, stringutils.UnsafeString(out)) +} diff --git a/cmd/e2e-test/e2etesting.go b/cmd/e2e-test/e2etesting.go index 38cca0e40..01495f23d 100644 --- a/cmd/e2e-test/e2etesting.go +++ b/cmd/e2e-test/e2etesting.go @@ -53,12 +53,13 @@ type Metric struct { } type RenderCheck struct { - Name string `toml:"name"` - Formats []client.FormatType `toml:"formats"` - From string `toml:"from"` - Until string `toml:"until"` - Targets []string `toml:"targets"` - Timeout time.Duration `toml:"timeout"` + Name string `toml:"name"` + Formats []client.FormatType `toml:"formats"` + From string `toml:"from"` + Until string `toml:"until"` + Targets []string `toml:"targets"` + Timeout time.Duration `toml:"timeout"` + DumpIfEmpty []string `toml:"dump_if_empty"` Optimize []string `toml:"optimize"` // optimize tables before run tests @@ -85,6 +86,8 @@ type MetricsFindCheck struct { Query string `toml:"query"` Timeout time.Duration `toml:"timeout"` + DumpIfEmpty []string `toml:"dump_if_empty"` + InCache bool `toml:"in_cache"` // already in cache CacheTTL int `toml:"cache_ttl"` @@ -109,6 +112,8 @@ type TagsCheck struct { Limits uint64 `toml:"limits"` Timeout time.Duration `toml:"timeout"` + DumpIfEmpty []string `toml:"dump_if_empty"` + InCache bool `toml:"in_cache"` // already in cache CacheTTL int `toml:"cache_ttl"` @@ -220,7 +225,7 @@ func verifyGraphiteClickhouse(test *TestSchema, gch *GraphiteClickhouse, clickho if len(check.Formats) == 0 { check.Formats = []client.FormatType{client.FormatPb_v3} } - if errs := verifyMetricsFind(gch.URL(), check); len(errs) > 0 { + if errs := verifyMetricsFind(clickhouse, gch, check); len(errs) > 0 { verifyFailed++ for _, e := range errs { fmt.Fprintln(os.Stderr, e) @@ -265,7 +270,7 @@ func verifyGraphiteClickhouse(test *TestSchema, gch *GraphiteClickhouse, clickho if len(check.Formats) == 0 { check.Formats = []client.FormatType{client.FormatJSON} } - if errs := verifyTags(gch.URL(), check); len(errs) > 0 { + if errs := verifyTags(clickhouse, gch, check); len(errs) > 0 { verifyFailed++ for _, e := range errs { fmt.Fprintln(os.Stderr, e) @@ -333,7 +338,7 @@ func verifyGraphiteClickhouse(test *TestSchema, gch *GraphiteClickhouse, clickho } } } - if errs := verifyRender(gch.URL(), check, test.Precision); len(errs) > 0 { + if errs := verifyRender(clickhouse, gch, check, test.Precision); len(errs) > 0 { verifyFailed++ for _, e := range errs { fmt.Fprintln(os.Stderr, e) @@ -445,20 +450,41 @@ func testGraphiteClickhouse(test *TestSchema, clickhouse *Clickhouse, testDir, r zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouseDir), ) - time.Sleep(2 * time.Second) - - // Populate test data - err = sendPlain("tcp", test.Cch.address, test.Input) - if err != nil { - logger.Error("send plain to carbon-clickhouse", + time.Sleep(500 * time.Millisecond) + for i := 200; i < 2000; i += 200 { + if clickhouse.IsUp() { + break + } + time.Sleep(time.Duration(i) * time.Millisecond) + } + if !clickhouse.IsUp() { + logger.Error("starting clickhouse", zap.String("config", test.name), - zap.String("clickhouse version", clickhouse.Version), + zap.Any("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouseDir), - zap.Error(err), + zap.String("error", "clickhouse is down"), ) + clickhouse.Logs() testSuccess = false } + if testSuccess { + // Populate test data + err = sendPlain("tcp", test.Cch.address, test.Input) + if err != nil { + logger.Error("send plain to carbon-clickhouse", + zap.String("config", test.name), + zap.String("clickhouse version", clickhouse.Version), + zap.String("clickhouse config", clickhouseDir), + zap.Error(err), + ) + testSuccess = false + } + if testSuccess { + time.Sleep(2 * time.Second) + } + } + if testSuccess { for _, gch := range test.Gch { stepSuccess, vCount, vFailed := verifyGraphiteClickhouse(test, &gch, clickhouse, testDir, clickhouseDir, verbose, breakOnError, logger) diff --git a/cmd/e2e-test/graphite-clickhouse.go b/cmd/e2e-test/graphite-clickhouse.go index 17428217c..891638d8a 100644 --- a/cmd/e2e-test/graphite-clickhouse.go +++ b/cmd/e2e-test/graphite-clickhouse.go @@ -2,6 +2,7 @@ package main import ( "errors" + "fmt" "io/ioutil" "net/http" "os" @@ -13,6 +14,7 @@ import ( "text/template" "github.com/lomik/graphite-clickhouse/helper/client" + "github.com/msaf1980/go-stringutils" ) type GraphiteClickhouse struct { @@ -143,3 +145,8 @@ func (c *GraphiteClickhouse) URL() string { func (c *GraphiteClickhouse) Cmd() string { return strings.Join(c.cmd.Args, " ") } + +func (c *GraphiteClickhouse) Grep(s string) { + out, _ := exec.Command("grep", "-F", s, c.storeDir+"/graphite-clickhouse.log").Output() + fmt.Fprintf(os.Stderr, "GREP %s", stringutils.UnsafeString(out)) +} diff --git a/config/config.go b/config/config.go index cec856cbc..81bf5bfb0 100644 --- a/config/config.go +++ b/config/config.go @@ -18,6 +18,7 @@ import ( "go.uber.org/zap" "github.com/lomik/graphite-clickhouse/cache" + "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/helper/rollup" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/zapwriter" @@ -129,6 +130,7 @@ type ClickHouse struct { URL string `toml:"url" json:"url" comment:"default url, see https://clickhouse.tech/docs/en/interfaces/http. Can be overwritten with query-params"` DataTimeout time.Duration `toml:"data-timeout" json:"data-timeout" comment:"default total timeout to fetch data, can be overwritten with query-params"` QueryParams []QueryParam `toml:"query-params" json:"query-params" comment:"customized query params (url, data timeout) for durations greater or equal"` + DateFormat string `toml:"date-format" json:"date-format" comment:"Date format (default, utc, both)"` IndexTable string `toml:"index-table" json:"index-table" comment:"see doc/index-table.md"` IndexUseDaily bool `toml:"index-use-daily" json:"index-use-daily"` IndexReverse string `toml:"index-reverse" json:"index-reverse" comment:"see doc/config.md"` @@ -539,6 +541,17 @@ func Unmarshal(body []byte, noLog bool) (*Config, error) { } } + switch strings.ToLower(cfg.ClickHouse.DateFormat) { + case "utc": + date.SetUTC() + case "both": + date.SetBoth() + default: + if cfg.ClickHouse.DateFormat != "" && cfg.ClickHouse.DateFormat != "default" { + return nil, fmt.Errorf("unsupported date-format: %s", cfg.ClickHouse.DateFormat) + } + } + cfg.setupGraphiteMetrics() return cfg, nil diff --git a/doc/config.md b/doc/config.md index d51bbc58d..f6f9e257b 100644 --- a/doc/config.md +++ b/doc/config.md @@ -230,6 +230,8 @@ Send internal metrics to graphite relay url = "http://localhost:8123?cancel_http_readonly_queries_on_client_close=1" # default total timeout to fetch data, can be overwritten with query-params data-timeout = "1m0s" + # Date format (default, utc, both) + date-format = "" # see doc/index-table.md index-table = "graphite_index" index-use-daily = true diff --git a/finder/date_reverse.go b/finder/date_reverse.go index 5bdaf095d..018580349 100644 --- a/finder/date_reverse.go +++ b/finder/date_reverse.go @@ -3,9 +3,9 @@ package finder import ( "context" "fmt" - "time" "github.com/lomik/graphite-clickhouse/helper/clickhouse" + "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/pkg/where" ) @@ -25,16 +25,21 @@ func NewDateFinderV3(url string, table string, opts clickhouse.Options) Finder { return &DateFinderV3{b} } -func (f *DateFinderV3) Execute(ctx context.Context, query string, from int64, until int64, stat *FinderStat) (err error) { +func (f *DateFinderV3) whereFilter(query string, from int64, until int64) (*where.Where, *where.Where) { w := f.where(ReverseString(query)) dateWhere := where.New() dateWhere.Andf( "Date >='%s' AND Date <= '%s'", - time.Unix(from, 0).Format("2006-01-02"), - time.Unix(until, 0).Format("2006-01-02"), + date.FromTimestampToDaysFormat(from), + date.UntilTimestampToDaysFormat(until), ) + return w, dateWhere +} + +func (f *DateFinderV3) Execute(ctx context.Context, query string, from int64, until int64, stat *FinderStat) (err error) { + w, dateWhere := f.whereFilter(query, from, until) f.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query( scope.WithTable(ctx, f.table), f.url, diff --git a/finder/date_reverse_test.go b/finder/date_reverse_test.go new file mode 100644 index 000000000..38e9b40b1 --- /dev/null +++ b/finder/date_reverse_test.go @@ -0,0 +1,49 @@ +package finder + +import ( + "testing" + "time" + + "github.com/lomik/graphite-clickhouse/helper/clickhouse" + "github.com/lomik/graphite-clickhouse/helper/date" +) + +func TestDateFinderV3_whereFilter(t *testing.T) { + tests := []struct { + name string + query string + from int64 + until int64 + want string + wantDate string + }{ + { + name: "midnight at utc (direct)", + query: "test.metric*", + from: 1668124800, // 2022-11-11 00:00:00 UTC + until: 1668124810, // 2022-11-11 00:00:10 UTC + want: "(Level=2) AND (Path LIKE 'metric%' AND match(Path, '^metric([^.]*?)[.]test[.]?$'))", + wantDate: "Date >='" + date.FromTimestampToDaysFormat(1668124800) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(1668124810) + "'", + }, + { + name: "midnight at utc (reverse)", + query: "*test.metric", + from: 1668124800, // 2022-11-11 00:00:00 UTC + until: 1668124810, // 2022-11-11 00:00:10 UTC + want: "(Level=2) AND (Path LIKE 'metric.%' AND match(Path, '^metric[.]([^.]*?)test[.]?$'))", + wantDate: "Date >='" + date.FromTimestampToDaysFormat(1668124800) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(1668124810) + "'", + }, + } + for _, tt := range tests { + t.Run(tt.name+" "+time.Unix(tt.from, 0).Format(time.RFC3339), func(t *testing.T) { + f := NewDateFinderV3("http://localhost:8123/", "graphite_index", clickhouse.Options{}).(*DateFinderV3) + got, gotDate := f.whereFilter(tt.query, tt.from, tt.until) + if got.String() != tt.want { + t.Errorf("DateFinderV3.whereFilter()[0] = %v, want %v", got, tt.want) + } + if gotDate.String() != tt.wantDate { + t.Errorf("DateFinderV3.whereFilter()[1] = %v, want %v", gotDate, tt.wantDate) + } + }) + } +} diff --git a/finder/index.go b/finder/index.go index 1e3895967..800ccedab 100644 --- a/finder/index.go +++ b/finder/index.go @@ -5,10 +5,10 @@ import ( "context" "fmt" "strings" - "time" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" + "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/pkg/where" ) @@ -117,8 +117,11 @@ func (idx *IndexFinder) useReverse(query string) bool { return idx.useReverse(query) } -func (idx *IndexFinder) Execute(ctx context.Context, query string, from int64, until int64, stat *FinderStat) (err error) { - idx.useReverse(query) +func (idx *IndexFinder) whereFilter(query string, from int64, until int64) *where.Where { + reverse := idx.useReverse(query) + if reverse { + query = ReverseString(query) + } if idx.dailyEnabled && from > 0 && until > 0 { idx.useDaily = true @@ -128,32 +131,30 @@ func (idx *IndexFinder) Execute(ctx context.Context, query string, from int64, u var levelOffset int if idx.useDaily { - if idx.useReverse(query) { + if reverse { levelOffset = ReverseLevelOffset } + } else if reverse { + levelOffset = ReverseTreeLevelOffset } else { - if idx.useReverse(query) { - levelOffset = ReverseTreeLevelOffset - } else { - levelOffset = TreeLevelOffset - } - } - - if idx.useReverse(query) { - query = ReverseString(query) + levelOffset = TreeLevelOffset } w := idx.where(query, levelOffset) - if idx.useDaily { w.Andf( "Date >='%s' AND Date <= '%s'", - time.Unix(from, 0).Format("2006-01-02"), - time.Unix(until, 0).Format("2006-01-02"), + date.FromTimestampToDaysFormat(from), + date.UntilTimestampToDaysFormat(until), ) } else { w.And(where.Eq("Date", DefaultTreeDate)) } + return w +} + +func (idx *IndexFinder) Execute(ctx context.Context, query string, from int64, until int64, stat *FinderStat) (err error) { + w := idx.whereFilter(query, from, until) idx.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query( scope.WithTable(ctx, idx.table), diff --git a/finder/index_test.go b/finder/index_test.go index 61a1fabab..3e9610dd8 100644 --- a/finder/index_test.go +++ b/finder/index_test.go @@ -3,8 +3,11 @@ package finder import ( "fmt" "testing" + "time" "github.com/lomik/graphite-clickhouse/config" + "github.com/lomik/graphite-clickhouse/helper/clickhouse" + "github.com/lomik/graphite-clickhouse/helper/date" "github.com/stretchr/testify/assert" ) @@ -134,3 +137,62 @@ func Benchmark_useReverseDepthRegex(b *testing.B) { _ = idx.checkReverses("a.b.c*.d.max") } } + +func TestIndexFinder_whereFilter(t *testing.T) { + tests := []struct { + name string + query string + from int64 + until int64 + dailyEnabled bool + indexReverse string + indexReverses config.IndexReverses + want string + }{ + { + name: "nodaily (direct)", + query: "test.metric*", + from: 1668106860, + until: 1668106870, + dailyEnabled: false, + want: "((Level=20002) AND (Path LIKE 'test.metric%')) AND (Date='1970-02-12')", + }, + { + name: "nodaily (reverse)", + query: "*test.metric", + from: 1668106860, + until: 1668106870, + dailyEnabled: false, + want: "((Level=30002) AND (Path LIKE 'metric.%' AND match(Path, '^metric[.]([^.]*?)test[.]?$'))) AND (Date='1970-02-12')", + }, + { + name: "midnight at utc (direct)", + query: "test.metric*", + from: 1668124800, // 2022-11-11 00:00:00 UTC + until: 1668124810, // 2022-11-11 00:00:10 UTC + dailyEnabled: true, + want: "((Level=2) AND (Path LIKE 'test.metric%')) AND (Date >='" + + date.FromTimestampToDaysFormat(1668124800) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(1668124810) + "')", + }, + { + name: "midnight at utc (reverse)", + query: "*test.metric", + from: 1668124800, // 2022-11-11 00:00:00 UTC + until: 1668124810, // 2022-11-11 00:00:10 UTC + dailyEnabled: true, + want: "((Level=10002) AND (Path LIKE 'metric.%' AND match(Path, '^metric[.]([^.]*?)test[.]?$'))) AND (Date >='" + + date.FromTimestampToDaysFormat(1668124800) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(1668124810) + "')", + }, + } + for _, tt := range tests { + t.Run(tt.name+" "+time.Unix(tt.from, 0).Format(time.RFC3339), func(t *testing.T) { + if tt.indexReverse == "" { + tt.indexReverse = "auto" + } + idx := NewIndex("http://localhost:8123/", "graphite_index", tt.dailyEnabled, tt.indexReverse, tt.indexReverses, clickhouse.Options{}, false).(*IndexFinder) + if got := idx.whereFilter(tt.query, tt.from, tt.until); got.String() != tt.want { + t.Errorf("IndexFinder.whereFilter() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/finder/tagged.go b/finder/tagged.go index c4146b76b..c3ad4cf2e 100644 --- a/finder/tagged.go +++ b/finder/tagged.go @@ -8,10 +8,10 @@ import ( "net/http" "sort" "strings" - "time" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" + "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/helper/errs" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/pkg/where" @@ -412,20 +412,27 @@ func (t *TaggedFinder) Execute(ctx context.Context, query string, from int64, un return t.ExecutePrepared(ctx, terms, from, until, stat) } -func (t *TaggedFinder) ExecutePrepared(ctx context.Context, terms []TaggedTerm, from int64, until int64, stat *FinderStat) error { +func (t *TaggedFinder) whereFilter(terms []TaggedTerm, from int64, until int64) (*where.Where, *where.Where, error) { w, pw, err := TaggedWhere(terms) if err != nil { - return err + return nil, nil, err } if t.dailyEnabled { w.Andf( "Date >='%s' AND Date <= '%s'", - time.Unix(from, 0).Format("2006-01-02"), - time.Unix(until, 0).Format("2006-01-02"), + date.FromTimestampToDaysFormat(from), + date.UntilTimestampToDaysFormat(until), ) } + return w, pw, nil +} +func (t *TaggedFinder) ExecutePrepared(ctx context.Context, terms []TaggedTerm, from int64, until int64, stat *FinderStat) error { + w, pw, err := t.whereFilter(terms, from, until) + if err != nil { + return err + } // TODO: consider consistent query generator sql := fmt.Sprintf("SELECT Path FROM %s %s %s GROUP BY Path FORMAT TabSeparatedRaw", t.table, pw.PreWhereSQL(), w.SQL()) t.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query(scope.WithTable(ctx, t.table), t.url, sql, t.opts, nil) diff --git a/finder/tagged_test.go b/finder/tagged_test.go index 8504bda6a..7fcb722b4 100644 --- a/finder/tagged_test.go +++ b/finder/tagged_test.go @@ -4,8 +4,11 @@ import ( "fmt" "strconv" "testing" + "time" "github.com/lomik/graphite-clickhouse/config" + "github.com/lomik/graphite-clickhouse/helper/clickhouse" + "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/pkg/where" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -260,3 +263,55 @@ func BenchmarkParseSeriesByTag(b *testing.B) { }) } } + +func TestTaggedFinder_whereFilter(t *testing.T) { + tests := []struct { + name string + query string + from int64 + until int64 + dailyEnabled bool + taggedCosts map[string]*config.Costs + want string + wantPre string + }{ + { + name: "nodaily", + query: "seriesByTag('name=metric')", + from: 1668106860, // 2022-11-11 00:01:00 +05:00 + until: 1668106870, // 2022-11-11 00:01:10 +05:00 + dailyEnabled: false, + want: "Tag1='__name__=metric'", + wantPre: "", + }, + { + name: "midnight at utc (direct)", + query: "seriesByTag('name=metric')", + from: 1668124800, // 2022-11-11 00:00:00 UTC + until: 1668124810, // 2022-11-11 00:00:10 UTC + dailyEnabled: true, + want: "(Tag1='__name__=metric') AND (Date >='" + + date.FromTimestampToDaysFormat(1668124800) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(1668124810) + "')", + wantPre: "", + }, + } + for _, tt := range tests { + t.Run(tt.name+" "+time.Unix(tt.from, 0).Format(time.RFC3339), func(t *testing.T) { + terms, err := ParseSeriesByTag(tt.query, tt.taggedCosts) + if err != nil { + t.Fatal(err) + } + f := NewTagged("http://localhost:8123/", "graphite_tags", tt.dailyEnabled, false, clickhouse.Options{}, tt.taggedCosts) + got, gotDate, err := f.whereFilter(terms, tt.from, tt.until) + if err != nil { + t.Fatal(err) + } + if got.String() != tt.want { + t.Errorf("TaggedFinder.whereFilter()[0] = %v, want %v", got, tt.want) + } + if gotDate.String() != tt.wantPre { + t.Errorf("TaggedFinder.whereFilter()[1] = %v, want %v", gotDate, tt.wantPre) + } + }) + } +} diff --git a/helper/date/date.go b/helper/date/date.go new file mode 100644 index 000000000..4ce4c6718 --- /dev/null +++ b/helper/date/date.go @@ -0,0 +1,103 @@ +package date + +import "time" + +var FromTimestampToDaysFormat func(int64) string +var FromTimeToDaysFormat func(time.Time) string +var UntilTimestampToDaysFormat func(int64) string +var UntilTimeToDaysFormat func(time.Time) string + +// SetDefault() is for broken SlowTimestampToDays in carbon-clickhouse +func SetDefault() { + FromTimestampToDaysFormat = DefaultTimestampToDaysFormat + FromTimeToDaysFormat = DefaultTimeToDaysFormat + UntilTimestampToDaysFormat = DefaultTimestampToDaysFormat + UntilTimeToDaysFormat = DefaultTimeToDaysFormat +} + +// SetUTC() is for UTCTimestampToDays in carbon-clickhouse (see https://github.com/go-graphite/carbon-clickhouse/pull/114) +func SetUTC() { + FromTimestampToDaysFormat = UTCTimestampToDaysFormat + FromTimeToDaysFormat = UTCTimeToDaysFormat + UntilTimestampToDaysFormat = UTCTimestampToDaysFormat + UntilTimeToDaysFormat = UTCTimeToDaysFormat +} + +// SetBoth() is for mixed SlowTimestampToDays/UTCTimestampToDays (before rebuild tables complete) +func SetBoth() { + FromTimestampToDaysFormat = MinTimestampToDaysFormat + FromTimeToDaysFormat = MinTimeToDaysFormat + UntilTimestampToDaysFormat = MaxTimestampToDaysFormat + UntilTimeToDaysFormat = MaxTimeToDaysFormat +} + +func init() { + SetDefault() +} + +// from carbon-clickhouse, port of SlowTimestampToDays, broken symmetic, not always UTC +func DefaultTimestampToDaysFormat(ts int64) string { + t := time.Unix(ts, 0) + return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC).Format("2006-01-02") +} + +// from carbon-clickhouse, port of SlowTimestampToDays, broken symmetic, not always UTC +func DefaultTimeToDaysFormat(t time.Time) string { + return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC).Format("2006-01-02") +} + +func UTCTimestampToDaysFormat(timestamp int64) string { + return time.Unix(timestamp, 0).UTC().Format("2006-01-02") +} + +func UTCTimeToDaysFormat(t time.Time) string { + return t.UTC().Format("2006-01-02") +} + +func defaultDate(t time.Time) time.Time { + return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) +} + +func minLocalAndUTC(t time.Time) time.Time { + tu := defaultDate(t.UTC()) + td := defaultDate(t) + if tu.Unix() < td.Unix() { + return tu + } else { + return td + } +} + +// MinTimestampToDaysFormat return formatted minimum (local, UTC) date +func MinTimestampToDaysFormat(ts int64) string { + t := minLocalAndUTC(time.Unix(ts, 0)) + return t.Format("2006-01-02") +} + +// MinTimeToDaysFormat return formatted minimum (local, UTC) date +func MinTimeToDaysFormat(t time.Time) string { + t = minLocalAndUTC(t) + return t.Format("2006-01-02") +} + +func maxLocalAndUTC(t time.Time) time.Time { + tu := defaultDate(t.UTC()) + td := defaultDate(t) + if tu.Unix() > td.Unix() { + return tu + } else { + return td + } +} + +// MaxTimestampToDaysFormat return formatted maximum (local, UTC) date +func MaxTimestampToDaysFormat(ts int64) string { + t := maxLocalAndUTC(time.Unix(ts, 0)) + return t.Format("2006-01-02") +} + +// MaxTimeToDaysFormat return formatted maximum (local, UTC) date +func MaxTimeToDaysFormat(t time.Time) string { + t = maxLocalAndUTC(t) + return t.Format("2006-01-02") +} diff --git a/helper/date/date_test.go b/helper/date/date_test.go new file mode 100644 index 000000000..25b544ce1 --- /dev/null +++ b/helper/date/date_test.go @@ -0,0 +1,254 @@ +package date + +import ( + "os" + "strconv" + "testing" + "time" +) + +var verbose bool + +func isVerbose() bool { + for _, arg := range os.Args { + if arg == "-test.v=true" { + return true + } + } + return false +} + +func init() { + verbose = isVerbose() +} + +// TimestampDaysFormat is broken symmetic with carbon-clickhouse of SlowTimestampToDays, not always UTC +// $ TZ=Etc/GMT-5 go test -v -timeout 30s -run ^TestTimestampDaysFormat$ github.com/lomik/graphite-clickhouse/helper/date +// === RUN TestTimestampDaysFormat +// === RUN TestTimestampDaysFormat/1668106870_2022-11-11T00:01:10+05:00_2022-11-10T19:01:10Z_[0] +// +// date_test.go:62: Warning (TimestampDaysFormat broken) TimestampDaysFormat(1668106870) = 2022-11-11, want UTC 2022-11-10 +// +// --- FAIL: TestTimestampDaysFormat (0.00s) +// +// --- FAIL: TestTimestampDaysFormat/1668106870_2022-11-11T00:01:10+05:00_2022-11-10T19:01:10Z_[0] (0.00s) +// +// FAIL +// FAIL github.com/lomik/graphite-clickhouse/helper/date 0.001s +// +// $ TZ=Etc/GMT+5 go test -v -timeout 30s -run ^TestTimestampDaysFormat$ github.com/lomik/graphite-clickhouse/helper/date +// === RUN TestTimestampDaysFormat +// === RUN TestTimestampDaysFormat/1668124800_2022-11-10T19:00:00-05:00_2022-11-11T00:00:00Z_[1] +// +// date_test.go:62: Warning (TimestampDaysFormat broken) TimestampDaysFormat(1668124800) = 2022-11-10, want UTC 2022-11-11 +// +// === RUN TestTimestampDaysFormat/1668142799_2022-11-10T23:59:59-05:00_2022-11-11T04:59:59Z_[2] +// +// date_test.go:62: Warning (TimestampDaysFormat broken) TimestampDaysFormat(1668142799) = 2022-11-10, want UTC 2022-11-11 +// +// === RUN TestTimestampDaysFormat/1650776160_2022-04-23T23:56:00-05:00_2022-04-24T04:56:00Z_[3] +// +// date_test.go:62: Warning (TimestampDaysFormat broken) TimestampDaysFormat(1650776160) = 2022-04-23, want UTC 2022-04-24 +// +// --- FAIL: TestTimestampDaysFormat (0.00s) +func TestDefaultTimestampToDaysFormat(t *testing.T) { + tests := []struct { + ts int64 + want string + }{ + { + ts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC + // select toDate(1650776160,'UTC') + // 2022-11-10 + want: time.Unix(1668106870, 0).Format("2006-01-02"), + }, + { + ts: 1668124800, // 2022-11-11 00:00:00 UTC + want: time.Unix(1668124800, 0).Format("2006-01-02"), + }, + { + ts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC + want: time.Unix(1668142799, 0).Format("2006-01-02"), + }, + { + ts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7) + // 2022-04-24 4:56:00 + // select toDate(1650776160,'UTC') + // 2022-04-24 + // select toDate(1650776160,'Etc/GMT+7') + // 2022-04-23 + want: time.Unix(1650776160, 0).Format("2006-01-02"), + }, + } + for i, tt := range tests { + t.Run(strconv.FormatInt(tt.ts, 10)+" "+time.Unix(tt.ts, 0).Format(time.RFC3339)+" "+time.Unix(tt.ts, 0).UTC().Format(time.RFC3339)+" ["+strconv.Itoa(i)+"]", func(t *testing.T) { + if got := DefaultTimestampToDaysFormat(tt.ts); got != tt.want { + t.Errorf("DefaultTimestampDaysFormat(%d) = %s, want %s", tt.ts, got, tt.want) + } else if gotUTC := UTCTimestampToDaysFormat(tt.ts); got != gotUTC { + // Run to see a warning + // go test -v -timeout 30s -run ^TestTimestampDaysFormat$ github.com/lomik/graphite-clickhouse/helper/date + if verbose { + t.Errorf("Warning (DefaultTimestampDaysFormat broken) DefaultTimestampDaysFormat(%d) = %s, want UTC %s", tt.ts, got, gotUTC) + } else { + t.Logf("Warning (DefaultTimestampDaysFormat broken) DefaultTimestampDaysFormat(%d) = %s, want UTC %s", tt.ts, got, gotUTC) + } + } + }) + } +} + +func TestDefaultTimeToDaysFormat(t *testing.T) { + tests := []struct { + ts int64 + want string + }{ + { + ts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC + // select toDate(1650776160,'UTC') + // 2022-11-10 + want: time.Unix(1668106870, 0).Format("2006-01-02"), + }, + { + ts: 1668124800, // 2022-11-11 00:00:00 UTC + want: time.Unix(1668124800, 0).Format("2006-01-02"), + }, + { + ts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC + want: time.Unix(1668142799, 0).Format("2006-01-02"), + }, + { + ts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7) + // 2022-04-24 4:56:00 + // select toDate(1650776160,'UTC') + // 2022-04-24 + // select toDate(1650776160,'Etc/GMT+7') + // 2022-04-23 + want: time.Unix(1650776160, 0).Format("2006-01-02"), + }, + } + for i, tt := range tests { + t.Run(strconv.FormatInt(tt.ts, 10)+" "+time.Unix(tt.ts, 0).UTC().Format(time.RFC3339)+" ["+strconv.Itoa(i)+"]", func(t *testing.T) { + if got := DefaultTimeToDaysFormat(time.Unix(tt.ts, 0)); got != tt.want { + t.Errorf("DefaultTimeDaysFormat() = %s, want %s", got, tt.want) + } + }) + } +} + +func TestUTCTimestampToDaysFormat(t *testing.T) { + tests := []struct { + ts int64 + want string + }{ + { + ts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC + // select toDate(1650776160,'UTC') + // 2022-11-10 + want: "2022-11-10", + }, + { + ts: 1668124800, // 2022-11-11 00:00:00 UTC + want: "2022-11-11", + }, + { + ts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC + want: "2022-11-11", + }, + { + ts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7) + // 2022-04-24 4:56:00 + // select toDate(1650776160,'UTC') + // 2022-04-24 + // select toDate(1650776160,'Etc/GMT+7') + // 2022-04-23 + want: "2022-04-24", + }, + } + for i, tt := range tests { + t.Run(strconv.FormatInt(tt.ts, 10)+" "+time.Unix(tt.ts, 0).Format(time.RFC3339)+" "+time.Unix(tt.ts, 0).UTC().Format(time.RFC3339)+" ["+strconv.Itoa(i)+"]", func(t *testing.T) { + if got := UTCTimestampToDaysFormat(tt.ts); got != tt.want { + t.Errorf("UTCTimestampDaysFormat(%d) = %s, want %s", tt.ts, got, tt.want) + } + }) + } +} + +func TestUTCTimeToDaysFormat(t *testing.T) { + tests := []struct { + ts int64 + want string + }{ + { + ts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC + // select toDate(1650776160,'UTC') + // 2022-11-10 + want: "2022-11-10", + }, + { + ts: 1668124800, // 2022-11-11 00:00:00 UTC + want: "2022-11-11", + }, + { + ts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC + want: "2022-11-11", + }, + { + ts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7) + // 2022-04-24 4:56:00 + // select toDate(1650776160,'UTC') + // 2022-04-24 + // select toDate(1650776160,'Etc/GMT+7') + // 2022-04-23 + want: "2022-04-24", + }, + } + for i, tt := range tests { + t.Run(strconv.FormatInt(tt.ts, 10)+" "+time.Unix(tt.ts, 0).UTC().Format(time.RFC3339)+" ["+strconv.Itoa(i)+"]", func(t *testing.T) { + if got := UTCTimeToDaysFormat(time.Unix(tt.ts, 0)); got != tt.want { + t.Errorf("UTCTimeDaysFormat() = %s, want %s", got, tt.want) + } + }) + } +} + +func TestMinMaxTimestampToDaysFormat(t *testing.T) { + tests := []struct { + ts int64 + }{ + { + ts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC + // select toDate(1650776160,'UTC') + // 2022-11-10 + }, + { + ts: 1668124800, // 2022-11-11 00:00:00 UTC + }, + { + ts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC + }, + { + ts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7) + // 2022-04-24 4:56:00 + // select toDate(1650776160,'UTC') + // 2022-04-24 + // select toDate(1650776160,'Etc/GMT+7') + // 2022-04-23 + }, + } + for i, tt := range tests { + t.Run(strconv.FormatInt(tt.ts, 10)+" "+time.Unix(tt.ts, 0).UTC().Format(time.RFC3339)+" ["+strconv.Itoa(i)+"]", func(t *testing.T) { + gotMin := MinTimestampToDaysFormat(tt.ts) + timeMin, _ := time.Parse("2006-01-02", gotMin) + gotMax := MaxTimestampToDaysFormat(tt.ts) + timeMax, _ := time.Parse("2006-01-02", gotMax) + got := DefaultTimestampToDaysFormat(tt.ts) + tm, _ := time.Parse("2006-01-02", got) + + if timeMin.UnixNano() > timeMax.UnixNano() || tm.UnixNano() > timeMax.UnixNano() || tm.UnixNano() < timeMin.UnixNano() { + t.Errorf("MinTimeDaysFormat() = %s > MaxTimeDaysFormat() = %s, DefaultTimeDaysFormat() = %s", gotMin, gotMax, got) + } else { + t.Logf("MinTimeDaysFormat() = %s, MaxTimeDaysFormat() = %s, DefaultTimeDaysFormat() = %s", gotMin, gotMax, got) + } + }) + } +} diff --git a/helper/datetime/datetime.go b/helper/datetime/datetime.go index 128d62104..01fce3200 100644 --- a/helper/datetime/datetime.go +++ b/helper/datetime/datetime.go @@ -98,7 +98,6 @@ func DateParamToEpoch(s string, tz *time.Location, now time.Time, truncate time. ds = s } else { ds = s[:delim] - ts = s[delim:] switch ds { case "now", "today": t = now @@ -117,16 +116,28 @@ func DateParamToEpoch(s string, tz *time.Location, now time.Time, truncate time. return 0 } - offset, err := parser.IntervalString(ts, 1) - if err != nil { - offset64, err := strconv.ParseInt(ts, 10, 32) + s = s[delim:] + for len(s) > 0 { + delim := strings.IndexAny(s[1:], "+-") + if delim == -1 { + ts = s + s = s[:0] + } else { + ts = s[:delim+1] + s = s[delim+1:] + } + offset, err := parser.IntervalString(ts, 1) if err != nil { - return 0 + offset64, err := strconv.ParseInt(ts, 10, 32) + if err != nil { + return 0 + } + offset = int32(offset64) } - offset = int32(offset64) + t = t.Add(time.Duration(offset) * time.Second) } - return t.Add(time.Duration(offset) * time.Second).Unix() + return t.Unix() } case len(split) == 2: ts, ds = split[0], split[1] diff --git a/helper/datetime/datetime_test.go b/helper/datetime/datetime_test.go index 991691b80..b77e8edf9 100644 --- a/helper/datetime/datetime_test.go +++ b/helper/datetime/datetime_test.go @@ -32,6 +32,7 @@ func TestDateParamToEpoch(t *testing.T) { {"midnight-10", "23:59:50 1994-Aug-15"}, {"midnight-1s", "23:59:59 1994-Aug-15"}, {"midnight-1day", "00:00:00 1994-Aug-15"}, + {"midnight-1day+1s", "00:00:01 1994-Aug-15"}, } for _, tt := range tests { diff --git a/issues/daytime/carbon-clickhouse.conf.tpl b/issues/daytime/carbon-clickhouse.conf.tpl new file mode 100644 index 000000000..41d7ce56d --- /dev/null +++ b/issues/daytime/carbon-clickhouse.conf.tpl @@ -0,0 +1,45 @@ +[common] + +[data] +path = "/etc/carbon-clickhouse/data" +chunk-interval = "1s" +chunk-auto-interval = "" + +[upload.graphite_index] +type = "index" +table = "graphite_index" +url = "{{ .CLICKHOUSE_URL }}/" +timeout = "2m30s" +cache-ttl = "1h" + +[upload.graphite_tags] +type = "tagged" +table = "graphite_tags" +threads = 3 +url = "{{ .CLICKHOUSE_URL }}/" +timeout = "2m30s" +cache-ttl = "1h" + +[upload.graphite_reverse] +type = "points-reverse" +table = "graphite_reverse" +url = "{{ .CLICKHOUSE_URL }}/" +timeout = "2m30s" +zero-timestamp = false + +[upload.graphite] +type = "points" +table = "graphite" +url = "{{ .CLICKHOUSE_URL }}/" +timeout = "2m30s" +zero-timestamp = false + +[tcp] +listen = ":2003" +enabled = true +drop-future = "0s" +drop-past = "0s" + +[logging] +file = "/etc/carbon-clickhouse/carbon-clickhouse.log" +level = "debug" diff --git a/issues/daytime/graphite-clickhouse-internal-aggr.conf.tpl b/issues/daytime/graphite-clickhouse-internal-aggr.conf.tpl new file mode 100644 index 000000000..2d15192f9 --- /dev/null +++ b/issues/daytime/graphite-clickhouse-internal-aggr.conf.tpl @@ -0,0 +1,35 @@ +[common] +listen = "{{ .GCH_ADDR }}" +max-cpu = 0 +max-metrics-in-render-answer = 10000 +max-metrics-per-target = 10000 +headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] + +[clickhouse] +url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" +data-timeout = "30s" + +index-table = "graphite_index" +index-use-daily = true +index-timeout = "1m" +internal-aggregation = true + +tagged-table = "graphite_tags" +tagged-autocomplete-days = 1 + +date-format = "both" + +[[data-table]] +# # clickhouse table name +table = "graphite" +# # points in table are stored with reverse path +reverse = false +rollup-conf = "auto" + +[[logging]] +logger = "" +file = "{{ .GCH_DIR }}/graphite-clickhouse.log" +level = "info" +encoding = "json" +encoding-time = "iso8601" +encoding-duration = "seconds" diff --git a/issues/daytime/graphite-clickhouse.conf.tpl b/issues/daytime/graphite-clickhouse.conf.tpl new file mode 100644 index 000000000..e4c68178c --- /dev/null +++ b/issues/daytime/graphite-clickhouse.conf.tpl @@ -0,0 +1,35 @@ +[common] +listen = "{{ .GCH_ADDR }}" +max-cpu = 0 +max-metrics-in-render-answer = 10000 +max-metrics-per-target = 10000 +headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] + +[clickhouse] +url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" +data-timeout = "30s" + +index-table = "graphite_index" +index-use-daily = true +index-timeout = "1m" +internal-aggregation = false + +tagged-table = "graphite_tags" +tagged-autocomplete-days = 1 + +date-format = "both" + +[[data-table]] +# # clickhouse table name +table = "graphite" +# # points in table are stored with reverse path +reverse = false +rollup-conf = "auto" + +[[logging]] +logger = "" +file = "{{ .GCH_DIR }}/graphite-clickhouse.log" +level = "info" +encoding = "json" +encoding-time = "iso8601" +encoding-duration = "seconds" diff --git a/issues/daytime/test.toml b/issues/daytime/test.toml new file mode 100644 index 000000000..74abc3a8c --- /dev/null +++ b/issues/daytime/test.toml @@ -0,0 +1,490 @@ +[test] +precision = "10s" + +[[test.clickhouse]] +version = "latest" +dir = "tests/clickhouse/rollup" + +[test.carbon_clickhouse] +#version = "v0.11.1" +image = "msaf1980/carbon-clickhouse" +version = "tz" +template = "carbon-clickhouse.conf.tpl" + +[[test.graphite_clickhouse]] +template = "graphite-clickhouse.conf.tpl" + +[[test.graphite_clickhouse]] +template = "graphite-clickhouse-internal-aggr.conf.tpl" + +[[test.input]] +name = "test.plain1" +points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow"}] + +[[test.input]] +name = "test.plain2" +points = [{value = 2.0, time = "rnow-30"}, {value = 1.0, time = "rnow-20"}, {value = 1.5, time = "rnow-10"}, {value = 2.5, time = "rnow"}] + +[[test.input]] +name = "test2.plain" +points = [{value = 1.0, time = "rnow-30"}, {value = 2.0, time = "rnow-20"}, {value = 2.5, time = "rnow-10"}, {value = 3.5, time = "rnow"}] + +[[test.input]] +name = "metric1;tag1=value1;tag2=value21;tag3=value3" +points = [{value = 2.0, time = "rnow-30"}, {value = 2.5, time = "rnow-20"}, {value = 2.0, time = "rnow-10"}, {value = 3.0, time = "rnow"}] + +[[test.input]] +name = "metric1;tag2=value22;tag4=value4" +points = [{value = 1.0, time = "rnow-30"}, {value = 2.0, time = "rnow-20"}, {value = 0.0, time = "rnow-10"}, {value = 1.0, time = "rnow"}] + +[[test.input]] +name = "metric1;tag1=value1;tag2=value23;tag3=value3" +points = [{value = 0.5, time = "rnow-30"}, {value = 1.5, time = "rnow-20"}, {value = 4.0, time = "rnow-10"}, {value = 3.0, time = "rnow"}] + +[[test.input]] +name = "metric2;tag2=value21;tag4=value4" +points = [{value = 2.0, time = "rnow-30"}, {value = 1.0, time = "rnow-20"}, {value = 0.0, time = "rnow-10"}, {value = 1.0, time = "rnow"}] + +###################################### +# Check metrics find + +[[test.find_checks]] +formats = [ "pickle", "protobuf", "carbonapi_v3_pb" ] +query = "test" +result = [ + { path = "test", is_leaf = false } +] + +[[test.find_checks]] +formats = [ "pickle", "protobuf", "carbonapi_v3_pb" ] +query = "test.pl*" +result = [ + { path = "test.plain1", is_leaf = true }, { path = "test.plain2", is_leaf = true } +] + +# End - Check metrics find +###################################### +# Check tags autocomplete + +[[test.tags_checks]] +query = "tag1;tag2=value21" +result = [ + "value1" +] + +[[test.tags_checks]] +query = "name;tag2=value21;tag1=~value" +result = [ + "metric1", +] + +# End - Check tags autocomplete +########################################################################## +# Plain metrics (carbonapi_v3_pb) + +# test.plain1 +# test.plain2 +# test2.plain + +[[test.render_checks]] +from = "rnow-10" +until = "rnow" +targets = [ + "test.plain*", + "test{1,2}.plain" +] + +[[test.render_checks.result]] +name = "test.plain1" +path = "test.plain*" +consolidation = "avg" +start = "rnow-10" +stop = "rnow+10" +step = 10 +req_start = "rnow-10" +req_stop = "rnow+10" +values = [1.0, 2.0] + +[[test.render_checks.result]] +name = "test.plain2" +path = "test.plain*" +consolidation = "avg" +start = "rnow-10" +stop = "rnow+10" +step = 10 +req_start = "rnow-10" +req_stop = "rnow+10" +values = [1.5, 2.5] + +[[test.render_checks.result]] +name = "test2.plain" +path = "test{1,2}.plain" +consolidation = "avg" +start = "rnow-10" +stop = "rnow+10" +step = 10 +req_start = "rnow-10" +req_stop = "rnow+10" +values = [2.5, 3.5] + +# End - Plain metrics (carbonapi_v3_pb) +########################################################################## +# Plain metrics (carbonapi_v2_pb) + +[[test.render_checks]] +formats = [ "protobuf", "carbonapi_v2_pb" ] +from = "rnow-10" +until = "rnow+1" +targets = [ + "test.plain*", + "test{1,2}.plain" +] + +[[test.render_checks.result]] +name = "test.plain1" +start = "rnow-10" +stop = "rnow+10" +step = 10 +values = [1.0, 2.0] + +[[test.render_checks.result]] +name = "test.plain2" +start = "rnow-10" +stop = "rnow+10" +step = 10 +values = [1.5, 2.5] + +[[test.render_checks.result]] +name = "test2.plain" +start = "rnow-10" +stop = "rnow+10" +step = 10 +values = [2.5, 3.5] + +# End - Plain metrics (carbonapi_v2_pb) +########################################################################## +# Plain metrics (pickle) + +[[test.render_checks]] +formats = [ "pickle" ] +from = "rnow-10" +until = "rnow+1" +targets = [ + "test.plain*", + "test{1,2}.plain" +] + +[[test.render_checks.result]] +name = "test.plain1" +path = "test.plain*" +start = "rnow-10" +stop = "rnow+10" +step = 10 +values = [1.0, 2.0] + +[[test.render_checks.result]] +name = "test.plain2" +path = "test.plain*" +start = "rnow-10" +stop = "rnow+10" +step = 10 +values = [1.5, 2.5] + +[[test.render_checks.result]] +name = "test2.plain" +path = "test{1,2}.plain" +start = "rnow-10" +stop = "rnow+10" +step = 10 +values = [2.5, 3.5] + +# End - Plain metrics (pickle) +########################################################################## +# Taged metrics (carbonapi_v3_pb) + +# metric1;tag1=value1;tag2=value21;tag3=value3 +# metric1;tag2=value22;tag4=value4 +# metric1;tag1=value1;tag2=value23;tag3=value3 +# metric2;tag2=value21;tag4=value4 + +[[test.render_checks]] +from = "rnow-10" +until = "rnow+1" +targets = [ + "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')", + "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" +] + +[[test.render_checks.result]] +name = "metric1;tag1=value1;tag2=value21;tag3=value3" +path = "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')" +consolidation = "avg" +start = "rnow-10" +stop = "rnow+10" +step = 10 +req_start = "rnow-10" +req_stop = "rnow+10" +values = [2.0, 3.0] + +[[test.render_checks.result]] +name = "metric1;tag1=value1;tag2=value23;tag3=value3" +path = "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')" +consolidation = "avg" +start = "rnow-10" +stop = "rnow+10" +step = 10 +req_start = "rnow-10" +req_stop = "rnow+10" +values = [4.0, 3.0] + +[[test.render_checks.result]] +name = "metric2;tag2=value21;tag4=value4" +path = "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" +consolidation = "avg" +start = "rnow-10" +stop = "rnow+10" +step = 10 +req_start = "rnow-10" +req_stop = "rnow+10" +values = [0.0, 1.0] + +# End - Tagged metrics (carbonapi_v3_pb) +########################################################################## +# Tagged metrics (carbonapi_v2_pb) + +[[test.render_checks]] +formats = [ "protobuf", "carbonapi_v2_pb" ] +from = "rnow-10" +until = "rnow+1" +targets = [ + "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')", + "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" +] + +[[test.render_checks.result]] +name = "metric1;tag1=value1;tag2=value21;tag3=value3" +start = "rnow-10" +stop = "rnow+10" +step = 10 +values = [2.0, 3.0] + +[[test.render_checks.result]] +name = "metric1;tag1=value1;tag2=value23;tag3=value3" +start = "rnow-10" +stop = "rnow+10" +step = 10 +values = [4.0, 3.0] + +[[test.render_checks.result]] +name = "metric2;tag2=value21;tag4=value4" +start = "rnow-10" +stop = "rnow+10" +step = 10 +values = [0.0, 1.0] + +# End - Tagged metrics (carbonapi_v2_pb) +########################################################################## +# Tagged metrics (pickle) + +[[test.render_checks]] +formats = [ "pickle" ] +from = "rnow-10" +until = "rnow+1" +targets = [ + "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')", + "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" +] + +[[test.render_checks.result]] +name = "metric1;tag1=value1;tag2=value21;tag3=value3" +path = "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')" +start = "rnow-10" +stop = "rnow+10" +step = 10 +values = [2.0, 3.0] + +[[test.render_checks.result]] +name = "metric1;tag1=value1;tag2=value23;tag3=value3" +path = "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')" +start = "rnow-10" +stop = "rnow+10" +step = 10 +values = [4.0, 3.0] + +[[test.render_checks.result]] +name = "metric2;tag2=value21;tag4=value4" +path = "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" +start = "rnow-10" +stop = "rnow+10" +step = 10 +values = [0.0, 1.0] + +# End - Tagged metrics (pickle) +########################################################################## +# Midnight + +# points for check https://github.com/go-graphite/graphite-clickhouse/issues/184 +[[test.input]] +name = "test.midnight" +points = [{value = 3.0, time = "midnight+60s"}] + +[[test.input]] +name = "now;scope=midnight" +points = [{value = 4.0, time = "midnight+60s"}] + +[[test.find_checks]] +name = "Midnight (direct)" +query = "test.midnight*" +result = [{ path = "test.midnight", is_leaf = true }] + +[[test.find_checks]] +name = "Midnight" +query = "test.midnight" +from = "midnight+60s" +until = "midnight+70s" +result = [{ path = "test.midnight", is_leaf = true }] + +[[test.find_checks]] +name = "Midnight (reverse)" +query = "*test.midnight" +result = [{ path = "test.midnight", is_leaf = true }] + +[[test.find_checks]] +name = "Midnight" +query = "test.midnight" +from = "midnight+60s" +until = "midnight+70s" +result = [{ path = "test.midnight", is_leaf = true }] + +[[test.tags_checks]] +name = "Midnight" +query = "name;scope=midnight" +result = [ + "now", +] + +[[test.render_checks]] +name = "Midnight (direct)" +formats = [ "protobuf" ] +from = "midnight+60s" +until = "midnight+70s" +targets = [ + "test.midnight*", + ] + +[[test.render_checks.result]] +name = "test.midnight" +start = "midnight+60s" +stop = "midnight+80s" +step = 10 +values = [3.0, nan] + +[[test.render_checks]] +name = "Midnight (reverse)" +formats = [ "protobuf" ] +from = "midnight+60s" +until = "midnight+70s" +targets = [ + "*test.midnight", +] +dump_if_empty = [ + "SELECT Date, Path FROM graphite_index WHERE ((Level=2) AND (Path LIKE 'test.midnight%')) GROUP BY Date, Path" +] + +[[test.render_checks.result]] +name = "test.midnight" +start = "midnight+60s" +stop = "midnight+80s" +step = 10 +values = [3.0, nan] + +[[test.render_checks]] +name = "Midnight" +formats = [ "protobuf" ] +from = "midnight+60s" +until = "midnight+70s" +targets = [ + "seriesByTag('name=now', 'scope=midnight')", + ] + +[[test.render_checks.result]] +name = "now;scope=midnight" +start = "midnight+60s" +stop = "midnight+80s" +step = 10 +values = [4.0, nan] + +# End - Midnight +########################################################################## +# Day end + +# points for check https://github.com/go-graphite/graphite-clickhouse/issues/184 +[[test.input]] +name = "test.23h" +points = [{value = 3.0, time = "midnight+1380m"}] + +[[test.input]] +name = "now;scope=23h" +points = [{value = 4.0, time = "midnight+1380m"}] + +[[test.find_checks]] +name = "Day end" +query = "test.23h" +from = "midnight+1380m" +until = "midnight+1381m" +result = [{ path = "test.23h", is_leaf = true }] + +[[test.find_checks]] +name = "Day end" +query = "test.23h" +from = "midnight+1380m" +until = "midnight+1381m" +result = [{ path = "test.23h", is_leaf = true }] + +[[test.tags_checks]] +name = "Day end" +query = "name;scope=23h" +result = [ + "now", +] +dump_if_empty = [ + "SELECT Date, Tags FROM graphite_tags WHERE ((Tag1='scope=23h') AND (arrayJoin(Tags) LIKE '__name__=%')) GROUP BY Date, Tags ORDER BY Date, Tags" +] + +[[test.render_checks]] +name = "Day end" +formats = [ "protobuf" ] +from = "midnight+1380m" +until = "midnight+1380m+10s" +targets = [ + "test.23h", +] +dump_if_empty = [ + "SELECT Date, Path FROM graphite_index WHERE ((Level=2) AND (Path IN ('test.23h','test.23h.'))) GROUP BY Date, Path" +] + +[[test.render_checks.result]] +name = "test.23h" +start = "midnight+1380m" +stop = "midnight+1380m+20s" +step = 10 +values = [3.0, nan] + +[[test.render_checks]] +name = "Day end" +formats = [ "protobuf" ] +from = "midnight+1380m" +until = "midnight+1380m+10s" +targets = [ + "seriesByTag('name=now', 'scope=23h')", + ] + +[[test.render_checks.result]] +name = "now;scope=23h" +start = "midnight+1380m" +stop = "midnight+1380m+20s" +step = 10 +values = [4.0, nan] + +# End - Day end +########################################################################## diff --git a/pkg/where/where.go b/pkg/where/where.go index 93ca01ce4..2efdb0067 100644 --- a/pkg/where/where.go +++ b/pkg/where/where.go @@ -7,6 +7,7 @@ import ( "strings" "unsafe" + "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/helper/errs" ) @@ -178,8 +179,8 @@ func InTable(field string, table string) string { func DateBetween(field string, from int64, until int64) string { return fmt.Sprintf( - "%s >= toDate(%d) AND %s <= toDate(%d)", - field, from, field, until, + "%s >= '%s' AND %s <= '%s'", + field, date.FromTimestampToDaysFormat(from), field, date.UntilTimestampToDaysFormat(until), ) } diff --git a/render/data/query_test.go b/render/data/query_test.go index b6f9cb130..229ef0927 100644 --- a/render/data/query_test.go +++ b/render/data/query_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/lomik/graphite-clickhouse/finder" + "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/helper/rollup" "github.com/lomik/graphite-clickhouse/pkg/alias" "github.com/lomik/graphite-clickhouse/pkg/reverse" @@ -423,19 +424,19 @@ func TestGenerateQuery(t *testing.T) { unaggregated string }{ { - in: in{1, 13, 1, "avg"}, - aggregated: ("WITH anyResample(1, 13, 1)(toUInt32(intDiv(Time, 1)*1), Time) AS mask\n" + + in: in{1668124800, 1668325322, 1, "avg"}, + aggregated: ("WITH anyResample(1668124800, 1668325322, 1)(toUInt32(intDiv(Time, 1)*1), Time) AS mask\n" + "SELECT Path,\n arrayFilter(m->m!=0, mask) AS times,\n" + - " arrayFilter((v,m)->m!=0, avgResample(1, 13, 1)(Value, Time), mask) AS values\n" + + " arrayFilter((v,m)->m!=0, avgResample(1668124800, 1668325322, 1)(Value, Time), mask) AS values\n" + "FROM graphite.table\n" + - "PREWHERE Date >= toDate(1) AND Date <= toDate(13)\n" + - "WHERE (Path in metrics_list) AND (Time >= 1 AND Time <= 13)\n" + + "PREWHERE Date >= '" + date.FromTimestampToDaysFormat(1668124800) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(1668325322) + "'\n" + + "WHERE (Path in metrics_list) AND (Time >= 1668124800 AND Time <= 1668325322)\n" + "GROUP BY Path\n" + "FORMAT RowBinary"), unaggregated: ("SELECT Path, groupArray(Time), groupArray(Value), groupArray(Timestamp)\n" + "FROM graphite.table\n" + - "PREWHERE Date >= toDate(1) AND Date <= toDate(13)\n" + - "WHERE (Path in metrics_list) AND (Time >= 1 AND Time <= 13)\n" + + "PREWHERE Date >= '" + date.FromTimestampToDaysFormat(1668124800) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(1668325322) + "'\n" + + "WHERE (Path in metrics_list) AND (Time >= 1668124800 AND Time <= 1668325322)\n" + "GROUP BY Path\n" + "FORMAT RowBinary"), }, @@ -445,13 +446,13 @@ func TestGenerateQuery(t *testing.T) { "SELECT Path,\n arrayFilter(m->m!=0, mask) AS times,\n" + " arrayFilter((v,m)->m!=0, minResample(11111, 33333, 11111)(Value, Time), mask) AS values\n" + "FROM graphite.table\n" + - "PREWHERE Date >= toDate(11111) AND Date <= toDate(33333)\n" + + "PREWHERE Date >= '" + date.FromTimestampToDaysFormat(11111) + "' AND Date <= '" + date.FromTimestampToDaysFormat(33333) + "'\n" + "WHERE (Path in metrics_list) AND (Time >= 11111 AND Time <= 33333)\n" + "GROUP BY Path\n" + "FORMAT RowBinary"), unaggregated: ("SELECT Path, groupArray(Time), groupArray(Value), groupArray(Timestamp)\n" + "FROM graphite.table\n" + - "PREWHERE Date >= toDate(11111) AND Date <= toDate(33333)\n" + + "PREWHERE Date >= '" + date.FromTimestampToDaysFormat(11111) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(33333) + "'\n" + "WHERE (Path in metrics_list) AND (Time >= 11111 AND Time <= 33333)\n" + "GROUP BY Path\n" + "FORMAT RowBinary"), diff --git a/tests/agg_oneblock/test.toml b/tests/agg_oneblock/test.toml index e96bb764b..7e6a237de 100644 --- a/tests/agg_oneblock/test.toml +++ b/tests/agg_oneblock/test.toml @@ -53,6 +53,9 @@ until = "rnow+10" targets = [ "test.{avg,min,max,sum}" ] +dump_if_empty = [ + "SELECT Date, Path FROM graphite_index WHERE ((Level=2) AND (Path LIKE 'test.%' AND match(Path, '^test[.](avg|min|max|sum)[.]?$'))) GROUP BY Date, Path" +] [[test.render_checks.result]] name = "test.avg" diff --git a/tests/one_table/test.toml b/tests/one_table/test.toml index d0a4f6063..f11588d9b 100644 --- a/tests/one_table/test.toml +++ b/tests/one_table/test.toml @@ -331,3 +331,161 @@ values = [0.0, 1.0] # End - Tagged metrics (pickle) ########################################################################## +# Midnight + +# points for check https://github.com/go-graphite/graphite-clickhouse/issues/184 +[[test.input]] +name = "test.midnight" +points = [{value = 3.0, time = "midnight+60s"}] + +[[test.input]] +name = "now;scope=midnight" +points = [{value = 4.0, time = "midnight+60s"}] + +[[test.find_checks]] +name = "Midnight (direct)" +query = "test.midnight*" +result = [{ path = "test.midnight", is_leaf = true }] + +[[test.find_checks]] +name = "Midnight" +query = "test.midnight" +from = "midnight+60s" +until = "midnight+70s" +result = [{ path = "test.midnight", is_leaf = true }] + +[[test.find_checks]] +name = "Midnight (reverse)" +query = "*test.midnight" +result = [{ path = "test.midnight", is_leaf = true }] + +[[test.find_checks]] +name = "Midnight" +query = "test.midnight" +from = "midnight+60s" +until = "midnight+70s" +result = [{ path = "test.midnight", is_leaf = true }] + +[[test.tags_checks]] +name = "Midnight" +query = "name;scope=midnight" +result = [ + "now", +] + +[[test.render_checks]] +name = "Midnight (direct)" +formats = [ "protobuf" ] +from = "midnight+60s" +until = "midnight+70s" +targets = [ + "test.midnight*", + ] + +[[test.render_checks.result]] +name = "test.midnight" +start = "midnight+60s" +stop = "midnight+80s" +step = 10 +values = [3.0, nan] + +[[test.render_checks]] +name = "Midnight (reverse)" +formats = [ "protobuf" ] +from = "midnight+60s" +until = "midnight+70s" +targets = [ + "*test.midnight", + ] + +[[test.render_checks.result]] +name = "test.midnight" +start = "midnight+60s" +stop = "midnight+80s" +step = 10 +values = [3.0, nan] + +[[test.render_checks]] +name = "Midnight" +formats = [ "protobuf" ] +from = "midnight+60s" +until = "midnight+70s" +targets = [ + "seriesByTag('name=now', 'scope=midnight')", + ] + +[[test.render_checks.result]] +name = "now;scope=midnight" +start = "midnight+60s" +stop = "midnight+80s" +step = 10 +values = [4.0, nan] + +# End - Midnight +########################################################################## +# Day end + +# points for check https://github.com/go-graphite/graphite-clickhouse/issues/184 +[[test.input]] +name = "test.23h" +points = [{value = 3.0, time = "midnight+1380m"}] + +[[test.input]] +name = "now;scope=23h" +points = [{value = 4.0, time = "midnight+1380m"}] + +[[test.find_checks]] +name = "Day end" +query = "test.23h" +from = "midnight+1380m" +until = "midnight+1381m" +result = [{ path = "test.23h", is_leaf = true }] + +[[test.find_checks]] +name = "Day end" +query = "test.23h" +from = "midnight+1380m" +until = "midnight+1381m" +result = [{ path = "test.23h", is_leaf = true }] + +[[test.tags_checks]] +name = "Day end" +query = "name;scope=23h" +result = [ + "now", +] + +[[test.render_checks]] +name = "Day end" +formats = [ "protobuf" ] +from = "midnight+1380m" +until = "midnight+1380m+10s" +targets = [ + "test.23h", + ] + +[[test.render_checks.result]] +name = "test.23h" +start = "midnight+1380m" +stop = "midnight+1380m+20s" +step = 10 +values = [3.0, nan] + +[[test.render_checks]] +name = "Day end" +formats = [ "protobuf" ] +from = "midnight+1380m" +until = "midnight+1380m+10s" +targets = [ + "seriesByTag('name=now', 'scope=23h')", + ] + +[[test.render_checks.result]] +name = "now;scope=23h" +start = "midnight+1380m" +stop = "midnight+1380m+20s" +step = 10 +values = [4.0, nan] + +# End - Day end +##########################################################################