Fix missing results on day start/end (depend on timezone)

.github/workflows/tests.yml

@@ -54,11 +54,39 @@ jobs:
       env:
         CGO_ENABLED: 1
 
+    - name: Test (with GMT-5)
+      run: |
+        go clean -testcache
+        TZ=Etc/GMT-5 make test
+      env:
+        CGO_ENABLED: 1
+
+    - name: Test (with GMT+5)
+      run: |
+        go clean -testcache
+        TZ=Etc/GMT+5 make test
+      env:
+        CGO_ENABLED: 1
+
     - name: Integration tests
       run: |
         make e2e-test
         ./e2e-test -config tests
 
+    # TODO (msaf1980): find a way to set TZ in carbon-clickhouse docker (or run locally)
+    # run with clickhouse.date-format = "both"
+    # - name: Integration tests (with Etc/GMT-5)
+    #   run: |
+    #     make e2e-test
+    #     sudo timedatectl set-timezone Etc/GMT-5
+    #     ./e2e-test -config issues/daytime
+
+    # - name: Integration tests (with Etc/GMT+5)
+    #   run: |
+    #     make e2e-test
+    #     sudo timedatectl set-timezone Etc/GMT+5
+    #     ./e2e-test -config issues/daytime
+
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:

autocomplete/autocomplete.go

@@ -13,6 +13,7 @@ import (
 	"github.com/lomik/graphite-clickhouse/config"
 	"github.com/lomik/graphite-clickhouse/finder"
 	"github.com/lomik/graphite-clickhouse/helper/clickhouse"
+	"github.com/lomik/graphite-clickhouse/helper/date"
 	"github.com/lomik/graphite-clickhouse/helper/utils"
 	"github.com/lomik/graphite-clickhouse/metrics"
 	"github.com/lomik/graphite-clickhouse/pkg/scope"
@@ -43,6 +44,12 @@ func NewValues(config *config.Config) *Handler {
 	return h
 }
 
+func dateString(autocompleteDays int, tm time.Time) (string, string) {
+	fromDate := date.FromTimeToDaysFormat(tm.AddDate(0, 0, -autocompleteDays))
+	untilDate := date.UntilTimeToDaysFormat(tm)
+	return fromDate, untilDate
+}
+
 func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	logger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("autocomplete")
 	r = r.WithContext(scope.WithLogger(r.Context(), logger))
@@ -172,9 +179,8 @@ func (h *Handler) ServeTags(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	now := time.Now()
-	fromDate := now.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Format("2006-01-02")
-	untilDate := now.Format("2006-01-02")
+	// TODO (msaf1980) fix for disable daily
+	fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, start)
 
 	var key string
 
@@ -358,8 +364,8 @@ func (h *Handler) ServeValues(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	fromDate := start.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Format("2006-01-02")
-	untilDate := start.Format("2006-01-02")
+	// TODO (msaf1980) fix for disable daily
+	fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, start)
 
 	var key string
 

autocomplete/autocomplete_test.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/lomik/graphite-clickhouse/config"
 	"github.com/lomik/graphite-clickhouse/helper/tests/clickhouse"
+	chtest "github.com/lomik/graphite-clickhouse/helper/tests/clickhouse"
 	"github.com/lomik/graphite-clickhouse/metrics"
-
 	"github.com/stretchr/testify/assert"
 )
 
@@ -52,19 +52,18 @@ func testResponce(t *testing.T, step int, h *Handler, tt *testStruct, wantCached
 
 func TestHandler_ServeValues(t *testing.T) {
 	metrics.DisableMetrics()
-	srv := clickhouse.NewTestServer()
+	srv := chtest.NewTestServer()
 	defer srv.Close()
 
 	cfg, _ := config.DefaultConfig()
 	cfg.ClickHouse.URL = srv.URL
 
 	h := NewTags(cfg)
 
-	from := "1636432127"
-	until := "1636442929"
-
-	fromDate := time.Now().AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Format("2006-01-02")
-	untilDate := time.Now().Format("2006-01-02")
+	now := time.Now()
+	until := strconv.FormatInt(now.Unix(), 10)
+	from := strconv.FormatInt(now.Add(-time.Minute).Unix(), 10)
+	fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
 
 	srv.AddResponce(
 		"SELECT substr(arrayJoin(Tags), 6) AS value FROM graphite_tagged  WHERE (((Tag1='environment=production') AND (arrayExists((x) -> x='project=web', Tags))) AND (arrayJoin(Tags) LIKE 'host=%')) AND "+
@@ -98,6 +97,7 @@ func TestHandler_ServeValues(t *testing.T) {
 }
 
 func TestTagsAutocomplete_ServeValuesCached(t *testing.T) {
+	metrics.DisableMetrics()
 	srv := clickhouse.NewTestServer()
 	defer srv.Close()
 
@@ -118,11 +118,10 @@ func TestTagsAutocomplete_ServeValuesCached(t *testing.T) {
 
 	h := NewTags(cfg)
 
-	from := "1636432127"
-	until := "1636442929"
-
-	fromDate := time.Now().AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Format("2006-01-02")
-	untilDate := time.Now().Format("2006-01-02")
+	now := time.Now()
+	until := strconv.FormatInt(now.Unix(), 10)
+	from := strconv.FormatInt(now.Add(-time.Minute).Unix(), 10)
+	fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
 
 	srv.AddResponce(
 		"SELECT substr(arrayJoin(Tags), 6) AS value FROM graphite_tagged  WHERE (((Tag1='environment=production') AND (arrayExists((x) -> x='project=web', Tags))) AND (arrayJoin(Tags) LIKE 'host=%')) AND "+

cmd/e2e-test/carbon-clickhouse.go

@@ -2,11 +2,13 @@ package main
 
 import (
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
+	"strings"
 	"text/template"
 )
 
@@ -79,22 +81,29 @@ func (c *CarbonClickhouse) Start(testDir, clickhouseURL, clickhouseContainer str
 		return "", err
 	}
 
-	// tz, _ := localTZLocationName()
+	tz := os.Getenv("TZ")
 
 	cchStart := []string{"run", "-d",
 		"--name", c.container,
 		"-p", c.address + ":2003",
 		"-v", c.storeDir + ":/etc/carbon-clickhouse",
+		// TZ, need to be same as graphite-clickhouse for prevent bugs, ike issue #184
+		"-v", "/etc/timezone:/etc/timezone:ro",
+		"-v", "/etc/localtime:/etc/localtime:ro",
+		"-e", "TZ=" + tz,
 		"--link", clickhouseContainer,
 	}
-	if c.TZ != "" {
-		cchStart = append(cchStart, "-e", "TZ="+c.TZ)
-	}
 
 	cchStart = append(cchStart, c.DockerImage+":"+c.Version)
 
 	cmd := exec.Command(DockerBinary, cchStart...)
 	out, err := cmd.CombinedOutput()
+	if err == nil {
+		dateLocal, _ := exec.Command("date").Output()
+		dateLocalStr := strings.TrimRight(string(dateLocal), "\n")
+		_, dateOnCCH := containerExec(c.container, []string{"date"})
+		fmt.Printf("date local %s, on carbon-clickhouse %s\n", dateLocalStr, dateOnCCH)
+	}
 
 	return string(out), err
 }

cmd/e2e-test/checks.go

@@ -63,18 +63,32 @@ func compareFindMatch(errors *[]string, name, url string, actual, expected []cli
 	}
 }
 
-func verifyMetricsFind(address string, check *MetricsFindCheck) []string {
+func verifyMetricsFind(ch *Clickhouse, gch *GraphiteClickhouse, check *MetricsFindCheck) []string {
 	var errors []string
 	httpClient := http.Client{
 		Timeout: check.Timeout,
 	}
+	address := gch.URL()
 	for _, format := range check.Formats {
 		name := ""
 		if url, result, respHeader, err := client.MetricsFind(&httpClient, address, format, check.Query, check.from, check.until); err == nil {
+			id := requestId(respHeader)
 			if check.ErrorRegexp != "" {
-				errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", requestId(respHeader), url, check.ErrorRegexp))
+				errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", id, url, check.ErrorRegexp))
 			}
 			compareFindMatch(&errors, name, url, result, check.Result, check.InCache, check.CacheTTL, respHeader)
+			if len(result) == 0 && len(check.Result) > 0 {
+				gch.Grep(id)
+				if len(check.DumpIfEmpty) > 0 {
+					for _, q := range check.DumpIfEmpty {
+						if out, err := ch.Query(q); err == nil {
+							fmt.Fprintf(os.Stderr, "%s\n%s", q, out)
+						} else {
+							fmt.Fprintf(os.Stderr, "%s: %s\n", err.Error(), q)
+						}
+					}
+				}
+			}
 
 			if check.CacheTTL > 0 && check.ErrorRegexp == "" {
 				// second query must be find-cached
@@ -124,11 +138,12 @@ func compareTags(errors *[]string, name, url string, actual, expected []string,
 	}
 }
 
-func verifyTags(address string, check *TagsCheck) []string {
+func verifyTags(ch *Clickhouse, gch *GraphiteClickhouse, check *TagsCheck) []string {
 	var errors []string
 	httpClient := http.Client{
 		Timeout: check.Timeout,
 	}
+	address := gch.URL()
 	for _, format := range check.Formats {
 		var (
 			result     []string
@@ -145,10 +160,23 @@ func verifyTags(address string, check *TagsCheck) []string {
 		}
 
 		if err == nil {
+			id := requestId(respHeader)
 			if check.ErrorRegexp != "" {
-				errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", requestId(respHeader), url, check.ErrorRegexp))
+				errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", id, url, check.ErrorRegexp))
 			}
 			compareTags(&errors, name, url, result, check.Result, check.InCache, check.CacheTTL, respHeader)
+			if len(result) == 0 && len(check.Result) > 0 {
+				gch.Grep(id)
+				if len(check.DumpIfEmpty) > 0 {
+					for _, q := range check.DumpIfEmpty {
+						if out, err := ch.Query(q); err == nil {
+							fmt.Fprintf(os.Stderr, "%s\n%s", q, out)
+						} else {
+							fmt.Fprintf(os.Stderr, "%s: %s\n", err.Error(), q)
+						}
+					}
+				}
+			}
 
 			if check.CacheTTL > 0 && check.ErrorRegexp == "" {
 				// second query must be find-cached
@@ -243,20 +271,34 @@ func compareRender(errors *[]string, name, url string, actual, expected []client
 	}
 }
 
-func verifyRender(address string, check *RenderCheck, defaultPreision time.Duration) []string {
+func verifyRender(ch *Clickhouse, gch *GraphiteClickhouse, check *RenderCheck, defaultPreision time.Duration) []string {
 	var errors []string
 	httpClient := http.Client{
 		Timeout: check.Timeout,
 	}
+	address := gch.URL()
 	from := datetime.TimestampTruncate(check.from, defaultPreision)
 	until := datetime.TimestampTruncate(check.until, defaultPreision)
 	for _, format := range check.Formats {
 		if url, result, respHeader, err := client.Render(&httpClient, address, format, check.Targets, from, until); err == nil {
+			id := requestId(respHeader)
 			name := ""
 			if check.ErrorRegexp != "" {
-				errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", requestId(respHeader), url, check.ErrorRegexp))
+				errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", id, url, check.ErrorRegexp))
 			}
 			compareRender(&errors, name, url, result, check.result, check.InCache, respHeader, check.CacheTTL)
+			if len(result) == 0 && len(check.result) > 0 {
+				gch.Grep(id)
+				if len(check.DumpIfEmpty) > 0 {
+					for _, q := range check.DumpIfEmpty {
+						if out, err := ch.Query(q); err == nil {
+							fmt.Fprintf(os.Stderr, "%s\n%s", q, out)
+						} else {
+							fmt.Fprintf(os.Stderr, "%s: %s\n", err.Error(), q)
+						}
+					}
+				}
+			}
 
 			if check.CacheTTL > 0 && check.ErrorRegexp == "" {
 				// second query must be find-cached

cmd/e2e-test/clickhouse.go

@@ -1,8 +1,17 @@
 package main
 
 import (
+	"bytes"
 	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
 	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/msaf1980/go-stringutils"
 )
 
 var ClickhouseContainerName = "clickhouse-server-gch-test"
@@ -107,3 +116,50 @@ func (c *Clickhouse) Container() string {
 func (c *Clickhouse) Exec(sql string) (bool, string) {
 	return containerExec(c.container, []string{"sh", "-c", "clickhouse-client -q '" + sql + "'"})
 }
+
+func (c *Clickhouse) Query(sql string) (string, error) {
+	reader := strings.NewReader(sql)
+	request, err := http.NewRequest("POST", c.URL(), reader)
+	if err != nil {
+		return "", err
+	}
+
+	httpClient := http.Client{
+		Timeout: time.Minute,
+	}
+	resp, err := httpClient.Do(request)
+	if err != nil {
+		return "", err
+	}
+	msg, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", errors.New(resp.Status + ": " + string(bytes.TrimRight(msg, "\n")))
+	}
+	return string(msg), nil
+}
+
+func (c *Clickhouse) IsUp() bool {
+	if len(c.container) == 0 {
+		return false
+	}
+	req, err := http.DefaultClient.Get(c.url)
+	if err != nil {
+		return false
+	}
+	return req.StatusCode == http.StatusOK
+}
+
+func (c *Clickhouse) Logs() {
+	if len(c.container) == 0 {
+		return
+	}
+
+	chArgs := []string{"logs", c.container}
+
+	cmd := exec.Command(DockerBinary, chArgs...)
+	out, _ := cmd.CombinedOutput()
+	fmt.Fprintln(os.Stderr, stringutils.UnsafeString(out))
+}