Control response cache behavoir (#3109)

Signed-off-by: Ben Ye <yb532204897@gmail.com>

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/cespare/xxhash v1.1.0
 	github.com/chromedp/cdproto v0.0.0-20200424080200-0de008e41fa0
 	github.com/chromedp/chromedp v0.5.3
-	github.com/cortexproject/cortex v1.2.1-0.20200812152417-a4aad5da5e4e
+	github.com/cortexproject/cortex v1.3.1-0.20200901115931-255ff3306960
 	github.com/davecgh/go-spew v1.1.1
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb
 	github.com/fatih/structtag v1.1.0
@@ -47,10 +47,10 @@ require (
 	github.com/prometheus/client_golang v1.7.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.11.1
-	github.com/prometheus/prometheus v1.8.2-0.20200811193703-869f1bc587e6
+	github.com/prometheus/prometheus v1.8.2-0.20200819132913-cb830b0a9c78
 	github.com/uber/jaeger-client-go v2.25.0+incompatible
 	github.com/uber/jaeger-lib v2.2.0+incompatible
-	github.com/weaveworks/common v0.0.0-20200625145055-4b1847531bc9
+	github.com/weaveworks/common v0.0.0-20200820123129-280614068c5e
 	go.elastic.co/apm v1.5.0
 	go.elastic.co/apm/module/apmot v1.5.0
 	go.uber.org/atomic v1.6.0
@@ -70,6 +70,9 @@ require (
 )
 
 replace (
+	// Using a 3rd-party branch for custom dialer - see https://github.com/bradfitz/gomemcache/pull/86.
+	// Required by Cortex https://github.com/cortexproject/cortex/pull/3051.
+	github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab
 	// Update to v1.1.1 to make sure windows CI pass.
 	github.com/elastic/go-sysinfo => github.com/elastic/go-sysinfo v1.1.1
 	// Make sure Prometheus version is pinned as Prometheus semver does not include Go APIs.

go.sum

@@ -153,8 +153,6 @@ github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
-github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0=
-github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
 github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cenkalti/backoff v0.0.0-20181003080854-62661b46c409/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
@@ -201,8 +199,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cortexproject/cortex v0.6.1-0.20200228110116-92ab6cbe0995/go.mod h1:3Xa3DjJxtpXqxcMGdk850lcIRb81M0fyY1MQ6udY134=
 github.com/cortexproject/cortex v1.2.1-0.20200805064754-d8edc95e2c91/go.mod h1:PVPxNLrxKH+yc8asaJOxuz7TiRmMizFfnSMOnRzM6oM=
-github.com/cortexproject/cortex v1.2.1-0.20200812152417-a4aad5da5e4e h1:cpXkRiRmZ16qHrP/LPlOC7thqSWB+bOwftnvyDInTm0=
-github.com/cortexproject/cortex v1.2.1-0.20200812152417-a4aad5da5e4e/go.mod h1:4qQ0CeiB+FH19L2xzWaVzrIodOfO9paqbf7SmUW4CDk=
+github.com/cortexproject/cortex v1.3.1-0.20200901115931-255ff3306960 h1:ztWDSohOmf46cJ+3WYmOrx722A97UlTLfwDaOsI3NtI=
+github.com/cortexproject/cortex v1.3.1-0.20200901115931-255ff3306960/go.mod h1:ub8BpRZrRa02BOM8NJTnI2YklxW/mGhEkJDrhsDfcfg=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8=
@@ -271,6 +269,8 @@ github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/structtag v1.1.0 h1:6j4mUV/ES2duvnAzKMFkN6/A5mCaNYPD3xfbAkLLOF8=
 github.com/fatih/structtag v1.1.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
+github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
+github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
@@ -937,6 +937,7 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/sony/gobreaker v0.4.1 h1:oMnRNZXX5j85zso6xCPRNPtmAycat+WcoKbklScLDgQ=
 github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
 github.com/soundcloud/go-runit v0.0.0-20150630195641-06ad41a06c4a/go.mod h1:LeFCbQYJ3KJlPs/FvPz2dy1tkpxyeNESVyCNNzRXFR0=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -965,6 +966,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/thanos-io/thanos v0.8.1-0.20200109203923-552ffa4c1a0d/go.mod h1:usT/TxtJQ7DzinTt+G9kinDQmRS5sxwu0unVKZ9vdcw=
 github.com/thanos-io/thanos v0.13.1-0.20200731083140-69b87607decf/go.mod h1:G8caR6G7pSDreRDvFm9wFuyjEBztmr8Ag3kBYpa/fEc=
 github.com/thanos-io/thanos v0.13.1-0.20200807203500-9b578afb4763/go.mod h1:KyW0a93tsh7v4hXAwo2CVAIRYuZT1Kkf4e04gisQjAg=
+github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU=
+github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY=
 github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
@@ -985,8 +988,9 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/weaveworks/common v0.0.0-20200206153930-760e36ae819a/go.mod h1:6enWAqfQBFrE8X/XdJwZr8IKgh1chStuFR0mjU/UOUw=
-github.com/weaveworks/common v0.0.0-20200625145055-4b1847531bc9 h1:dNVIG9aKQHR9T4uYAC4YxmkHHryOsfTwsL54WrS7u28=
 github.com/weaveworks/common v0.0.0-20200625145055-4b1847531bc9/go.mod h1:c98fKi5B9u8OsKGiWHLRKus6ToQ1Tubeow44ECO1uxY=
+github.com/weaveworks/common v0.0.0-20200820123129-280614068c5e h1:t/as1iFw9iI6s0q9ESR2tTn2qGhI42LjBkPuQLuLzM8=
+github.com/weaveworks/common v0.0.0-20200820123129-280614068c5e/go.mod h1:hz10LOsAdzC3K/iXaKoFxOKTDRgxJl+BTGX1GY+TzO4=
 github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M=
 github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA=
 github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
@@ -1039,7 +1043,6 @@ go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/automaxprocs v1.2.0 h1:+RUihKM+nmYUoB9w0D0Ov5TJ2PpFO2FgenTxMJiZBZA=
 go.uber.org/automaxprocs v1.2.0/go.mod h1:YfO3fm683kQpzETxlTGZhGIVmXAhaw3gxeBADbpZtnU=
-go.uber.org/goleak v1.0.0 h1:qsup4IcBdlmsnGfqyLl4Ntn3C2XCCuKAE7DwHpScyUo=
 go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
 go.uber.org/goleak v1.1.0 h1:MJDxhkyAAWXEJf/y4NSOPYD/bBx7JAzIjUbv12/4FFs=
 go.uber.org/goleak v1.1.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=

pkg/queryfrontend/codec.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"net/url"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/cortexproject/cortex/pkg/querier/queryrange"
@@ -21,6 +22,14 @@ import (
 	"github.com/thanos-io/thanos/pkg/store/storepb"
 )
 
+const (
+	// Name of the cache control header.
+	cacheControlHeader = "Cache-Control"
+
+	// Value that cacheControlHeader has if the response indicates that the results should not be cached.
+	noStoreValue = "no-store"
+)
+
 var (
 	errEndBeforeStart = httpgrpc.Errorf(http.StatusBadRequest, "end timestamp must not be before start time")
 	errNegativeStep   = httpgrpc.Errorf(http.StatusBadRequest, "zero or negative query resolution step widths are not accepted. Try a positive integer")
@@ -105,6 +114,14 @@ func (c codec) DecodeRequest(_ context.Context, r *http.Request) (queryrange.Req
 
 	result.Query = r.FormValue("query")
 	result.Path = r.URL.Path
+
+	for _, value := range r.Header.Values(cacheControlHeader) {
+		if strings.Contains(value, noStoreValue) {
+			result.CachingOptions.Disabled = true
+			break
+		}
+	}
+
 	return &result, nil
 }
 

pkg/queryfrontend/request.go

@@ -26,6 +26,7 @@ type ThanosRequest struct {
 	MaxSourceResolution int64
 	ReplicaLabels       []string
 	StoreMatchers       [][]storepb.LabelMatcher
+	CachingOptions      queryrange.CachingOptions
 }
 
 // GetStart returns the start timestamp of the request in milliseconds.
@@ -48,6 +49,10 @@ func (r *ThanosRequest) GetQuery() string {
 	return r.Query
 }
 
+func (r *ThanosRequest) GetCachingOptions() queryrange.CachingOptions {
+	return r.CachingOptions
+}
+
 // WithStartEnd clone the current request with different start and end timestamp.
 func (r *ThanosRequest) WithStartEnd(start int64, end int64) queryrange.Request {
 	q := *r

pkg/queryfrontend/roundtrip.go

@@ -51,10 +51,14 @@ func NewTripperWare(
 	)
 
 	if splitQueryInterval != 0 {
+		// TODO(yeya24): make interval dynamic in next pr.
+		queryIntervalFn := func(_ queryrange.Request) time.Duration {
+			return splitQueryInterval
+		}
 		queryRangeMiddleware = append(
 			queryRangeMiddleware,
 			queryrange.InstrumentMiddleware("split_by_interval", metrics),
-			queryrange.SplitByIntervalMiddleware(splitQueryInterval, limits, codec, reg),
+			queryrange.SplitByIntervalMiddleware(queryIntervalFn, limits, codec, reg),
 		)
 	}
 
@@ -72,6 +76,7 @@ func NewTripperWare(
 			codec,
 			cacheExtractor,
 			nil,
+			shouldCache,
 			reg,
 		)
 		if err != nil {
@@ -112,3 +117,14 @@ func NewTripperWare(
 		})
 	}, nil
 }
+
+// Don't go to response cache if StoreMatchers are set.
+func shouldCache(r queryrange.Request) bool {
+	if thanosReq, ok := r.(*ThanosRequest); ok {
+		if len(thanosReq.StoreMatchers) > 0 {
+			return false
+		}
+	}
+
+	return !r.GetCachingOptions().Disabled
+}

pkg/queryfrontend/roundtrip_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/prometheus/prometheus/promql/parser"
 	"github.com/weaveworks/common/user"
 
+	"github.com/thanos-io/thanos/pkg/store/storepb"
 	"github.com/thanos-io/thanos/pkg/testutil"
 )
 
@@ -260,7 +261,7 @@ func TestRoundTripCacheMiddleware(t *testing.T) {
 	}
 
 	// Non query range request, won't be cached.
-	testRequest2 := &ThanosRequest{
+	testRequestInstant := &ThanosRequest{
 		Path:  "/api/v1/query",
 		Start: 0,
 		End:   2 * hour,
@@ -269,7 +270,7 @@ func TestRoundTripCacheMiddleware(t *testing.T) {
 
 	// Same query params as testRequest, different maxSourceResolution
 	// but still in the same downsampling level, so it will be cached in this case.
-	testRequest3 := &ThanosRequest{
+	testRequestSameLevelDownsampling := &ThanosRequest{
 		Path:                "/api/v1/query_range",
 		Start:               0,
 		End:                 2 * hour,
@@ -279,14 +280,24 @@ func TestRoundTripCacheMiddleware(t *testing.T) {
 
 	// Same query params as testRequest, different maxSourceResolution
 	// and downsampling level so it won't be cached in this case.
-	testRequest4 := &ThanosRequest{
+	testRequestHigherLevelDownsampling := &ThanosRequest{
 		Path:                "/api/v1/query_range",
 		Start:               0,
 		End:                 2 * hour,
 		Step:                10 * seconds,
 		MaxSourceResolution: 1 * hour,
 	}
 
+	// Same query params as testRequest, but with storeMatchers
+	testRequestWithStoreMatchers := &ThanosRequest{
+		Path:                "/api/v1/query_range",
+		Start:               0,
+		End:                 2 * hour,
+		Step:                10 * seconds,
+		MaxSourceResolution: 1 * seconds,
+		StoreMatchers:       [][]storepb.LabelMatcher{{storepb.LabelMatcher{Type: storepb.LabelMatcher_EQ, Name: "foo", Value: "bar"}}},
+	}
+
 	cacheConf := &queryrange.ResultsCacheConfig{
 		CacheConfig: cortexcache.Config{
 			EnableFifoCache: true,
@@ -319,10 +330,11 @@ func TestRoundTripCacheMiddleware(t *testing.T) {
 	}{
 		{name: "first request", req: testRequest, expected: 1},
 		{name: "same request as the first one, directly use cache", req: testRequest, expected: 1},
-		{name: "non query range request won't be cached", req: testRequest2, expected: 2},
-		{name: "do it again", req: testRequest2, expected: 3},
-		{name: "different max source resolution but still same level", req: testRequest3, expected: 3},
-		{name: "different max source resolution and different level", req: testRequest4, expected: 4},
+		{name: "non query range request won't be cached", req: testRequestInstant, expected: 2},
+		{name: "do it again", req: testRequestInstant, expected: 3},
+		{name: "different max source resolution but still same level", req: testRequestSameLevelDownsampling, expected: 3},
+		{name: "different max source resolution and different level", req: testRequestHigherLevelDownsampling, expected: 4},
+		{name: "storeMatchers requests won't go to cache", req: testRequestWithStoreMatchers, expected: 5},
 		{
 			name: "request but will be partitioned",
 			req: &ThanosRequest{
@@ -331,7 +343,7 @@ func TestRoundTripCacheMiddleware(t *testing.T) {
 				End:   timestamp.FromTime(now.Add(time.Hour)),
 				Step:  10 * seconds,
 			},
-			expected: 5,
+			expected: 6,
 		},
 		{
 			name: "same query as the previous one",
@@ -341,7 +353,7 @@ func TestRoundTripCacheMiddleware(t *testing.T) {
 				End:   timestamp.FromTime(now.Add(time.Hour)),
 				Step:  10 * seconds,
 			},
-			expected: 6,
+			expected: 7,
 		},
 	} {