Merge release-0.19 to main. (#4008)

* Cut v0.19.0-rc.0 (#3860) Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Fixed missed changelog comments. (#3865) Missed from: #3860 Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * rename Metadata API to MetricMetadata API (#3877) Signed-off-by: yeya24 <yb532204897@gmail.com> * Fix parseStep bug introduced in PR #3740 (#3887) (#3889) * Fix bug introduced in PR #3740 Signed-off-by: Hitanshu Mehta <hitanshu99amehta@gmail.com> * Minor change in test Signed-off-by: Hitanshu Mehta <hitanshu99amehta@gmail.com> Co-authored-by: Hitanshu Mehta <44025541+hitanshu-mehta@users.noreply.github.com> * Tools: rewrite delete can delete series only if it matches all matchers (#3886) (#3890) * rewrite delete should delete a series only if it matches all matchers in a deletion request Signed-off-by: yeya24 <yb532204897@gmail.com> * add test case Signed-off-by: yeya24 <yb532204897@gmail.com> Co-authored-by: Ben Ye <yb532204897@gmail.com> * cmd/thanos/receive.go: Receive client infers TLS (#3899) Currently, the thanos Receiver infers whether it should use TLS for the gRPC clients that forwards time series to other receivers based on whether the remote-write HTTP server uses TLS. This is not correct, as the HTTP server may use TLS without the gRPC server using TLS. This commit fixes the inference. Longer-term, this will be taken care of by the receive/router split. Signed-off-by: Lucas Servén Marín <lserven@gmail.com> * Cut v0.19.0-rc.1 (#3900) Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * tools: Fix partial and empty matchers in rewrite (#3891) * Fix partial and empty matchers match Signed-off-by: yeya24 <yb532204897@gmail.com> * add testcase for non-equal matchers Signed-off-by: yeya24 <yb532204897@gmail.com> * v0.19.0 patch: Added receive benchmark; Fixed Receiver excessive mem usage introduced in 0.17 (#3943) * Added receive benchmark, baseline. ``` goos: linux goarch: amd64 pkg: github.com/thanos-io/thanos/pkg/receive BenchmarkHandlerReceiveHTTP BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them. BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them./OK BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them./OK-12 22260 1550152 ns/op 1380340 B/op 6093 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them./conflict_errors BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them./conflict_errors-12 6619 6430408 ns/op 4522487 B/op 26118 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them. BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them./OK BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them./OK-12 2695 17208794 ns/op 15072963 B/op 60441 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them./conflict_errors BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them./conflict_errors-12 474 72533286 ns/op 46396932 B/op 260141 allocs/op BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 270 137050518 ns/op 226595379 B/op 132 allocs/op BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 21 1616025443 ns/op 698724321 B/op 408 allocs/op PASS Process finished with exit code 0 ``` Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Copy labels. ``` GOROOT=/home/bwplotka/.gvm/gos/go1.15 #gosetup GOPATH=/home/bwplotka/Repos/thanosgopath #gosetup /home/bwplotka/.gvm/gos/go1.15/bin/go test -c -o /tmp/___BenchmarkHandlerReceiveHTTP_in_github_com_thanos_io_thanos_pkg_receive github.com/thanos-io/thanos/pkg/receive #gosetup /tmp/___BenchmarkHandlerReceiveHTTP_in_github_com_thanos_io_thanos_pkg_receive -test.v -test.bench ^\QBenchmarkHandlerReceiveHTTP\E$ -test.run ^$ -test.benchmem -test.benchtime=30s goos: linux goarch: amd64 pkg: github.com/thanos-io/thanos/pkg/receive BenchmarkHandlerReceiveHTTP BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 25887 1537262 ns/op 1380023 B/op 6092 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 4237 7547968 ns/op 4522583 B/op 26118 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 2205 16513380 ns/op 15071092 B/op 60420 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 525 67278233 ns/op 46396645 B/op 260141 allocs/op BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 285 148049189 ns/op 226596168 B/op 132 allocs/op BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 20 1731361499 ns/op 698722550 B/op 401 allocs/op PASS Process finished with exit code 0 ``` Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Addded bench., Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Fix. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Improved API. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Changelog. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Addressed Lucas comments. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * compact: clean up directories thoroughly (#3869) * compact: clean up directories properly I couldn't stop thinking about this code for some reason and I have figured that I had missed one case in #3031. We need to also clean up the directories in compaction groups. A compaction could fail leaving some new directory with a random ULID on the disk. Before attempting to do another compaction loop, we need to remove it as well because the compaction process always produces a new, unique directory. Signed-off-by: Giedrius Statkevičius <giedriuswork@gmail.com> * Remove all non expected dirs. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Addressed comment. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com> * Added benchmark, Moved minio-deps to fork without race fix we don't need. (#3968) Fixes: #3917 Long term fix: #3967 Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Cut v0.19.0-rc.2 (#3969) Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * pkg/rules: fix deduplication of equal alerts with different labels (#3960) (#3999) Currently, if an alerting rule having the same name with different severity labels is being returned from different replicas then they are being treated as separate alerts. Given the following alerts a1,a2 with severities s1,s2 returned from replicas r1,2: a1[s1,r1] a1[s2,r1] a1[s1,r2] a1[s2,r2] Then, currently, the algorithm deduplicates to: a1[s1] a1[s2] a1[s1] a1[s2] Instead of the intendet result: a1[s1] a1[s2] This fixes it by removing replica labels before sorting labels for deduplication. Signed-off-by: Sergiusz Urbaniak <sergiusz.urbaniak@gmail.com> # Conflicts: # CHANGELOG.md Co-authored-by: Sergiusz Urbaniak <sergiusz.urbaniak@gmail.com> * Cut v0.19.0 (#3998) Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Fixed VERSION file. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> Co-authored-by: Ben Ye <yb532204897@gmail.com> Co-authored-by: Hitanshu Mehta <44025541+hitanshu-mehta@users.noreply.github.com> Co-authored-by: Lucas Servén Marín <lserven@gmail.com> Co-authored-by: Giedrius Statkevičius <giedriuswork@gmail.com> Co-authored-by: Sergiusz Urbaniak <sergiusz.urbaniak@gmail.com>
thanos-io · Apr 9, 2021 · 7dc2f3d · 7dc2f3d
2 parents ebda62e + 801c7c1
commit 7dc2f3d
Show file tree

Hide file tree

Showing 31 changed files with 222 additions and 212 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,14 +10,12 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 
 We use _breaking :warning:_ to mark changes that are not backward compatible (relates only to v0.y.z releases.)
 
-## Unreleased
+
+### Added
 
 - [#4029](https://github.com/thanos-io/thanos/pull/4029) Mixin: Remove dependency on the rule dashboard when generating the compact dashboard
 - [#4019](https://github.com/thanos-io/thanos/pull/4019) Query: Adds query range histogram.
 - [#3350](https://github.com/thanos-io/thanos/pull/3350) Query/Sidecar: Added targets API support. You can now configure you Querier to fetch Prometheus targets from leaf Prometheus-es!
-
-### Added
-
 - [#3977](https://github.com/thanos-io/thanos/pull/3903) Expose exemplars for `http_request_duration_seconds` histogram if tracing is enabled.
 - [#3903](https://github.com/thanos-io/thanos/pull/3903) Store: Returning custom grpc code when reaching series/chunk limits.
 - [#3919](https://github.com/thanos-io/thanos/pull/3919) Allow to disable automatically setting CORS headers using `--web.disable-cors` flag in each component that exposes an API.
@@ -40,9 +38,7 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re
 
 ### Removed
 
-## [v0.19.0-rc.2](https://github.com/thanos-io/thanos/releases/tag/v0.19.0-rc.2) - 2021.03.24
-
-### Added
+## [v0.19.0](https://github.com/thanos-io/thanos/releases/tag/v0.19.0) - 2021.03.31
 
 - [#3700](https://github.com/thanos-io/thanos/pull/3700) Compact/Web: Make old bucket viewer UI work with vanilla Prometheus blocks.
 - [#3657](https://github.com/thanos-io/thanos/pull/3657) *: It's now possible to configure HTTP transport options for S3 client.
@@ -65,7 +61,8 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re
 - [#3815](https://github.com/thanos-io/thanos/pull/3815) Receive: Improve handling of empty time series from clients
 - [#3795](https://github.com/thanos-io/thanos/pull/3795) s3: A truncated "get object" response is reported as error.
 - [#3899](https://github.com/thanos-io/thanos/pull/3899) Receive: Correct the inference of client gRPC configuration.
-- [#3943](https://github.com/thanos-io/thanos/pull/3943): Receive: Fixed memory regression introduced in v0.17.0.
+- [#3943](https://github.com/thanos-io/thanos/pull/3943) Receive: Fixed memory regression introduced in v0.17.0.
+- [#3960](https://github.com/thanos-io/thanos/pull/3960) Query: Fixed deduplication of equal alerts with different labels.
 
 ### Changed
 

diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go
@@ -58,7 +58,7 @@ func registerReceive(app *extkingpin.App) {
 	rwClientCert := cmd.Flag("remote-write.client-tls-cert", "TLS Certificates to use to identify this client to the server.").Default("").String()
 	rwClientKey := cmd.Flag("remote-write.client-tls-key", "TLS Key for the client's certificate.").Default("").String()
 	rwClientServerCA := cmd.Flag("remote-write.client-tls-ca", "TLS CA Certificates to use to verify servers.").Default("").String()
-	rwClientServerName := cmd.Flag("remote-write.client-server-name", "Server name to verify the hostname on the returned gRPC certificates. See https://tools.ietf.org/html/rfc4366#section-3.1").Default("").String()
+	rwClientServerName := cmd.Flag("remote-write.client-server-name", "Server name to verify the hostname on the returned TLS certificates. See https://tools.ietf.org/html/rfc4366#section-3.1").Default("").String()
 
 	dataDir := cmd.Flag("tsdb.path", "Data directory of TSDB.").
 		Default("./data").String()
@@ -233,7 +233,17 @@ func runReceive(
 	if err != nil {
 		return err
 	}
-	dialOpts, err := extgrpc.StoreClientGRPCOpts(logger, reg, tracer, rwServerCert != "", rwServerClientCA == "", rwClientCert, rwClientKey, rwClientServerCA, rwClientServerName)
+	dialOpts, err := extgrpc.StoreClientGRPCOpts(
+		logger,
+		reg,
+		tracer,
+		rwServerCert != "",
+		rwServerClientCA == "",
+		rwClientCert,
+		rwClientKey,
+		rwClientServerCA,
+		rwClientServerName,
+	)
 	if err != nil {
 		return err
 	}

diff --git a/docs/components/receive.md b/docs/components/receive.md
@@ -129,7 +129,7 @@ Flags:
                                  TLS CA Certificates to use to verify servers.
       --remote-write.client-server-name=""
                                  Server name to verify the hostname on the
-                                 returned gRPC certificates. See
+                                 returned TLS certificates. See
                                  https://tools.ietf.org/html/rfc4366#section-3.1
       --tsdb.path="./data"       Data directory of TSDB.
       --label=key="value" ...    External labels to announce. This flag will be

diff --git a/pkg/api/query/v1.go b/pkg/api/query/v1.go
@@ -908,7 +908,7 @@ func NewMetricMetadataHandler(client metadata.UnaryClient, enablePartialResponse
 	}
 
 	return func(r *http.Request) (interface{}, []error, *api.ApiError) {
-		req := &metadatapb.MetadataRequest{
+		req := &metadatapb.MetricMetadataRequest{
 			// By default we use -1, which means no limit.
 			Limit:                   -1,
 			Metric:                  r.URL.Query().Get("metric"),
@@ -924,7 +924,7 @@ func NewMetricMetadataHandler(client metadata.UnaryClient, enablePartialResponse
 			req.Limit = int32(limit)
 		}
 
-		t, warnings, err := client.Metadata(r.Context(), req)
+		t, warnings, err := client.MetricMetadata(r.Context(), req)
 		if err != nil {
 			return nil, nil, &api.ApiError{Typ: api.ErrorInternal, Err: errors.Wrap(err, "retrieving metadata")}
 		}

diff --git a/pkg/metadata/metadata.go b/pkg/metadata/metadata.go
@@ -13,10 +13,10 @@ import (
 
 var _ UnaryClient = &GRPCClient{}
 
-// UnaryClient is gRPC metadatapb.Metadata client which expands streaming metadata API. Useful for consumers that does not
+// UnaryClient is a gRPC metadatapb.Metadata client which expands streaming metadata API. Useful for consumers that does not
 // support streaming.
 type UnaryClient interface {
-	Metadata(ctx context.Context, req *metadatapb.MetadataRequest) (map[string][]metadatapb.Meta, storage.Warnings, error)
+	MetricMetadata(ctx context.Context, req *metadatapb.MetricMetadataRequest) (map[string][]metadatapb.Meta, storage.Warnings, error)
 }
 
 // GRPCClient allows to retrieve metadata from local gRPC streaming server implementation.
@@ -31,7 +31,7 @@ func NewGRPCClient(ts metadatapb.MetadataServer) *GRPCClient {
 	}
 }
 
-func (rr *GRPCClient) Metadata(ctx context.Context, req *metadatapb.MetadataRequest) (map[string][]metadatapb.Meta, storage.Warnings, error) {
+func (rr *GRPCClient) MetricMetadata(ctx context.Context, req *metadatapb.MetricMetadataRequest) (map[string][]metadatapb.Meta, storage.Warnings, error) {
 	srv := &metadataServer{ctx: ctx, metric: req.Metric, limit: int(req.Limit)}
 
 	if req.Limit >= 0 {
@@ -46,16 +46,16 @@ func (rr *GRPCClient) Metadata(ctx context.Context, req *metadatapb.MetadataRequ
 		srv.metadataMap = make(map[string][]metadatapb.Meta)
 	}
 
-	if err := rr.proxy.Metadata(req, srv); err != nil {
-		return nil, nil, errors.Wrap(err, "proxy Metadata")
+	if err := rr.proxy.MetricMetadata(req, srv); err != nil {
+		return nil, nil, errors.Wrap(err, "proxy MetricMetadata")
 	}
 
 	return srv.metadataMap, srv.warnings, nil
 }
 
 type metadataServer struct {
 	// This field just exist to pseudo-implement the unused methods of the interface.
-	metadatapb.Metadata_MetadataServer
+	metadatapb.Metadata_MetricMetadataServer
 	ctx context.Context
 
 	metric string
@@ -65,7 +65,7 @@ type metadataServer struct {
 	metadataMap map[string][]metadatapb.Meta
 }
 
-func (srv *metadataServer) Send(res *metadatapb.MetadataResponse) error {
+func (srv *metadataServer) Send(res *metadatapb.MetricMetadataResponse) error {
 	if res.GetWarning() != "" {
 		srv.warnings = append(srv.warnings, errors.New(res.GetWarning()))
 		return nil

diff --git a/pkg/metadata/metadatapb/custom.go b/pkg/metadata/metadatapb/custom.go
@@ -7,17 +7,17 @@ import (
 	"unsafe"
 )
 
-func NewMetadataResponse(metadata *MetricMetadata) *MetadataResponse {
-	return &MetadataResponse{
-		Result: &MetadataResponse_Metadata{
+func NewMetricMetadataResponse(metadata *MetricMetadata) *MetricMetadataResponse {
+	return &MetricMetadataResponse{
+		Result: &MetricMetadataResponse_Metadata{
 			Metadata: metadata,
 		},
 	}
 }
 
-func NewWarningMetadataResponse(warning error) *MetadataResponse {
-	return &MetadataResponse{
-		Result: &MetadataResponse_Warning{
+func NewWarningMetadataResponse(warning error) *MetricMetadataResponse {
+	return &MetricMetadataResponse{
+		Result: &MetricMetadataResponse_Warning{
 			Warning: warning.Error(),
 		},
 	}