Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Bring 0.19.0 rc fixes to main. (#3978)
* tools: Fix partial and empty matchers in rewrite (#3891) * Fix partial and empty matchers match Signed-off-by: yeya24 <yb532204897@gmail.com> * add testcase for non-equal matchers Signed-off-by: yeya24 <yb532204897@gmail.com> * v0.19.0 patch: Added receive benchmark; Fixed Receiver excessive mem usage introduced in 0.17 (#3943) * Added receive benchmark, baseline. ``` goos: linux goarch: amd64 pkg: github.com/thanos-io/thanos/pkg/receive BenchmarkHandlerReceiveHTTP BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them. BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them./OK BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them./OK-12 22260 1550152 ns/op 1380340 B/op 6093 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them./conflict_errors BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them./conflict_errors-12 6619 6430408 ns/op 4522487 B/op 26118 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them. BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them./OK BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them./OK-12 2695 17208794 ns/op 15072963 B/op 60441 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them./conflict_errors BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them./conflict_errors-12 474 72533286 ns/op 46396932 B/op 260141 allocs/op BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 270 137050518 ns/op 226595379 B/op 132 allocs/op BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 21 1616025443 ns/op 698724321 B/op 408 allocs/op PASS Process finished with exit code 0 ``` Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Copy labels. ``` GOROOT=/home/bwplotka/.gvm/gos/go1.15 #gosetup GOPATH=/home/bwplotka/Repos/thanosgopath #gosetup /home/bwplotka/.gvm/gos/go1.15/bin/go test -c -o /tmp/___BenchmarkHandlerReceiveHTTP_in_github_com_thanos_io_thanos_pkg_receive github.com/thanos-io/thanos/pkg/receive #gosetup /tmp/___BenchmarkHandlerReceiveHTTP_in_github_com_thanos_io_thanos_pkg_receive -test.v -test.bench ^\QBenchmarkHandlerReceiveHTTP\E$ -test.run ^$ -test.benchmem -test.benchtime=30s goos: linux goarch: amd64 pkg: github.com/thanos-io/thanos/pkg/receive BenchmarkHandlerReceiveHTTP BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/OK-12 25887 1537262 ns/op 1380023 B/op 6092 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_500_of_them/conflict_errors-12 4237 7547968 ns/op 4522583 B/op 26118 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/OK-12 2205 16513380 ns/op 15071092 B/op 60420 allocs/op BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors BenchmarkHandlerReceiveHTTP/typical_labels_under_1KB,_5000_of_them/conflict_errors-12 525 67278233 ns/op 46396645 B/op 260141 allocs/op BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/OK-12 285 148049189 ns/op 226596168 B/op 132 allocs/op BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors BenchmarkHandlerReceiveHTTP/extremely_large_label_value_10MB,_10_of_them/conflict_errors-12 20 1731361499 ns/op 698722550 B/op 401 allocs/op PASS Process finished with exit code 0 ``` Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Addded bench., Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Fix. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Improved API. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Changelog. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Addressed Lucas comments. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> # Conflicts: # CHANGELOG.md * compact: clean up directories thoroughly (#3869) * compact: clean up directories properly I couldn't stop thinking about this code for some reason and I have figured that I had missed one case in #3031. We need to also clean up the directories in compaction groups. A compaction could fail leaving some new directory with a random ULID on the disk. Before attempting to do another compaction loop, we need to remove it as well because the compaction process always produces a new, unique directory. Signed-off-by: Giedrius Statkevičius <giedriuswork@gmail.com> * Remove all non expected dirs. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Addressed comment. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> Co-authored-by: Bartlomiej Plotka <bwplotka@gmail.com> * Added benchmark, Moved minio-deps to fork without race fix we don't need. (#3968) Fixes: #3917 Long term fix: #3967 Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> # Conflicts: # go.sum * Changelog fix. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * Build fix. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> Co-authored-by: Ben Ye <yb532204897@gmail.com> Co-authored-by: Giedrius Statkevičius <giedriuswork@gmail.com>
- Loading branch information
3 people
committed
Mar 26, 2021
1 parent
4c3e1aa
commit 7e9958d
Showing
19 changed files
with
550 additions
and
69 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| // Copyright (c) The Thanos Authors. | ||
| // Licensed under the Apache License 2.0. | ||
|
|
||
| package s3_test | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/cortexproject/cortex/integration/e2e" | ||
| e2edb "github.com/cortexproject/cortex/integration/e2e/db" | ||
| "github.com/go-kit/kit/log" | ||
| "github.com/thanos-io/thanos/pkg/objstore/s3" | ||
| "github.com/thanos-io/thanos/test/e2e/e2ethanos" | ||
|
|
||
| "github.com/thanos-io/thanos/pkg/testutil" | ||
| ) | ||
|
|
||
| // Regression benchmark for https://github.com/thanos-io/thanos/issues/3917. | ||
| func BenchmarkUpload(b *testing.B) { | ||
| b.ReportAllocs() | ||
| ctx := context.Background() | ||
|
|
||
| s, err := e2e.NewScenario("e2e_bench_mino_client") | ||
| testutil.Ok(b, err) | ||
| b.Cleanup(e2ethanos.CleanScenario(b, s)) | ||
|
|
||
| const bucket = "test" | ||
| m := e2edb.NewMinio(8080, bucket) | ||
| testutil.Ok(b, s.StartAndWaitReady(m)) | ||
|
|
||
| bkt, err := s3.NewBucketWithConfig(log.NewNopLogger(), s3.Config{ | ||
| Bucket: bucket, | ||
| AccessKey: e2edb.MinioAccessKey, | ||
| SecretKey: e2edb.MinioSecretKey, | ||
| Endpoint: m.HTTPEndpoint(), | ||
| Insecure: true, | ||
| }, "test-feed") | ||
| testutil.Ok(b, err) | ||
|
|
||
| buf := bytes.Buffer{} | ||
| buf.Grow(1028 * 1028 * 100) // 100MB. | ||
| word := "abcdefghij" | ||
| for i := 0; i < buf.Cap()/len(word); i++ { | ||
| _, _ = buf.WriteString(word) | ||
| } | ||
| str := buf.String() | ||
|
|
||
| b.ResetTimer() | ||
| for i := 0; i < b.N; i++ { | ||
| testutil.Ok(b, bkt.Upload(ctx, "test", strings.NewReader(str))) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.