From ec73ed7935880e020fcef1fdcea77f859850db4b Mon Sep 17 00:00:00 2001 From: Moad Zardab Date: Fri, 21 Jan 2022 12:27:22 +0000 Subject: [PATCH] Compactor: Adding minimum retention flag validation for downsampling retention (#5059) * Renaming `DownsampleRange` -> `ResLevel1DownsampleRange` to specify the mapping between the downsample ranges and their respective resolutions Signed-off-by: mzardab * Adding compactor downsample retention flag validation... ...if the following conditions are met: * Downsampling is enabled, raw retention must be > 10 days * Downsampling is enabled, 5m resolution retention must be at least 40h * Downsampling is enabled, 1h resolution must be at least 10d This is to avoid unexpected deletions of downsampled/raw data as compactor will only downsample raw blocks after there is enough data to form 2 whole blocks, it will not downsample immediately. Signed-off-by: mzardab * Added more clarifying docs on impact of retention on downsampling Signed-off-by: mzardab * Resolving conflicts Signed-off-by: mzardab * Updated CHANGELOG.md Signed-off-by: mzardab * Addressing PR comments Signed-off-by: mzardab --- CHANGELOG.md | 1 + cmd/thanos/compact.go | 8 ++++++++ cmd/thanos/downsample.go | 4 ++-- cmd/thanos/main_test.go | 8 ++++---- docs/components/compact.md | 13 +++++++++---- pkg/compact/compact.go | 8 ++++---- pkg/compact/compact_test.go | 18 +++++++++--------- pkg/compact/downsample/downsample.go | 4 ++-- 8 files changed, 39 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29ac58e900..d22969300c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#4946](https://github.com/thanos-io/thanos/pull/4946) Store: Support tls_config configuration for the s3 minio client. - [#4974](https://github.com/thanos-io/thanos/pull/4974) Store: Support tls_config configuration for connecting with Azure storage. - [#4999](https://github.com/thanos-io/thanos/pull/4999) COS: Support `endpoint` configuration for vpc internal endpoint. +- [#5059](https://github.com/thanos-io/thanos/pull/5059) Compactor: Adding minimum retention flag validation for downsampling retention. ### Fixed diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 2b75a88ede..bad0c7f4a2 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -388,9 +388,17 @@ func runCompact( } if retentionByResolution[compact.ResolutionLevelRaw].Seconds() != 0 { + // If downsampling is enabled, error if raw retention is not sufficient for downsampling to occur (upper bound 10 days for 1h resolution) + if !conf.disableDownsampling && retentionByResolution[compact.ResolutionLevelRaw].Seconds() < downsample.ResLevel1DownsampleRange { + return errors.New("raw resolution must be higher than the minimum block size after which 5m resolution downsampling will occur (40 hours)") + } level.Info(logger).Log("msg", "retention policy of raw samples is enabled", "duration", retentionByResolution[compact.ResolutionLevelRaw]) } if retentionByResolution[compact.ResolutionLevel5m].Seconds() != 0 { + // If retention is lower than minimum downsample range, then no downsampling at this resolution will be persisted + if !conf.disableDownsampling && retentionByResolution[compact.ResolutionLevel5m].Seconds() < downsample.ResLevel2DownsampleRange { + return errors.New("5m resolution retention must be higher than the minimum block size after which 1h resolution downsampling will occur (10 days)") + } level.Info(logger).Log("msg", "retention policy of 5 min aggregated samples is enabled", "duration", retentionByResolution[compact.ResolutionLevel5m]) } if retentionByResolution[compact.ResolutionLevel1h].Seconds() != 0 { diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 5a9eaa2cf0..ef1893ea0d 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -283,7 +283,7 @@ metaSendLoop: // Only downsample blocks once we are sure to get roughly 2 chunks out of it. // NOTE(fabxc): this must match with at which block size the compactor creates downsampled // blocks. Otherwise we may never downsample some data. - if m.MaxTime-m.MinTime < downsample.DownsampleRange0 { + if m.MaxTime-m.MinTime < downsample.ResLevel1DownsampleRange { continue } @@ -301,7 +301,7 @@ metaSendLoop: // Only downsample blocks once we are sure to get roughly 2 chunks out of it. // NOTE(fabxc): this must match with at which block size the compactor creates downsampled // blocks. Otherwise we may never downsample some data. - if m.MaxTime-m.MinTime < downsample.DownsampleRange1 { + if m.MaxTime-m.MinTime < downsample.ResLevel2DownsampleRange { continue } } diff --git a/cmd/thanos/main_test.go b/cmd/thanos/main_test.go index 18dd566aa6..5f2f8844de 100644 --- a/cmd/thanos/main_test.go +++ b/cmd/thanos/main_test.go @@ -121,7 +121,7 @@ func TestRegression4960_Deadlock(t *testing.T) { ctx, dir, []labels.Labels{{{Name: "a", Value: "1"}}}, - 1, 0, downsample.DownsampleRange0+1, // Pass the minimum DownsampleRange0 check. + 1, 0, downsample.ResLevel1DownsampleRange+1, // Pass the minimum ResLevel1DownsampleRange check. labels.Labels{{Name: "e1", Value: "1"}}, downsample.ResLevel0, metadata.NoneFunc) testutil.Ok(t, err) @@ -132,7 +132,7 @@ func TestRegression4960_Deadlock(t *testing.T) { ctx, dir, []labels.Labels{{{Name: "a", Value: "2"}}}, - 1, 0, downsample.DownsampleRange0+1, // Pass the minimum DownsampleRange0 check. + 1, 0, downsample.ResLevel1DownsampleRange+1, // Pass the minimum ResLevel1DownsampleRange check. labels.Labels{{Name: "e1", Value: "2"}}, downsample.ResLevel0, metadata.NoneFunc) testutil.Ok(t, err) @@ -143,7 +143,7 @@ func TestRegression4960_Deadlock(t *testing.T) { ctx, dir, []labels.Labels{{{Name: "a", Value: "2"}}}, - 1, 0, downsample.DownsampleRange0+1, // Pass the minimum DownsampleRange0 check. + 1, 0, downsample.ResLevel1DownsampleRange+1, // Pass the minimum ResLevel1DownsampleRange check. labels.Labels{{Name: "e1", Value: "2"}}, downsample.ResLevel0, metadata.NoneFunc) testutil.Ok(t, err) @@ -183,7 +183,7 @@ func TestCleanupDownsampleCacheFolder(t *testing.T) { ctx, dir, []labels.Labels{{{Name: "a", Value: "1"}}}, - 1, 0, downsample.DownsampleRange0+1, // Pass the minimum DownsampleRange0 check. + 1, 0, downsample.ResLevel1DownsampleRange+1, // Pass the minimum ResLevel1DownsampleRange check. labels.Labels{{Name: "e1", Value: "1"}}, downsample.ResLevel0, metadata.NoneFunc) testutil.Ok(t, err) diff --git a/docs/components/compact.md b/docs/components/compact.md index b9ba48a1ae..4038885486 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -2,10 +2,9 @@ The `thanos compact` command applies the compaction procedure of the Prometheus 2.0 storage engine to block data stored in object storage. It is generally not semantically concurrency safe and must be deployed as a singleton against a bucket. -Compactor is also responsible for downsampling of data: - -* Creating 5m downsampling for blocks larger than **40 hours** (2d, 2w) -* Creating 1h downsampling for blocks larger than **10 days** (2w) +Compactor is also responsible for downsampling of data. There is a time delay before downsampling at a given resolution is possible. This is necessary because downsampled chunks will have fewer samples in them, and as chunks are fixed size, data spanning more time will be required to fill them. +* Creating 5m downsampling for blocks older than **40 hours** (2d) +* Creating 1h downsampling for blocks older than **10 days** (2w) Example: @@ -161,6 +160,12 @@ Resolution is a distance between data points on your graphs. E.g. * `5 minutes` - data point is every 5 minutes * `1 hour` - data point is every 1h +Compactor downsampling is done in two passes: +1) All raw resolution metrics that are older than **40 hours** are downsampled at a 5m resolution +2) All 5m resolution metrics older than **10 days** are downsampled at a 1h resolution + +> **NOTE:** If retention at each resolution is lower than minimum age for the successive downsampling pass, data will be deleted before downsampling can be completed. As a rule of thumb retention for each downsampling level should be the same, and should be greater than the maximum date range (10 days for 5m to 1h downsampling). + Keep in mind, that the initial goal of downsampling is not saving disk or object storage space. In fact, downsampling doesn't save you **any** space but instead, it adds 2 more blocks for each raw block which are only slightly smaller or relatively similar size to raw block. This is done by internal downsampling implementation which to be mathematically correct holds various aggregations. This means that downsampling can increase the size of your storage a bit (~3x), if you choose to store all resolutions (recommended and by default). The goal of downsampling is to provide an opportunity to get fast results for range queries of big time intervals like months or years. In other words, if you set `--retention.resolution-raw` less than `--retention.resolution-5m` and `--retention.resolution-1h` - you might run into a problem of not being able to "zoom in" to your historical data. diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index c8e9945ba7..4ed4585f24 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -123,9 +123,9 @@ func UntilNextDownsampling(m *metadata.Meta) (time.Duration, error) { case downsample.ResLevel2: return time.Duration(0), errors.New("no downsampling") case downsample.ResLevel1: - return time.Duration(downsample.DownsampleRange1*time.Millisecond) - timeRange, nil + return time.Duration(downsample.ResLevel2DownsampleRange*time.Millisecond) - timeRange, nil case downsample.ResLevel0: - return time.Duration(downsample.DownsampleRange0*time.Millisecond) - timeRange, nil + return time.Duration(downsample.ResLevel1DownsampleRange*time.Millisecond) - timeRange, nil default: panic(errors.Errorf("invalid resolution %v", m.Thanos.Downsample.Resolution)) } @@ -637,7 +637,7 @@ func (ds *DownsampleProgressCalculator) ProgressCalculate(ctx context.Context, g continue } - if m.MaxTime-m.MinTime < downsample.DownsampleRange0 { + if m.MaxTime-m.MinTime < downsample.ResLevel1DownsampleRange { continue } groupBlocks[group.key]++ @@ -653,7 +653,7 @@ func (ds *DownsampleProgressCalculator) ProgressCalculate(ctx context.Context, g continue } - if m.MaxTime-m.MinTime < downsample.DownsampleRange1 { + if m.MaxTime-m.MinTime < downsample.ResLevel2DownsampleRange { continue } groupBlocks[group.key]++ diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 8fc5555839..a1b3984506 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -509,10 +509,10 @@ func TestDownsampleProgressCalculate(t *testing.T) { // This test case has blocks from multiple groups and resolution levels. Only the blocks in the second group should be downsampled since the others either have time differences not in the range for their resolution, or a resolution which should not be downsampled. testName: "multi_group_test", input: []*metadata.Meta{ - createBlockMeta(6, 1, downsample.DownsampleRange0, map[string]string{"a": "1"}, downsample.ResLevel0, []uint64{7, 8}), - createBlockMeta(7, 0, downsample.DownsampleRange1, map[string]string{"b": "2"}, downsample.ResLevel1, []uint64{8, 9}), - createBlockMeta(9, 0, downsample.DownsampleRange1, map[string]string{"b": "2"}, downsample.ResLevel1, []uint64{8, 11}), - createBlockMeta(8, 0, downsample.DownsampleRange1, map[string]string{"a": "1", "b": "2"}, downsample.ResLevel2, []uint64{9, 10}), + createBlockMeta(6, 1, downsample.ResLevel1DownsampleRange, map[string]string{"a": "1"}, downsample.ResLevel0, []uint64{7, 8}), + createBlockMeta(7, 0, downsample.ResLevel2DownsampleRange, map[string]string{"b": "2"}, downsample.ResLevel1, []uint64{8, 9}), + createBlockMeta(9, 0, downsample.ResLevel2DownsampleRange, map[string]string{"b": "2"}, downsample.ResLevel1, []uint64{8, 11}), + createBlockMeta(8, 0, downsample.ResLevel2DownsampleRange, map[string]string{"a": "1", "b": "2"}, downsample.ResLevel2, []uint64{9, 10}), }, expected: map[string]float64{ keys[0]: 0.0, @@ -524,7 +524,7 @@ func TestDownsampleProgressCalculate(t *testing.T) { // This block should be downsampled. testName: "res_level0_test", input: []*metadata.Meta{ - createBlockMeta(9, 0, downsample.DownsampleRange0, map[string]string{"a": "1"}, downsample.ResLevel0, []uint64{10, 11}), + createBlockMeta(9, 0, downsample.ResLevel1DownsampleRange, map[string]string{"a": "1"}, downsample.ResLevel0, []uint64{10, 11}), }, expected: map[string]float64{ keys[0]: 1.0, @@ -534,7 +534,7 @@ func TestDownsampleProgressCalculate(t *testing.T) { // This block should be downsampled. testName: "res_level1_test", input: []*metadata.Meta{ - createBlockMeta(9, 0, downsample.DownsampleRange1, map[string]string{"b": "2"}, downsample.ResLevel1, []uint64{10, 11}), + createBlockMeta(9, 0, downsample.ResLevel2DownsampleRange, map[string]string{"b": "2"}, downsample.ResLevel1, []uint64{10, 11}), }, expected: map[string]float64{ keys[1]: 1.0, @@ -545,7 +545,7 @@ func TestDownsampleProgressCalculate(t *testing.T) { // Blocks with this resolution should not be downsampled. testName: "res_level2_test", input: []*metadata.Meta{ - createBlockMeta(10, 0, downsample.DownsampleRange1, map[string]string{"a": "1", "b": "2"}, downsample.ResLevel2, []uint64{11, 12}), + createBlockMeta(10, 0, downsample.ResLevel2DownsampleRange, map[string]string{"a": "1", "b": "2"}, downsample.ResLevel2, []uint64{11, 12}), }, expected: map[string]float64{ keys[2]: 0.0, @@ -555,7 +555,7 @@ func TestDownsampleProgressCalculate(t *testing.T) { // This block should be downsampled. testName: "res_level0_test_incorrect", input: []*metadata.Meta{ - createBlockMeta(9, 1, downsample.DownsampleRange0, map[string]string{"a": "1"}, downsample.ResLevel0, []uint64{10, 11}), + createBlockMeta(9, 1, downsample.ResLevel1DownsampleRange, map[string]string{"a": "1"}, downsample.ResLevel0, []uint64{10, 11}), }, expected: map[string]float64{ keys[0]: 0.0, @@ -566,7 +566,7 @@ func TestDownsampleProgressCalculate(t *testing.T) { // This block should be downsampled. testName: "res_level1_test", input: []*metadata.Meta{ - createBlockMeta(9, 1, downsample.DownsampleRange1, map[string]string{"b": "2"}, downsample.ResLevel1, []uint64{10, 11}), + createBlockMeta(9, 1, downsample.ResLevel2DownsampleRange, map[string]string{"b": "2"}, downsample.ResLevel1, []uint64{10, 11}), }, expected: map[string]float64{ keys[1]: 0.0, diff --git a/pkg/compact/downsample/downsample.go b/pkg/compact/downsample/downsample.go index ea66b29571..f656a796c5 100644 --- a/pkg/compact/downsample/downsample.go +++ b/pkg/compact/downsample/downsample.go @@ -35,8 +35,8 @@ const ( // Downsampling ranges i.e. minimum block size after which we start to downsample blocks (in seconds). const ( - DownsampleRange0 = 40 * 60 * 60 * 1000 // 40 hours. - DownsampleRange1 = 10 * 24 * 60 * 60 * 1000 // 10 days. + ResLevel1DownsampleRange = 40 * 60 * 60 * 1000 // 40 hours. + ResLevel2DownsampleRange = 10 * 24 * 60 * 60 * 1000 // 10 days. ) // Downsample downsamples the given block. It writes a new block into dir and returns its ID.