Skip to content

Commit 181d133

Browse files
devanbenzdevanbenz
authored
feat: Add CompactPointsPerBlock config opt (#26100) (#27149)
1 parent 9a3dc55 commit 181d133

5 files changed

Lines changed: 195 additions & 32 deletions

File tree

‎tsdb/config.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,9 @@ const (
5151
// block in a TSM file
5252
DefaultMaxPointsPerBlock = 1000
5353

54-
// AggressiveMaxPointsPerBlock is used when we want to further compact blocks
55-
// it is 100 times the default amount of points we use per block
56-
AggressiveMaxPointsPerBlock = DefaultMaxPointsPerBlock * 100
54+
// DefaultAggressiveMaxPointsPerBlock is used when we want to further compact blocks
55+
// it is 10 times the default amount of points we use per block
56+
DefaultAggressiveMaxPointsPerBlock = DefaultMaxPointsPerBlock * 10
5757

5858
// DefaultMaxValuesPerTag is the maximum number of values a tag can have within a measurement.
5959
DefaultMaxValuesPerTag = 100000
@@ -84,7 +84,7 @@ var SingleGenerationReasonText string = SingleGenerationReason()
8484
// when checked for full compaction.
8585
// 1048576000 is a magic number for bytes per gigabyte.
8686
func SingleGenerationReason() string {
87-
return fmt.Sprintf("not fully compacted and not idle because single generation with more than 2 files under %d GB and more than 1 file(s) under aggressive compaction points per block count (%d points)", int(MaxTSMFileSize/1048576000), AggressiveMaxPointsPerBlock)
87+
return fmt.Sprintf("not fully compacted and not idle because single generation with more than 2 files under %d GB and more than 1 file(s) under aggressive compaction points per block count (default: %d points)", int(MaxTSMFileSize/1048576000), DefaultAggressiveMaxPointsPerBlock)
8888
}
8989

9090
// Config holds the configuration for the tsbd package.
@@ -131,6 +131,7 @@ type Config struct {
131131
CompactFullWriteColdDuration toml.Duration `toml:"compact-full-write-cold-duration"`
132132
CompactThroughput toml.Size `toml:"compact-throughput"`
133133
CompactThroughputBurst toml.Size `toml:"compact-throughput-burst"`
134+
AggressivePointsPerBlock toml.Size `toml:"aggressive-points-per-block"`
134135

135136
// Limits
136137

@@ -181,6 +182,7 @@ func NewConfig() Config {
181182
CompactFullWriteColdDuration: toml.Duration(DefaultCompactFullWriteColdDuration),
182183
CompactThroughput: toml.Size(DefaultCompactThroughput),
183184
CompactThroughputBurst: toml.Size(DefaultCompactThroughputBurst),
185+
AggressivePointsPerBlock: toml.Size(DefaultAggressiveMaxPointsPerBlock),
184186

185187
MaxConcurrentCompactions: DefaultMaxConcurrentCompactions,
186188

‎tsdb/engine/tsm1/compact.go‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,10 @@ type CompactionPlanner interface {
124124
ForceFull()
125125

126126
SetFileStore(fs *FileStore)
127+
128+
SetAggressiveCompactionPointsPerBlock(aggressiveCompactionPointsPerBlock int)
129+
130+
GetAggressiveCompactionPointsPerBlock() int
127131
}
128132

129133
// DefaultPlanner implements CompactionPlanner using a strategy to roll up
@@ -157,6 +161,10 @@ type DefaultPlanner struct {
157161
// filesInUse is the set of files that have been returned as part of a plan and might
158162
// be being compacted. Two plans should not return the same file at any given time.
159163
filesInUse map[string]struct{}
164+
165+
// aggressiveCompactionPointsPerBlock is the amount of points that should be
166+
// packed in to a TSM file block during aggressive compaction
167+
aggressiveCompactionPointsPerBlock int
160168
}
161169

162170
type fileStore interface {
@@ -168,9 +176,10 @@ type fileStore interface {
168176

169177
func NewDefaultPlanner(fs fileStore, writeColdDuration time.Duration) *DefaultPlanner {
170178
return &DefaultPlanner{
171-
FileStore: fs,
172-
compactFullWriteColdDuration: writeColdDuration,
173-
filesInUse: make(map[string]struct{}),
179+
FileStore: fs,
180+
compactFullWriteColdDuration: writeColdDuration,
181+
filesInUse: make(map[string]struct{}),
182+
aggressiveCompactionPointsPerBlock: tsdb.DefaultAggressiveMaxPointsPerBlock,
174183
}
175184
}
176185

@@ -223,6 +232,14 @@ func (t *tsmGeneration) hasTombstones() bool {
223232
return false
224233
}
225234

235+
func (c *DefaultPlanner) SetAggressiveCompactionPointsPerBlock(aggressiveCompactionPointsPerBlock int) {
236+
c.aggressiveCompactionPointsPerBlock = aggressiveCompactionPointsPerBlock
237+
}
238+
239+
func (c *DefaultPlanner) GetAggressiveCompactionPointsPerBlock() int {
240+
return c.aggressiveCompactionPointsPerBlock
241+
}
242+
226243
func (c *DefaultPlanner) SetFileStore(fs *FileStore) {
227244
c.FileStore = fs
228245
}
@@ -248,7 +265,7 @@ func (c *DefaultPlanner) FullyCompacted() (bool, string) {
248265
aggressivePointsPerBlockCount := 0
249266
filesUnderMaxTsmSizeCount := 0
250267
for _, tsmFile := range gens[0].files {
251-
if c.FileStore.BlockCount(tsmFile.Path, 1) >= tsdb.AggressiveMaxPointsPerBlock {
268+
if c.FileStore.BlockCount(tsmFile.Path, 1) >= c.aggressiveCompactionPointsPerBlock {
252269
aggressivePointsPerBlockCount++
253270
}
254271
if tsmFile.Size < tsdb.MaxTSMFileSize {

‎tsdb/engine/tsm1/compact_test.go‎

Lines changed: 160 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2411,7 +2411,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
24112411
// > 2 GB total group size
24122412
// 50% of files are at aggressive max block size
24132413
{
2414-
"Small group size with single generation 50% at DefaultMaxPointsPerBlock and 50% at AggressiveMaxPointsPerBlock",
2414+
"Small group size with single generation 50% at DefaultMaxPointsPerBlock and 50% at DefaultAggressiveMaxPointsPerBlock",
24152415
[]tsm1.FileStat{
24162416
{
24172417
Path: "01-05.tsm1",
@@ -2447,10 +2447,10 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
24472447
},
24482448
},
24492449
[]int{
2450-
tsdb.AggressiveMaxPointsPerBlock,
2451-
tsdb.AggressiveMaxPointsPerBlock,
2452-
tsdb.AggressiveMaxPointsPerBlock,
2453-
tsdb.AggressiveMaxPointsPerBlock,
2450+
tsdb.DefaultAggressiveMaxPointsPerBlock,
2451+
tsdb.DefaultAggressiveMaxPointsPerBlock,
2452+
tsdb.DefaultAggressiveMaxPointsPerBlock,
2453+
tsdb.DefaultAggressiveMaxPointsPerBlock,
24542454
tsdb.DefaultMaxPointsPerBlock,
24552455
tsdb.DefaultMaxPointsPerBlock,
24562456
tsdb.DefaultMaxPointsPerBlock,
@@ -2475,7 +2475,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
24752475
Size: 450 * 1024 * 1024,
24762476
},
24772477
}, []int{
2478-
tsdb.AggressiveMaxPointsPerBlock,
2478+
tsdb.DefaultAggressiveMaxPointsPerBlock,
24792479
tsdb.DefaultMaxPointsPerBlock,
24802480
tsdb.DefaultMaxPointsPerBlock,
24812481
},
@@ -2593,16 +2593,16 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
25932593
Size: 400 * 1024 * 1024,
25942594
},
25952595
}, []int{
2596-
tsdb.AggressiveMaxPointsPerBlock,
2597-
tsdb.AggressiveMaxPointsPerBlock,
2598-
tsdb.AggressiveMaxPointsPerBlock,
2596+
tsdb.DefaultAggressiveMaxPointsPerBlock,
2597+
tsdb.DefaultAggressiveMaxPointsPerBlock,
2598+
tsdb.DefaultAggressiveMaxPointsPerBlock,
25992599

2600-
tsdb.AggressiveMaxPointsPerBlock,
2601-
tsdb.AggressiveMaxPointsPerBlock,
2600+
tsdb.DefaultAggressiveMaxPointsPerBlock,
2601+
tsdb.DefaultAggressiveMaxPointsPerBlock,
26022602
tsdb.DefaultMaxPointsPerBlock,
26032603

2604-
tsdb.AggressiveMaxPointsPerBlock,
2605-
tsdb.AggressiveMaxPointsPerBlock,
2604+
tsdb.DefaultAggressiveMaxPointsPerBlock,
2605+
tsdb.DefaultAggressiveMaxPointsPerBlock,
26062606
tsdb.DefaultMaxPointsPerBlock,
26072607

26082608
tsdb.DefaultMaxPointsPerBlock,
@@ -2685,7 +2685,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
26852685
// This test is added to account for a single generation that has a group size
26862686
// over 2 GB with 1 file under 2 GB all at max points per block with aggressive compaction.
26872687
// It should not compact any further.
2688-
"TSM files at AggressiveMaxPointsPerBlock",
2688+
"TSM files at DefaultAggressiveMaxPointsPerBlock",
26892689
[]tsm1.FileStat{
26902690
{
26912691
Path: "01-13.tsm1",
@@ -2696,8 +2696,8 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
26962696
Size: 691 * 1024 * 1024,
26972697
},
26982698
}, []int{
2699-
tsdb.AggressiveMaxPointsPerBlock,
2700-
tsdb.AggressiveMaxPointsPerBlock,
2699+
tsdb.DefaultAggressiveMaxPointsPerBlock,
2700+
tsdb.DefaultAggressiveMaxPointsPerBlock,
27012701
}, "", 0,
27022702
},
27032703
{
@@ -2716,7 +2716,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
27162716
Size: 691 * 1024 * 1024,
27172717
},
27182718
}, []int{
2719-
tsdb.AggressiveMaxPointsPerBlock,
2719+
tsdb.DefaultAggressiveMaxPointsPerBlock,
27202720
tsdb.DefaultMaxPointsPerBlock,
27212721
},
27222722
"",
@@ -2725,7 +2725,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
27252725
{
27262726
// This test is added to account for a single generation that has a group size
27272727
// over 2 GB and multiple files under 2 GB all at max points per block for aggressive compaction.
2728-
"Group size over 2 with multiple files under 2GB and at AggressiveMaxPointsPerBlock",
2728+
"Group size over 2 with multiple files under 2GB and at DefaultAggressiveMaxPointsPerBlock",
27292729
[]tsm1.FileStat{
27302730
{
27312731
Path: "01-13.tsm1",
@@ -2740,9 +2740,9 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
27402740
Size: 450 * 1024 * 1024,
27412741
},
27422742
}, []int{
2743-
tsdb.AggressiveMaxPointsPerBlock,
2744-
tsdb.AggressiveMaxPointsPerBlock,
2745-
tsdb.AggressiveMaxPointsPerBlock,
2743+
tsdb.DefaultAggressiveMaxPointsPerBlock,
2744+
tsdb.DefaultAggressiveMaxPointsPerBlock,
2745+
tsdb.DefaultAggressiveMaxPointsPerBlock,
27462746
}, "", 0,
27472747
},
27482748
}
@@ -2797,6 +2797,145 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
27972797
expectedFullyCompacted(cp, test.expectedFullyCompactedReasonExp)
27982798
})
27992799
}
2800+
2801+
type PlanOptimizeMixedTests struct {
2802+
name string
2803+
fs []tsm1.FileStat
2804+
bc []int
2805+
expectedFullyCompactedReasonExp string
2806+
expectedgenerationCount int64
2807+
fullyCompacted bool
2808+
}
2809+
2810+
mixedPlanOptimizeTests := []PlanOptimizeMixedTests{
2811+
{
2812+
// This test is added to account for halting state after
2813+
// TestDefaultPlanner_FullyCompacted_SmallSingleGeneration
2814+
// will need to ensure that once we have single TSM file under 2 GB we stop
2815+
"Single TSM file with increase block count",
2816+
[]tsm1.FileStat{
2817+
{
2818+
Path: "01-09.tsm1",
2819+
Size: 650 * 1024 * 1024,
2820+
},
2821+
},
2822+
[]int{},
2823+
"", 0, true,
2824+
},
2825+
{
2826+
// This test is added to account for a single generation that has a group size
2827+
// over 2 GB with 1 file under 2 GB all at max points per block with aggressive compaction.
2828+
// It should not compact any further.
2829+
"TSM files at DefaultAggressiveMaxPointsPerBlock with increased block count",
2830+
[]tsm1.FileStat{
2831+
{
2832+
Path: "01-13.tsm1",
2833+
Size: 2048 * 1024 * 1024,
2834+
},
2835+
{
2836+
Path: "01-14.tsm1",
2837+
Size: 691 * 1024 * 1024,
2838+
},
2839+
}, []int{
2840+
tsdb.DefaultAggressiveMaxPointsPerBlock,
2841+
tsdb.DefaultAggressiveMaxPointsPerBlock,
2842+
}, "", 0, true,
2843+
},
2844+
{
2845+
// This test is added to account for a single generation that has a group size
2846+
// over 2 GB at max points per block with aggressive compaction, and, 1 file
2847+
// under 2 GB at default max points per block.
2848+
// It should not compact any further.
2849+
"TSM files cannot compact further, single file under 2G and at DefaultMaxPointsPerBlock with increased block count",
2850+
[]tsm1.FileStat{
2851+
{
2852+
Path: "01-13.tsm1",
2853+
Size: 2048 * 1024 * 1024,
2854+
},
2855+
{
2856+
Path: "01-14.tsm1",
2857+
Size: 691 * 1024 * 1024,
2858+
},
2859+
}, []int{
2860+
tsdb.DefaultAggressiveMaxPointsPerBlock,
2861+
tsdb.DefaultMaxPointsPerBlock,
2862+
},
2863+
"",
2864+
0, true,
2865+
},
2866+
{
2867+
// This test is added to account for a single generation that has a group size
2868+
// over 2 GB and multiple files under 2 GB all at max points per block for aggressive compaction.
2869+
"Group size over 2 with multiple files under 2GB and at DefaultAggressiveMaxPointsPerBlock with increased block count",
2870+
[]tsm1.FileStat{
2871+
{
2872+
Path: "01-13.tsm1",
2873+
Size: 2048 * 1024 * 1024,
2874+
},
2875+
{
2876+
Path: "01-14.tsm1",
2877+
Size: 650 * 1024 * 1024,
2878+
},
2879+
{
2880+
Path: "01-15.tsm1",
2881+
Size: 450 * 1024 * 1024,
2882+
},
2883+
}, []int{
2884+
tsdb.DefaultAggressiveMaxPointsPerBlock,
2885+
tsdb.DefaultAggressiveMaxPointsPerBlock,
2886+
tsdb.DefaultAggressiveMaxPointsPerBlock,
2887+
}, tsdb.SingleGenerationReasonText, 1, false,
2888+
},
2889+
}
2890+
2891+
mixedPlanOptimizeTestRunner := func(cp *tsm1.DefaultPlanner, reasonExp string, fullyCompacted bool) {
2892+
compacted, reason := cp.FullyCompacted()
2893+
require.Equal(t, reason, reasonExp, "fullyCompacted reason")
2894+
require.Equal(t, compacted, fullyCompacted, "is fully compacted")
2895+
2896+
// Ensure that no level planning takes place
2897+
_, cgLen := cp.PlanLevel(1)
2898+
require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
2899+
_, cgLen = cp.PlanLevel(2)
2900+
require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
2901+
_, cgLen = cp.PlanLevel(3)
2902+
require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
2903+
}
2904+
2905+
// These tests will decrease the max points per block for aggressive compaction.
2906+
// For SetAggressiveCompactionPointsPerBlock we are using 10x the default to
2907+
// mock an administrator setting the max points per block to 100_000 and overriding
2908+
// the default of 10_000.
2909+
for _, test := range mixedPlanOptimizeTests {
2910+
t.Run(test.name, func(t *testing.T) {
2911+
ffs := &fakeFileStore{
2912+
PathsFn: func() []tsm1.FileStat {
2913+
return test.fs
2914+
},
2915+
}
2916+
2917+
if len(test.bc) > 0 {
2918+
err := ffs.SetBlockCounts(test.bc)
2919+
require.NoError(t, err, "setting block counts")
2920+
}
2921+
2922+
cp := tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
2923+
cp.SetAggressiveCompactionPointsPerBlock(tsdb.DefaultAggressiveMaxPointsPerBlock * 10)
2924+
mixedPlanOptimizeTestRunner(cp, test.expectedFullyCompactedReasonExp, test.fullyCompacted)
2925+
2926+
// Reverse test files and re-run tests
2927+
slices.Reverse(test.fs)
2928+
if len(test.bc) > 0 {
2929+
slices.Reverse(test.bc)
2930+
err := ffs.SetBlockCounts(test.bc)
2931+
require.NoError(t, err, "setting reverse block counts")
2932+
}
2933+
2934+
cp = tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
2935+
cp.SetAggressiveCompactionPointsPerBlock(tsdb.DefaultAggressiveMaxPointsPerBlock * 10)
2936+
mixedPlanOptimizeTestRunner(cp, test.expectedFullyCompactedReasonExp, test.fullyCompacted)
2937+
})
2938+
}
28002939
}
28012940

28022941
func TestDefaultPlanner_PlanOptimize_Tombstones(t *testing.T) {

‎tsdb/engine/tsm1/engine.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,8 @@ func NewEngine(id uint64, idx tsdb.Index, path string, walPath string, sfile *ts
186186
c.RateLimit = opt.CompactionThroughputLimiter
187187

188188
var planner CompactionPlanner = NewDefaultPlanner(fs, time.Duration(opt.Config.CompactFullWriteColdDuration))
189+
planner.SetAggressiveCompactionPointsPerBlock(int(opt.Config.AggressivePointsPerBlock))
190+
189191
if opt.CompactionPlannerCreator != nil {
190192
planner = opt.CompactionPlannerCreator(opt.Config).(CompactionPlanner)
191193
planner.SetFileStore(fs)
@@ -2112,14 +2114,14 @@ func (e *Engine) compact(wg *sync.WaitGroup) {
21122114
level3Groups = level3Groups[1:]
21132115
}
21142116
case 4:
2115-
// This is a heuristic. 100_000 points per block is suitable for when we have a
2117+
// This is a heuristic. The 10_000 points per block default is suitable for when we have a
21162118
// single generation with multiple files at max block size under 2 GB.
21172119
if genLen == 1 {
21182120
// Log TSM files that will have an increased points per block count.
21192121
for _, f := range level4Groups[0] {
2120-
e.logger.Info("TSM optimized compaction on single generation running, increasing total points per block to 100_000.", zap.String("path", f))
2122+
e.logger.Info("TSM optimized compaction on single generation running, increasing total points per block.", zap.String("path", f), zap.Int("points-per-block", e.CompactionPlan.GetAggressiveCompactionPointsPerBlock()))
21212123
}
2122-
e.Compactor.Size = tsdb.AggressiveMaxPointsPerBlock
2124+
e.Compactor.Size = e.CompactionPlan.GetAggressiveCompactionPointsPerBlock()
21232125
} else {
21242126
e.Compactor.Size = tsdb.DefaultMaxPointsPerBlock
21252127
}

‎tsdb/engine/tsm1/engine_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2866,6 +2866,9 @@ func MustParsePointString(buf string) models.Point { return MustParsePointsStrin
28662866

28672867
type mockPlanner struct{}
28682868

2869+
func (m *mockPlanner) GetAggressiveCompactionPointsPerBlock() int { return 0 }
2870+
func (m *mockPlanner) SetAggressiveCompactionPointsPerBlock(aggressiveCompactionPointsPerBlock int) {
2871+
}
28692872
func (m *mockPlanner) Plan(lastWrite time.Time) ([]tsm1.CompactionGroup, int64) { return nil, 0 }
28702873
func (m *mockPlanner) PlanLevel(level int) ([]tsm1.CompactionGroup, int64) { return nil, 0 }
28712874
func (m *mockPlanner) PlanOptimize() ([]tsm1.CompactionGroup, int64, int64) { return nil, 0, 0 }

0 commit comments

Comments
 (0)