Skip to content

Commit e5fcf2f

Browse files
committed
db: fix delete-only compaction event log
Previously for delete-only compaction event logs, there was no label for those tables getting excised vs those getting fully deleted. This commit now labels excised tables during delete-only compactions. Fixes #4803
1 parent dff3eec commit e5fcf2f

File tree

3 files changed

+35
-7
lines changed

3 files changed

+35
-7
lines changed

compaction.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@ package pebble
66

77
import (
88
"bytes"
9+
stdcmp "cmp"
910
"context"
1011
"fmt"
1112
"iter"
13+
"maps"
1214
"math"
1315
"runtime/pprof"
1416
"slices"
@@ -331,6 +333,8 @@ type tableCompaction struct {
331333

332334
tableFormat sstable.TableFormat
333335
objCreateOpts objstorage.CreateOptions
336+
337+
annotations []string
334338
}
335339

336340
// Assert that tableCompaction implements the compaction interface.
@@ -2681,6 +2685,7 @@ func (d *DB) compact1(jobID JobID, c *tableCompaction) (err error) {
26812685

26822686
ve, stats, err := d.runCompaction(jobID, c)
26832687

2688+
info.Annotations = append(info.Annotations, c.annotations...)
26842689
info.Duration = d.timeNow().Sub(startTime)
26852690
if err == nil {
26862691
validateVersionEdit(ve, d.opts.Comparer.ValidateKey, d.opts.Comparer.FormatKey, d.opts.Logger)
@@ -3114,6 +3119,13 @@ func (d *DB) runDeleteOnlyCompaction(
31143119
}
31153120
return false
31163121
})
3122+
sort.Slice(ve.NewTables, func(i, j int) bool {
3123+
return ve.NewTables[i].Meta.TableNum < ve.NewTables[j].Meta.TableNum
3124+
})
3125+
deletedTableEntries := slices.Collect(maps.Keys(ve.DeletedTables))
3126+
slices.SortFunc(deletedTableEntries, func(a, b manifest.DeletedTableEntry) int {
3127+
return stdcmp.Compare(a.FileNum, b.FileNum)
3128+
})
31173129
// Remove any entries from CreatedBackingTables that are not used in any
31183130
// NewFiles.
31193131
usedBackingFiles := make(map[base.DiskFileNum]struct{})
@@ -3126,6 +3138,22 @@ func (d *DB) runDeleteOnlyCompaction(
31263138
_, used := usedBackingFiles[b.DiskFileNum]
31273139
return !used
31283140
})
3141+
3142+
// Iterate through the deleted tables and new tables to annotate excised tables.
3143+
// If a new table is virtual and the base.DiskFileNum is the same as a deleted table, then
3144+
// our deleted table was excised.
3145+
for _, table := range deletedTableEntries {
3146+
for _, newEntry := range ve.NewTables {
3147+
if newEntry.Meta.Virtual &&
3148+
newEntry.Meta.TableBacking.DiskFileNum == ve.DeletedTables[table].TableBacking.DiskFileNum {
3149+
c.annotations = append(c.annotations,
3150+
fmt.Sprintf("(excised: %s)", ve.DeletedTables[table].TableNum))
3151+
break
3152+
}
3153+
}
3154+
3155+
}
3156+
31293157
// Refresh the disk available statistic whenever a compaction/flush
31303158
// completes, before re-acquiring the mutex.
31313159
d.calculateDiskAvailableBytes()

testdata/compaction_delete_only_hints

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ maybe-compact
8888
Deletion hints:
8989
L0.000004 b-r seqnums(tombstone=200-230, file-smallest=30, type=point-key-only)
9090
Compactions:
91-
[JOB 100] compacted(delete-only) multilevel L2 [000005] (769B) Score=0.00 + L3 [000006] (769B) Score=0.00 + L4 [000007] (769B) Score=0.00 -> L6 [000008] (94B), in 1.0s (2.0s total), output rate 94B/s
91+
[JOB 100] compacted(delete-only) multilevel (excised: 000007) L2 [000005] (769B) Score=0.00 + L3 [000006] (769B) Score=0.00 + L4 [000007] (769B) Score=0.00 -> L6 [000008] (94B), in 1.0s (2.0s total), output rate 94B/s
9292

9393
# Verify that compaction correctly handles the presence of multiple
9494
# overlapping hints which might delete a file multiple times. All of the
@@ -127,7 +127,7 @@ maybe-compact
127127
Deletion hints:
128128
L1.000005 b-r seqnums(tombstone=200-230, file-smallest=30, type=point-key-only)
129129
Compactions:
130-
[JOB 100] compacted(delete-only) multilevel L2 [000005] (769B) Score=0.00 + L3 [000006] (769B) Score=0.00 + L4 [000007] (769B) Score=0.00 -> L6 [000008] (94B), in 1.0s (2.0s total), output rate 94B/s
130+
[JOB 100] compacted(delete-only) multilevel (excised: 000007) L2 [000005] (769B) Score=0.00 + L3 [000006] (769B) Score=0.00 + L4 [000007] (769B) Score=0.00 -> L6 [000008] (94B), in 1.0s (2.0s total), output rate 94B/s
131131

132132
# Test a range tombstone that is already compacted into L6.
133133

@@ -206,7 +206,7 @@ maybe-compact
206206
Deletion hints:
207207
(none)
208208
Compactions:
209-
[JOB 100] compacted(delete-only) multilevel L2 [000005] (769B) Score=0.00 + L3 [000006] (769B) Score=0.00 + L4 [000007] (769B) Score=0.00 -> L6 [000009] (94B), in 1.0s (2.0s total), output rate 94B/s
209+
[JOB 100] compacted(delete-only) multilevel (excised: 000007) L2 [000005] (769B) Score=0.00 + L3 [000006] (769B) Score=0.00 + L4 [000007] (769B) Score=0.00 -> L6 [000009] (94B), in 1.0s (2.0s total), output rate 94B/s
210210

211211
# A deletion hint present on an sstable in a higher level should NOT result in a
212212
# deletion-only compaction incorrectly removing an sstable in L6 following an
@@ -474,7 +474,7 @@ maybe-compact
474474
Deletion hints:
475475
(none)
476476
Compactions:
477-
[JOB 100] compacted(delete-only) multilevel L1 [000005] (756B) Score=0.00 + L2 [000006] (769B) Score=0.00 + L3 [000007] (769B) Score=0.00 + L4 [000008] (769B) Score=0.00 -> L6 [000009 000010] (95B), in 1.0s (2.0s total), output rate 95B/s
477+
[JOB 100] compacted(delete-only) multilevel (excised: 000005) (excised: 000008) L1 [000005] (756B) Score=0.00 + L2 [000006] (769B) Score=0.00 + L3 [000007] (769B) Score=0.00 + L4 [000008] (769B) Score=0.00 -> L6 [000009 000010] (95B), in 1.0s (2.0s total), output rate 95B/s
478478

479479
describe-lsm
480480
----
@@ -608,7 +608,7 @@ maybe-compact
608608
Deletion hints:
609609
(none)
610610
Compactions:
611-
[JOB 100] compacted(delete-only) L6 [000004] (928B) Score=0.00 -> L6 [000007 000008] (186B), in 1.0s (2.0s total), output rate 186B/s
611+
[JOB 100] compacted(delete-only) (excised: 000004) L6 [000004] (928B) Score=0.00 -> L6 [000007 000008] (186B), in 1.0s (2.0s total), output rate 186B/s
612612

613613
describe-lsm
614614
----

testdata/compaction_tombstones

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -375,7 +375,7 @@ range-deletions-bytes-estimate: 8380
375375
# numbers, so the test overwrites them to 0.
376376
maybe-compact
377377
----
378-
[JOB 100] compacted(delete-only) L6 [000007] (13KB) Score=0.00 -> L6 [000000] (8.2KB), in 1.0s (2.0s total), output rate 8.2KB/s
378+
[JOB 100] compacted(delete-only) (excised: 000007) L6 [000007] (13KB) Score=0.00 -> L6 [000000] (8.2KB), in 1.0s (2.0s total), output rate 8.2KB/s
379379
[JOB 101] compacted(default) L5 [000004] (798B) Score=3.10 + L6 [000006] (13KB) Score=0.00 -> L6 [000000] (4.8KB), in 1.0s (2.0s total), output rate 4.8KB/s
380380

381381
# The same LSM as above. However, this time, with point tombstone weighting at
@@ -421,7 +421,7 @@ range-deletions-bytes-estimate: 8380
421421
# numbers, so the test overwrites them to 0.
422422
maybe-compact
423423
----
424-
[JOB 100] compacted(delete-only) L6 [000007] (13KB) Score=0.00 -> L6 [000000] (8.2KB), in 1.0s (2.0s total), output rate 8.2KB/s
424+
[JOB 100] compacted(delete-only) (excised: 000007) L6 [000007] (13KB) Score=0.00 -> L6 [000000] (8.2KB), in 1.0s (2.0s total), output rate 8.2KB/s
425425
[JOB 101] compacted(default) L5 [000004] (798B) Score=3.10 + L6 [000006] (13KB) Score=0.00 -> L6 [000000] (4.8KB), in 1.0s (2.0s total), output rate 4.8KB/s
426426

427427
# These tests demonstrate the behavior of the tombstone density compaction feature

0 commit comments

Comments
 (0)