Skip to content

Commit 92cba40

Browse files
RaduBerindeRaduBerinde
committed
db: move disk usage code to separate file
1 parent 1916ebf commit 92cba40

File tree

2 files changed

+87
-76
lines changed

2 files changed

+87
-76
lines changed

‎db.go‎

Lines changed: 0 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -2146,82 +2146,6 @@ func (d *DB) SSTables(opts ...SSTablesOption) ([][]SSTableInfo, error) {
21462146
return destLevels, nil
21472147
}
21482148

2149-
// makeFileSizeAnnotator returns an annotator that computes the total
2150-
// storage size of files that meet some criteria defined by filter. When
2151-
// applicable, this includes both the sstable size and the size of any
2152-
// referenced blob files.
2153-
func (d *DB) makeFileSizeAnnotator(
2154-
filter func(f *manifest.TableMetadata) bool,
2155-
) *manifest.TableAnnotator[uint64] {
2156-
return manifest.NewTableAnnotator[uint64](manifest.SumAggregator{
2157-
AccumulateFunc: func(f *manifest.TableMetadata) (uint64, bool) {
2158-
if filter(f) {
2159-
return f.Size + f.EstimatedReferenceSize(), true
2160-
}
2161-
return 0, true
2162-
},
2163-
AccumulatePartialOverlapFunc: func(f *manifest.TableMetadata, bounds base.UserKeyBounds) uint64 {
2164-
if filter(f) {
2165-
overlappingFileSize, err := d.fileCache.estimateSize(f, bounds.Start, bounds.End.Key)
2166-
if err != nil {
2167-
return 0
2168-
}
2169-
overlapFraction := float64(overlappingFileSize) / float64(f.Size)
2170-
// Scale the blob reference size proportionally to the file
2171-
// overlap from the bounds to approximate only the blob
2172-
// references that overlap with the requested bounds.
2173-
return overlappingFileSize + uint64(float64(f.EstimatedReferenceSize())*overlapFraction)
2174-
}
2175-
return 0
2176-
},
2177-
})
2178-
}
2179-
2180-
// EstimateDiskUsage returns the estimated filesystem space used in bytes for
2181-
// storing the range `[start, end]`. The estimation is computed as follows:
2182-
//
2183-
// - For sstables fully contained in the range the whole file size is included.
2184-
// - For sstables partially contained in the range the overlapping data block sizes
2185-
// are included. Even if a data block partially overlaps, or we cannot determine
2186-
// overlap due to abbreviated index keys, the full data block size is included in
2187-
// the estimation. Note that unlike fully contained sstables, none of the
2188-
// meta-block space is counted for partially overlapped files.
2189-
// - For virtual sstables, we use the overlap between start, end and the virtual
2190-
// sstable bounds to determine disk usage.
2191-
// - There may also exist WAL entries for unflushed keys in this range. This
2192-
// estimation currently excludes space used for the range in the WAL.
2193-
func (d *DB) EstimateDiskUsage(start, end []byte) (uint64, error) {
2194-
bytes, _, _, err := d.EstimateDiskUsageByBackingType(start, end)
2195-
return bytes, err
2196-
}
2197-
2198-
// EstimateDiskUsageByBackingType is like EstimateDiskUsage but additionally
2199-
// returns the subsets of that size in remote ane external files.
2200-
func (d *DB) EstimateDiskUsageByBackingType(
2201-
start, end []byte,
2202-
) (totalSize, remoteSize, externalSize uint64, _ error) {
2203-
if err := d.closed.Load(); err != nil {
2204-
panic(err)
2205-
}
2206-
2207-
bounds := base.UserKeyBoundsInclusive(start, end)
2208-
if !bounds.Valid(d.cmp) {
2209-
return 0, 0, 0, errors.New("invalid key-range specified (start > end)")
2210-
}
2211-
2212-
// Grab and reference the current readState. This prevents the underlying
2213-
// files in the associated version from being deleted if there is a concurrent
2214-
// compaction.
2215-
readState := d.loadReadState()
2216-
defer readState.unref()
2217-
2218-
totalSize = *d.mu.annotators.totalFileSize.VersionRangeAnnotation(readState.current, bounds)
2219-
remoteSize = *d.mu.annotators.remoteSize.VersionRangeAnnotation(readState.current, bounds)
2220-
externalSize = *d.mu.annotators.externalSize.VersionRangeAnnotation(readState.current, bounds)
2221-
2222-
return
2223-
}
2224-
22252149
func (d *DB) walPreallocateSize() int {
22262150
// Set the WAL preallocate size to 110% of the memtable size. Note that there
22272151
// is a bit of apples and oranges in units here as the memtabls size

‎disk_usage.go‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
// Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use
2+
// of this source code is governed by a BSD-style license that can be found in
3+
// the LICENSE file.
4+
5+
package pebble
6+
7+
import (
8+
"github.com/cockroachdb/errors"
9+
"github.com/cockroachdb/pebble/internal/base"
10+
"github.com/cockroachdb/pebble/internal/manifest"
11+
)
12+
13+
// EstimateDiskUsage returns the estimated filesystem space used in bytes for
14+
// storing the range `[start, end]`. The estimation is computed as follows:
15+
//
16+
// - For sstables fully contained in the range the whole file size is included.
17+
// - For sstables partially contained in the range the overlapping data block sizes
18+
// are included. Even if a data block partially overlaps, or we cannot determine
19+
// overlap due to abbreviated index keys, the full data block size is included in
20+
// the estimation. Note that unlike fully contained sstables, none of the
21+
// meta-block space is counted for partially overlapped files.
22+
// - For virtual sstables, we use the overlap between start, end and the virtual
23+
// sstable bounds to determine disk usage.
24+
// - There may also exist WAL entries for unflushed keys in this range. This
25+
// estimation currently excludes space used for the range in the WAL.
26+
func (d *DB) EstimateDiskUsage(start, end []byte) (uint64, error) {
27+
bytes, _, _, err := d.EstimateDiskUsageByBackingType(start, end)
28+
return bytes, err
29+
}
30+
31+
// EstimateDiskUsageByBackingType is like EstimateDiskUsage but additionally
32+
// returns the subsets of that size in remote ane external files.
33+
func (d *DB) EstimateDiskUsageByBackingType(
34+
start, end []byte,
35+
) (totalSize, remoteSize, externalSize uint64, _ error) {
36+
if err := d.closed.Load(); err != nil {
37+
panic(err)
38+
}
39+
40+
bounds := base.UserKeyBoundsInclusive(start, end)
41+
if !bounds.Valid(d.cmp) {
42+
return 0, 0, 0, errors.New("invalid key-range specified (start > end)")
43+
}
44+
45+
// Grab and reference the current readState. This prevents the underlying
46+
// files in the associated version from being deleted if there is a concurrent
47+
// compaction.
48+
readState := d.loadReadState()
49+
defer readState.unref()
50+
51+
totalSize = *d.mu.annotators.totalFileSize.VersionRangeAnnotation(readState.current, bounds)
52+
remoteSize = *d.mu.annotators.remoteSize.VersionRangeAnnotation(readState.current, bounds)
53+
externalSize = *d.mu.annotators.externalSize.VersionRangeAnnotation(readState.current, bounds)
54+
55+
return
56+
}
57+
58+
// makeFileSizeAnnotator returns an annotator that computes the total
59+
// storage size of files that meet some criteria defined by filter. When
60+
// applicable, this includes both the sstable size and the size of any
61+
// referenced blob files.
62+
func (d *DB) makeFileSizeAnnotator(
63+
filter func(f *manifest.TableMetadata) bool,
64+
) *manifest.TableAnnotator[uint64] {
65+
return manifest.NewTableAnnotator[uint64](manifest.SumAggregator{
66+
AccumulateFunc: func(f *manifest.TableMetadata) (uint64, bool) {
67+
if filter(f) {
68+
return f.Size + f.EstimatedReferenceSize(), true
69+
}
70+
return 0, true
71+
},
72+
AccumulatePartialOverlapFunc: func(f *manifest.TableMetadata, bounds base.UserKeyBounds) uint64 {
73+
if filter(f) {
74+
overlappingFileSize, err := d.fileCache.estimateSize(f, bounds.Start, bounds.End.Key)
75+
if err != nil {
76+
return 0
77+
}
78+
overlapFraction := float64(overlappingFileSize) / float64(f.Size)
79+
// Scale the blob reference size proportionally to the file
80+
// overlap from the bounds to approximate only the blob
81+
// references that overlap with the requested bounds.
82+
return overlappingFileSize + uint64(float64(f.EstimatedReferenceSize())*overlapFraction)
83+
}
84+
return 0
85+
},
86+
})
87+
}

0 commit comments

Comments
 (0)