Skip to content

Commit 6715148

Browse files
jbowensjbowens
committed
manifest: explicitly encode BlobFileID in manifest
Adapt the NewBlobFiles section of a version edit to explicitly encode a BlobFileID rather than inferring it from the disk file number. This is in preparation for blob file replacement. Informs #4802.
1 parent 491b500 commit 6715148

20 files changed

+223
-209
lines changed

‎compaction.go‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2493,13 +2493,13 @@ func (d *DB) cleanupVersionEdit(ve *manifest.VersionEdit) {
24932493
deletedTables[key.FileNum] = struct{}{}
24942494
}
24952495
for i := range ve.NewBlobFiles {
2496-
obsoleteFiles.AddBlob(ve.NewBlobFiles[i])
2496+
obsoleteFiles.AddBlob(ve.NewBlobFiles[i].Physical)
24972497
d.mu.versions.zombieBlobs.Add(objectInfo{
24982498
fileInfo: fileInfo{
2499-
FileNum: ve.NewBlobFiles[i].FileNum,
2500-
FileSize: ve.NewBlobFiles[i].Size,
2499+
FileNum: ve.NewBlobFiles[i].Physical.FileNum,
2500+
FileSize: ve.NewBlobFiles[i].Physical.Size,
25012501
},
2502-
isLocal: objstorage.IsLocalBlobFile(d.objProvider, ve.NewBlobFiles[i].FileNum),
2502+
isLocal: objstorage.IsLocalBlobFile(d.objProvider, ve.NewBlobFiles[i].Physical.FileNum),
25032503
})
25042504
}
25052505
for i := range ve.NewTables {
@@ -3294,9 +3294,12 @@ func (c *compaction) makeVersionEdit(result compact.Result) (*manifest.VersionEd
32943294
}
32953295
}
32963296
// Add any newly constructed blob files to the version edit.
3297-
ve.NewBlobFiles = make([]*manifest.PhysicalBlobFile, len(result.Blobs))
3297+
ve.NewBlobFiles = make([]manifest.BlobFileMetadata, len(result.Blobs))
32983298
for i := range result.Blobs {
3299-
ve.NewBlobFiles[i] = result.Blobs[i].Metadata
3299+
ve.NewBlobFiles[i] = manifest.BlobFileMetadata{
3300+
FileID: base.BlobFileID(result.Blobs[i].Metadata.FileNum),
3301+
Physical: result.Blobs[i].Metadata,
3302+
}
33003303
}
33013304

33023305
startLevelBytes := c.startLevel.files.TableSizeSum()

‎data_test.go‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,9 @@ import (
1010
crand "crypto/rand"
1111
"fmt"
1212
"io"
13-
"maps"
1413
"math"
1514
"math/rand/v2"
1615
"regexp"
17-
"slices"
1816
"strconv"
1917
"strings"
2018
"testing"
@@ -1177,7 +1175,13 @@ func runDBDefineCmdReuseFS(td *datadriven.TestData, opts *Options) (*DB, error)
11771175
valueSeparator.metas[base.BlobFileID(f)].ValueSize = stats.UncompressedValueBytes
11781176
}
11791177

1180-
ve.NewBlobFiles = slices.Collect(maps.Values(valueSeparator.metas))
1178+
ve.NewBlobFiles = make([]manifest.BlobFileMetadata, 0, len(valueSeparator.metas))
1179+
for blobFileID, m := range valueSeparator.metas {
1180+
ve.NewBlobFiles = append(ve.NewBlobFiles, manifest.BlobFileMetadata{
1181+
FileID: blobFileID,
1182+
Physical: m,
1183+
})
1184+
}
11811185

11821186
jobID := d.newJobIDLocked()
11831187
err = d.mu.versions.UpdateVersionLocked(func() (versionUpdate, error) {

‎internal/manifest/blob_metadata.go‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ type BlobFileMetadata struct {
9999

100100
// SafeFormat implements redact.SafeFormatter.
101101
func (m BlobFileMetadata) SafeFormat(w redact.SafePrinter, _ rune) {
102-
w.Printf("%s -> %s", m.FileID, m.Physical)
102+
w.Printf("%s physical:{%s}", m.FileID, m.Physical)
103103
}
104104

105105
// String implements fmt.Stringer.
@@ -183,15 +183,17 @@ func ParseBlobFileMetadataDebug(s string) (_ BlobFileMetadata, err error) {
183183
}()
184184

185185
// Input format:
186-
// 000102 -> 000000: size:[206536 (201KiB)], vals:[393256 (384KiB)]
186+
// 000102 physical:{000000: size:[206536 (201KiB)], vals:[393256 (384KiB)]}
187187
p := strparse.MakeParser(debugParserSeparators, s)
188188
fileID := base.BlobFileID(p.Int())
189-
p.Expect("-")
190-
p.Expect(">")
189+
p.Expect("physical")
190+
p.Expect(":")
191+
p.Expect("{")
191192
physical, err := parsePhysicalBlobFileDebug(&p)
192193
if err != nil {
193194
return BlobFileMetadata{}, err
194195
}
196+
p.Expect("}")
195197
return BlobFileMetadata{FileID: fileID, Physical: physical}, nil
196198
}
197199

@@ -227,7 +229,7 @@ func parsePhysicalBlobFileDebug(p *strparse.Parser) (*PhysicalBlobFile, error) {
227229
}
228230
}
229231

230-
for !p.Done() {
232+
for !p.Done() && p.Peek() != "}" {
231233
field := p.Next()
232234
p.Expect(":")
233235
switch field {
@@ -487,17 +489,17 @@ func (s *CurrentBlobFileSet) Metadatas() []BlobFileMetadata {
487489
// edit is modified to record the blob file removal.
488490
func (s *CurrentBlobFileSet) ApplyAndUpdateVersionEdit(ve *VersionEdit) error {
489491
// Insert new blob files into the set.
490-
for _, nf := range ve.NewBlobFiles {
491-
if _, ok := s.files[base.BlobFileID(nf.FileNum)]; ok {
492-
return errors.AssertionFailedf("pebble: new blob file %d already exists", nf.FileNum)
492+
for _, m := range ve.NewBlobFiles {
493+
if _, ok := s.files[m.FileID]; ok {
494+
return errors.AssertionFailedf("pebble: new blob file %d already exists", m.FileID)
493495
}
494-
blobFileID := base.BlobFileID(nf.FileNum)
496+
blobFileID := m.FileID
495497
cbf := &currentBlobFile{references: make(map[*TableMetadata]struct{})}
496-
cbf.metadata = BlobFileMetadata{FileID: blobFileID, Physical: nf}
498+
cbf.metadata = BlobFileMetadata{FileID: blobFileID, Physical: m.Physical}
497499
s.files[blobFileID] = cbf
498500
s.stats.Count++
499-
s.stats.PhysicalSize += nf.Size
500-
s.stats.ValueSize += nf.ValueSize
501+
s.stats.PhysicalSize += m.Physical.Size
502+
s.stats.ValueSize += m.Physical.ValueSize
501503
}
502504

503505
// Update references to blob files from new tables. Any referenced blob

‎internal/manifest/blob_metadata_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,17 +57,17 @@ func TestBlobFileMetadata_ParseRoundTrip(t *testing.T) {
5757
}{
5858
{
5959
name: "verbatim",
60-
input: "000002 -> 000001 size:[903530 (882KB)] vals:[39531 (39KB)]",
60+
input: "000002 physical:{000001 size:[903530 (882KB)] vals:[39531 (39KB)]}",
6161
},
6262
{
6363
name: "whitespace is insignificant",
64-
input: "000002 -> 000001 size : [ 903530 (882KB )] vals: [ 39531 ( 39KB ) ]",
65-
output: "000002 -> 000001 size:[903530 (882KB)] vals:[39531 (39KB)]",
64+
input: "000002 physical : {000001 size : [ 903530 (882KB )] vals: [ 39531 ( 39KB ) ] }",
65+
output: "000002 physical:{000001 size:[903530 (882KB)] vals:[39531 (39KB)]}",
6666
},
6767
{
6868
name: "humanized sizes are optional",
69-
input: "000002 -> 000001 size:[903530] vals:[39531]",
70-
output: "000002 -> 000001 size:[903530 (882KB)] vals:[39531 (39KB)]",
69+
input: "000002 physical:{000001 size:[903530] vals:[39531]}",
70+
output: "000002 physical:{000001 size:[903530 (882KB)] vals:[39531 (39KB)]}",
7171
},
7272
}
7373
for _, tc := range testCases {

‎internal/manifest/table_metadata.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -726,7 +726,7 @@ func (m *TableMetadata) DebugString(format base.FormatKey, verbose bool) string
726726
return b.String()
727727
}
728728

729-
const debugParserSeparators = ":-[]();"
729+
const debugParserSeparators = ":-[]();{}"
730730

731731
// errFromPanic can be used in a recover block to convert panics into errors.
732732
func errFromPanic(r any) error {

‎internal/manifest/testdata/current_blob_file_set‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,11 @@ Files:{Count: 0, Size: 0, ValueSize: 0}, References:{ValueSize: 0, Count: 0}
1616

1717
applyAndUpdateVersionEdit
1818
add-table: L3 000011:[d#1,SET-e#1,SET] blobrefs:[(000012: 25935); depth:1]
19-
add-blob-file: 000012 size:[20535 (20KB)] vals:[25935 (25KB)]
19+
add-blob-file: 000012 physical:{000012 size:[20535 (20KB)] vals:[25935 (25KB)]}
2020
----
2121
modified version edit:
2222
add-table: L3 000011:[d#1,SET-e#1,SET] seqnums:[0-0] points:[d#1,SET-e#1,SET] blobrefs:[(000012: 25935); depth:1]
23-
add-blob-file: 000012 size:[20535 (20KB)] vals:[25935 (25KB)]
23+
add-blob-file: 000012 physical:{000012 size:[20535 (20KB)] vals:[25935 (25KB)]}
2424
current blob file set:
2525
Files:{Count: 1, Size: 20535, ValueSize: 25935}, References:{ValueSize: 25935, Count: 1}
2626

‎internal/manifest/testdata/version_edit_apply‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ error during Accumulate: blob file 000003 referenced by L0.000004 not found
239239
# that references its entirety.
240240

241241
apply v8 name=v9
242-
add-blob-file: 000003 size:[20535 (20KB)] vals:[25935 (25KB)]
242+
add-blob-file: 000003 physical:{000003 size:[20535 (20KB)] vals:[25935 (25KB)]}
243243
add-table: L0 000004:[a#9,SET-z#9,DEL] seqnums:[9-9] points:[a#9,SET-z#9,DEL] blobrefs:[(000003: 25935); depth:1]
244244
----
245245
L0.0:
@@ -249,7 +249,7 @@ L1:
249249
L2:
250250
000002:[c#1,SET-f#1,SET] seqnums:[0-0] points:[c#1,SET-f#1,SET]
251251
Blob files:
252-
000003 -> 000003 size:[20535 (20KB)] vals:[25935 (25KB)]
252+
000003 physical:{000003 size:[20535 (20KB)] vals:[25935 (25KB)]}
253253

254254
# Try to add another table referencing the blob file. Since the file is not
255255
# added in the same version edit, and no referencing table is deleted in the
@@ -275,7 +275,7 @@ L1:
275275
L2:
276276
000002:[c#1,SET-f#1,SET] seqnums:[0-0] points:[c#1,SET-f#1,SET]
277277
Blob files:
278-
000003 -> 000003 size:[20535 (20KB)] vals:[25935 (25KB)]
278+
000003 physical:{000003 size:[20535 (20KB)] vals:[25935 (25KB)]}
279279

280280
# Repeat the above but this time applying the entire lifecycle of a blob file in
281281
# a single bulk version edit. This simulates a manifest replay during startup.
@@ -292,7 +292,7 @@ L2:
292292
000002:[c#1,SET-f#1,SET] seqnums:[0-0] points:[c#1,SET-f#1,SET]
293293

294294
apply v10
295-
add-blob-file: 000003 size:[20535 (20KB)] vals:[25935 (25KB)]
295+
add-blob-file: 000003 physical:{000003 size:[20535 (20KB)] vals:[25935 (25KB)]}
296296
add-table: L0 000004:[a#9,SET-z#9,DEL] seqnums:[9-9] points:[a#9,SET-z#9,DEL] blobrefs:[(000003: 25935); depth:1]
297297
new version edit
298298
del-table: L0 000004

‎internal/manifest/testdata/version_edit_decode‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -56,37 +56,38 @@ decode
5656
0366 # <tagNextFileNumber>, 42
5757
048410 # <tagLastSequence>, 2052
5858
6b # <tagNewBlobFile>
59-
29 # . FileNum = 000029
59+
29 # . FileID = 000029
60+
29 # . DiskFileNum = 000029
6061
d8a301 # . Size = 20952
6162
f9a006 # . ValueSize = 102521
6263
9f9485bd06 # . CreationTime = 1738623519
6364
6c # <tagDeleteBlobFile>
6465
21 # . FileNum = 000021
6566
----
66-
021903660484106b29d8a301f9a0069f9485bd066c21
67+
021903660484106b2929d8a301f9a0069f9485bd066c21
6768
log-num: 25
6869
next-file-num: 102
6970
last-seq-num: 2052
70-
add-blob-file: 000041 size:[20952 (20KB)] vals:[102521 (100KB)]
71+
add-blob-file: 000041 physical:{000041 size:[20952 (20KB)] vals:[102521 (100KB)]}
7172
del-blob-file: 000033
7273

7374
encode
7475
add-table: L6 000029:[bar#14,DEL-foo#13,SET] blobrefs:[(000041: 20952); depth:1]
75-
add-blob-file: 000041 size:[20952 (20KB)] vals:[102521 (100KB)]
76+
add-blob-file: 000943 physical:{000041 size:[20952 (20KB)] vals:[102521 (100KB)]}
7677
del-blob-file: 000033
7778
----
7879
67061d000b626172000e0000000000000b666f6f010d0000000000000000
79-
45010129d8a301016b29d8a301f9a006006c21
80+
45010129d8a301016baf0729d8a301f9a006006c21
8081
add-table: L6 000029:[bar#14,DEL-foo#13,SET] seqnums:[0-0] points:[bar#14,DEL-foo#13,SET] blobrefs:[(000041: 20952); depth:1]
81-
add-blob-file: 000041 size:[20952 (20KB)] vals:[102521 (100KB)]
82+
add-blob-file: 000943 physical:{000041 size:[20952 (20KB)] vals:[102521 (100KB)]}
8283
del-blob-file: 000033
8384

8485
decode
8586
67061d000b626172000e0000000000000b666f6f010d0000000000000000
86-
45010129d8a301016b29d8a301f9a006006c21
87+
45010129d8a301016baf0729d8a301f9a006006c21
8788
----
8889
67061d000b626172000e0000000000000b666f6f010d0000000000000000
89-
45010129d8a301016b29d8a301f9a006006c21
90+
45010129d8a301016baf0729d8a301f9a006006c21
9091
add-table: L6 000029:[bar#14,DEL-foo#13,SET]
91-
add-blob-file: 000041 size:[20952 (20KB)] vals:[102521 (100KB)]
92+
add-blob-file: 000943 physical:{000041 size:[20952 (20KB)] vals:[102521 (100KB)]}
9293
del-blob-file: 000033

‎internal/manifest/version_edit.go‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ type VersionEdit struct {
157157
RemovedBackingTables []base.DiskFileNum
158158
// NewBlobFiles holds the metadata for all new blob files introduced within
159159
// the version edit.
160-
NewBlobFiles []*PhysicalBlobFile
160+
NewBlobFiles []BlobFileMetadata
161161
// DeletedBlobFiles holds all blob files that became unreferenced during the
162162
// version edit. These blob files must not be referenced by any sstable in
163163
// the resulting Version.
@@ -507,7 +507,11 @@ func (v *VersionEdit) Decode(r io.Reader) error {
507507
v.NewTables = append(v.NewTables, nfe)
508508

509509
case tagNewBlobFile:
510-
fileNum, err := d.readFileNum()
510+
fileID, err := d.readUvarint()
511+
if err != nil {
512+
return err
513+
}
514+
diskFileNum, err := d.readFileNum()
511515
if err != nil {
512516
return err
513517
}
@@ -523,11 +527,14 @@ func (v *VersionEdit) Decode(r io.Reader) error {
523527
if err != nil {
524528
return err
525529
}
526-
v.NewBlobFiles = append(v.NewBlobFiles, &PhysicalBlobFile{
527-
FileNum: base.DiskFileNum(fileNum),
528-
Size: size,
529-
ValueSize: valueSize,
530-
CreationTime: creationTime,
530+
v.NewBlobFiles = append(v.NewBlobFiles, BlobFileMetadata{
531+
FileID: base.BlobFileID(fileID),
532+
Physical: &PhysicalBlobFile{
533+
FileNum: base.DiskFileNum(diskFileNum),
534+
Size: size,
535+
ValueSize: valueSize,
536+
CreationTime: creationTime,
537+
},
531538
})
532539

533540
case tagDeletedBlobFile:
@@ -680,7 +687,7 @@ func ParseVersionEditDebug(s string) (_ *VersionEdit, err error) {
680687
ve.RemovedBackingTables = append(ve.RemovedBackingTables, n)
681688

682689
case "add-blob-file":
683-
meta, err := ParsePhysicalBlobFileDebug(p.Remaining())
690+
meta, err := ParseBlobFileMetadataDebug(p.Remaining())
684691
if err != nil {
685692
return nil, err
686693
}
@@ -822,10 +829,11 @@ func (v *VersionEdit) Encode(w io.Writer) error {
822829
}
823830
for _, x := range v.NewBlobFiles {
824831
e.writeUvarint(tagNewBlobFile)
825-
e.writeUvarint(uint64(x.FileNum))
826-
e.writeUvarint(x.Size)
827-
e.writeUvarint(x.ValueSize)
828-
e.writeUvarint(x.CreationTime)
832+
e.writeUvarint(uint64(x.FileID))
833+
e.writeUvarint(uint64(x.Physical.FileNum))
834+
e.writeUvarint(x.Physical.Size)
835+
e.writeUvarint(x.Physical.ValueSize)
836+
e.writeUvarint(x.Physical.CreationTime)
829837
}
830838
for x := range v.DeletedBlobFiles {
831839
e.writeUvarint(tagDeletedBlobFile)
@@ -1019,7 +1027,7 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
10191027
if b.BlobFiles.Added == nil {
10201028
b.BlobFiles.Added = make(map[base.BlobFileID]*PhysicalBlobFile)
10211029
}
1022-
b.BlobFiles.Added[base.BlobFileID(nbf.FileNum)] = nbf
1030+
b.BlobFiles.Added[nbf.FileID] = nbf.Physical
10231031
}
10241032

10251033
b.BlobFiles.Deleted = make(map[base.BlobFileID]*PhysicalBlobFile, len(ve.DeletedBlobFiles))

‎internal/manifest/version_edit_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -482,7 +482,7 @@ func TestParseVersionEditDebugRoundTrip(t *testing.T) {
482482
input: ` del-blob-file: 000001`,
483483
},
484484
{
485-
input: ` add-blob-file: 000005 size:[20535 (20KB)] vals:[25935 (25KB)]`,
485+
input: ` add-blob-file: 000925 physical:{000005 size:[20535 (20KB)] vals:[25935 (25KB)]}`,
486486
},
487487
{
488488
input: ` add-table: L1 000001:[a#0,SET-z#0,DEL] seqnums:[0-0] points:[a#0,SET-z#0,DEL] size:1`,
@@ -508,7 +508,7 @@ func TestParseVersionEditDebugRoundTrip(t *testing.T) {
508508
` del-table: L3 000003`,
509509
` add-table: L1 000001:[a#0,SET-z#0,DEL] seqnums:[0-0] points:[a#0,SET-z#0,DEL] size:1`,
510510
` add-table: L2 000002:[a#0,SET-z#0,DEL] seqnums:[0-0] points:[a#0,SET-z#0,DEL] size:2`,
511-
` add-blob-file: 000005 size:[20535 (20KB)] vals:[25935 (25KB)]`,
511+
` add-blob-file: 000005 physical:{000005 size:[20535 (20KB)] vals:[25935 (25KB)]}`,
512512
` del-blob-file: 000004`,
513513
` del-blob-file: 000006`,
514514
}, "\n"),

0 commit comments

Comments
 (0)