Skip to content

Commit 8866088

Browse files
jbowensjbowens
committed
manifest: explicitly encode deleted physical blob file numbers
Adapt the version edit encoding of blob file deletions to encode both the BlobFileID and the deleted physical blob file's file number. This reduces ambiguity and makes explicit the intention when we encode a blob file replacement. Informs #112.
1 parent 2ca2aeb commit 8866088

File tree

12 files changed

+162
-56
lines changed

12 files changed

+162
-56
lines changed

‎checkpoint.go‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -364,12 +364,15 @@ func (d *DB) Checkpoint(
364364
// When we write the MANIFEST of the checkpoint, we'll include a final
365365
// VersionEdit that removes these blob files so that the checkpointed
366366
// manifest is consistent.
367-
var excludedBlobFiles map[base.BlobFileID]*manifest.PhysicalBlobFile
367+
var excludedBlobFiles map[manifest.DeletedBlobFileEntry]*manifest.PhysicalBlobFile
368368
if len(includedBlobFiles) < len(versionBlobFiles) {
369-
excludedBlobFiles = make(map[base.BlobFileID]*manifest.PhysicalBlobFile, len(versionBlobFiles)-len(includedBlobFiles))
369+
excludedBlobFiles = make(map[manifest.DeletedBlobFileEntry]*manifest.PhysicalBlobFile, len(versionBlobFiles)-len(includedBlobFiles))
370370
for _, meta := range versionBlobFiles {
371371
if _, ok := includedBlobFiles[meta.FileID]; !ok {
372-
excludedBlobFiles[meta.FileID] = meta.Physical
372+
excludedBlobFiles[manifest.DeletedBlobFileEntry{
373+
FileID: meta.FileID,
374+
FileNum: meta.Physical.FileNum,
375+
}] = meta.Physical
373376
}
374377
}
375378
}
@@ -480,7 +483,7 @@ func (d *DB) writeCheckpointManifest(
480483
manifestSize int64,
481484
excludedTables map[manifest.DeletedTableEntry]*manifest.TableMetadata,
482485
removeBackingTables []base.DiskFileNum,
483-
excludedBlobFiles map[base.BlobFileID]*manifest.PhysicalBlobFile,
486+
excludedBlobFiles map[manifest.DeletedBlobFileEntry]*manifest.PhysicalBlobFile,
484487
) error {
485488
// Copy the MANIFEST, and create a pointer to it. We copy rather
486489
// than link because additional version edits added to the

‎internal/manifest/blob_metadata.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -614,10 +614,13 @@ func (s *CurrentBlobFileSet) ApplyAndUpdateVersionEdit(ve *VersionEdit) error {
614614
cbf.referencedValueSize, cbf.metadata.FileID)
615615
}
616616
if ve.DeletedBlobFiles == nil {
617-
ve.DeletedBlobFiles = make(map[base.BlobFileID]*PhysicalBlobFile)
617+
ve.DeletedBlobFiles = make(map[DeletedBlobFileEntry]*PhysicalBlobFile)
618618
}
619619

620-
ve.DeletedBlobFiles[cbf.metadata.FileID] = cbf.metadata.Physical
620+
ve.DeletedBlobFiles[DeletedBlobFileEntry{
621+
FileID: cbf.metadata.FileID,
622+
FileNum: cbf.metadata.Physical.FileNum,
623+
}] = cbf.metadata.Physical
621624
s.stats.Count--
622625
s.stats.PhysicalSize -= cbf.metadata.Physical.Size
623626
s.stats.ValueSize -= cbf.metadata.Physical.ValueSize

‎internal/manifest/testdata/current_blob_file_set‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,6 @@ applyAndUpdateVersionEdit
7070
----
7171
modified version edit:
7272
del-table: L5 000013
73-
del-blob-file: B000012
73+
del-blob-file: B000012 000012
7474
current blob file set:
7575
Files:{Count: 0, Size: 0, ValueSize: 0}, References:{ValueSize: 0, Count: 0}

‎internal/manifest/testdata/version_edit_apply‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,7 @@ new version edit
302302
new version edit
303303
del-table: L1 000005
304304
del-table: L1 000006
305-
del-blob-file: B000003
305+
del-blob-file: B000003 000003
306306
----
307307
L2:
308308
000002:[c#1,SET-f#1,SET] seqnums:[0-0] points:[c#1,SET-f#1,SET]
@@ -331,10 +331,10 @@ Blob files:
331331

332332
apply v11
333333
del-table: L0 000005
334-
del-blob-file: B000003
334+
del-blob-file: B000003 000003
335335
new version edit
336336
del-table: L0 000006
337-
del-blob-file: B000004
337+
del-blob-file: B000004 000004
338338
----
339339
L1:
340340
000001:[a#2,SET-e#2,SET] seqnums:[0-0] points:[a#2,SET-e#2,SET]

‎internal/manifest/testdata/version_edit_decode‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -62,32 +62,33 @@ d8a301 # . Size = 20952
6262
f9a006 # . ValueSize = 102521
6363
9f9485bd06 # . CreationTime = 1738623519
6464
6c # <tagDeleteBlobFile>
65-
21 # . FileNum = 000021
65+
21 # . BlobFileID = 000033
66+
21 # . FileNum = 000033
6667
----
67-
021903660484106b2929d8a301f9a0069f9485bd066c21
68+
021903660484106b2929d8a301f9a0069f9485bd066c2121
6869
log-num: 25
6970
next-file-num: 102
7071
last-seq-num: 2052
7172
add-blob-file: B000041 physical:{000041 size:[20952 (20KB)] vals:[102521 (100KB)]}
72-
del-blob-file: B000033
73+
del-blob-file: B000033 000033
7374

7475
encode
7576
add-table: L6 000029:[bar#14,DEL-foo#13,SET] blobrefs:[(B000041: 20952); depth:1]
7677
add-blob-file: B000943 physical:{000041 size:[20952 (20KB)] vals:[102521 (100KB)]}
77-
del-blob-file: B000033
78+
del-blob-file: B000033 000033
7879
----
7980
67061d000b626172000e0000000000000b666f6f010d0000000000000000
80-
45010129d8a301016baf0729d8a301f9a006006c21
81+
45010129d8a301016baf0729d8a301f9a006006c2121
8182
add-table: L6 000029:[bar#14,DEL-foo#13,SET] seqnums:[0-0] points:[bar#14,DEL-foo#13,SET] blobrefs:[(B000041: 20952); depth:1]
8283
add-blob-file: B000943 physical:{000041 size:[20952 (20KB)] vals:[102521 (100KB)]}
83-
del-blob-file: B000033
84+
del-blob-file: B000033 000033
8485

8586
decode
8687
67061d000b626172000e0000000000000b666f6f010d0000000000000000
87-
45010129d8a301016baf0729d8a301f9a006006c21
88+
45010129d8a301016baf0729d8a301f9a006006c2121
8889
----
8990
67061d000b626172000e0000000000000b666f6f010d0000000000000000
90-
45010129d8a301016baf0729d8a301f9a006006c21
91+
45010129d8a301016baf0729d8a301f9a006006c2121
9192
add-table: L6 000029:[bar#14,DEL-foo#13,SET]
9293
add-blob-file: B000943 physical:{000041 size:[20952 (20KB)] vals:[102521 (100KB)]}
93-
del-blob-file: B000033
94+
del-blob-file: B000033 000033

‎internal/manifest/version_edit.go‎

Lines changed: 55 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,13 @@ type DeletedTableEntry struct {
8181
FileNum base.FileNum
8282
}
8383

84+
// DeletedBlobFileEntry holds the state for a blob file deletion. The blob file
85+
// ID may still be in-use with a different physical blob file.
86+
type DeletedBlobFileEntry struct {
87+
FileID base.BlobFileID
88+
FileNum base.DiskFileNum
89+
}
90+
8491
// NewTableEntry holds the state for a new sstable or one moved from a different
8592
// level.
8693
type NewTableEntry struct {
@@ -158,13 +165,20 @@ type VersionEdit struct {
158165
// NewBlobFiles holds the metadata for all new blob files introduced within
159166
// the version edit.
160167
NewBlobFiles []BlobFileMetadata
161-
// DeletedBlobFiles holds all blob files that became unreferenced during the
162-
// version edit. These blob files must not be referenced by any sstable in
163-
// the resulting Version.
168+
// DeletedBlobFiles holds all physical blob files that became unused during
169+
// the version edit.
170+
//
171+
// A physical blob file may become unused if the corresponding BlobFileID
172+
// becomes unreferenced during the version edit. In this case the BlobFileID
173+
// is not referenced by any sstable in the resulting Version.
174+
//
175+
// A physical blob file may also become unused if it is being replaced by a
176+
// new physical blob file. In this case NewBlobFiles must contain a
177+
// BlobFileMetadata with the same BlobFileID.
164178
//
165179
// While replaying a MANIFEST, the values are nil. Otherwise the values must
166180
// not be nil.
167-
DeletedBlobFiles map[base.BlobFileID]*PhysicalBlobFile
181+
DeletedBlobFiles map[DeletedBlobFileEntry]*PhysicalBlobFile
168182
}
169183

170184
// Decode decodes an edit from the specified reader.
@@ -538,14 +552,21 @@ func (v *VersionEdit) Decode(r io.Reader) error {
538552
})
539553

540554
case tagDeletedBlobFile:
555+
fileID, err := d.readUvarint()
556+
if err != nil {
557+
return err
558+
}
541559
fileNum, err := d.readFileNum()
542560
if err != nil {
543561
return err
544562
}
545563
if v.DeletedBlobFiles == nil {
546-
v.DeletedBlobFiles = make(map[base.BlobFileID]*PhysicalBlobFile)
564+
v.DeletedBlobFiles = make(map[DeletedBlobFileEntry]*PhysicalBlobFile)
547565
}
548-
v.DeletedBlobFiles[base.BlobFileID(fileNum)] = nil
566+
v.DeletedBlobFiles[DeletedBlobFileEntry{
567+
FileID: base.BlobFileID(fileID),
568+
FileNum: base.DiskFileNum(fileNum),
569+
}] = nil
549570

550571
case tagPrevLogNumber:
551572
n, err := d.readUvarint()
@@ -610,10 +631,15 @@ func (v *VersionEdit) string(verbose bool, fmtKey base.FormatKey) string {
610631
for _, f := range v.NewBlobFiles {
611632
fmt.Fprintf(&buf, " add-blob-file: %s\n", f.String())
612633
}
613-
deletedBlobFiles := slices.Collect(maps.Keys(v.DeletedBlobFiles))
614-
slices.Sort(deletedBlobFiles)
615-
for _, df := range deletedBlobFiles {
616-
fmt.Fprintf(&buf, " del-blob-file: %s\n", df)
634+
deletedBlobFileEntries := slices.Collect(maps.Keys(v.DeletedBlobFiles))
635+
slices.SortFunc(deletedBlobFileEntries, func(a, b DeletedBlobFileEntry) int {
636+
if v := stdcmp.Compare(a.FileID, b.FileID); v != 0 {
637+
return v
638+
}
639+
return stdcmp.Compare(a.FileNum, b.FileNum)
640+
})
641+
for _, df := range deletedBlobFileEntries {
642+
fmt.Fprintf(&buf, " del-blob-file: %s %s\n", df.FileID, df.FileNum)
617643
}
618644
return buf.String()
619645
}
@@ -695,9 +721,12 @@ func ParseVersionEditDebug(s string) (_ *VersionEdit, err error) {
695721

696722
case "del-blob-file":
697723
if ve.DeletedBlobFiles == nil {
698-
ve.DeletedBlobFiles = make(map[base.BlobFileID]*PhysicalBlobFile)
724+
ve.DeletedBlobFiles = make(map[DeletedBlobFileEntry]*PhysicalBlobFile)
699725
}
700-
ve.DeletedBlobFiles[p.BlobFileID()] = nil
726+
ve.DeletedBlobFiles[DeletedBlobFileEntry{
727+
FileID: p.BlobFileID(),
728+
FileNum: p.DiskFileNum(),
729+
}] = nil
701730

702731
default:
703732
return nil, errors.Errorf("field %q not implemented", field)
@@ -837,7 +866,8 @@ func (v *VersionEdit) Encode(w io.Writer) error {
837866
}
838867
for x := range v.DeletedBlobFiles {
839868
e.writeUvarint(tagDeletedBlobFile)
840-
e.writeUvarint(uint64(x))
869+
e.writeUvarint(uint64(x.FileID))
870+
e.writeUvarint(uint64(x.FileNum))
841871
}
842872
_, err := w.Write(e.Bytes())
843873
return err
@@ -1030,20 +1060,24 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
10301060
b.BlobFiles.Added[nbf.FileID] = nbf.Physical
10311061
}
10321062

1033-
for blobFileID, physicalBlobFile := range ve.DeletedBlobFiles {
1034-
if b.BlobFiles.Deleted == nil {
1035-
b.BlobFiles.Deleted = make(map[base.BlobFileID]*PhysicalBlobFile)
1036-
}
1063+
for entry, physicalBlobFile := range ve.DeletedBlobFiles {
10371064
// If the blob file was added in a prior, accumulated version edit we
10381065
// can resolve the deletion by removing it from the added files map.
1066+
if b.BlobFiles.Added != nil {
1067+
added := b.BlobFiles.Added[entry.FileID]
1068+
if added != nil && added.FileNum == entry.FileNum {
1069+
delete(b.BlobFiles.Added, entry.FileID)
1070+
continue
1071+
}
1072+
}
10391073
// Otherwise the blob file deleted was added prior to this bulk edit,
10401074
// and we insert it into BlobFiles.Deleted so that Apply may remove it
10411075
// from the resulting version.
1042-
if b.BlobFiles.Added != nil && b.BlobFiles.Added[blobFileID] != nil {
1043-
delete(b.BlobFiles.Added, blobFileID)
1044-
} else {
1045-
b.BlobFiles.Deleted[blobFileID] = physicalBlobFile
1076+
if b.BlobFiles.Deleted == nil {
1077+
b.BlobFiles.Deleted = make(map[base.BlobFileID]*PhysicalBlobFile)
10461078
}
1079+
b.BlobFiles.Deleted[entry.FileID] = physicalBlobFile
1080+
10471081
}
10481082

10491083
for df, m := range ve.DeletedTables {

‎internal/manifest/version_edit_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -479,7 +479,7 @@ func TestParseVersionEditDebugRoundTrip(t *testing.T) {
479479
output string
480480
}{
481481
{
482-
input: ` del-blob-file: B000001`,
482+
input: ` del-blob-file: B000001 000001`,
483483
},
484484
{
485485
input: ` add-blob-file: B000925 physical:{000005 size:[20535 (20KB)] vals:[25935 (25KB)]}`,
@@ -509,8 +509,8 @@ func TestParseVersionEditDebugRoundTrip(t *testing.T) {
509509
` add-table: L1 000001:[a#0,SET-z#0,DEL] seqnums:[0-0] points:[a#0,SET-z#0,DEL] size:1`,
510510
` add-table: L2 000002:[a#0,SET-z#0,DEL] seqnums:[0-0] points:[a#0,SET-z#0,DEL] size:2`,
511511
` add-blob-file: B000005 physical:{000005 size:[20535 (20KB)] vals:[25935 (25KB)]}`,
512-
` del-blob-file: B000004`,
513-
` del-blob-file: B000006`,
512+
` del-blob-file: B000004 000004`,
513+
` del-blob-file: B000006 000006`,
514514
}, "\n"),
515515
},
516516
{

‎testdata/version_set‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -434,12 +434,12 @@ no obsolete blob files
434434

435435
apply
436436
del-table: L3 000012
437-
del-blob-file: B000011
437+
del-blob-file: B000011 000011
438438
----
439439
applied:
440440
last-seq-num: 99
441441
del-table: L3 000012
442-
del-blob-file: B000011
442+
del-blob-file: B000011 000011
443443
current version:
444444
L2:
445445
000001:[a#1,SET-c#1,SET] seqnums:[0-0] points:[a#1,SET-c#1,SET] size:100
@@ -521,13 +521,13 @@ no obsolete blob files
521521
apply
522522
del-table: L3 000015
523523
del-table: L4 000014
524-
del-blob-file: B000013
524+
del-blob-file: B000013 000013
525525
----
526526
applied:
527527
last-seq-num: 99
528528
del-table: L3 000015
529529
del-table: L4 000014
530-
del-blob-file: B000013
530+
del-blob-file: B000013 000013
531531
current version:
532532
L2:
533533
000001:[a#1,SET-c#1,SET] seqnums:[0-0] points:[a#1,SET-c#1,SET] size:100

‎tool/testdata/find-val-sep-db/MANIFEST-000001‎

0 Bytes
Binary file not shown.

‎tool/testdata/find-val-sep-db/OPTIONS-000003‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,3 +61,63 @@
6161
filter_type=table
6262
index_block_size=4096
6363
target_file_size=2097152
64+
65+
[Level "1"]
66+
block_restart_interval=16
67+
block_size=4096
68+
block_size_threshold=90
69+
compression=Snappy
70+
filter_policy=none
71+
filter_type=table
72+
index_block_size=4096
73+
target_file_size=4194304
74+
75+
[Level "2"]
76+
block_restart_interval=16
77+
block_size=4096
78+
block_size_threshold=90
79+
compression=Snappy
80+
filter_policy=none
81+
filter_type=table
82+
index_block_size=4096
83+
target_file_size=8388608
84+
85+
[Level "3"]
86+
block_restart_interval=16
87+
block_size=4096
88+
block_size_threshold=90
89+
compression=Snappy
90+
filter_policy=none
91+
filter_type=table
92+
index_block_size=4096
93+
target_file_size=16777216
94+
95+
[Level "4"]
96+
block_restart_interval=16
97+
block_size=4096
98+
block_size_threshold=90
99+
compression=Snappy
100+
filter_policy=none
101+
filter_type=table
102+
index_block_size=4096
103+
target_file_size=33554432
104+
105+
[Level "5"]
106+
block_restart_interval=16
107+
block_size=4096
108+
block_size_threshold=90
109+
compression=Snappy
110+
filter_policy=none
111+
filter_type=table
112+
index_block_size=4096
113+
target_file_size=67108864
114+
115+
[Level "6"]
116+
block_restart_interval=16
117+
block_size=4096
118+
block_size_threshold=90
119+
compression=Snappy
120+
filter_policy=none
121+
filter_type=table
122+
index_block_size=4096
123+
target_file_size=134217728

0 commit comments

Comments
 (0)