Skip to content

Commit 3747415

Browse files
jbowensjbowens
committed
manifest: support blob file replacement
Adapt CurrentBlobFileSet and VersionEdit application to support version edits that remove a physical blob file and add a new physical blob file under the same BlobFileID. Informs #112.
1 parent 8866088 commit 3747415

File tree

5 files changed

+177
-5
lines changed

5 files changed

+177
-5
lines changed

‎internal/manifest/blob_metadata.go‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -540,9 +540,24 @@ func (s *CurrentBlobFileSet) Metadatas() []BlobFileMetadata {
540540
func (s *CurrentBlobFileSet) ApplyAndUpdateVersionEdit(ve *VersionEdit) error {
541541
// Insert new blob files into the set.
542542
for _, m := range ve.NewBlobFiles {
543-
if _, ok := s.files[m.FileID]; ok {
544-
return errors.AssertionFailedf("pebble: new blob file %d already exists", m.FileID)
543+
// Check whether we already have a blob file with this ID. This is
544+
// possible if the blob file is being replaced.
545+
if cbf, ok := s.files[m.FileID]; ok {
546+
// There should be a DeletedBlobFileEntry for this file ID and the
547+
// FileNum currently in the set.
548+
dbfe := DeletedBlobFileEntry{FileID: m.FileID, FileNum: cbf.metadata.Physical.FileNum}
549+
if _, ok := ve.DeletedBlobFiles[dbfe]; !ok {
550+
return errors.AssertionFailedf("pebble: new blob file %d already exists", m.FileID)
551+
}
552+
// The file is being replaced. Update the statistics and cbf.Physical.
553+
s.stats.PhysicalSize -= cbf.metadata.Physical.Size
554+
s.stats.ValueSize -= cbf.metadata.Physical.ValueSize
555+
cbf.metadata.Physical = m.Physical
556+
s.stats.PhysicalSize += m.Physical.Size
557+
s.stats.ValueSize += m.Physical.ValueSize
558+
continue
545559
}
560+
546561
blobFileID := m.FileID
547562
cbf := &currentBlobFile{references: make(map[*TableMetadata]struct{})}
548563
cbf.metadata = BlobFileMetadata{FileID: blobFileID, Physical: m.Physical}

‎internal/manifest/testdata/current_blob_file_set‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,3 +73,25 @@ modified version edit:
7373
del-blob-file: B000012 000012
7474
current blob file set:
7575
Files:{Count: 0, Size: 0, ValueSize: 0}, References:{ValueSize: 0, Count: 0}
76+
77+
applyAndUpdateVersionEdit
78+
add-table: L3 000015:[d#1,SET-e#1,SET] blobrefs:[(B000016: 25935); depth:1]
79+
add-blob-file: B000016 physical:{000016 size:[20535 (20KB)] vals:[25935 (25KB)]}
80+
----
81+
modified version edit:
82+
add-table: L3 000015:[d#1,SET-e#1,SET] seqnums:[0-0] points:[d#1,SET-e#1,SET] blobrefs:[(B000016: 25935); depth:1]
83+
add-blob-file: B000016 physical:{000016 size:[20535 (20KB)] vals:[25935 (25KB)]}
84+
current blob file set:
85+
Files:{Count: 1, Size: 20535, ValueSize: 25935}, References:{ValueSize: 25935, Count: 1}
86+
87+
# Perform a blob file replacement.
88+
89+
applyAndUpdateVersionEdit
90+
del-blob-file: B000016 000016
91+
add-blob-file: B000016 physical:{000017 size:[10535 (10KB)] vals:[10535 (10KB)]}
92+
----
93+
modified version edit:
94+
add-blob-file: B000016 physical:{000017 size:[10535 (10KB)] vals:[10535 (10KB)]}
95+
del-blob-file: B000016 000016
96+
current blob file set:
97+
Files:{Count: 1, Size: 10535, ValueSize: 10535}, References:{ValueSize: 25935, Count: 1}

‎internal/manifest/testdata/version_edit_apply‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,3 +340,21 @@ L1:
340340
000001:[a#2,SET-e#2,SET] seqnums:[0-0] points:[a#2,SET-e#2,SET]
341341
L2:
342342
000002:[c#1,SET-f#1,SET] seqnums:[0-0] points:[c#1,SET-f#1,SET]
343+
344+
# Test a version edit that replaces a blob file.
345+
346+
apply v11
347+
del-blob-file: B000003 000003
348+
add-blob-file: B000003 physical:{000052 size:[10000] vals:[10000]}
349+
----
350+
L0.1:
351+
000006:[a#9,SET-z#9,DEL] seqnums:[9-9] points:[a#9,SET-z#9,DEL] blobrefs:[(B000004: 25935); depth:1]
352+
L0.0:
353+
000005:[a#9,SET-z#9,DEL] seqnums:[9-9] points:[a#9,SET-z#9,DEL] blobrefs:[(B000003: 25935); depth:1]
354+
L1:
355+
000001:[a#2,SET-e#2,SET] seqnums:[0-0] points:[a#2,SET-e#2,SET]
356+
L2:
357+
000002:[c#1,SET-f#1,SET] seqnums:[0-0] points:[c#1,SET-f#1,SET]
358+
Blob files:
359+
B000003 physical:{000052 size:[10000 (9.8KB)] vals:[10000 (9.8KB)]}
360+
B000004 physical:{000004 size:[20535 (20KB)] vals:[25935 (25KB)]}

‎internal/manifest/version_edit.go‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1223,6 +1223,13 @@ func (b *BulkVersionEdit) Apply(curr *Version, readCompactionRate int64) (*Versi
12231223
// BlobFileSet ensures any physical blob files that are referenced by the
12241224
// version remain on storage until they're no longer referenced by any
12251225
// version.
1226+
//
1227+
// We remove deleted blob files first, because during a blob file
1228+
// replacement the BlobFileID is reused. The B-Tree insert will fail if the
1229+
// old blob file is still present in the tree.
1230+
for blobFileID := range b.BlobFiles.Deleted {
1231+
v.BlobFiles.remove(BlobFileMetadata{FileID: blobFileID})
1232+
}
12261233
for blobFileID, physical := range b.BlobFiles.Added {
12271234
if err := v.BlobFiles.insert(BlobFileMetadata{
12281235
FileID: blobFileID,
@@ -1231,9 +1238,6 @@ func (b *BulkVersionEdit) Apply(curr *Version, readCompactionRate int64) (*Versi
12311238
return nil, err
12321239
}
12331240
}
1234-
for blobFileID := range b.BlobFiles.Deleted {
1235-
v.BlobFiles.remove(BlobFileMetadata{FileID: blobFileID})
1236-
}
12371241

12381242
for level := range v.Levels {
12391243
v.Levels[level] = curr.Levels[level].clone()

‎testdata/version_set‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -570,3 +570,116 @@ no zombie tables
570570
no obsolete tables
571571
no zombie blob files
572572
no obsolete blob files
573+
574+
# Test a blob file replacement.
575+
576+
apply
577+
add-blob-file: B000016 physical:{000016 size:[10000 (9.8KB)] vals:[10000 (9.8KB)]}
578+
add-table: L4 000017:[f#20,SET-g#20,SET] blobrefs:[(B000016: 10000); depth:1]
579+
----
580+
applied:
581+
last-seq-num: 99
582+
add-table: L4 000017:[f#20,SET-g#20,SET]
583+
add-blob-file: B000016 physical:{000016 size:[10000 (9.8KB)] vals:[10000 (9.8KB)]}
584+
current version:
585+
L2:
586+
000001:[a#1,SET-c#1,SET] seqnums:[0-0] points:[a#1,SET-c#1,SET] size:100
587+
L3:
588+
000008:[a#1,SET-c#1,SET] seqnums:[0-0] points:[a#1,SET-c#1,SET] size:800
589+
000010:[d#1,SET-e#1,SET] seqnums:[0-0] points:[d#1,SET-e#1,SET] size:1000
590+
L4:
591+
000017:[f#20,SET-g#20,SET] seqnums:[0-0] points:[f#20,SET-g#20,SET] size:1700 blobrefs:[(B000016: 10000); depth:1]
592+
Blob files:
593+
B000016 physical:{000016 size:[10000 (9.8KB)] vals:[10000 (9.8KB)]}
594+
no virtual backings
595+
no zombie tables
596+
no obsolete tables
597+
no zombie blob files
598+
no obsolete blob files
599+
600+
ref-version r3
601+
----
602+
current version:
603+
L2:
604+
000001:[a#1,SET-c#1,SET] seqnums:[0-0] points:[a#1,SET-c#1,SET] size:100
605+
L3:
606+
000008:[a#1,SET-c#1,SET] seqnums:[0-0] points:[a#1,SET-c#1,SET] size:800
607+
000010:[d#1,SET-e#1,SET] seqnums:[0-0] points:[d#1,SET-e#1,SET] size:1000
608+
L4:
609+
000017:[f#20,SET-g#20,SET] seqnums:[0-0] points:[f#20,SET-g#20,SET] size:1700 blobrefs:[(B000016: 10000); depth:1]
610+
Blob files:
611+
B000016 physical:{000016 size:[10000 (9.8KB)] vals:[10000 (9.8KB)]}
612+
no virtual backings
613+
no zombie tables
614+
no obsolete tables
615+
no zombie blob files
616+
no obsolete blob files
617+
618+
# Apply the blob file replacement. Because we ref'd the previous version, the
619+
# previous physical blob file should be considered a zombie.
620+
621+
apply
622+
del-blob-file: B000016 000016
623+
add-blob-file: B000016 physical:{000018 size:[5000 (4.9KB)] vals:[5000 (4.9KB)]}
624+
----
625+
applied:
626+
last-seq-num: 99
627+
add-blob-file: B000016 physical:{000018 size:[5000 (4.9KB)] vals:[5000 (4.9KB)]}
628+
del-blob-file: B000016 000016
629+
current version:
630+
L2:
631+
000001:[a#1,SET-c#1,SET] seqnums:[0-0] points:[a#1,SET-c#1,SET] size:100
632+
L3:
633+
000008:[a#1,SET-c#1,SET] seqnums:[0-0] points:[a#1,SET-c#1,SET] size:800
634+
000010:[d#1,SET-e#1,SET] seqnums:[0-0] points:[d#1,SET-e#1,SET] size:1000
635+
L4:
636+
000017:[f#20,SET-g#20,SET] seqnums:[0-0] points:[f#20,SET-g#20,SET] size:1700 blobrefs:[(B000016: 10000); depth:1]
637+
Blob files:
638+
B000016 physical:{000018 size:[5000 (4.9KB)] vals:[5000 (4.9KB)]}
639+
no virtual backings
640+
no zombie tables
641+
no obsolete tables
642+
zombie blob files: 000016
643+
no obsolete blob files
644+
645+
# Once we unref the version, the old 000016 physical blob file should become
646+
# obsolete. The new physical blob file 000018 remains under the 000016
647+
# BlobFileID.
648+
649+
unref-version r3
650+
----
651+
current version:
652+
L2:
653+
000001:[a#1,SET-c#1,SET] seqnums:[0-0] points:[a#1,SET-c#1,SET] size:100
654+
L3:
655+
000008:[a#1,SET-c#1,SET] seqnums:[0-0] points:[a#1,SET-c#1,SET] size:800
656+
000010:[d#1,SET-e#1,SET] seqnums:[0-0] points:[d#1,SET-e#1,SET] size:1000
657+
L4:
658+
000017:[f#20,SET-g#20,SET] seqnums:[0-0] points:[f#20,SET-g#20,SET] size:1700 blobrefs:[(B000016: 10000); depth:1]
659+
Blob files:
660+
B000016 physical:{000018 size:[5000 (4.9KB)] vals:[5000 (4.9KB)]}
661+
no virtual backings
662+
no zombie tables
663+
no obsolete tables
664+
no zombie blob files
665+
obsolete blob files: 000016
666+
667+
# Reopening should recover the previous state.
668+
669+
reopen
670+
----
671+
current version:
672+
L2:
673+
000001:[a#1,SET-c#1,SET] seqnums:[0-0] points:[a#1,SET-c#1,SET] size:100
674+
L3:
675+
000008:[a#1,SET-c#1,SET] seqnums:[0-0] points:[a#1,SET-c#1,SET] size:800
676+
000010:[d#1,SET-e#1,SET] seqnums:[0-0] points:[d#1,SET-e#1,SET] size:1000
677+
L4:
678+
000017:[f#20,SET-g#20,SET] seqnums:[0-0] points:[f#20,SET-g#20,SET] size:1700 blobrefs:[(B000016: 10000); depth:1]
679+
Blob files:
680+
B000016 physical:{000018 size:[5000 (4.9KB)] vals:[5000 (4.9KB)]}
681+
no virtual backings
682+
no zombie tables
683+
no obsolete tables
684+
no zombie blob files
685+
no obsolete blob files

0 commit comments

Comments
 (0)