Skip to content

Commit 4ee17c2

Browse files
xinhaozxinhaoz
committed
pebble: add FMV for writing BackingValueSize to manifest
This field will be added for virtual tables with blob refs.
1 parent 414d942 commit 4ee17c2

File tree

8 files changed

+88
-60
lines changed

8 files changed

+88
-60
lines changed

‎format_major_version.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,11 @@ const (
243243
// file format (which adds compression statistics).
244244
FormatV2BlobFiles
245245

246+
// FormatBackingValueSize is a format major version that adds support for
247+
// persisting the value size of the backing sst for virtual sstables in the
248+
// manifest (VersionEdit).
249+
FormatBackingValueSize
250+
246251
// -- Add new versions here --
247252

248253
// FormatNewest is the most recent format major version.
@@ -375,6 +380,9 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
375380
FormatV2BlobFiles: func(d *DB) error {
376381
return d.finalizeFormatVersUpgrade(FormatV2BlobFiles)
377382
},
383+
FormatBackingValueSize: func(d *DB) error {
384+
return d.finalizeFormatVersUpgrade(FormatBackingValueSize)
385+
},
378386
}
379387

380388
const formatVersionMarkerName = `format-version`

‎format_major_version_test.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,12 @@ func TestFormatMajorVersionStableValues(t *testing.T) {
3636
require.Equal(t, FormatValueSeparation, FormatMajorVersion(24))
3737
require.Equal(t, FormatExciseBoundsRecord, FormatMajorVersion(25))
3838
require.Equal(t, FormatV2BlobFiles, FormatMajorVersion(26))
39+
require.Equal(t, FormatBackingValueSize, FormatMajorVersion(27))
3940

4041
// When we add a new version, we should add a check for the new version above
4142
// in addition to updating the expected values below.
42-
require.Equal(t, FormatNewest, FormatMajorVersion(26))
43-
require.Equal(t, internalFormatNewest, FormatMajorVersion(26))
43+
require.Equal(t, FormatNewest, FormatMajorVersion(27))
44+
require.Equal(t, internalFormatNewest, FormatMajorVersion(27))
4445
}
4546

4647
func TestFormatMajorVersion_MigrationDefined(t *testing.T) {
@@ -229,6 +230,7 @@ func TestFormatMajorVersions_TableFormat(t *testing.T) {
229230
FormatValueSeparation: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
230231
FormatExciseBoundsRecord: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
231232
FormatV2BlobFiles: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
233+
FormatBackingValueSize: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
232234
}
233235

234236
// Valid versions.
@@ -254,6 +256,7 @@ func TestFormatMajorVersions_BlobFileFormat(t *testing.T) {
254256
FormatValueSeparation: blob.FileFormatV1,
255257
FormatExciseBoundsRecord: blob.FileFormatV1,
256258
FormatV2BlobFiles: blob.FileFormatV2,
259+
FormatBackingValueSize: blob.FileFormatV2,
257260
}
258261

259262
// Valid versions.

‎open_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -503,7 +503,7 @@ func TestNewDBFilenames(t *testing.T) {
503503
"LOCK",
504504
"MANIFEST-000001",
505505
"OPTIONS-000003",
506-
"marker.format-version.000013.026",
506+
"marker.format-version.000014.027",
507507
"marker.manifest.000001.MANIFEST-000001",
508508
},
509509
}

‎testdata/checkpoint‎

Lines changed: 37 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,10 @@ create: db/marker.format-version.000013.026
7171
close: db/marker.format-version.000013.026
7272
remove: db/marker.format-version.000012.025
7373
sync: db
74+
create: db/marker.format-version.000014.027
75+
close: db/marker.format-version.000014.027
76+
remove: db/marker.format-version.000013.026
77+
sync: db
7478
create: db/temporary.000003.dbtmp
7579
sync: db/temporary.000003.dbtmp
7680
close: db/temporary.000003.dbtmp
@@ -137,9 +141,9 @@ sync-data: checkpoints/checkpoint1/OPTIONS-000003
137141
close: checkpoints/checkpoint1/OPTIONS-000003
138142
close: db/OPTIONS-000003
139143
open-dir: checkpoints/checkpoint1
140-
create: checkpoints/checkpoint1/marker.format-version.000001.026
141-
sync-data: checkpoints/checkpoint1/marker.format-version.000001.026
142-
close: checkpoints/checkpoint1/marker.format-version.000001.026
144+
create: checkpoints/checkpoint1/marker.format-version.000001.027
145+
sync-data: checkpoints/checkpoint1/marker.format-version.000001.027
146+
close: checkpoints/checkpoint1/marker.format-version.000001.027
143147
sync: checkpoints/checkpoint1
144148
close: checkpoints/checkpoint1
145149
link: db/000005.sst -> checkpoints/checkpoint1/000005.sst
@@ -181,9 +185,9 @@ sync-data: checkpoints/checkpoint2/OPTIONS-000003
181185
close: checkpoints/checkpoint2/OPTIONS-000003
182186
close: db/OPTIONS-000003
183187
open-dir: checkpoints/checkpoint2
184-
create: checkpoints/checkpoint2/marker.format-version.000001.026
185-
sync-data: checkpoints/checkpoint2/marker.format-version.000001.026
186-
close: checkpoints/checkpoint2/marker.format-version.000001.026
188+
create: checkpoints/checkpoint2/marker.format-version.000001.027
189+
sync-data: checkpoints/checkpoint2/marker.format-version.000001.027
190+
close: checkpoints/checkpoint2/marker.format-version.000001.027
187191
sync: checkpoints/checkpoint2
188192
close: checkpoints/checkpoint2
189193
link: db/000007.sst -> checkpoints/checkpoint2/000007.sst
@@ -220,9 +224,9 @@ sync-data: checkpoints/checkpoint3/OPTIONS-000003
220224
close: checkpoints/checkpoint3/OPTIONS-000003
221225
close: db/OPTIONS-000003
222226
open-dir: checkpoints/checkpoint3
223-
create: checkpoints/checkpoint3/marker.format-version.000001.026
224-
sync-data: checkpoints/checkpoint3/marker.format-version.000001.026
225-
close: checkpoints/checkpoint3/marker.format-version.000001.026
227+
create: checkpoints/checkpoint3/marker.format-version.000001.027
228+
sync-data: checkpoints/checkpoint3/marker.format-version.000001.027
229+
close: checkpoints/checkpoint3/marker.format-version.000001.027
226230
sync: checkpoints/checkpoint3
227231
close: checkpoints/checkpoint3
228232
link: db/000005.sst -> checkpoints/checkpoint3/000005.sst
@@ -306,7 +310,7 @@ list db
306310
LOCK
307311
MANIFEST-000001
308312
OPTIONS-000003
309-
marker.format-version.000013.026
313+
marker.format-version.000014.027
310314
marker.manifest.000001.MANIFEST-000001
311315

312316
list checkpoints/checkpoint1
@@ -316,7 +320,7 @@ list checkpoints/checkpoint1
316320
000007.sst
317321
MANIFEST-000001
318322
OPTIONS-000003
319-
marker.format-version.000001.026
323+
marker.format-version.000001.027
320324
marker.manifest.000001.MANIFEST-000001
321325

322326
open checkpoints/checkpoint1 readonly
@@ -383,7 +387,7 @@ list checkpoints/checkpoint2
383387
000007.sst
384388
MANIFEST-000001
385389
OPTIONS-000003
386-
marker.format-version.000001.026
390+
marker.format-version.000001.027
387391
marker.manifest.000001.MANIFEST-000001
388392

389393
open checkpoints/checkpoint2 readonly
@@ -425,7 +429,7 @@ list checkpoints/checkpoint3
425429
000007.sst
426430
MANIFEST-000001
427431
OPTIONS-000003
428-
marker.format-version.000001.026
432+
marker.format-version.000001.027
429433
marker.manifest.000001.MANIFEST-000001
430434

431435
open checkpoints/checkpoint3 readonly
@@ -544,9 +548,9 @@ sync-data: checkpoints/checkpoint4/OPTIONS-000003
544548
close: checkpoints/checkpoint4/OPTIONS-000003
545549
close: db/OPTIONS-000003
546550
open-dir: checkpoints/checkpoint4
547-
create: checkpoints/checkpoint4/marker.format-version.000001.026
548-
sync-data: checkpoints/checkpoint4/marker.format-version.000001.026
549-
close: checkpoints/checkpoint4/marker.format-version.000001.026
551+
create: checkpoints/checkpoint4/marker.format-version.000001.027
552+
sync-data: checkpoints/checkpoint4/marker.format-version.000001.027
553+
close: checkpoints/checkpoint4/marker.format-version.000001.027
550554
sync: checkpoints/checkpoint4
551555
close: checkpoints/checkpoint4
552556
link: db/000010.sst -> checkpoints/checkpoint4/000010.sst
@@ -634,7 +638,7 @@ list db
634638
LOCK
635639
MANIFEST-000001
636640
OPTIONS-000003
637-
marker.format-version.000013.026
641+
marker.format-version.000014.027
638642
marker.manifest.000001.MANIFEST-000001
639643

640644

@@ -653,9 +657,9 @@ sync-data: checkpoints/checkpoint5/OPTIONS-000003
653657
close: checkpoints/checkpoint5/OPTIONS-000003
654658
close: db/OPTIONS-000003
655659
open-dir: checkpoints/checkpoint5
656-
create: checkpoints/checkpoint5/marker.format-version.000001.026
657-
sync-data: checkpoints/checkpoint5/marker.format-version.000001.026
658-
close: checkpoints/checkpoint5/marker.format-version.000001.026
660+
create: checkpoints/checkpoint5/marker.format-version.000001.027
661+
sync-data: checkpoints/checkpoint5/marker.format-version.000001.027
662+
close: checkpoints/checkpoint5/marker.format-version.000001.027
659663
sync: checkpoints/checkpoint5
660664
close: checkpoints/checkpoint5
661665
link: db/000010.sst -> checkpoints/checkpoint5/000010.sst
@@ -755,9 +759,9 @@ sync-data: checkpoints/checkpoint6/OPTIONS-000003
755759
close: checkpoints/checkpoint6/OPTIONS-000003
756760
close: db/OPTIONS-000003
757761
open-dir: checkpoints/checkpoint6
758-
create: checkpoints/checkpoint6/marker.format-version.000001.026
759-
sync-data: checkpoints/checkpoint6/marker.format-version.000001.026
760-
close: checkpoints/checkpoint6/marker.format-version.000001.026
762+
create: checkpoints/checkpoint6/marker.format-version.000001.027
763+
sync-data: checkpoints/checkpoint6/marker.format-version.000001.027
764+
close: checkpoints/checkpoint6/marker.format-version.000001.027
761765
sync: checkpoints/checkpoint6
762766
close: checkpoints/checkpoint6
763767
link: db/000011.sst -> checkpoints/checkpoint6/000011.sst
@@ -942,6 +946,10 @@ create: valsepdb/marker.format-version.000013.026
942946
close: valsepdb/marker.format-version.000013.026
943947
remove: valsepdb/marker.format-version.000012.025
944948
sync: valsepdb
949+
create: valsepdb/marker.format-version.000014.027
950+
close: valsepdb/marker.format-version.000014.027
951+
remove: valsepdb/marker.format-version.000013.026
952+
sync: valsepdb
945953
create: valsepdb/temporary.000003.dbtmp
946954
sync: valsepdb/temporary.000003.dbtmp
947955
close: valsepdb/temporary.000003.dbtmp
@@ -990,9 +998,9 @@ sync-data: checkpoints/checkpoint8/OPTIONS-000003
990998
close: checkpoints/checkpoint8/OPTIONS-000003
991999
close: valsepdb/OPTIONS-000003
9921000
open-dir: checkpoints/checkpoint8
993-
create: checkpoints/checkpoint8/marker.format-version.000001.026
994-
sync-data: checkpoints/checkpoint8/marker.format-version.000001.026
995-
close: checkpoints/checkpoint8/marker.format-version.000001.026
1001+
create: checkpoints/checkpoint8/marker.format-version.000001.027
1002+
sync-data: checkpoints/checkpoint8/marker.format-version.000001.027
1003+
close: checkpoints/checkpoint8/marker.format-version.000001.027
9961004
sync: checkpoints/checkpoint8
9971005
close: checkpoints/checkpoint8
9981006
link: valsepdb/000006.blob -> checkpoints/checkpoint8/000006.blob
@@ -1104,9 +1112,9 @@ sync-data: checkpoints/checkpoint9/OPTIONS-000003
11041112
close: checkpoints/checkpoint9/OPTIONS-000003
11051113
close: valsepdb/OPTIONS-000003
11061114
open-dir: checkpoints/checkpoint9
1107-
create: checkpoints/checkpoint9/marker.format-version.000001.026
1108-
sync-data: checkpoints/checkpoint9/marker.format-version.000001.026
1109-
close: checkpoints/checkpoint9/marker.format-version.000001.026
1115+
create: checkpoints/checkpoint9/marker.format-version.000001.027
1116+
sync-data: checkpoints/checkpoint9/marker.format-version.000001.027
1117+
close: checkpoints/checkpoint9/marker.format-version.000001.027
11101118
sync: checkpoints/checkpoint9
11111119
close: checkpoints/checkpoint9
11121120
link: valsepdb/000006.blob -> checkpoints/checkpoint9/000006.blob

‎testdata/checkpoint_shared‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,10 @@ create: db/marker.format-version.000010.026
5959
close: db/marker.format-version.000010.026
6060
remove: db/marker.format-version.000009.025
6161
sync: db
62+
create: db/marker.format-version.000011.027
63+
close: db/marker.format-version.000011.027
64+
remove: db/marker.format-version.000010.026
65+
sync: db
6266
create: db/temporary.000003.dbtmp
6367
sync: db/temporary.000003.dbtmp
6468
close: db/temporary.000003.dbtmp
@@ -125,9 +129,9 @@ sync-data: checkpoints/checkpoint1/OPTIONS-000003
125129
close: checkpoints/checkpoint1/OPTIONS-000003
126130
close: db/OPTIONS-000003
127131
open-dir: checkpoints/checkpoint1
128-
create: checkpoints/checkpoint1/marker.format-version.000001.026
129-
sync-data: checkpoints/checkpoint1/marker.format-version.000001.026
130-
close: checkpoints/checkpoint1/marker.format-version.000001.026
132+
create: checkpoints/checkpoint1/marker.format-version.000001.027
133+
sync-data: checkpoints/checkpoint1/marker.format-version.000001.027
134+
close: checkpoints/checkpoint1/marker.format-version.000001.027
131135
sync: checkpoints/checkpoint1
132136
close: checkpoints/checkpoint1
133137
open: db/MANIFEST-000001 (options: *vfs.sequentialReadsOption)
@@ -178,9 +182,9 @@ sync-data: checkpoints/checkpoint2/OPTIONS-000003
178182
close: checkpoints/checkpoint2/OPTIONS-000003
179183
close: db/OPTIONS-000003
180184
open-dir: checkpoints/checkpoint2
181-
create: checkpoints/checkpoint2/marker.format-version.000001.026
182-
sync-data: checkpoints/checkpoint2/marker.format-version.000001.026
183-
close: checkpoints/checkpoint2/marker.format-version.000001.026
185+
create: checkpoints/checkpoint2/marker.format-version.000001.027
186+
sync-data: checkpoints/checkpoint2/marker.format-version.000001.027
187+
close: checkpoints/checkpoint2/marker.format-version.000001.027
184188
sync: checkpoints/checkpoint2
185189
close: checkpoints/checkpoint2
186190
open: db/MANIFEST-000001 (options: *vfs.sequentialReadsOption)
@@ -227,9 +231,9 @@ sync-data: checkpoints/checkpoint3/OPTIONS-000003
227231
close: checkpoints/checkpoint3/OPTIONS-000003
228232
close: db/OPTIONS-000003
229233
open-dir: checkpoints/checkpoint3
230-
create: checkpoints/checkpoint3/marker.format-version.000001.026
231-
sync-data: checkpoints/checkpoint3/marker.format-version.000001.026
232-
close: checkpoints/checkpoint3/marker.format-version.000001.026
234+
create: checkpoints/checkpoint3/marker.format-version.000001.027
235+
sync-data: checkpoints/checkpoint3/marker.format-version.000001.027
236+
close: checkpoints/checkpoint3/marker.format-version.000001.027
233237
sync: checkpoints/checkpoint3
234238
close: checkpoints/checkpoint3
235239
open: db/MANIFEST-000001 (options: *vfs.sequentialReadsOption)
@@ -286,7 +290,7 @@ LOCK
286290
MANIFEST-000001
287291
OPTIONS-000003
288292
REMOTE-OBJ-CATALOG-000001
289-
marker.format-version.000010.026
293+
marker.format-version.000011.027
290294
marker.manifest.000001.MANIFEST-000001
291295
marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001
292296

@@ -296,7 +300,7 @@ list checkpoints/checkpoint1
296300
MANIFEST-000001
297301
OPTIONS-000003
298302
REMOTE-OBJ-CATALOG-000001
299-
marker.format-version.000001.026
303+
marker.format-version.000001.027
300304
marker.manifest.000001.MANIFEST-000001
301305
marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001
302306

@@ -348,7 +352,7 @@ list checkpoints/checkpoint2
348352
MANIFEST-000001
349353
OPTIONS-000003
350354
REMOTE-OBJ-CATALOG-000001
351-
marker.format-version.000001.026
355+
marker.format-version.000001.027
352356
marker.manifest.000001.MANIFEST-000001
353357
marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001
354358

‎testdata/event_listener‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,11 @@ close: db/marker.format-version.000013.026
9292
remove: db/marker.format-version.000012.025
9393
sync: db
9494
upgraded to format version: 026
95+
create: db/marker.format-version.000014.027
96+
close: db/marker.format-version.000014.027
97+
remove: db/marker.format-version.000013.026
98+
sync: db
99+
upgraded to format version: 027
95100
create: db/temporary.000003.dbtmp
96101
sync: db/temporary.000003.dbtmp
97102
close: db/temporary.000003.dbtmp
@@ -480,9 +485,9 @@ sync-data: checkpoint/OPTIONS-000003
480485
close: checkpoint/OPTIONS-000003
481486
close: db/OPTIONS-000003
482487
open-dir: checkpoint
483-
create: checkpoint/marker.format-version.000001.026
484-
sync-data: checkpoint/marker.format-version.000001.026
485-
close: checkpoint/marker.format-version.000001.026
488+
create: checkpoint/marker.format-version.000001.027
489+
sync-data: checkpoint/marker.format-version.000001.027
490+
close: checkpoint/marker.format-version.000001.027
486491
sync: checkpoint
487492
close: checkpoint
488493
link: db/000013.sst -> checkpoint/000013.sst

0 commit comments

Comments
 (0)