Skip to content

Commit 0e42024

Browse files
jbowensjbowens
committed
db: export FormatTableFormatV6
Export the new format major version that enables use of TableFormatPebblev6, and update FormatNewest to point to this format major version. This new table format adds the checksum over the sstable footer, and allows the encoding of blob handles within the value column.
1 parent 7fde12a commit 0e42024

File tree

9 files changed

+291
-276
lines changed

9 files changed

+291
-276
lines changed

‎compaction_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1225,8 +1225,8 @@ func TestManualCompaction(t *testing.T) {
12251225
},
12261226
{
12271227
testData: "testdata/manual_compaction_set_with_del_sstable_Pebblev6",
1228-
minVersion: formatTableFormatV6,
1229-
maxVersion: formatTableFormatV6,
1228+
minVersion: FormatTableFormatV6,
1229+
maxVersion: FormatTableFormatV6,
12301230
},
12311231
}
12321232

‎format_major_version.go‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,16 @@ const (
204204
// as a commitment that the WAL should have been synced up until the offset.
205205
FormatWALSyncChunks
206206

207+
// FormatTableFormatV6 is a format major version enabling the sstable table
208+
// format TableFormatPebblev6.
209+
//
210+
// The TableFormatPebblev6 sstable format introduces a checksum within the
211+
// sstable footer, and allows inclusion of blob handle references within the
212+
// value column of a sstable block.
213+
//
214+
// This format major version does not yet enable use of value separation.
215+
FormatTableFormatV6
216+
207217
// -- Add new versions here --
208218

209219
// FormatNewest is the most recent format major version.
@@ -214,16 +224,6 @@ const (
214224

215225
// -- Add experimental versions here --
216226

217-
// formatTableFormatV6 is a format major version enabling the sstable table
218-
// format TableFormatPebblev6.
219-
//
220-
// The TableFormatPebblev6 sstable format introduces a checksum within the
221-
// sstable footer, and allows inclusion of blob handle references within the
222-
// value column of a sstable block.
223-
//
224-
// This format major version does not yet enable use of value separation.
225-
formatTableFormatV6
226-
227227
// internalFormatNewest is the most recent, possibly experimental format major
228228
// version.
229229
internalFormatNewest FormatMajorVersion = iota - 2
@@ -254,7 +254,7 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
254254
return sstable.TableFormatPebblev4
255255
case FormatColumnarBlocks, FormatWALSyncChunks:
256256
return sstable.TableFormatPebblev5
257-
case formatTableFormatV6:
257+
case FormatTableFormatV6:
258258
return sstable.TableFormatPebblev6
259259
default:
260260
panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
@@ -268,7 +268,7 @@ func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat {
268268
case FormatDefault, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted,
269269
FormatDeleteSizedAndObsolete, FormatVirtualSSTables, FormatSyntheticPrefixSuffix,
270270
FormatFlushableIngestExcises, FormatColumnarBlocks, FormatWALSyncChunks,
271-
formatTableFormatV6:
271+
FormatTableFormatV6:
272272
return sstable.TableFormatPebblev1
273273
default:
274274
panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
@@ -314,8 +314,8 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
314314
FormatWALSyncChunks: func(d *DB) error {
315315
return d.finalizeFormatVersUpgrade(FormatWALSyncChunks)
316316
},
317-
formatTableFormatV6: func(d *DB) error {
318-
return d.finalizeFormatVersUpgrade(formatTableFormatV6)
317+
FormatTableFormatV6: func(d *DB) error {
318+
return d.finalizeFormatVersUpgrade(FormatTableFormatV6)
319319
},
320320
}
321321

‎format_major_version_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ func TestFormatMajorVersionStableValues(t *testing.T) {
3030

3131
// When we add a new version, we should add a check for the new version in
3232
// addition to updating these expected values.
33-
require.Equal(t, FormatNewest, FormatMajorVersion(20))
33+
require.Equal(t, FormatNewest, FormatMajorVersion(21))
3434
require.Equal(t, internalFormatNewest, FormatMajorVersion(21))
3535
}
3636

@@ -64,8 +64,8 @@ func TestRatchetFormat(t *testing.T) {
6464
require.Equal(t, FormatColumnarBlocks, d.FormatMajorVersion())
6565
require.NoError(t, d.RatchetFormatMajorVersion(FormatWALSyncChunks))
6666
require.Equal(t, FormatWALSyncChunks, d.FormatMajorVersion())
67-
require.NoError(t, d.RatchetFormatMajorVersion(formatTableFormatV6))
68-
require.Equal(t, formatTableFormatV6, d.FormatMajorVersion())
67+
require.NoError(t, d.RatchetFormatMajorVersion(FormatTableFormatV6))
68+
require.Equal(t, FormatTableFormatV6, d.FormatMajorVersion())
6969

7070
require.NoError(t, d.Close())
7171

@@ -224,7 +224,7 @@ func TestFormatMajorVersions_TableFormat(t *testing.T) {
224224
FormatFlushableIngestExcises: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4},
225225
FormatColumnarBlocks: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev5},
226226
FormatWALSyncChunks: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev5},
227-
formatTableFormatV6: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev6},
227+
FormatTableFormatV6: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev6},
228228
}
229229

230230
// Valid versions.

‎range_del_test.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,7 @@ func TestRangeDelCompactionTruncation(t *testing.T) {
259259
FS: vfs.NewMem(),
260260
Levels: []LevelOptions{
261261
{TargetFileSize: 100},
262-
{TargetFileSize: 100},
262+
{TargetFileSize: 80},
263263
{TargetFileSize: 1},
264264
},
265265
DebugCheck: DebugCheckLevels,
@@ -381,9 +381,13 @@ L3:
381381
}
382382
}
383383

384+
// TODO(jackson): Create a datadriven test and exercise it on
385+
// TableFormatPebblev6 and later format major versions. This test is tightly
386+
// coupled to the current estimated sizes and won't produce the necessary
387+
// input LSM structure on later format major versions.
384388
versions := []FormatMajorVersion{
385389
FormatMinSupported,
386-
FormatNewest,
390+
FormatWALSyncChunks,
387391
}
388392
for _, version := range versions {
389393
t.Run(fmt.Sprintf("version-%s", version), func(t *testing.T) {

‎testdata/compaction_picker_pick_file‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@ L2:
1616
file-sizes
1717
----
1818
L1:
19-
000004:[b#11,SET-c#11,SET]: 743 bytes (743B)
19+
000004:[b#11,SET-c#11,SET]: 747 bytes (747B)
2020
L2:
21-
000005:[c#0,SET-d#0,SET]: 730 bytes (730B)
21+
000005:[c#0,SET-d#0,SET]: 734 bytes (734B)
2222

2323
pick-file L1
2424
----
@@ -131,12 +131,12 @@ L6:
131131
file-sizes
132132
----
133133
L5:
134-
000004:[c#11,SET-e#11,SET]: 99369 bytes (97KB)
135-
000005:[f#11,SET-f#11,SET]: 58090 bytes (57KB)
134+
000004:[c#11,SET-e#11,SET]: 99373 bytes (97KB)
135+
000005:[f#11,SET-f#11,SET]: 58094 bytes (57KB)
136136
L6:
137-
000006:[c#0,SET-c#0,SET]: 66284 bytes (65KB)
138-
000007:[e#0,SET-e#0,SET]: 66284 bytes (65KB)
139-
000008:[f#0,SET-f#0,SET]: 66284 bytes (65KB)
137+
000006:[c#0,SET-c#0,SET]: 66288 bytes (65KB)
138+
000007:[e#0,SET-e#0,SET]: 66288 bytes (65KB)
139+
000008:[f#0,SET-f#0,SET]: 66288 bytes (65KB)
140140

141141
# Sst 5 is picked since 65KB/57KB is less than 130KB/97KB.
142142
pick-file L5
@@ -168,12 +168,12 @@ file-sizes
168168
L5:
169169
000010:[c#11,SET-c#11,SET]: 32862 bytes (32KB)
170170
000011:[e#11,SET-e#11,SET]: 191 bytes (191B)
171-
000005:[f#11,SET-f#11,SET]: 58090 bytes (57KB)
171+
000005:[f#11,SET-f#11,SET]: 58094 bytes (57KB)
172172
L6:
173-
000006:[c#0,SET-c#0,SET]: 66284 bytes (65KB)
174-
000009:[d#13,SET-d#13,SET]: 728 bytes (728B)
175-
000007:[e#0,SET-e#0,SET]: 66284 bytes (65KB)
176-
000008:[f#0,SET-f#0,SET]: 66284 bytes (65KB)
173+
000006:[c#0,SET-c#0,SET]: 66288 bytes (65KB)
174+
000009:[d#13,SET-d#13,SET]: 732 bytes (732B)
175+
000007:[e#0,SET-e#0,SET]: 66288 bytes (65KB)
176+
000008:[f#0,SET-f#0,SET]: 66288 bytes (65KB)
177177

178178
# Superficially, sst 10 causes write amp of 65KB/32KB which is worse than sst
179179
# 5. But the garbage of ~64KB in the backing sst 4 is equally distributed
@@ -207,12 +207,12 @@ file-sizes
207207
----
208208
L5:
209209
000011:[e#11,SET-e#11,SET]: 191 bytes (191B)
210-
000005:[f#11,SET-f#11,SET]: 58090 bytes (57KB)
210+
000005:[f#11,SET-f#11,SET]: 58094 bytes (57KB)
211211
L6:
212-
000012:[c#15,SET-c#15,SET]: 728 bytes (728B)
213-
000009:[d#13,SET-d#13,SET]: 728 bytes (728B)
214-
000007:[e#0,SET-e#0,SET]: 66284 bytes (65KB)
215-
000008:[f#0,SET-f#0,SET]: 66284 bytes (65KB)
212+
000012:[c#15,SET-c#15,SET]: 732 bytes (732B)
213+
000009:[d#13,SET-d#13,SET]: 732 bytes (732B)
214+
000007:[e#0,SET-e#0,SET]: 66288 bytes (65KB)
215+
000008:[f#0,SET-f#0,SET]: 66288 bytes (65KB)
216216

217217
# Even though picking sst 11 seems to cause poor write amp of 65KB/126B, it is
218218
# picked because it is blamed for all the garbage in backing sst 4 (~96KB),

‎testdata/compaction_picker_scores‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ L1 0B 0.0
2424
L2 0B 0.0
2525
L3 0B 0.0
2626
L4 0B 0.0
27-
L5 720B 0.0
27+
L5 724B 0.0
2828
L6 321KB -
2929

3030
enable-table-stats
@@ -37,7 +37,7 @@ num-entries: 1
3737
num-deletions: 1
3838
num-range-key-sets: 0
3939
point-deletions-bytes-estimate: 0
40-
range-deletions-bytes-estimate: 328865
40+
range-deletions-bytes-estimate: 328869
4141

4242
scores
4343
----
@@ -47,7 +47,7 @@ L1 0B 0.0
4747
L2 0B 0.0
4848
L3 0B 0.0
4949
L4 0B 0.0
50-
L5 720B 4.5
50+
L5 724B 4.5
5151
L6 321KB -
5252

5353
# Ensure that point deletions in a higher level result in a compensated level
@@ -80,7 +80,7 @@ L1 0B 0.0
8080
L2 0B 0.0
8181
L3 0B 0.0
8282
L4 0B 0.0
83-
L5 774B 0.0
83+
L5 778B 0.0
8484
L6 321KB -
8585

8686
enable-table-stats
@@ -92,7 +92,7 @@ wait-pending-table-stats
9292
num-entries: 5
9393
num-deletions: 5
9494
num-range-key-sets: 0
95-
point-deletions-bytes-estimate: 164784
95+
point-deletions-bytes-estimate: 164788
9696
range-deletions-bytes-estimate: 0
9797

9898
scores
@@ -103,7 +103,7 @@ L1 0B 0.0
103103
L2 0B 0.0
104104
L3 0B 0.0
105105
L4 0B 0.0
106-
L5 774B 2.3
106+
L5 778B 2.3
107107
L6 321KB -
108108

109109
# Run a similar test as above, but this time the table containing the DELs is
@@ -145,7 +145,7 @@ wait-pending-table-stats
145145
num-entries: 5
146146
num-deletions: 5
147147
num-range-key-sets: 0
148-
point-deletions-bytes-estimate: 164792
148+
point-deletions-bytes-estimate: 164796
149149
range-deletions-bytes-estimate: 0
150150

151151
maybe-compact
@@ -212,11 +212,11 @@ L6 386KB -
212212
lsm verbose
213213
----
214214
L5:
215-
000004:[aa#2,SET-dd#2,SET] seqnums:[2-2] points:[aa#2,SET-dd#2,SET] size:525373
216-
000005:[e#2,SET-e#2,SET] seqnums:[2-2] points:[e#2,SET-e#2,SET] size:131828
215+
000004:[aa#2,SET-dd#2,SET] seqnums:[2-2] points:[aa#2,SET-dd#2,SET] size:525377
216+
000005:[e#2,SET-e#2,SET] seqnums:[2-2] points:[e#2,SET-e#2,SET] size:131832
217217
L6:
218-
000006:[a#1,SET-d#1,SET] seqnums:[1-1] points:[a#1,SET-d#1,SET] size:263225
219-
000007:[e#1,SET-e#1,SET] seqnums:[1-1] points:[e#1,SET-e#1,SET] size:131828
218+
000006:[a#1,SET-d#1,SET] seqnums:[1-1] points:[a#1,SET-d#1,SET] size:263229
219+
000007:[e#1,SET-e#1,SET] seqnums:[1-1] points:[e#1,SET-e#1,SET] size:131832
220220

221221
# Attempting to schedule a compaction should begin a L5->L6 compaction.
222222

‎testdata/ingest_external‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -330,8 +330,8 @@ gi: (foo, .)
330330
lsm verbose
331331
----
332332
L6:
333-
000004(000004):[gc#10,DELSIZED-gf#inf,RANGEDEL] seqnums:[10-10] points:[gc#10,DELSIZED-gf#inf,RANGEDEL] size:1329
334-
000005(000005):[gg#11,DELSIZED-gj#inf,RANGEDEL] seqnums:[11-11] points:[gg#11,DELSIZED-gj#inf,RANGEDEL] size:907
333+
000004(000004):[gc#10,DELSIZED-gf#inf,RANGEDEL] seqnums:[10-10] points:[gc#10,DELSIZED-gf#inf,RANGEDEL] size:1333
334+
000005(000005):[gg#11,DELSIZED-gj#inf,RANGEDEL] seqnums:[11-11] points:[gg#11,DELSIZED-gj#inf,RANGEDEL] size:911
335335

336336
download g h via-backing-file-download
337337
----
@@ -341,8 +341,8 @@ ok
341341
lsm verbose
342342
----
343343
L6:
344-
000006(000006):[gc#10,DELSIZED-gf#inf,RANGEDEL] seqnums:[10-10] points:[gc#10,DELSIZED-gf#inf,RANGEDEL] size:976
345-
000007(000007):[gg#11,DELSIZED-gj#inf,RANGEDEL] seqnums:[11-11] points:[gg#11,DELSIZED-gj#inf,RANGEDEL] size:891
344+
000006(000006):[gc#10,DELSIZED-gf#inf,RANGEDEL] seqnums:[10-10] points:[gc#10,DELSIZED-gf#inf,RANGEDEL] size:980
345+
000007(000007):[gg#11,DELSIZED-gj#inf,RANGEDEL] seqnums:[11-11] points:[gg#11,DELSIZED-gj#inf,RANGEDEL] size:895
346346

347347
reopen
348348
----

0 commit comments

Comments
 (0)