Skip to content

Commit d7e89db

Browse files
RaduBerindeRaduBerinde
committed
sstable: use a single compressor in columnar writer
The columnar writer has its own compressor and the layout writer has one too. Using both leads to incomplete stats. Consolidate and use only the layout writer.
1 parent 36d0909 commit d7e89db

31 files changed

+502
-499
lines changed

‎replay/testdata/corpus/simple_val_sep‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,6 @@ stat simple_val_sep/MANIFEST-000013 simple_val_sep/000015.sst simple_val_sep/000
103103
simple_val_sep/MANIFEST-000013:
104104
size: 250
105105
simple_val_sep/000015.sst:
106-
size: 865
106+
size: 878
107107
simple_val_sep/000016.blob:
108108
size: 97

‎replay/testdata/replay_val_sep‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,12 @@ tree
77
build/
88
906 000005.sst
99
101 000006.blob
10-
871 000008.sst
10+
884 000008.sst
1111
97 000009.blob
1212
92 000011.log
13-
756 000012.sst
13+
769 000012.sst
1414
63 000014.log
15-
865 000015.sst
15+
878 000015.sst
1616
97 000016.blob
1717
0 LOCK
1818
152 MANIFEST-000010
@@ -21,16 +21,16 @@ tree
2121
0 marker.format-version.000011.024
2222
0 marker.manifest.000003.MANIFEST-000013
2323
simple_val_sep/
24-
865 000015.sst
24+
878 000015.sst
2525
97 000016.blob
2626
250 MANIFEST-000013
2727
checkpoint/
2828
906 000005.sst
2929
101 000006.blob
30-
871 000008.sst
30+
884 000008.sst
3131
97 000009.blob
3232
64 000011.log
33-
756 000012.sst
33+
769 000012.sst
3434
187 MANIFEST-000013
3535
2936 OPTIONS-000003
3636
0 marker.format-version.000001.024

‎sstable/colblk_writer.go‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -111,9 +111,6 @@ type RawColumnWriter struct {
111111
validator invariants.Value[*colblk.DataBlockValidator]
112112
disableKeyOrderChecks bool
113113
cpuMeasurer base.CPUMeasurer
114-
115-
// RawColumnWriter writes data sequentially so each writer can have a compressor
116-
compressor block.Compressor
117114
}
118115

119116
// Assert that *RawColumnWriter implements RawWriter.
@@ -150,7 +147,7 @@ func newColumnarWriter(
150147
if !o.DisableValueBlocks {
151148
w.valueBlock = valblk.NewWriter(
152149
block.MakeFlushGovernor(o.BlockSize, o.BlockSizeThreshold, o.SizeClassAwareThreshold, o.AllocatorSizeClasses),
153-
&w.compressor, w.opts.Checksum, func(compressedSize int) {})
150+
&w.layout.compressor, w.opts.Checksum, func(compressedSize int) {})
154151
}
155152
if o.FilterPolicy != base.NoFilterPolicy {
156153
switch o.FilterType {
@@ -196,7 +193,6 @@ func newColumnarWriter(
196193
w.cpuMeasurer = cpuMeasurer
197194
go w.drainWriteQueue()
198195

199-
w.compressor = block.MakeCompressor(w.opts.Compression)
200196
return w
201197
}
202198

@@ -729,7 +725,7 @@ func (w *RawColumnWriter) enqueueDataBlock(
729725
&cb.blockBuf.dataBuf,
730726
serializedBlock,
731727
blockkind.SSTableData,
732-
&w.compressor,
728+
&w.layout.compressor,
733729
&cb.blockBuf.checksummer,
734730
)
735731
return w.enqueuePhysicalBlock(cb, separator)
@@ -1047,7 +1043,7 @@ func (w *RawColumnWriter) Close() (err error) {
10471043
}
10481044
}
10491045

1050-
w.props.CompressionStats = w.compressor.Stats().String()
1046+
w.props.CompressionStats = w.layout.compressor.Stats().String()
10511047
var toWrite []byte
10521048
w.props.CompressionOptions = rocksDBCompressionOptions
10531049
if w.opts.TableFormat >= TableFormatPebblev7 {
@@ -1081,7 +1077,6 @@ func (w *RawColumnWriter) Close() (err error) {
10811077
return err
10821078
}
10831079
w.meta.Properties = w.props
1084-
w.compressor.Close()
10851080
// Release any held memory and make any future calls error.
10861081
*w = RawColumnWriter{meta: w.meta, err: errWriterClosed}
10871082
return nil
@@ -1101,7 +1096,7 @@ func (w *RawColumnWriter) rewriteSuffixes(
11011096
return errors.Wrap(err, "reading layout")
11021097
}
11031098
// Copy data blocks in parallel, rewriting suffixes as we go.
1104-
blocks, err := rewriteDataBlocksInParallel(r, sstBytes, wo, l.Data, from, to, concurrency, w.compressor.Stats(), func() blockRewriter {
1099+
blocks, err := rewriteDataBlocksInParallel(r, sstBytes, wo, l.Data, from, to, concurrency, w.layout.compressor.Stats(), func() blockRewriter {
11051100
return colblk.NewDataBlockRewriter(wo.KeySchema, w.comparer)
11061101
})
11071102
if err != nil {
@@ -1270,7 +1265,7 @@ func (w *RawColumnWriter) addDataBlock(b, sep []byte, bhp block.HandleWithProper
12701265
&cb.blockBuf.dataBuf,
12711266
b,
12721267
blockkind.SSTableData,
1273-
&w.compressor,
1268+
&w.layout.compressor,
12741269
&cb.blockBuf.checksummer,
12751270
)
12761271
if err := w.enqueuePhysicalBlock(cb, sep); err != nil {

‎sstable/testdata/columnar_writer/simple_binary‎

Lines changed: 89 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -80,39 +80,39 @@ sstable
8080
│ ├── 00000 block:0/88
8181
│ │
8282
│ └── trailer [compression=none checksum=0x6147006e]
83-
├── properties offset: 149 length: 659
83+
├── properties offset: 149 length: 661
8484
│ ├── 00000 obsolete-key (16) [restart]
8585
│ ├── 00016 pebble.colblk.schema (68)
86-
│ ├── 00084 pebble.compression_stats (37)
87-
│ ├── 00121 pebble.internal.testkeys.suffixes (41)
88-
│ ├── 00162 pebble.raw.point-tombstone.key.size (32)
89-
│ ├── 00194 rocksdb.block.based.table.index.type (43)
90-
│ ├── 00237 rocksdb.comparator (37)
91-
│ ├── 00274 rocksdb.compression (23)
92-
│ ├── 00297 rocksdb.compression_options (106)
93-
│ ├── 00403 rocksdb.data.size (13)
94-
│ ├── 00416 rocksdb.deleted.keys (15)
95-
│ ├── 00431 rocksdb.filter.size (15)
96-
│ ├── 00446 rocksdb.index.size (14)
97-
│ ├── 00460 rocksdb.merge.operands (18)
98-
│ ├── 00478 rocksdb.merge.operator (24)
99-
│ ├── 00502 rocksdb.num.data.blocks (19)
100-
│ ├── 00521 rocksdb.num.entries (11)
101-
│ ├── 00532 rocksdb.num.range-deletions (19)
102-
│ ├── 00551 rocksdb.property.collectors (70)
103-
│ ├── 00621 rocksdb.raw.key.size (16)
104-
│ ├── 00637 rocksdb.raw.value.size (14)
86+
│ ├── 00084 pebble.compression_stats (39)
87+
│ ├── 00123 pebble.internal.testkeys.suffixes (41)
88+
│ ├── 00164 pebble.raw.point-tombstone.key.size (32)
89+
│ ├── 00196 rocksdb.block.based.table.index.type (43)
90+
│ ├── 00239 rocksdb.comparator (37)
91+
│ ├── 00276 rocksdb.compression (23)
92+
│ ├── 00299 rocksdb.compression_options (106)
93+
│ ├── 00405 rocksdb.data.size (13)
94+
│ ├── 00418 rocksdb.deleted.keys (15)
95+
│ ├── 00433 rocksdb.filter.size (15)
96+
│ ├── 00448 rocksdb.index.size (14)
97+
│ ├── 00462 rocksdb.merge.operands (18)
98+
│ ├── 00480 rocksdb.merge.operator (24)
99+
│ ├── 00504 rocksdb.num.data.blocks (19)
100+
│ ├── 00523 rocksdb.num.entries (11)
101+
│ ├── 00534 rocksdb.num.range-deletions (19)
102+
│ ├── 00553 rocksdb.property.collectors (70)
103+
│ ├── 00623 rocksdb.raw.key.size (16)
104+
│ ├── 00639 rocksdb.raw.value.size (14)
105105
│ ├── restart points
106-
│ │ └── 00651 [restart 0]
107-
│ └── trailer [compression=none checksum=0x73801b9f]
108-
├── meta-index offset: 813 length: 33
109-
│ ├── 0000 rocksdb.properties block:149/659 [restart]
106+
│ │ └── 00653 [restart 0]
107+
│ └── trailer [compression=none checksum=0x78d91a9f]
108+
├── meta-index offset: 815 length: 33
109+
│ ├── 0000 rocksdb.properties block:149/661 [restart]
110110
│ ├── restart points
111111
│ │ └── 00025 [restart 0]
112-
│ └── trailer [compression=none checksum=0x842c7bfd]
113-
└── footer offset: 851 length: 53
112+
│ └── trailer [compression=none checksum=0x590655dc]
113+
└── footer offset: 853 length: 53
114114
├── 000 checksum type: crc32c
115-
├── 001 meta: offset=813, length=33
115+
├── 001 meta: offset=815, length=33
116116
├── 004 index: offset=93, length=51
117117
├── 041 version: 5
118118
└── 045 magic number: 0xf09faab3f09faab3
@@ -137,7 +137,7 @@ pebble.colblk.schema: DefaultKeySchema(pebble.internal.testkeys,16)
137137
rocksdb.merge.operator: pebble.concatenate
138138
rocksdb.merge.operands: 0
139139
rocksdb.property.collectors: [pebble.internal.testkeys.suffixes,obsolete-key]
140-
pebble.compression_stats: NoCompression:88/88
140+
pebble.compression_stats: NoCompression:139/139
141141
obsolete-key: hex:01
142142
pebble.internal.testkeys.suffixes: hex:0000ffffffffffffffffff01
143143

@@ -253,7 +253,7 @@ sstable
253253
│ ├── 00607 rocksdb.raw.value.size (14)
254254
│ ├── restart points
255255
│ │ └── 00621 [restart 0]
256-
│ └── trailer [compression=none checksum=0x17600a6]
256+
│ └── trailer [compression=none checksum=0xf4e8ad4]
257257
├── meta-index offset: 811 length: 33
258258
│ ├── 0000 rocksdb.properties block:177/629 [restart]
259259
│ ├── restart points
@@ -649,7 +649,7 @@ sstable
649649
│ ├── 00609 rocksdb.raw.value.size (15)
650650
│ ├── restart points
651651
│ │ └── 00624 [restart 0]
652-
│ └── trailer [compression=none checksum=0xaeac6223]
652+
│ └── trailer [compression=none checksum=0xfdfb788c]
653653
├── meta-index offset: 1334 length: 33
654654
│ ├── 0000 rocksdb.properties block:697/632 [restart]
655655
│ ├── restart points
@@ -685,7 +685,7 @@ pebble.colblk.schema: DefaultKeySchema(pebble.internal.testkeys,16)
685685
rocksdb.merge.operator: pebble.concatenate
686686
rocksdb.merge.operands: 0
687687
rocksdb.property.collectors: [pebble.internal.testkeys.suffixes,obsolete-key]
688-
pebble.compression_stats: NoCompression:562/562
688+
pebble.compression_stats: NoCompression:672/672
689689
obsolete-key: hex:01
690690
pebble.internal.testkeys.suffixes: hex:0000ffffffffffffffffff01
691691

@@ -1042,7 +1042,7 @@ sstable
10421042
│ ├── 00609 rocksdb.raw.value.size (15)
10431043
│ ├── restart points
10441044
│ │ └── 00624 [restart 0]
1045-
│ └── trailer [compression=none checksum=0xaeac6223]
1045+
│ └── trailer [compression=none checksum=0xfdfb788c]
10461046
├── meta-index offset: 1334 length: 33
10471047
│ ├── 0000 rocksdb.properties block:697/632 [restart]
10481048
│ ├── restart points
@@ -1117,39 +1117,40 @@ sstable
11171117
│ │ │ └── 50-50: x # data[0]:
11181118
│ │ └── 50-51: x 00 # block padding byte
11191119
│ └── trailer [compression=none checksum=0xb1e3982b]
1120-
├── properties offset: 89 length: 580
1120+
├── properties offset: 89 length: 617
11211121
│ ├── 00000 obsolete-key (17) [restart]
11221122
│ ├── 00017 pebble.colblk.schema (68)
1123-
│ ├── 00085 pebble.internal.testkeys.suffixes (30)
1124-
│ ├── 00115 rocksdb.block.based.table.index.type (43)
1125-
│ ├── 00158 rocksdb.comparator (37)
1126-
│ ├── 00195 rocksdb.compression (23)
1127-
│ ├── 00218 rocksdb.compression_options (106)
1128-
│ ├── 00324 rocksdb.data.size (13)
1129-
│ ├── 00337 rocksdb.deleted.keys (15)
1130-
│ ├── 00352 rocksdb.filter.size (15)
1131-
│ ├── 00367 rocksdb.index.size (14)
1132-
│ ├── 00381 rocksdb.merge.operands (18)
1133-
│ ├── 00399 rocksdb.merge.operator (24)
1134-
│ ├── 00423 rocksdb.num.data.blocks (19)
1135-
│ ├── 00442 rocksdb.num.entries (11)
1136-
│ ├── 00453 rocksdb.num.range-deletions (19)
1137-
│ ├── 00472 rocksdb.property.collectors (70)
1138-
│ ├── 00542 rocksdb.raw.key.size (16)
1139-
│ ├── 00558 rocksdb.raw.value.size (14)
1123+
│ ├── 00085 pebble.compression_stats (37)
1124+
│ ├── 00122 pebble.internal.testkeys.suffixes (30)
1125+
│ ├── 00152 rocksdb.block.based.table.index.type (43)
1126+
│ ├── 00195 rocksdb.comparator (37)
1127+
│ ├── 00232 rocksdb.compression (23)
1128+
│ ├── 00255 rocksdb.compression_options (106)
1129+
│ ├── 00361 rocksdb.data.size (13)
1130+
│ ├── 00374 rocksdb.deleted.keys (15)
1131+
│ ├── 00389 rocksdb.filter.size (15)
1132+
│ ├── 00404 rocksdb.index.size (14)
1133+
│ ├── 00418 rocksdb.merge.operands (18)
1134+
│ ├── 00436 rocksdb.merge.operator (24)
1135+
│ ├── 00460 rocksdb.num.data.blocks (19)
1136+
│ ├── 00479 rocksdb.num.entries (11)
1137+
│ ├── 00490 rocksdb.num.range-deletions (19)
1138+
│ ├── 00509 rocksdb.property.collectors (70)
1139+
│ ├── 00579 rocksdb.raw.key.size (16)
1140+
│ ├── 00595 rocksdb.raw.value.size (14)
11401141
│ ├── restart points
1141-
│ │ └── 00572 [restart 0]
1142-
│ └── trailer [compression=none checksum=0x50f07e7e]
1143-
├── meta-index offset: 674 length: 59
1144-
│ ├── 0000 rocksdb.properties block:89/580 [restart]
1142+
│ │ └── 00609 [restart 0]
1143+
│ └── trailer [compression=none checksum=0x9dc2b838]
1144+
├── meta-index offset: 711 length: 59
1145+
│ ├── 0000 rocksdb.properties block:89/617 [restart]
11451146
│ ├── 0024 rocksdb.range_del2 block:33/51 [restart]
11461147
│ ├── restart points
11471148
│ │ ├── 00047 [restart 0]
11481149
│ │ └── 00051 [restart 24]
1149-
│ └── trailer [compression=none checksum=0xb95e4caa]
1150-
└── footer offset: 738 length: 53
1150+
│ └── trailer [compression=none checksum=0xa7304de4]
1151+
└── footer offset: 775 length: 53
11511152
├── 000 checksum type: crc32c
1152-
├── 001 meta: offset=674, length=59
1153+
├── 001 meta: offset=711, length=59
11531154
├── 004 index: offset=0, length=28
11541155
├── 041 version: 5
11551156
└── 045 magic number: 0xf09faab3f09faab3
@@ -1230,44 +1231,45 @@ sstable
12301231
│ │ │ └── 67-67: x # data[1]:
12311232
│ │ └── 67-68: x 00 # block padding byte
12321233
│ └── trailer [compression=none checksum=0x45325be7]
1233-
├── properties offset: 106 length: 662
1234+
├── properties offset: 106 length: 699
12341235
│ ├── 00000 obsolete-key (17) [restart]
12351236
│ ├── 00017 pebble.colblk.schema (68)
1236-
│ ├── 00085 pebble.internal.testkeys.suffixes (32)
1237-
│ ├── 00117 pebble.num.range-key-dels (22)
1238-
│ ├── 00139 pebble.num.range-key-sets (8)
1239-
│ ├── 00147 pebble.num.range-key-unsets (10)
1240-
│ ├── 00157 pebble.raw.range-key.key.size (26)
1241-
│ ├── 00183 pebble.raw.range-key.value.size (14)
1242-
│ ├── 00197 rocksdb.block.based.table.index.type (43)
1243-
│ ├── 00240 rocksdb.comparator (37)
1244-
│ ├── 00277 rocksdb.compression (23)
1245-
│ ├── 00300 rocksdb.compression_options (106)
1246-
│ ├── 00406 rocksdb.data.size (13)
1247-
│ ├── 00419 rocksdb.deleted.keys (15)
1248-
│ ├── 00434 rocksdb.filter.size (15)
1249-
│ ├── 00449 rocksdb.index.size (14)
1250-
│ ├── 00463 rocksdb.merge.operands (18)
1251-
│ ├── 00481 rocksdb.merge.operator (24)
1252-
│ ├── 00505 rocksdb.num.data.blocks (19)
1253-
│ ├── 00524 rocksdb.num.entries (11)
1254-
│ ├── 00535 rocksdb.num.range-deletions (19)
1255-
│ ├── 00554 rocksdb.property.collectors (70)
1256-
│ ├── 00624 rocksdb.raw.key.size (16)
1257-
│ ├── 00640 rocksdb.raw.value.size (14)
1237+
│ ├── 00085 pebble.compression_stats (37)
1238+
│ ├── 00122 pebble.internal.testkeys.suffixes (32)
1239+
│ ├── 00154 pebble.num.range-key-dels (22)
1240+
│ ├── 00176 pebble.num.range-key-sets (8)
1241+
│ ├── 00184 pebble.num.range-key-unsets (10)
1242+
│ ├── 00194 pebble.raw.range-key.key.size (26)
1243+
│ ├── 00220 pebble.raw.range-key.value.size (14)
1244+
│ ├── 00234 rocksdb.block.based.table.index.type (43)
1245+
│ ├── 00277 rocksdb.comparator (37)
1246+
│ ├── 00314 rocksdb.compression (23)
1247+
│ ├── 00337 rocksdb.compression_options (106)
1248+
│ ├── 00443 rocksdb.data.size (13)
1249+
│ ├── 00456 rocksdb.deleted.keys (15)
1250+
│ ├── 00471 rocksdb.filter.size (15)
1251+
│ ├── 00486 rocksdb.index.size (14)
1252+
│ ├── 00500 rocksdb.merge.operands (18)
1253+
│ ├── 00518 rocksdb.merge.operator (24)
1254+
│ ├── 00542 rocksdb.num.data.blocks (19)
1255+
│ ├── 00561 rocksdb.num.entries (11)
1256+
│ ├── 00572 rocksdb.num.range-deletions (19)
1257+
│ ├── 00591 rocksdb.property.collectors (70)
1258+
│ ├── 00661 rocksdb.raw.key.size (16)
1259+
│ ├── 00677 rocksdb.raw.value.size (14)
12581260
│ ├── restart points
1259-
│ │ └── 00654 [restart 0]
1260-
│ └── trailer [compression=none checksum=0x29bae4b8]
1261-
├── meta-index offset: 773 length: 57
1261+
│ │ └── 00691 [restart 0]
1262+
│ └── trailer [compression=none checksum=0xc9180526]
1263+
├── meta-index offset: 810 length: 57
12621264
│ ├── 0000 pebble.range_key block:33/68 [restart]
1263-
│ ├── 0021 rocksdb.properties block:106/662 [restart]
1265+
│ ├── 0021 rocksdb.properties block:106/699 [restart]
12641266
│ ├── restart points
12651267
│ │ ├── 00045 [restart 0]
12661268
│ │ └── 00049 [restart 21]
1267-
│ └── trailer [compression=none checksum=0x1a6d95f4]
1268-
└── footer offset: 835 length: 53
1269+
│ └── trailer [compression=none checksum=0x996f900b]
1270+
└── footer offset: 872 length: 53
12691271
├── 000 checksum type: crc32c
1270-
├── 001 meta: offset=773, length=57
1272+
├── 001 meta: offset=810, length=57
12711273
├── 004 index: offset=0, length=28
12721274
├── 041 version: 5
12731275
└── 045 magic number: 0xf09faab3f09faab3

0 commit comments

Comments
 (0)