Skip to content

Commit f318b2d

Browse files
committed
sstable: use single physicalBlockMaker in columnar writer
The columnar writer has its own `physicalBlockMaker` and the `layoutWriter` has one as well. Using two for different purposes results in incomplete statistics. Consolidate and only use the one in `layoutWriter`.
1 parent a9841e6 commit f318b2d

33 files changed

+500
-498
lines changed

replay/testdata/corpus/simple_ingest

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,6 @@ simple_ingest/MANIFEST-000008:
101101
size: 209
102102
simple_ingest/MANIFEST-000010: stat simple_ingest/MANIFEST-000010: file does not exist
103103
simple_ingest/000007.sst:
104-
size: 757
104+
size: 770
105105
simple_ingest/000009.sst:
106-
size: 763
106+
size: 765

replay/testdata/corpus/simple_val_sep

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,6 @@ stat simple_val_sep/MANIFEST-000013 simple_val_sep/000015.sst simple_val_sep/000
103103
simple_val_sep/MANIFEST-000013:
104104
size: 250
105105
simple_val_sep/000015.sst:
106-
size: 865
106+
size: 878
107107
simple_val_sep/000016.blob:
108108
size: 97

replay/testdata/replay_ingest

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,24 +6,24 @@ tree
66
/
77
build/
88
36 000002.log
9-
757 000005.sst
10-
757 000007.sst
11-
763 000009.sst
9+
770 000005.sst
10+
770 000007.sst
11+
765 000009.sst
1212
0 LOCK
1313
172 MANIFEST-000001
1414
209 MANIFEST-000008
1515
2800 OPTIONS-000003
1616
0 marker.format-version.000012.025
1717
0 marker.manifest.000002.MANIFEST-000008
1818
simple_ingest/
19-
757 000007.sst
20-
763 000009.sst
19+
770 000007.sst
20+
765 000009.sst
2121
172 MANIFEST-000001
2222
209 MANIFEST-000008
2323
checkpoint/
2424
25 000002.log
25-
774 000004.sst
26-
757 000005.sst
25+
787 000004.sst
26+
770 000005.sst
2727
172 MANIFEST-000001
2828
2800 OPTIONS-000003
2929
0 marker.format-version.000001.025

replay/testdata/replay_val_sep

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,12 @@ tree
77
build/
88
906 000005.sst
99
101 000006.blob
10-
871 000008.sst
10+
884 000008.sst
1111
97 000009.blob
1212
92 000011.log
13-
756 000012.sst
13+
769 000012.sst
1414
63 000014.log
15-
865 000015.sst
15+
878 000015.sst
1616
97 000016.blob
1717
0 LOCK
1818
152 MANIFEST-000010
@@ -21,16 +21,16 @@ tree
2121
0 marker.format-version.000011.024
2222
0 marker.manifest.000003.MANIFEST-000013
2323
simple_val_sep/
24-
865 000015.sst
24+
878 000015.sst
2525
97 000016.blob
2626
250 MANIFEST-000013
2727
checkpoint/
2828
906 000005.sst
2929
101 000006.blob
30-
871 000008.sst
30+
884 000008.sst
3131
97 000009.blob
3232
64 000011.log
33-
756 000012.sst
33+
769 000012.sst
3434
187 MANIFEST-000013
3535
2936 OPTIONS-000003
3636
0 marker.format-version.000001.024

sstable/colblk_writer.go

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -111,10 +111,6 @@ type RawColumnWriter struct {
111111
validator invariants.Value[*colblk.DataBlockValidator]
112112
disableKeyOrderChecks bool
113113
cpuMeasurer base.CPUMeasurer
114-
115-
// RawColumnWriter writes data sequentially so each writer can have a
116-
// physical block maker.
117-
physBlockMaker block.PhysicalBlockMaker
118114
}
119115

120116
// Assert that *RawColumnWriter implements RawWriter.
@@ -148,12 +144,11 @@ func newColumnarWriter(
148144
w.topLevelIndexBlock.Init()
149145
w.rangeDelBlock.Init(w.comparer.Equal)
150146
w.rangeKeyBlock.Init(w.comparer.Equal)
151-
w.physBlockMaker.Init(w.opts.Compression, w.opts.Checksum)
152147
if !o.DisableValueBlocks {
153148
flushGovernor := block.MakeFlushGovernor(o.BlockSize, o.BlockSizeThreshold, o.SizeClassAwareThreshold, o.AllocatorSizeClasses)
154149
// We use the value block writer in the same goroutine so it's safe to share
155150
// the physBlockMaker.
156-
w.valueBlock = valblk.NewWriter(flushGovernor, &w.physBlockMaker, func(compressedSize int) {})
151+
w.valueBlock = valblk.NewWriter(flushGovernor, &w.layout.physBlockMaker, func(compressedSize int) {})
157152
}
158153
if o.FilterPolicy != base.NoFilterPolicy {
159154
switch o.FilterType {
@@ -714,7 +709,7 @@ func (w *RawColumnWriter) enqueueDataBlock(
714709
}
715710

716711
// Compress and checksum the data block and send it to the write queue.
717-
pb := w.physBlockMaker.Make(serializedBlock, blockkind.SSTableData, block.NoFlags)
712+
pb := w.layout.physBlockMaker.Make(serializedBlock, blockkind.SSTableData, block.NoFlags)
718713
return w.enqueuePhysicalBlock(pb, separator)
719714
}
720715

@@ -1035,7 +1030,7 @@ func (w *RawColumnWriter) Close() (err error) {
10351030
}
10361031
}
10371032

1038-
w.props.CompressionStats = w.physBlockMaker.Compressor.Stats().String()
1033+
w.props.CompressionStats = w.layout.physBlockMaker.Compressor.Stats().String()
10391034
var toWrite []byte
10401035
w.props.CompressionOptions = rocksDBCompressionOptions
10411036
if w.opts.TableFormat >= TableFormatPebblev7 {
@@ -1069,7 +1064,6 @@ func (w *RawColumnWriter) Close() (err error) {
10691064
return err
10701065
}
10711066
w.meta.Properties = w.props
1072-
w.physBlockMaker.Close()
10731067
// Release any held memory and make any future calls error.
10741068
*w = RawColumnWriter{meta: w.meta, err: errWriterClosed}
10751069
return nil
@@ -1089,7 +1083,7 @@ func (w *RawColumnWriter) rewriteSuffixes(
10891083
return errors.Wrap(err, "reading layout")
10901084
}
10911085
// Copy data blocks in parallel, rewriting suffixes as we go.
1092-
blocks, err := rewriteDataBlocksInParallel(r, sstBytes, wo, l.Data, from, to, concurrency, w.physBlockMaker.Compressor.Stats(), func() blockRewriter {
1086+
blocks, err := rewriteDataBlocksInParallel(r, sstBytes, wo, l.Data, from, to, concurrency, w.layout.physBlockMaker.Compressor.Stats(), func() blockRewriter {
10931087
return colblk.NewDataBlockRewriter(wo.KeySchema, w.comparer)
10941088
})
10951089
if err != nil {
@@ -1247,7 +1241,7 @@ func (w *RawColumnWriter) copyDataBlocks(
12471241
// using CopySpan().
12481242
func (w *RawColumnWriter) addDataBlock(b, sep []byte, bhp block.HandleWithProperties) error {
12491243
// Compress and checksum the data block and send it to the write queue.
1250-
pb := w.physBlockMaker.Make(b, blockkind.SSTableData, block.NoFlags)
1244+
pb := w.layout.physBlockMaker.Make(b, blockkind.SSTableData, block.NoFlags)
12511245
if err := w.enqueuePhysicalBlock(pb, sep); err != nil {
12521246
return err
12531247
}

0 commit comments

Comments
 (0)