Skip to content

Commit fac9d89

Browse files
committed
sstable: introduce Attributes bitset field for Reader
This commit introduces the bitset type `Attributes` to track which features are in use in an sstable. The field is derived from the sstable properties on Reader creation. Initially, we include the following attributes: - AttributeValueBlocks - AttributeRangeKeySets - AttributeRangeKeyUnsets - AttributeRangeKeyDels - AttributeRangeDels - AttributeBlobValues Closes: #4557
1 parent f81f12c commit fac9d89

File tree

12 files changed

+176
-23
lines changed

12 files changed

+176
-23
lines changed

external_iterator.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,7 @@ func openExternalTables(
310310
if err != nil {
311311
return readers, err
312312
}
313-
if r.Properties.NumValuesInBlobFiles > 0 {
313+
if r.Attributes.Has(sstable.AttributeBlobValues) {
314314
return readers, errors.Newf("pebble: NewExternalIter does not support blob references")
315315
}
316316
readers = append(readers, r)

file_cache.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -759,7 +759,7 @@ func newRangeKeyIter(
759759
// file's range key blocks may surface deleted range keys below. This is
760760
// done here, rather than deferring to the block-property collector in order
761761
// to maintain parity with point keys and the treatment of RANGEDELs.
762-
if r.Properties.NumRangeKeyDels == 0 && len(opts.RangeKeyFilters) > 0 {
762+
if !r.Attributes.Has(sstable.AttributeRangeKeyDels) && len(opts.RangeKeyFilters) > 0 {
763763
ok, _, err := checkAndIntersectFilters(r, opts.RangeKeyFilters, nil, transforms.SyntheticSuffix())
764764
if err != nil {
765765
return nil, err

internal/problemspans/doc.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
// This package is designed for efficiently tracking key ranges that may need
1111
// special handling.
1212
//
13-
// Key Features:
13+
// Key Attributes:
1414
//
1515
// - **Span Registration:**
1616
// Add spans with specified expiration times so that they automatically

sstable/attributes.go

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use
2+
// of this source code is governed by a BSD-style license that can be found in
3+
// the LICENSE file.
4+
5+
package sstable
6+
7+
import "strings"
8+
9+
// Attributes is a bitset containing features in use in an sstable.
10+
type Attributes uint32
11+
12+
const (
13+
AttributeValueBlocks Attributes = 1 << iota
14+
AttributeRangeKeySets
15+
AttributeRangeKeyUnsets
16+
AttributeRangeKeyDels
17+
AttributeRangeDels
18+
AttributeTwoLevelIndex
19+
AttributeBlobValues
20+
)
21+
22+
// Intersects checks if any bits in attr are set in a.
23+
func (a Attributes) Intersects(attr Attributes) bool {
24+
return a&attr != 0
25+
}
26+
27+
// Has checks if all bits in attr are set in a.
28+
func (a Attributes) Has(attr Attributes) bool {
29+
return a&attr == attr
30+
}
31+
32+
// Add sets the bits in attr to a.
33+
func (a *Attributes) Add(attr Attributes) {
34+
*a = *a | attr
35+
}
36+
37+
// String converts the Attributes fs to a string representation for testing.
38+
func (a Attributes) String() string {
39+
var attributes []string
40+
if a.Has(AttributeValueBlocks) {
41+
attributes = append(attributes, "ValueBlocks")
42+
}
43+
if a.Has(AttributeRangeKeySets) {
44+
attributes = append(attributes, "RangeKeySets")
45+
}
46+
if a.Has(AttributeRangeKeyUnsets) {
47+
attributes = append(attributes, "RangeKeyUnsets")
48+
}
49+
if a.Has(AttributeRangeKeyDels) {
50+
attributes = append(attributes, "RangeKeyDels")
51+
}
52+
if a.Has(AttributeRangeDels) {
53+
attributes = append(attributes, "RangeDels")
54+
}
55+
if a.Has(AttributeTwoLevelIndex) {
56+
attributes = append(attributes, "TwoLevelIndex")
57+
}
58+
if a.Has(AttributeBlobValues) {
59+
attributes = append(attributes, "BlobValues")
60+
}
61+
return "[" + strings.Join(attributes, ",") + "]"
62+
}

sstable/copier.go

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import (
88
"context"
99

1010
"github.com/cockroachdb/errors"
11-
"github.com/cockroachdb/pebble/internal/base"
1211
"github.com/cockroachdb/pebble/internal/bytealloc"
1312
"github.com/cockroachdb/pebble/objstorage"
1413
"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
@@ -51,7 +50,8 @@ func CopySpan(
5150
) (size uint64, _ error) {
5251
defer func() { _ = input.Close() }()
5352

54-
if r.Properties.NumValueBlocks > 0 || r.Properties.NumRangeKeys() > 0 || r.Properties.NumRangeDeletions > 0 {
53+
const unsupportedCopyFeatures = AttributeValueBlocks | AttributeRangeKeySets | AttributeRangeKeyUnsets | AttributeRangeKeyDels | AttributeRangeDels
54+
if r.Attributes.Intersects(unsupportedCopyFeatures) {
5555
return copyWholeFileBecauseOfUnsupportedFeature(ctx, input, output) // Finishes/Aborts output.
5656
}
5757

@@ -77,11 +77,6 @@ func CopySpan(
7777
}
7878
}()
7979

80-
if r.Properties.NumValueBlocks > 0 || r.Properties.NumRangeKeys() > 0 || r.Properties.NumRangeDeletions > 0 {
81-
// We just checked for these conditions above.
82-
return 0, base.AssertionFailedf("cannot CopySpan sstables with value blocks or range keys")
83-
}
84-
8580
var preallocRH objstorageprovider.PreallocatedReadHandle
8681
// ReadBeforeForIndexAndFilter attempts to read the top-level index, filter
8782
// and lower-level index blocks with one read.
@@ -224,7 +219,7 @@ func intersectingIndexEntries(
224219
if err != nil {
225220
return nil, err
226221
}
227-
if r.Properties.IndexType != twoLevelIndex {
222+
if !r.Attributes.Has(AttributeTwoLevelIndex) {
228223
entry := indexEntry{bh: bh, sep: top.Separator()}
229224
alloc, entry.bh.Props = alloc.Copy(entry.bh.Props)
230225
alloc, entry.sep = alloc.Copy(entry.sep)

sstable/reader.go

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ type Reader struct {
6565

6666
Properties Properties
6767
tableFormat TableFormat
68+
Attributes Attributes
6869
}
6970

7071
type ReadEnv struct {
@@ -130,7 +131,7 @@ func (r *Reader) newPointIter(ctx context.Context, opts IterOptions) (Iterator,
130131
// until the final iterator closes.
131132
var res Iterator
132133
var err error
133-
if r.Properties.IndexType == twoLevelIndex {
134+
if r.Attributes.Has(AttributeTwoLevelIndex) {
134135
if r.tableFormat.BlockColumnar() {
135136
res, err = newColumnBlockTwoLevelIterator(
136137
ctx, r, opts)
@@ -204,7 +205,7 @@ func (r *Reader) newCompactionIter(
204205
BlobContext: blobContext,
205206
}
206207

207-
if r.Properties.IndexType == twoLevelIndex {
208+
if r.Attributes.Has(AttributeTwoLevelIndex) {
208209
if !r.tableFormat.BlockColumnar() {
209210
i, err := newRowBlockTwoLevelIterator(ctx, r, opts)
210211
if err != nil {
@@ -897,6 +898,29 @@ func NewReader(ctx context.Context, f objstorage.Readable, o ReaderOptions) (*Re
897898
return nil, r.Close()
898899
}
899900

901+
// Set which attributes are in use based on property values.
902+
if r.Properties.NumValueBlocks > 0 || r.Properties.NumValuesInValueBlocks > 0 {
903+
r.Attributes.Add(AttributeValueBlocks)
904+
}
905+
if r.Properties.NumRangeKeySets > 0 {
906+
r.Attributes.Add(AttributeRangeKeySets)
907+
}
908+
if r.Properties.NumRangeKeyUnsets > 0 {
909+
r.Attributes.Add(AttributeRangeKeyUnsets)
910+
}
911+
if r.Properties.NumRangeKeyDels > 0 {
912+
r.Attributes.Add(AttributeRangeKeyDels)
913+
}
914+
if r.Properties.NumRangeDeletions > 0 {
915+
r.Attributes.Add(AttributeRangeDels)
916+
}
917+
if r.Properties.IndexType == twoLevelIndex {
918+
r.Attributes.Add(AttributeTwoLevelIndex)
919+
}
920+
if r.Properties.NumValuesInBlobFiles > 0 {
921+
r.Attributes.Add(AttributeBlobValues)
922+
}
923+
900924
if r.Properties.ComparerName == "" || o.Comparer.Name == r.Properties.ComparerName {
901925
r.Comparer = o.Comparer
902926
} else if comparer, ok := o.Comparers[r.Properties.ComparerName]; ok {

sstable/reader_iter_single_lvl.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ func newColumnBlockSingleLevelIterator(
209209
}
210210
i := singleLevelIterColumnBlockPool.Get().(*singleLevelIteratorColumnBlocks)
211211
i.init(ctx, r, opts)
212-
if r.Properties.NumValueBlocks > 0 {
212+
if r.Attributes.Has(AttributeValueBlocks) {
213213
i.internalValueConstructor.vbReader = valblk.MakeReader(i, opts.ReaderProvider, r.valueBIH, opts.Env.Block.Stats)
214214
i.vbRH = r.blockReader.UsePreallocatedReadHandle(objstorage.NoReadBefore, &i.vbRHPrealloc)
215215
}
@@ -244,7 +244,7 @@ func newRowBlockSingleLevelIterator(
244244
i := singleLevelIterRowBlockPool.Get().(*singleLevelIteratorRowBlocks)
245245
i.init(ctx, r, opts)
246246
if r.tableFormat >= TableFormatPebblev3 {
247-
if r.Properties.NumValueBlocks > 0 {
247+
if r.Attributes.Has(AttributeValueBlocks) {
248248
i.internalValueConstructor.vbReader = valblk.MakeReader(i, opts.ReaderProvider, r.valueBIH, opts.Env.Block.Stats)
249249
// We can set the GetLazyValuer directly to the vbReader because
250250
// rowblk sstables never contain blob value handles.

sstable/reader_iter_two_lvl.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ func newColumnBlockTwoLevelIterator(
166166
i.useFilterBlock = i.secondLevel.useFilterBlock
167167
i.secondLevel.useFilterBlock = false
168168

169-
if r.Properties.NumValueBlocks > 0 {
169+
if r.Attributes.Has(AttributeValueBlocks) {
170170
// NB: we cannot avoid this ~248 byte allocation, since valueBlockReader
171171
// can outlive the singleLevelIterator due to be being embedded in a
172172
// LazyValue. This consumes ~2% in microbenchmark CPU profiles, but we
@@ -215,7 +215,7 @@ func newRowBlockTwoLevelIterator(
215215
i.secondLevel.useFilterBlock = false
216216

217217
if r.tableFormat >= TableFormatPebblev3 {
218-
if r.Properties.NumValueBlocks > 0 {
218+
if r.Attributes.Has(AttributeValueBlocks) {
219219
// NB: we cannot avoid this ~248 byte allocation, since valueBlockReader
220220
// can outlive the singleLevelIterator due to be being embedded in a
221221
// LazyValue. This consumes ~2% in microbenchmark CPU profiles, but we

sstable/reader_test.go

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -558,6 +558,18 @@ func TestReaderWithBlockPropertyFilter(t *testing.T) {
558558
})
559559
}
560560

561+
func TestReaderAttributes(t *testing.T) {
562+
defer leaktest.AfterTest(t)()
563+
opts := WriterOptions{
564+
// Use format supporting blob handles.
565+
TableFormat: TableFormatPebblev6,
566+
BlockSize: math.MaxInt32,
567+
IndexBlockSize: math.MaxInt32,
568+
Comparer: testkeys.Comparer,
569+
}
570+
runTestReader(t, opts, "testdata/reader_attributes", nil /* Reader */, false)
571+
}
572+
561573
func TestInjectedErrors(t *testing.T) {
562574
defer leaktest.AfterTest(t)()
563575
for _, fixture := range TestFixtures {
@@ -702,11 +714,12 @@ func runTestReader(t *testing.T, o WriterOptions, dir string, r *Reader, printVa
702714
r = nil
703715
}
704716
var cacheSize int
705-
var printLayout bool
717+
var printLayout, printAttributes bool
706718
d.MaybeScanArgs(t, "cache-size", &cacheSize)
707719
d.MaybeScanArgs(t, "print-layout", &printLayout)
708720
d.MaybeScanArgs(t, "block-size", &o.BlockSize)
709721
d.MaybeScanArgs(t, "index-block-size", &o.IndexBlockSize)
722+
d.MaybeScanArgs(t, "print-attributes", &printAttributes)
710723

711724
closeCache()
712725
c = cache.New(int64(cacheSize))
@@ -716,10 +729,15 @@ func runTestReader(t *testing.T, o WriterOptions, dir string, r *Reader, printVa
716729
if err != nil {
717730
return err.Error()
718731
}
732+
var buf bytes.Buffer
719733
if printLayout {
720-
return indexLayoutString(t, r)
734+
buf.WriteString(indexLayoutString(t, r))
721735
}
722-
return ""
736+
if printAttributes {
737+
buf.WriteString("attributes: ")
738+
buf.WriteString(r.Attributes.String())
739+
}
740+
return buf.String()
723741

724742
case "iter":
725743
var globalSeqNum uint64

sstable/suffix_rewriter.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ func rewriteKeySuffixesInBlocks(
8787
switch {
8888
case concurrency < 1:
8989
return nil, TableFormatUnspecified, errors.New("concurrency must be >= 1")
90-
case r.Properties.NumValueBlocks > 0 || r.Properties.NumValuesInValueBlocks > 0:
90+
case r.Attributes.Has(AttributeValueBlocks):
9191
return nil, TableFormatUnspecified,
9292
errors.New("sstable with a single suffix should not have value blocks")
9393
case r.Properties.ComparerName != o.Comparer.Name:
@@ -308,7 +308,7 @@ func RewriteKeySuffixesViaWriter(
308308
if o.Comparer == nil || o.Comparer.Split == nil {
309309
return nil, errors.New("a valid splitter is required to rewrite suffixes")
310310
}
311-
if r.Properties.NumValuesInBlobFiles > 0 {
311+
if r.Attributes.Has(AttributeBlobValues) {
312312
return nil, errors.New("cannot rewrite suffixes of sstable with blob values")
313313
}
314314

0 commit comments

Comments
 (0)