sstable: track value separation policy props used

Add ValueSeparationKind and ValueSeparationMinSize to sstable properties to
track the value separation policy used when writing the table.

Author: xinhaoz

internal/compact/run.go

@@ -124,6 +124,11 @@ type ValueSeparation interface {
 	// sstable. It can be used to configure value separation specifically for
 	// the next compaction output.
 	SetNextOutputConfig(config ValueSeparationOutputConfig)
+	// Kind returns the kind of value separation strategy being used.
+	Kind() sstable.ValueSeparationKind
+	// MinimumSize returns the minimum size a value must be in order to be
+	// separated into a blob file.
+	MinimumSize() int
 	// EstimatedFileSize returns an estimate of the disk space consumed by the
 	// current, pending blob file if it were closed now. If no blob file has
 	// been created, it returns 0.
@@ -233,6 +238,10 @@ func (r *Runner) WriteTable(
 	if r.err != nil {
 		panic("error already encountered")
 	}
+
+	// Set the value separation kind on the writer based on the strategy being used.
+	tw.SetValueSeparationProps(valueSeparation.Kind(), uint64(valueSeparation.MinimumSize()))
+
 	r.tables = append(r.tables, OutputTable{
 		CreationTime: time.Now(),
 		ObjMeta:      objMeta,
@@ -498,6 +507,14 @@ var _ ValueSeparation = NeverSeparateValues{}
 // SetNextOutputConfig implements the ValueSeparation interface.
 func (NeverSeparateValues) SetNextOutputConfig(config ValueSeparationOutputConfig) {}
 
+// Kind implements the ValueSeparation interface.
+func (NeverSeparateValues) Kind() sstable.ValueSeparationKind {
+	return sstable.ValueSeparationNone
+}
+
+// MinimumSize implements the ValueSeparation interface.
+func (v NeverSeparateValues) MinimumSize() int { return 0 }
+
 // EstimatedFileSize implements the ValueSeparation interface.
 func (NeverSeparateValues) EstimatedFileSize() uint64 { return 0 }
 

replay/testdata/corpus/simple_val_sep

@@ -103,6 +103,6 @@ stat simple_val_sep/MANIFEST-000013 simple_val_sep/000015.sst simple_val_sep/000
 simple_val_sep/MANIFEST-000013:
   size: 250
 simple_val_sep/000015.sst:
-  size: 760
+  size: 802
 simple_val_sep/000016.blob:
   size: 97

replay/testdata/replay_val_sep

@@ -5,14 +5,14 @@ tree
 ----
           /
             build/
-     787      000005.sst
+     829      000005.sst
      101      000006.blob
-     766      000008.sst
+     808      000008.sst
       97      000009.blob
       92      000011.log
-     660      000012.sst
+     707      000012.sst
       63      000014.log
-     760      000015.sst
+     802      000015.sst
       97      000016.blob
        0      LOCK
      152      MANIFEST-000010
@@ -21,16 +21,16 @@ tree
        0      marker.format-version.000011.024
        0      marker.manifest.000003.MANIFEST-000013
             simple_val_sep/
-     760      000015.sst
+     802      000015.sst
       97      000016.blob
      250      MANIFEST-000013
               checkpoint/
-     787        000005.sst
+     829        000005.sst
      101        000006.blob
-     766        000008.sst
+     808        000008.sst
       97        000009.blob
       11        000011.log
-     660        000012.sst
+     707        000012.sst
      187        MANIFEST-000013
     2947        OPTIONS-000003
        0        marker.format-version.000001.024

sstable/colblk_writer.go

@@ -1272,6 +1272,12 @@ func (w *RawColumnWriter) copyProperties(props Properties) {
 	w.props.IndexType = 0
 }
 
+// SetValueSeparationProps implements RawWriter.
+func (w *RawColumnWriter) SetValueSeparationProps(kind ValueSeparationKind, minValueSize uint64) {
+	w.props.ValueSeparationKind = uint8(kind)
+	w.props.ValueSeparationMinSize = minValueSize
+}
+
 // IsLikelyMVCCGarbage determines whether the given user key is likely MVCC
 // garbage.
 //

sstable/internal/genprops/gen_props.go

@@ -24,7 +24,7 @@ import (
 type Field struct {
 	Name        string // Go identifier
 	Tag         string // tag value
-	Kind        string // bool | uint32 | uint64 | string
+	Kind        string // bool | uint8 | uint32 | uint64 | string
 	EncodeEmpty bool   // whether to encode empty a zero value
 	Intern      bool   // whether to intern the string value
 }
@@ -56,6 +56,8 @@ func (p *Properties) load(i iter.Seq2[[]byte, []byte]) error {
       p.Loaded |= 1 << _bit_{{ .Name }}
   {{- if eq .Kind "bool" }}
       p.{{ .Name }} = len(v) == 1 && v[0] == '1'
+  {{- else if eq .Kind "uint8" }}
+      p.{{ .Name }} = v[0]
   {{- else if eq .Kind "uint32" }}
       p.{{ .Name }} = binary.LittleEndian.Uint32(v)
   {{- else if eq .Kind "uint64" }}
@@ -106,6 +108,9 @@ func (p *Properties) encodeAll() map[string][]byte {
     if p.{{ .Name }} {
       val[0] = '1'
     }
+{{- else if eq .Kind "uint8" }}
+    val := alloc(1)
+    val[0] = p.{{ .Name }}
 {{- else if eq .Kind "uint32" }}
     val := alloc(4)
     binary.LittleEndian.PutUint32(val, p.{{ .Name }})
@@ -163,7 +168,7 @@ func zeroVal(kind string) string {
 	switch kind {
 	case "bool":
 		return "false"
-	case "uint32", "uint64":
+	case "uint8", "uint32", "uint64":
 		return "0"
 	case "string":
 		return `""`
@@ -217,6 +222,8 @@ func main() {
 						switch typ {
 						case "bool":
 							kind = "bool"
+						case "uint8":
+							kind = "uint8"
 						case "uint32":
 							kind = "uint32"
 						case "uint64":

sstable/properties.go

@@ -122,6 +122,13 @@ type Properties struct {
 	// The compression statistics encoded as a string. The format is:
 	// "<setting1>:<compressed1>/<uncompressed1>,<setting2>:<compressed2>/<uncompressed2>,..."
 	CompressionStats string `prop:"pebble.compression_stats"`
+	// The value separation policy that was used when writing this sstable.
+	// This indicates how values were handled during writing (separated into
+	// blob files, preserved references, or kept in-place). See ValueSeparationKind type
+	// for possible values.
+	ValueSeparationKind uint8 `prop:"pebble.value-separation.kind"`
+	// The minimum size a value must be to be separated into a blob file during writing.
+	ValueSeparationMinSize uint64 `prop:"pebble.value-separation.min-size"`
 	// User collected properties. Currently, we only use them to store block
 	// properties aggregated at the table level.
 	UserProperties map[string]string
@@ -224,6 +231,28 @@ func (p *Properties) toAttributes() Attributes {
 	return attributes
 }
 
+// ValueSeparationKind indicates the value separation policy that was used
+// when writing an sstable.
+type ValueSeparationKind uint8
+
+const (
+	// ValueSeparationNone indicates that no value separation was used.
+	// All values are stored in-place within the sstable.
+	ValueSeparationNone ValueSeparationKind = iota
+
+	// ValueSeparationDefault indicates that values were separated into
+	// new blob files during writing.
+	ValueSeparationDefault
+
+	// ValueSeparationSpanPolicy indicates that values were separated
+	// based on a span policy during writing.
+	ValueSeparationSpanPolicy
+
+	// ValueSeparationPreservedRefs indicates that existing blob references
+	// were preserved without creating new blob files.
+	ValueSeparationPreservedRefs
+)
+
 var (
 	singleZeroSlice = []byte{0x00}
 	maxInt32Slice   = binary.AppendUvarint([]byte(nil), math.MaxInt32)

sstable/properties_gen.go

@@ -1966,6 +1966,11 @@ func (w *RawRowWriter) setFilter(fw filterWriter) {
 	w.filter = fw
 }
 
+// SetValueSeparationProps implements RawWriter.
+func (w *RawRowWriter) SetValueSeparationProps(_kind ValueSeparationKind, _minValueSize uint64) {
+	// Value separation requires TableFormatPebblev7+ which uses column writers.
+}
+
 // SetSnapshotPinnedProperties sets the properties for pinned keys. Should only
 // be used internally by Pebble.
 func (w *RawRowWriter) SetSnapshotPinnedProperties(

sstable/rowblk_writer.go

@@ -362,6 +362,10 @@ type RawWriter interface {
 	// Must not be called after Writer is closed.
 	IsPrefixEqualPrev(k []byte) bool
 
+	// SetValueSeparationProps sets the value separation props that were used when
+	// writing this sstable. This is recorded in the sstable properties.
+	SetValueSeparationProps(kind ValueSeparationKind, minValueSize uint64)
+
 	// rewriteSuffixes rewrites the table's data blocks to all contain the
 	// provided suffix. It's specifically used for the implementation of
 	// RewriteKeySuffixesAndReturnFormat. See that function's documentation for

sstable/writer.go

@@ -1055,9 +1055,9 @@ close: checkpoints/checkpoint8/000004.log
 scan checkpoints/checkpoint8
 ----
 open: checkpoints/checkpoint8/000005.sst (options: *vfs.randomReadsOption)
-read-at(735, 61): checkpoints/checkpoint8/000005.sst
-read-at(658, 77): checkpoints/checkpoint8/000005.sst
-read-at(197, 461): checkpoints/checkpoint8/000005.sst
+read-at(785, 61): checkpoints/checkpoint8/000005.sst
+read-at(708, 77): checkpoints/checkpoint8/000005.sst
+read-at(197, 511): checkpoints/checkpoint8/000005.sst
 read-at(131, 41): checkpoints/checkpoint8/000005.sst
 read-at(0, 131): checkpoints/checkpoint8/000005.sst
 open: checkpoints/checkpoint8/000006.blob (options: *vfs.randomReadsOption)
@@ -1165,9 +1165,9 @@ close: checkpoints/checkpoint9/000007.log
 scan checkpoints/checkpoint9
 ----
 open: checkpoints/checkpoint9/000005.sst (options: *vfs.randomReadsOption)
-read-at(735, 61): checkpoints/checkpoint9/000005.sst
-read-at(658, 77): checkpoints/checkpoint9/000005.sst
-read-at(197, 461): checkpoints/checkpoint9/000005.sst
+read-at(785, 61): checkpoints/checkpoint9/000005.sst
+read-at(708, 77): checkpoints/checkpoint9/000005.sst
+read-at(197, 511): checkpoints/checkpoint9/000005.sst
 read-at(131, 41): checkpoints/checkpoint9/000005.sst
 read-at(0, 131): checkpoints/checkpoint9/000005.sst
 open: checkpoints/checkpoint9/000006.blob (options: *vfs.randomReadsOption)