sstable: mangle some more buffers in invariants builds

Mangle some more byte buffers when they become unused to help shake out memory
safety bugs.

Author: jbowens

internal/invariants/off.go

@@ -50,6 +50,12 @@ func (*Value[V]) Get() V {
 // Set the value; no-op in non-invariant builds.
 func (*Value[V]) Set(v V) {}
 
+// MaybeMangle mangles a byte slice sometimes in invariant builds.
+func MaybeMangle(b []byte) {}
+
+// Mangle mangles a byte slice in invariant builds.
+func Mangle(b []byte) {}
+
 // BufMangler is a utility that can be used to test that the caller doesn't use
 type BufMangler struct{}
 

internal/invariants/on.go

@@ -72,6 +72,20 @@ func (v *Value[V]) Get() V {
 	return v.v
 }
 
+// MaybeMangle mangles a byte slice sometimes in invariant builds.
+func MaybeMangle(b []byte) {
+	if Sometimes(10) {
+		Mangle(b)
+	}
+}
+
+// Mangle mangles a byte slice in invariant builds.
+func Mangle(b []byte) {
+	for i := range b {
+		b[i] = 0xCC
+	}
+}
+
 // BufMangler is a utility that can be used to test that the caller doesn't use
 type BufMangler struct {
 	lastReturnedBuf []byte

sstable/block/buffer_pool.go

@@ -8,6 +8,7 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/cache"
+	"github.com/cockroachdb/pebble/internal/invariants"
 )
 
 // Alloc allocates a new Value for a block of length n (excluding the block
@@ -235,6 +236,7 @@ func (b *Buf) Release() {
 	if b.p == nil {
 		return
 	}
+	invariants.MaybeMangle(b.p.pool[b.i].b)
 	// Clear the allocedBuffer's byte slice. This signals the allocated buffer
 	// is no longer in use and a future call to BufferPool.Alloc may reuse this
 	// buffer.

sstable/block/compression.go

@@ -305,9 +305,7 @@ func (b *PhysicalBlock) WriteTo(w objstorage.Writable) (n int, err error) {
 	// WriteTo is allowed to mangle the data. Mangle it ourselves some of the time
 	// in invariant builds to catch callers that don't handle this.
 	if invariants.Enabled && invariants.Sometimes(1) {
-		for i := range b.data {
-			b.data[i] = 0xFF
-		}
+		invariants.Mangle(b.data)
 	}
 	return len(b.data) + len(b.trailer), nil
 }
@@ -429,12 +427,7 @@ func (tb *TempBuffer) Release() {
 	// maximum to the pool. This avoids holding on to occasional large buffers
 	// necessary for e.g. singular large values.
 	if tb.b != nil && len(tb.b) < tempBufferMaxReusedSize {
-		if invariants.Sometimes(20) {
-			// Mangle the buffer data.
-			for i := range tb.b {
-				tb.b[i] = 0xCC
-			}
-		}
+		invariants.MaybeMangle(tb.b)
 		tb.b = tb.b[:0]
 		tempBufferPool.Put(tb)
 	}

sstable/colblk/block.go

@@ -141,6 +141,7 @@ import (
 	"github.com/cockroachdb/crlib/crbytes"
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/binfmt"
+	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/treeprinter"
 )
 
@@ -209,6 +210,7 @@ type BlockEncoder struct {
 
 // Reset resets an encoder for reuse.
 func (e *BlockEncoder) Reset() {
+	invariants.MaybeMangle(e.buf)
 	if cap(e.buf) > maxBlockRetainedSize {
 		e.buf = nil
 	}

sstable/colblk/prefix_bytes.go

@@ -721,6 +721,7 @@ func (b *PrefixBytesBuilder) DataType(int) DataType { return DataTypePrefixBytes
 // size.
 func (b *PrefixBytesBuilder) Reset() {
 	const maxRetainedData = 512 << 10 // 512 KB
+	invariants.MaybeMangle(b.data)
 	*b = PrefixBytesBuilder{
 		bundleCalc: b.bundleCalc,
 		data:       b.data[:0],

sstable/colblk/raw_bytes.go

@@ -153,6 +153,7 @@ func (b *RawBytesBuilder) Init() {
 // Reset resets the builder to an empty state.
 func (b *RawBytesBuilder) Reset() {
 	b.rows = 0
+	invariants.MaybeMangle(b.data)
 	b.data = b.data[:0]
 	b.offsets.Reset()
 	// Add an initial offset of zero to streamline the logic in RawBytes.At() to

sstable/runlength_bitmap.go

@@ -7,6 +7,8 @@ package sstable
 import (
 	"encoding/binary"
 	"iter"
+
+	"github.com/cockroachdb/pebble/internal/invariants"
 )
 
 // BitmapRunLengthEncoder encodes a bitmap. It uses a run-length encoding for
@@ -42,6 +44,7 @@ type BitmapRunLengthEncoder struct {
 
 // Init initializes or resets the encoder to its initial state.
 func (bb *BitmapRunLengthEncoder) Init() {
+	invariants.MaybeMangle(bb.buf)
 	bb.buf = bb.buf[:0]
 	bb.allSetRunLength = 0
 	bb.currByte = 0