base: move BlobReferenceID, BlobFileMapping

jbowens · jbowens · commit cbfea02d22fd · 2025-08-07T11:04:05.000-04:00
Move blob.ReferenceID and blob.FileMapping types into the base package,
renaming them BlobReferenceID and BlobFileMapping. This removes any dependency
from internal/manifest onto the sstable/blob package.
diff --git a/blob_rewrite_test.go b/blob_rewrite_test.go
@@ -296,7 +296,7 @@ func TestBlobRewriteRandomized(t *testing.T) {
 			base.MakeInternalKey(keys[i], base.SeqNum(i), base.InternalKeyKindSet),
 			blob.InlineHandle{
 				InlineHandlePreface: blob.InlineHandlePreface{
-					ReferenceID: blob.ReferenceID(0),
+					ReferenceID: base.BlobReferenceID(0),
 					ValueLen:    uint32(len(values[i])),
 				},
 				HandleSuffix: blob.HandleSuffix{
@@ -415,11 +415,11 @@ func TestBlobRewriteRandomized(t *testing.T) {
 	}
 }
 
-// constantFileMapping implements blob.FileMapping and always maps to itself.
+// constantFileMapping implements base.BlobFileMapping and always maps to itself.
 type constantFileMapping base.DiskFileNum
 
-// Assert that (*inputFileMapping) implements blob.FileMapping.
-var _ blob.FileMapping = constantFileMapping(0)
+// Assert that (*inputFileMapping) implements base.BlobFileMapping.
+var _ base.BlobFileMapping = constantFileMapping(0)
 
 func (m constantFileMapping) Lookup(fileID base.BlobFileID) (base.DiskFileNum, bool) {
 	return base.DiskFileNum(m), true
diff --git a/internal/base/filenames.go b/internal/base/filenames.go
@@ -59,6 +59,20 @@ func (id BlobFileID) SafeFormat(w redact.SafePrinter, _ rune) {
 	w.Printf("B%06d", redact.SafeUint(id))
 }
 
+// BlobReferenceID identifies a particular blob reference within a table. It's
+// implemented as an index into the slice of the BlobReferences recorded in the
+// manifest.
+type BlobReferenceID uint32
+
+// BlobFileMapping defines the mapping between blob file IDs and disk file numbers.
+// It's implemented by *manifest.BlobFileSet.
+type BlobFileMapping interface {
+	// Lookup returns the disk file number for the given blob file ID. It
+	// returns false for the second return value if the blob file ID is not
+	// present in the mapping.
+	Lookup(BlobFileID) (DiskFileNum, bool)
+}
+
 // A DiskFileNum identifies a file or object with exists on disk.
 type DiskFileNum uint64
 
diff --git a/internal/blobtest/handles.go b/internal/blobtest/handles.go
@@ -297,13 +297,13 @@ type References struct {
 }
 
 // MapToReferenceID maps the given file number to a reference ID.
-func (b *References) MapToReferenceID(fileID base.BlobFileID) blob.ReferenceID {
+func (b *References) MapToReferenceID(fileID base.BlobFileID) base.BlobReferenceID {
 	for i, fn := range b.fileIDs {
 		if fn == fileID {
-			return blob.ReferenceID(i)
+			return base.BlobReferenceID(i)
 		}
 	}
 	i := uint32(len(b.fileIDs))
 	b.fileIDs = append(b.fileIDs, fileID)
-	return blob.ReferenceID(i)
+	return base.BlobReferenceID(i)
 }
diff --git a/internal/manifest/blob_metadata.go b/internal/manifest/blob_metadata.go
@@ -21,7 +21,6 @@ import (
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/strparse"
 	"github.com/cockroachdb/pebble/sstable"
-	"github.com/cockroachdb/pebble/sstable/blob"
 	"github.com/cockroachdb/redact"
 )
 
@@ -291,20 +290,20 @@ type BlobReferences []BlobReference
 var _ sstable.BlobReferences = (*BlobReferences)(nil)
 
 // BlobFileIDByID returns the BlobFileID for the identified BlobReference.
-func (br *BlobReferences) BlobFileIDByID(i blob.ReferenceID) base.BlobFileID {
+func (br *BlobReferences) BlobFileIDByID(i base.BlobReferenceID) base.BlobFileID {
 	return (*br)[i].FileID
 }
 
 // IDByBlobFileID returns the reference ID for the given BlobFileID. If the
 // blob file ID is not found, the second return value is false.
 // IDByBlobFileID is linear in the length of the BlobReferences slice.
-func (br *BlobReferences) IDByBlobFileID(fileID base.BlobFileID) (blob.ReferenceID, bool) {
+func (br *BlobReferences) IDByBlobFileID(fileID base.BlobFileID) (base.BlobReferenceID, bool) {
 	for i, ref := range *br {
 		if ref.FileID == fileID {
-			return blob.ReferenceID(i), true
+			return base.BlobReferenceID(i), true
 		}
 	}
-	return blob.ReferenceID(len(*br)), false
+	return base.BlobReferenceID(len(*br)), false
 }
 
 // BlobFileSet contains a set of blob files that are referenced by a version.
@@ -386,8 +385,8 @@ func (s *BlobFileSet) LookupPhysical(fileID base.BlobFileID) (*PhysicalBlobFile,
 	return nil, false
 }
 
-// Assert that (*BlobFileSet) implements blob.FileMapping.
-var _ blob.FileMapping = (*BlobFileSet)(nil)
+// Assert that (*BlobFileSet) implements base.BlobFileMapping.
+var _ base.BlobFileMapping = (*BlobFileSet)(nil)
 
 // clone returns a copy-on-write clone of the blob file set.
 func (s *BlobFileSet) clone() BlobFileSet {
diff --git a/level_checker.go b/level_checker.go
@@ -786,7 +786,7 @@ func validateBlobValueLiveness(
 		}
 		if err := fc.withReader(ctx, readEnv, t, func(r *sstable.Reader, readEnv sstable.ReadEnv) error {
 			// For this sstable, gather all the blob handles -- tracking
-			// each blob.ReferenceID + blob.BlockID's referenced
+			// each base.BlobReferenceID + blob.BlockID's referenced
 			// blob.BlockValueIDs.
 			referenced, err := gatherBlobHandles(ctx, r, t.BlobReferences, valueFetcher)
 			if err != nil {
diff --git a/sstable/blob/fetcher.go b/sstable/blob/fetcher.go
@@ -38,15 +38,6 @@ type ValueReader interface {
 	ReadIndexBlock(context.Context, block.ReadEnv, objstorage.ReadHandle) (block.BufferHandle, error)
 }
 
-// A FileMapping defines the mapping between blob file IDs and disk file numbers.
-// It's implemented by *manifest.BlobFileSet.
-type FileMapping interface {
-	// Lookup returns the disk file number for the given blob file ID. It
-	// returns false for the second return value if the blob file ID is not
-	// present in the mapping.
-	Lookup(base.BlobFileID) (base.DiskFileNum, bool)
-}
-
 // A ReaderProvider is an interface that can be used to retrieve a ValueReader
 // for a given file number.
 type ReaderProvider interface {
@@ -64,7 +55,7 @@ type ReaderProvider interface {
 // When finished with a ValueFetcher, one must call Close to release all cached
 // readers and block buffers.
 type ValueFetcher struct {
-	fileMapping    FileMapping
+	fileMapping    base.BlobFileMapping
 	readerProvider ReaderProvider
 	env            block.ReadEnv
 	fetchCount     int
@@ -78,7 +69,7 @@ type ValueFetcher struct {
 var _ base.ValueFetcher = (*ValueFetcher)(nil)
 
 // Init initializes the ValueFetcher.
-func (r *ValueFetcher) Init(fm FileMapping, rp ReaderProvider, env block.ReadEnv) {
+func (r *ValueFetcher) Init(fm base.BlobFileMapping, rp ReaderProvider, env block.ReadEnv) {
 	r.fileMapping = fm
 	r.readerProvider = rp
 	r.env = env
diff --git a/sstable/blob/fetcher_test.go b/sstable/blob/fetcher_test.go
@@ -25,12 +25,12 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// identityFileMapping is an implementation of FileMapping that casts a
+// identityFileMapping is an implementation of base.BlobFileMapping that casts a
 // BlobFileID to a DiskFileNum.
 type identityFileMapping struct{}
 
-// Assert that (identityFileMapping) implements FileMapping.
-var _ FileMapping = identityFileMapping{}
+// Assert that (identityFileMapping) implements base.BlobFileMapping.
+var _ base.BlobFileMapping = identityFileMapping{}
 
 func (identityFileMapping) Lookup(blobFileID base.BlobFileID) (base.DiskFileNum, bool) {
 	return base.DiskFileNum(blobFileID), true
diff --git a/sstable/blob/handle.go b/sstable/blob/handle.go
@@ -67,15 +67,10 @@ type InlineHandle struct {
 	HandleSuffix
 }
 
-// ReferenceID identifies a particular blob reference within a table. It's
-// implemented as an index into the slice of the BlobReferences recorded in the
-// manifest.
-type ReferenceID uint32
-
 // InlineHandlePreface is the prefix of an inline handle. It's eagerly decoded
 // when returning an InternalValue to higher layers.
 type InlineHandlePreface struct {
-	ReferenceID ReferenceID
+	ReferenceID base.BlobReferenceID
 	ValueLen    uint32
 }
 
@@ -158,7 +153,7 @@ func DecodeInlineHandlePreface(src []byte) (InlineHandlePreface, []byte) {
 	}
 
 	return InlineHandlePreface{
-		ReferenceID: ReferenceID(refIdx),
+		ReferenceID: base.BlobReferenceID(refIdx),
 		ValueLen:    valueLen,
 	}, src
 }
diff --git a/sstable/blob/rewrite.go b/sstable/blob/rewrite.go
@@ -110,11 +110,11 @@ func (rw *FileRewriter) Close() (FileWriterStats, error) {
 	return stats, errors.CombineErrors(err, rw.f.Close())
 }
 
-// inputFileMapping implements blob.FileMapping and always maps to itself.
+// inputFileMapping implements base.BlobFileMapping and always maps to itself.
 type inputFileMapping base.DiskFileNum
 
-// Assert that (*inputFileMapping) implements blob.FileMapping.
-var _ FileMapping = inputFileMapping(0)
+// Assert that (*inputFileMapping) implements base.BlobFileMapping.
+var _ base.BlobFileMapping = inputFileMapping(0)
 
 func (m inputFileMapping) Lookup(fileID base.BlobFileID) (base.DiskFileNum, bool) {
 	return base.DiskFileNum(m), true
diff --git a/sstable/blob_reference_index.go b/sstable/blob_reference_index.go
@@ -65,7 +65,7 @@ func (s *blobReferenceValues) finishCurrentBlock() {
 // blocks for a sstable's blob references. It maintains a refState, a slice of
 // blobRefValueLivenessState. This tracks the in-progress value liveness for
 // each blob value block for our sstable's blob references. The index of the
-// slice corresponds to the blob.ReferenceID.
+// slice corresponds to the base.BlobReferenceID.
 type blobRefValueLivenessWriter struct {
 	refState []blobReferenceValues
 }
@@ -87,10 +87,10 @@ func (w *blobRefValueLivenessWriter) numReferences() int {
 // blockID, a new state is created.
 //
 // addLiveValue adds a new state for the provided refID if one does
-// not already exist. It assumes that any new blob.ReferenceIDs are visited in
+// not already exist. It assumes that any new base.BlobReferenceIDs are visited in
 // monotonically increasing order.
 func (w *blobRefValueLivenessWriter) addLiveValue(
-	refID blob.ReferenceID, blockID blob.BlockID, valueID blob.BlockValueID, valueSize uint64,
+	refID base.BlobReferenceID, blockID blob.BlockID, valueID blob.BlockValueID, valueSize uint64,
 ) error {
 	// Compute the minimum expected length of the state slice in order for our
 	// refID to be indexable.
@@ -123,12 +123,12 @@ func (w *blobRefValueLivenessWriter) addLiveValue(
 
 // finish finishes encoding the per-blob reference liveness encodings, and
 // returns an in-order sequence of (referenceID, encoding) pairs.
-func (w *blobRefValueLivenessWriter) finish() iter.Seq2[blob.ReferenceID, []byte] {
-	return func(yield func(blob.ReferenceID, []byte) bool) {
-		// N.B. `i` is equivalent to blob.ReferenceID.
+func (w *blobRefValueLivenessWriter) finish() iter.Seq2[base.BlobReferenceID, []byte] {
+	return func(yield func(base.BlobReferenceID, []byte) bool) {
+		// N.B. `i` is equivalent to base.BlobReferenceID.
 		for i, state := range w.refState {
 			state.finishCurrentBlock()
-			if !yield(blob.ReferenceID(i), state.encodedFinishedBlocks) {
+			if !yield(base.BlobReferenceID(i), state.encodedFinishedBlocks) {
 				return
 			}
 		}
diff --git a/sstable/blob_reference_index_test.go b/sstable/blob_reference_index_test.go
@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/testutils"
 	"github.com/cockroachdb/pebble/sstable/blob"
 	"github.com/stretchr/testify/require"
@@ -22,7 +23,7 @@ func TestBlobRefValueLivenessWriter(t *testing.T) {
 	t.Run("basic", func(t *testing.T) {
 		w := &blobRefValueLivenessWriter{}
 		w.init()
-		refID := blob.ReferenceID(0)
+		refID := base.BlobReferenceID(0)
 		blockID := blob.BlockID(0)
 
 		require.NoError(t, w.addLiveValue(refID, blockID, 0 /* valueID */, 24 /* valueSize */))
@@ -60,7 +61,7 @@ func TestBlobRefValueLivenessWriter(t *testing.T) {
 	t.Run("all-ones", func(t *testing.T) {
 		w := &blobRefValueLivenessWriter{}
 		w.init()
-		refID := blob.ReferenceID(0)
+		refID := base.BlobReferenceID(0)
 		blockID := blob.BlockID(0)
 
 		// Add only live values.
@@ -87,7 +88,7 @@ func TestBlobRefLivenessEncoding_Randomized(t *testing.T) {
 	prng := rand.New(rand.NewPCG(uint64(time.Now().UnixNano()), 0))
 	w := blobRefValueLivenessWriter{}
 
-	collectSlice := func(i iter.Seq2[blob.ReferenceID, []byte]) [][]byte {
+	collectSlice := func(i iter.Seq2[base.BlobReferenceID, []byte]) [][]byte {
 		var s [][]byte
 		for _, enc := range i {
 			s = append(s, enc)
@@ -113,7 +114,7 @@ func TestBlobRefLivenessEncoding_Randomized(t *testing.T) {
 				currentValueID := blob.BlockValueID(testutils.RandIntInRange(prng, 0, 4))
 				for range numValues {
 					require.NoError(t, w.addLiveValue(
-						blob.ReferenceID(refID),
+						base.BlobReferenceID(refID),
 						currentBlockID,
 						currentValueID,
 						valueSize,
@@ -133,7 +134,7 @@ func TestBlobRefLivenessEncoding_Randomized(t *testing.T) {
 				// the writer.
 				for valueID := range IterSetBitsInRunLengthBitmap(block.Bitmap) {
 					require.NoError(t, w.addLiveValue(
-						blob.ReferenceID(refID),
+						base.BlobReferenceID(refID),
 						block.BlockID,
 						blob.BlockValueID(valueID),
 						valueSize,
diff --git a/sstable/colblk/value_liveness_block.go b/sstable/colblk/value_liveness_block.go
@@ -22,7 +22,7 @@ const (
 // ReferenceLivenessBlockEncoder encodes a reference liveness block.
 // A reference liveness block is a columnar block that contains a single column:
 // an array of bytes encoding the liveness of values within a sstable's blob
-// references. The indexes into this array are the blob.ReferenceIDs contained
+// references. The indexes into this array are the base.BlobReferenceIDs contained
 // within the sstable.
 type ReferenceLivenessBlockEncoder struct {
 	values RawBytesBuilder
diff --git a/sstable/values.go b/sstable/values.go
@@ -52,7 +52,7 @@ var DebugHandlesBlobContext = TableBlobContext{
 // sstable iterator to fetch the value stored in a blob file. It is the
 // caller's responsibility to close the ValueFetcher returned.
 func LoadValBlobContext(
-	fm blob.FileMapping, rp blob.ReaderProvider, blobRefs BlobReferences,
+	fm base.BlobFileMapping, rp blob.ReaderProvider, blobRefs BlobReferences,
 ) (*blob.ValueFetcher, TableBlobContext) {
 	vf := &blob.ValueFetcher{}
 	vf.Init(fm, rp, block.ReadEnv{})
@@ -67,10 +67,10 @@ func LoadValBlobContext(
 // manifest.BlobReferences.
 type BlobReferences interface {
 	// BlobFileIDByID returns the BlobFileID for the identified BlobReference.
-	BlobFileIDByID(i blob.ReferenceID) base.BlobFileID
+	BlobFileIDByID(i base.BlobReferenceID) base.BlobFileID
 	// IDByBlobFileID returns the reference ID for the given BlobFileID. If the
 	// blob file ID is not found, the second return value is false.
-	IDByBlobFileID(fileID base.BlobFileID) (blob.ReferenceID, bool)
+	IDByBlobFileID(fileID base.BlobFileID) (base.BlobReferenceID, bool)
 }
 
 // TableBlobContext configures how values that reference external blob files
diff --git a/tool/blob_files.go b/tool/blob_files.go
@@ -24,7 +24,7 @@ import (
 // file.
 //
 // An sstable alone does not contain sufficient information to read a value from
-// a blob file. The sstable encodes a ReferenceID which first must be translated
+// a blob file. The sstable encodes a BlobReferenceID which first must be translated
 // to a BlobFileID and then from a BlobFileID to a DiskFileNum. The manifest
 // contains this metadata. The blobFileMappings type contains this state, loaded
 // from the manifest.
diff --git a/value_separation.go b/value_separation.go
@@ -362,7 +362,7 @@ type preserveBlobReferences struct {
 	buf []byte
 	// currReferences holds the pending references that have been referenced by
 	// the current output sstable. The index of a reference with a given blob
-	// file ID is the value of the blob.ReferenceID used by its value handles
+	// file ID is the value of the base.BlobReferenceID used by its value handles
 	// within the output sstable.
 	currReferences []pendingReference
 	// totalValueSize is the sum of currReferenceValueSizes.
@@ -428,13 +428,13 @@ func (vs *preserveBlobReferences) Add(
 	lv := kv.V.LazyValue()
 	fileID := lv.Fetcher.BlobFileID
 
-	var refID blob.ReferenceID
+	var refID base.BlobReferenceID
 	if refIdx := slices.IndexFunc(vs.currReferences, func(ref pendingReference) bool {
 		return ref.blobFileID == fileID
 	}); refIdx != -1 {
-		refID = blob.ReferenceID(refIdx)
+		refID = base.BlobReferenceID(refIdx)
 	} else {
-		refID = blob.ReferenceID(len(vs.currReferences))
+		refID = base.BlobReferenceID(len(vs.currReferences))
 		vs.currReferences = append(vs.currReferences, pendingReference{
 			blobFileID: fileID,
 			valueSize:  0,

Original file line number	Diff line number	Diff line change
`@@ -297,13 +297,13 @@ type References struct {`
`297`	`297`	`}`
`298`	`298`
`299`	`299`	`// MapToReferenceID maps the given file number to a reference ID.`
`300`		`-func (b *References) MapToReferenceID(fileID base.BlobFileID) blob.ReferenceID {`
	`300`	`+func (b *References) MapToReferenceID(fileID base.BlobFileID) base.BlobReferenceID {`
`301`	`301`	`for i, fn := range b.fileIDs {`
`302`	`302`	`if fn == fileID {`
`303`		`- return blob.ReferenceID(i)`
	`303`	`+ return base.BlobReferenceID(i)`
`304`	`304`	`}`
`305`	`305`	`}`
`306`	`306`	`i := uint32(len(b.fileIDs))`
`307`	`307`	`b.fileIDs = append(b.fileIDs, fileID)`
`308`		`- return blob.ReferenceID(i)`
	`308`	`+ return base.BlobReferenceID(i)`
`309`	`309`	`}`
Original file line number	Diff line number	Diff line change
`@@ -786,7 +786,7 @@ func validateBlobValueLiveness(`
`786`	`786`	`}`
`787`	`787`	`if err := fc.withReader(ctx, readEnv, t, func(r *sstable.Reader, readEnv sstable.ReadEnv) error {`
`788`	`788`	`// For this sstable, gather all the blob handles -- tracking`
`789`		`- // each blob.ReferenceID + blob.BlockID's referenced`
	`789`	`+ // each base.BlobReferenceID + blob.BlockID's referenced`
`790`	`790`	`// blob.BlockValueIDs.`
`791`	`791`	`referenced, err := gatherBlobHandles(ctx, r, t.BlobReferences, valueFetcher)`
`792`	`792`	`if err != nil {`
Original file line number	Diff line number	Diff line change
`@@ -67,15 +67,10 @@ type InlineHandle struct {`
`67`	`67`	`HandleSuffix`
`68`	`68`	`}`
`69`	`69`
`70`		`-// ReferenceID identifies a particular blob reference within a table. It's`
`71`		`-// implemented as an index into the slice of the BlobReferences recorded in the`
`72`		`-// manifest.`
`73`		`-type ReferenceID uint32`
`74`		`-`
`75`	`70`	`// InlineHandlePreface is the prefix of an inline handle. It's eagerly decoded`
`76`	`71`	`// when returning an InternalValue to higher layers.`
`77`	`72`	`type InlineHandlePreface struct {`
`78`		`- ReferenceID ReferenceID`
	`73`	`+ ReferenceID base.BlobReferenceID`
`79`	`74`	`ValueLen uint32`
`80`	`75`	`}`
`81`	`76`
`@@ -158,7 +153,7 @@ func DecodeInlineHandlePreface(src []byte) (InlineHandlePreface, []byte) {`
`158`	`153`	`}`
`159`	`154`
`160`	`155`	`return InlineHandlePreface{`
`161`		`- ReferenceID: ReferenceID(refIdx),`
	`156`	`+ ReferenceID: base.BlobReferenceID(refIdx),`
`162`	`157`	`ValueLen: valueLen,`
`163`	`158`	`}, src`
`164`	`159`	`}`