internal/manifest: add BlobReferences type

Define a BlobReferences type for a slice of blob references, and use it on the
TableMetadata for its BlobReferences field. When yielding an InternalValue
backed by a blob value handle to higher levels, an iterator will need to map
the blob handle's file reference index to the file number. This type will
implement the interface to perform this mapping.

Author: jbowens

internal/blobtest/handles.go

@@ -190,8 +190,8 @@ func (bv *Values) ParseInlineHandle(
 	}
 	return blob.InlineHandle{
 		InlineHandlePreface: blob.InlineHandlePreface{
-			ReferenceIndex: references.MapToReferenceIndex(fullHandle.FileNum),
-			ValueLen:       fullHandle.ValueLen,
+			ReferenceID: references.MapToReferenceID(fullHandle.FileNum),
+			ValueLen:    fullHandle.ValueLen,
 		},
 		HandleSuffix: blob.HandleSuffix{
 			BlockNum:      fullHandle.BlockNum,
@@ -261,14 +261,14 @@ type References struct {
 	fileNums []base.DiskFileNum
 }
 
-// MapToReferenceIndex maps the given file number to a reference index.
-func (b *References) MapToReferenceIndex(fileNum base.DiskFileNum) uint32 {
+// MapToReferenceID maps the given file number to a reference ID.
+func (b *References) MapToReferenceID(fileNum base.DiskFileNum) blob.ReferenceID {
 	for i, fn := range b.fileNums {
 		if fn == fileNum {
-			return uint32(i)
+			return blob.ReferenceID(i)
 		}
 	}
 	i := uint32(len(b.fileNums))
 	b.fileNums = append(b.fileNums, fileNum)
-	return i
+	return blob.ReferenceID(i)
 }

internal/compact/run.go

@@ -48,7 +48,7 @@ type OutputTable struct {
 	// failure (see Result), WriterMeta might not be set.
 	WriterMeta sstable.WriterMetadata
 	// BlobReferences is the list of blob references for the table.
-	BlobReferences []manifest.BlobReference
+	BlobReferences manifest.BlobReferences
 	// BlobReferenceDepth is the depth of the blob references for the table.
 	BlobReferenceDepth manifest.BlobReferenceDepth
 }
@@ -138,7 +138,7 @@ type ValueSeparation interface {
 // ValueSeparationMetadata describes metadata about a table's blob references,
 // and optionally a newly constructed blob file.
 type ValueSeparationMetadata struct {
-	BlobReferences     []manifest.BlobReference
+	BlobReferences     manifest.BlobReferences
 	BlobReferenceSize  uint64
 	BlobReferenceDepth manifest.BlobReferenceDepth
 

internal/manifest/blob_metadata.go

@@ -12,6 +12,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/humanize"
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/strparse"
+	"github.com/cockroachdb/pebble/sstable/blob"
 	"github.com/cockroachdb/redact"
 )
 
@@ -216,3 +217,26 @@ func ParseBlobFileMetadataDebug(s string) (_ *BlobFileMetadata, err error) {
 // is actually better than in this analysis because more of the keys will be
 // from the lower level.
 type BlobReferenceDepth int
+
+// BlobReferences is a slice of BlobReference. The order of the slice is
+// significant and should be maintained. In practice, a sstable's BlobReferences
+// are ordered by earliest appearance within the sstable. The ordering is
+// persisted to the manifest.
+type BlobReferences []BlobReference
+
+// FileNumByID returns the FileNum for the identified BlobReference.
+func (br BlobReferences) FileNumByID(i blob.ReferenceID) base.DiskFileNum {
+	return br[i].FileNum
+}
+
+// IDByFileNum returns the reference ID for the given FileNum. If the file
+// number is not found, the second return value is false. IDByFileNum is linear
+// in the length of the BlobReferences slice.
+func (br BlobReferences) IDByFileNum(fileNum base.DiskFileNum) (blob.ReferenceID, bool) {
+	for i, ref := range br {
+		if ref.FileNum == fileNum {
+			return blob.ReferenceID(i), true
+		}
+	}
+	return blob.ReferenceID(len(br)), false
+}

internal/manifest/version.go

@@ -238,7 +238,7 @@ type TableMetadata struct {
 	Largest  InternalKey
 	// BlobReferences is a list of blob files containing values that are
 	// referenced by this sstable.
-	BlobReferences []BlobReference
+	BlobReferences BlobReferences
 	// BlobReferenceDepth is the stack depth of blob files referenced by this
 	// sstable. See the comment on the BlobReferenceDepth type for more details.
 	//

internal/manifest/version_edit.go

@@ -357,7 +357,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 			}{}
 			var syntheticPrefix sstable.SyntheticPrefix
 			var syntheticSuffix sstable.SyntheticSuffix
-			var blobReferences []BlobReference
+			var blobReferences BlobReferences
 			var blobReferenceDepth BlobReferenceDepth
 			if tag == tagNewFile4 || tag == tagNewFile5 {
 				for {

sstable/blob/blob_test.go

@@ -87,8 +87,8 @@ func TestHandleRoundtrip(t *testing.T) {
 	handles := []InlineHandle{
 		{
 			InlineHandlePreface: InlineHandlePreface{
-				ReferenceIndex: 0,
-				ValueLen:       29357353,
+				ReferenceID: 0,
+				ValueLen:    29357353,
 			},
 			HandleSuffix: HandleSuffix{
 				BlockNum:      194,
@@ -97,8 +97,8 @@ func TestHandleRoundtrip(t *testing.T) {
 		},
 		{
 			InlineHandlePreface: InlineHandlePreface{
-				ReferenceIndex: 129,
-				ValueLen:       205,
+				ReferenceID: 129,
+				ValueLen:    205,
 			},
 			HandleSuffix: HandleSuffix{
 				BlockNum:      2,

sstable/blob/handle.go

@@ -54,11 +54,16 @@ type InlineHandle struct {
 	HandleSuffix
 }
 
+// ReferenceID identifies a particular blob reference within a table. It's
+// implemented as an index into the slice of the BlobReferences recorded in the
+// manifest.
+type ReferenceID uint32
+
 // InlineHandlePreface is the prefix of an inline handle. It's eagerly decoded
 // when returning an InternalValue to higher layers.
 type InlineHandlePreface struct {
-	ReferenceIndex uint32
-	ValueLen       uint32
+	ReferenceID ReferenceID
+	ValueLen    uint32
 }
 
 // HandleSuffix is the suffix of an inline handle. It's decoded only when the
@@ -85,14 +90,14 @@ func (h InlineHandle) String() string {
 // SafeFormat implements redact.SafeFormatter.
 func (h InlineHandle) SafeFormat(w redact.SafePrinter, _ rune) {
 	w.Printf("(f%d,blk%d[%d:%d])",
-		h.ReferenceIndex, h.BlockNum, h.OffsetInBlock, h.OffsetInBlock+h.ValueLen)
+		h.ReferenceID, h.BlockNum, h.OffsetInBlock, h.OffsetInBlock+h.ValueLen)
 }
 
 // Encode encodes the inline handle into the provided buffer, returning the
 // number of bytes encoded.
 func (h InlineHandle) Encode(b []byte) int {
 	n := 0
-	n += binary.PutUvarint(b[n:], uint64(h.ReferenceIndex))
+	n += binary.PutUvarint(b[n:], uint64(h.ReferenceID))
 	n += valblk.EncodeHandle(b[n:], valblk.Handle{
 		BlockNum:      h.BlockNum,
 		OffsetInBlock: h.OffsetInBlock,
@@ -145,8 +150,8 @@ func DecodeInlineHandlePrefix(src []byte) (InlineHandlePreface, []byte) {
 	}
 
 	return InlineHandlePreface{
-		ReferenceIndex: refIdx,
-		ValueLen:       valueLen,
+		ReferenceID: ReferenceID(refIdx),
+		ValueLen:    valueLen,
 	}, src
 }
 

sstable/data_test.go

@@ -204,8 +204,8 @@ func decodeBlobInlineHandleAndAttribute(
 	}
 	return blob.InlineHandle{
 		InlineHandlePreface: blob.InlineHandlePreface{
-			ReferenceIndex: uint32(refIdx),
-			ValueLen:       uint32(valLen),
+			ReferenceID: blob.ReferenceID(refIdx),
+			ValueLen:    uint32(valLen),
 		},
 		HandleSuffix: blob.HandleSuffix{
 			BlockNum:      uint32(blockNum),

value_separation.go

@@ -143,8 +143,8 @@ func (vs *writeNewBlobFiles) Add(
 			// between sstables and blob files, the reference index is always 0
 			// here. Only compactions that don't rewrite blob files will produce
 			// handles with nonzero reference indices.
-			ReferenceIndex: 0,
-			ValueLen:       handle.ValueLen,
+			ReferenceID: 0,
+			ValueLen:    handle.ValueLen,
 		},
 		HandleSuffix: blob.HandleSuffix{
 			BlockNum:      handle.BlockNum,
@@ -166,7 +166,7 @@ func (vs *writeNewBlobFiles) FinishOutput() (compact.ValueSeparationMetadata, er
 	}
 	vs.writer = nil
 	return compact.ValueSeparationMetadata{
-		BlobReferences: []manifest.BlobReference{{
+		BlobReferences: manifest.BlobReferences{{
 			FileNum:   vs.objMeta.DiskFileNum,
 			ValueSize: stats.UncompressedValueBytes,
 		}},
@@ -197,7 +197,7 @@ type preserveBlobReferences struct {
 
 	// state
 	buf            []byte
-	currReferences []manifest.BlobReference
+	currReferences manifest.BlobReferences
 	// totalValueSize is the sum of the sizes of all ValueSizes in currReferences.
 	totalValueSize uint64
 }
@@ -250,24 +250,16 @@ func (vs *preserveBlobReferences) Add(
 	// sstable, taking note of the reference for the table metadata.
 	lv := kv.V.LazyValue()
 	fn := lv.Fetcher.BlobFileNum
-	var found bool
-	refIdx := 0
-	for refIdx = range vs.currReferences {
-		if vs.currReferences[refIdx].FileNum == fn {
-			// This sstable contains an existing reference to this blob file.
-			// Record the reference.
-			found = true
-			break
-		}
-	}
+
+	refIdx, found := vs.currReferences.IDByFileNum(fn)
 	if !found {
 		// This is the first time we're seeing this blob file for this sstable.
 		// Find the blob file metadata for this file among the input metadatas.
 		idx, found := vs.findInputBlobMetadata(fn)
 		if !found {
 			return errors.AssertionFailedf("pebble: blob file %s not found among input sstables", fn)
 		}
-		refIdx = len(vs.currReferences)
+		refIdx = blob.ReferenceID(len(vs.currReferences))
 		vs.currReferences = append(vs.currReferences, manifest.BlobReference{
 			FileNum:  fn,
 			Metadata: vs.inputBlobMetadatas[idx],
@@ -281,8 +273,8 @@ func (vs *preserveBlobReferences) Add(
 	handleSuffix := blob.DecodeHandleSuffix(lv.ValueOrHandle)
 	inlineHandle := blob.InlineHandle{
 		InlineHandlePreface: blob.InlineHandlePreface{
-			ReferenceIndex: uint32(refIdx),
-			ValueLen:       lv.Fetcher.Attribute.ValueLen,
+			ReferenceID: refIdx,
+			ValueLen:    lv.Fetcher.Attribute.ValueLen,
 		},
 		HandleSuffix: handleSuffix,
 	}