db: preserve blob reference metadata across excise

During excise, we now preserve the top-level blob references
across virtual ssts created. The value size of each blob
reference is approximated by the proportion of the virtual table's
size to the original table's size.

Fixes: #4595

Author: annrpom

compaction_test.go

@@ -1004,6 +1004,13 @@ func TestCompaction(t *testing.T) {
 				}
 				return s
 
+			case "excise-dryrun":
+				ve, err := runExciseDryRunCmd(td, d)
+				if err != nil {
+					td.Fatalf(t, err.Error())
+				}
+				return fmt.Sprintf("would excise %d files.\n%s", len(ve.DeletedTables), ve.DebugString(base.DefaultFormatter))
+
 			case "file-sizes":
 				return runTableFileSizesCmd(td, d)
 

data_test.go

@@ -1410,6 +1410,42 @@ func runExciseCmd(td *datadriven.TestData, d *DB) error {
 	return d.Excise(context.Background(), exciseSpan)
 }
 
+func runExciseDryRunCmd(td *datadriven.TestData, d *DB) (*versionEdit, error) {
+	ve := &versionEdit{
+		DeletedTables: map[deletedFileEntry]*tableMetadata{},
+	}
+	var exciseSpan KeyRange
+	if len(td.CmdArgs) != 2 {
+		panic("insufficient args for excise-dryrun command")
+	}
+	exciseSpan.Start = []byte(td.CmdArgs[0].Key)
+	exciseSpan.End = []byte(td.CmdArgs[1].Key)
+
+	d.mu.Lock()
+	d.mu.versions.logLock()
+	defer func() {
+		d.mu.Lock()
+		d.mu.versions.logUnlock()
+		d.mu.Unlock()
+	}()
+	d.mu.Unlock()
+	current := d.mu.versions.currentVersion()
+
+	exciseBounds := exciseSpan.UserKeyBounds()
+	for l, ls := range current.AllLevelsAndSublevels() {
+		iter := ls.Iter()
+		for m := iter.SeekGE(d.cmp, exciseSpan.Start); m != nil && d.cmp(m.Smallest().UserKey, exciseSpan.End) < 0; m = iter.Next() {
+			leftTable, rightTable, err := d.exciseTable(context.Background(), exciseBounds, m, l.Level(), tightExciseBounds)
+			if err != nil {
+				return nil, errors.Errorf("error when excising %s: %s", m.FileNum, err.Error())
+			}
+			applyExciseToVersionEdit(ve, m, leftTable, rightTable, l.Level())
+		}
+	}
+
+	return ve, nil
+}
+
 func runIngestAndExciseCmd(td *datadriven.TestData, d *DB) error {
 	var exciseSpan KeyRange
 	paths := make([]string, 0, len(td.CmdArgs))

excise.go

@@ -135,7 +135,7 @@ func (d *DB) exciseTable(
 	// TODO(bilal): Some of this work can happen without grabbing the manifest
 	// lock; we could grab one currentVersion, release the lock, calculate excised
 	// files, then grab the lock again and recalculate for just the files that
-	// have changed since our previous calculation. Do this optimiaztino as part of
+	// have changed since our previous calculation. Do this optimization as part of
 	// https://github.com/cockroachdb/pebble/issues/2112 .
 	if d.cmp(m.Smallest().UserKey, exciseBounds.Start) < 0 {
 		leftTable = &tableMetadata{
@@ -147,6 +147,7 @@ func (d *DB) exciseTable(
 			LargestSeqNum:            m.LargestSeqNum,
 			LargestSeqNumAbsolute:    m.LargestSeqNumAbsolute,
 			SyntheticPrefixAndSuffix: m.SyntheticPrefixAndSuffix,
+			BlobReferenceDepth:       m.BlobReferenceDepth,
 		}
 		if looseBounds {
 			looseLeftTableBounds(d.cmp, m, leftTable, exciseBounds.Start)
@@ -159,6 +160,7 @@ func (d *DB) exciseTable(
 			if err := determineExcisedTableSize(d.fileCache, m, leftTable); err != nil {
 				return nil, nil, err
 			}
+			determineExcisedTableBlobReferences(m.BlobReferences, m.Size, leftTable)
 			if err := leftTable.Validate(d.cmp, d.opts.Comparer.FormatKey); err != nil {
 				return nil, nil, err
 			}
@@ -182,6 +184,7 @@ func (d *DB) exciseTable(
 			LargestSeqNum:            m.LargestSeqNum,
 			LargestSeqNumAbsolute:    m.LargestSeqNumAbsolute,
 			SyntheticPrefixAndSuffix: m.SyntheticPrefixAndSuffix,
+			BlobReferenceDepth:       m.BlobReferenceDepth,
 		}
 		if looseBounds {
 			// We already checked that the end bound is exclusive.
@@ -194,6 +197,7 @@ func (d *DB) exciseTable(
 			if err := determineExcisedTableSize(d.fileCache, m, rightTable); err != nil {
 				return nil, nil, err
 			}
+			determineExcisedTableBlobReferences(m.BlobReferences, m.Size, rightTable)
 			if err := rightTable.Validate(d.cmp, d.opts.Comparer.FormatKey); err != nil {
 				return nil, nil, err
 			}
@@ -434,6 +438,21 @@ func determineExcisedTableSize(
 	return nil
 }
 
+// determineExcisedTableBlobReferences copies blob references from the original
+// table to the excised table, scaling each blob reference's value size
+// proportionally based on the ratio of the excised table's size to the original
+// table's size.
+func determineExcisedTableBlobReferences(
+	originalBlobReferences manifest.BlobReferences, originalSize uint64, excisedTable *tableMetadata,
+) {
+	newBlobReferences := make(manifest.BlobReferences, len(originalBlobReferences))
+	for i, bf := range originalBlobReferences {
+		bf.ValueSize = bf.ValueSize * excisedTable.Size / originalSize
+		newBlobReferences[i] = bf
+	}
+	excisedTable.BlobReferences = newBlobReferences
+}
+
 // applyExciseToVersionEdit updates ve with a table deletion for the original
 // table and table additions for the left and/or right table.
 //

excise_test.go

@@ -282,36 +282,10 @@ func TestExcise(t *testing.T) {
 			}
 			return ""
 		case "excise-dryrun":
-			ve := &versionEdit{
-				DeletedTables: map[deletedFileEntry]*tableMetadata{},
-			}
-			var exciseSpan KeyRange
-			if len(td.CmdArgs) != 2 {
-				panic("insufficient args for compact command")
-			}
-			exciseSpan.Start = []byte(td.CmdArgs[0].Key)
-			exciseSpan.End = []byte(td.CmdArgs[1].Key)
-
-			d.mu.Lock()
-			d.mu.versions.logLock()
-			d.mu.Unlock()
-			current := d.mu.versions.currentVersion()
-
-			exciseBounds := exciseSpan.UserKeyBounds()
-			for l, ls := range current.AllLevelsAndSublevels() {
-				iter := ls.Iter()
-				for m := iter.SeekGE(d.cmp, exciseSpan.Start); m != nil && d.cmp(m.Smallest().UserKey, exciseSpan.End) < 0; m = iter.Next() {
-					leftTable, rightTable, err := d.exciseTable(context.Background(), exciseBounds, m, l.Level(), tightExciseBounds)
-					if err != nil {
-						td.Fatalf(t, "error when excising %s: %s", m.FileNum, err.Error())
-					}
-					applyExciseToVersionEdit(ve, m, leftTable, rightTable, l.Level())
-				}
+			ve, err := runExciseDryRunCmd(td, d)
+			if err != nil {
+				td.Fatalf(t, err.Error())
 			}
-
-			d.mu.Lock()
-			d.mu.versions.logUnlock()
-			d.mu.Unlock()
 			return fmt.Sprintf("would excise %d files, use ingest-and-excise to excise.\n%s", len(ve.DeletedTables), ve.DebugString(base.DefaultFormatter))
 		case "confirm-backing":
 			// Confirms that the files have the same FileBacking.

internal/manifest/blob_metadata.go

@@ -25,7 +25,9 @@ type BlobReference struct {
 	// FileNum identifies the referenced blob file.
 	FileNum base.DiskFileNum
 	// ValueSize is the sum of the lengths of the uncompressed values within the
-	// blob file for which there exists a reference in the sstable.
+	// blob file for which there exists a reference in the sstable. Note that if
+	// any of the referencing tables are virtualized tables, the ValueSize may
+	// be approximate.
 	//
 	// INVARIANT: ValueSize <= Metadata.ValueSize
 	ValueSize uint64

testdata/compaction/value_separation

@@ -342,3 +342,12 @@ Blob files:
   000038: 1686698 physical bytes, 1681728 value bytes
   000040: 1691574 physical bytes, 1686592 value bytes
   000042: 113626 physical bytes, 113280 value bytes
+
+
+excise-dryrun b c
+----
+would excise 1 files.
+  del-table:     L6 000044
+  add-table:     L6 000053(000044):[a@1#0,SET-azzz@1#0,SET] seqnums:[0-0] points:[a@1#0,SET-azzz@1#0,SET] size:238596 blobrefs:[(000006: 566234), (000008: 564260), (000010: 37624); depth:1]
+  add-table:     L6 000054(000044):[c@1#0,SET-czks@1#0,SET] seqnums:[0-0] points:[c@1#0,SET-czks@1#0,SET] size:235737 blobrefs:[(000006: 559450), (000008: 557499), (000010: 37173); depth:1]
+  add-backing:   000044