sstable: set up proper iterator transforms during EstimateDiskUsage

Previously, we did not set up any iterator transforms during
EstimateDiskUsage -- leading to an inaccurate disk usage estimation
that was eventually used in compaction heuristics. This patch fixes
this bug by grabbing the IterTransforms from the table we are reading
and using it during disk usage estimation.

Fixes: #4597

Author: annrpom

file_cache.go

@@ -328,7 +328,7 @@ func (h *fileCacheHandle) estimateSize(
 	meta *manifest.TableMetadata, lower, upper []byte,
 ) (size uint64, err error) {
 	err = h.withReader(context.TODO(), block.NoReadEnv, meta, func(r *sstable.Reader, env sstable.ReadEnv) error {
-		size, err = r.EstimateDiskUsage(lower, upper, env)
+		size, err = r.EstimateDiskUsage(lower, upper, env, meta.IterTransforms())
 		return err
 	})
 	return size, err

ingest.go

@@ -1172,7 +1172,7 @@ type ExternalFile struct {
 	// ingestion.
 	HasPointKey, HasRangeKey bool
 
-	// SyntheticPrefix will prepend this suffix to all keys in the file during
+	// SyntheticPrefix will prepend this prefix to all keys in the file during
 	// iteration. Note that the backing file itself is not modified.
 	//
 	// SyntheticPrefix must be a prefix of both Bounds.Start and Bounds.End.

sstable/reader.go

@@ -796,19 +796,20 @@ func (r *Reader) ValidateBlockChecksums() error {
 // TODO(ajkr): account for metablock space usage. Perhaps look at the fraction of
 // data blocks overlapped and add that same fraction of the metadata blocks to the
 // estimate.
-func (r *Reader) EstimateDiskUsage(start []byte, end []byte, env ReadEnv) (uint64, error) {
+func (r *Reader) EstimateDiskUsage(
+	start []byte, end []byte, env ReadEnv, transforms IterTransforms,
+) (uint64, error) {
 	if env.Virtual != nil {
 		_, start, end = env.Virtual.ConstrainBounds(start, end, false, r.Comparer.Compare)
 	}
-
 	if !r.tableFormat.BlockColumnar() {
-		return estimateDiskUsage[rowblk.IndexIter, *rowblk.IndexIter](r, start, end)
+		return estimateDiskUsage[rowblk.IndexIter, *rowblk.IndexIter](r, start, end, transforms)
 	}
-	return estimateDiskUsage[colblk.IndexIter, *colblk.IndexIter](r, start, end)
+	return estimateDiskUsage[colblk.IndexIter, *colblk.IndexIter](r, start, end, transforms)
 }
 
 func estimateDiskUsage[I any, PI indexBlockIterator[I]](
-	r *Reader, start, end []byte,
+	r *Reader, start, end []byte, transforms IterTransforms,
 ) (uint64, error) {
 	if r.err != nil {
 		return 0, r.err
@@ -830,13 +831,13 @@ func estimateDiskUsage[I any, PI indexBlockIterator[I]](
 	var startIdxIter, endIdxIter PI
 	if !r.Attributes.Has(AttributeTwoLevelIndex) {
 		startIdxIter = new(I)
-		if err := startIdxIter.InitHandle(r.Comparer, indexH, NoTransforms); err != nil {
+		if err := startIdxIter.InitHandle(r.Comparer, indexH, transforms); err != nil {
 			return 0, err
 		}
 		endIdxIter = startIdxIter
 	} else {
 		var topIter PI = new(I)
-		if err := topIter.InitHandle(r.Comparer, indexH, NoTransforms); err != nil {
+		if err := topIter.InitHandle(r.Comparer, indexH, transforms); err != nil {
 			return 0, err
 		}
 		if !topIter.SeekGE(start) {
@@ -853,7 +854,7 @@ func estimateDiskUsage[I any, PI indexBlockIterator[I]](
 		}
 		defer startIdxBlock.Release()
 		startIdxIter = new(I)
-		err = startIdxIter.InitHandle(r.Comparer, startIdxBlock, NoTransforms)
+		err = startIdxIter.InitHandle(r.Comparer, startIdxBlock, transforms)
 		if err != nil {
 			return 0, err
 		}
@@ -869,7 +870,7 @@ func estimateDiskUsage[I any, PI indexBlockIterator[I]](
 			}
 			defer endIdxBlock.Release()
 			endIdxIter = new(I)
-			err = endIdxIter.InitHandle(r.Comparer, endIdxBlock, NoTransforms)
+			err = endIdxIter.InitHandle(r.Comparer, endIdxBlock, transforms)
 			if err != nil {
 				return 0, err
 			}

sstable/reader_test.go

@@ -199,6 +199,13 @@ func runVirtualReaderTest(t *testing.T, path string, blockSize, indexBlockSize i
 
 			showProps := td.HasArg("show-props")
 
+			var syntheticPrefix []byte
+			if td.HasArg("prefix") {
+				var synthPrefixStr string
+				td.ScanArgs(t, "prefix", &synthPrefixStr)
+				syntheticPrefix = []byte(synthPrefixStr)
+			}
+
 			syntheticSuffix = nil
 			if td.HasArg("suffix") {
 				var synthSuffixStr string
@@ -210,7 +217,10 @@ func runVirtualReaderTest(t *testing.T, path string, blockSize, indexBlockSize i
 			env.Virtual = &params
 
 			var err error
-			tableSize, err := r.EstimateDiskUsage(params.Lower.UserKey, params.Upper.UserKey, env)
+			transforms := IterTransforms{
+				SyntheticPrefixAndSuffix: block.MakeSyntheticPrefixAndSuffix(syntheticPrefix, syntheticSuffix),
+			}
+			tableSize, err := r.EstimateDiskUsage(params.Lower.UserKey, params.Upper.UserKey, env, transforms)
 			if err != nil {
 				return err.Error()
 			}
@@ -590,7 +600,7 @@ func TestInjectedErrors(t *testing.T) {
 			}
 			defer func() { reterr = firstError(reterr, r.Close()) }()
 
-			_, err = r.EstimateDiskUsage([]byte("borrower"), []byte("lender"), NoReadEnv)
+			_, err = r.EstimateDiskUsage([]byte("borrower"), []byte("lender"), NoReadEnv, NoTransforms)
 			if err != nil {
 				return err
 			}

sstable/testdata/virtual_reader_props

@@ -345,3 +345,25 @@ props:
   rocksdb.num.data.blocks: 1
   rocksdb.compression: Snappy
   rocksdb.compression_options: window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0;
+
+# Test virtual sstable with a synthetic prefix.
+build
+a.SET.1:a
+b.SET.1:b
+c.SET.1:c
+d.SET.1:d
+----
+point:    [a#1,SET-d#1,SET]
+seqnums:  [1-1]
+
+virtualize lower=poi-b.SET.1 upper=poi-c.SET.1 prefix=poi- show-props
+----
+bounds:  [poi-b#1,SET-poi-c#1,SET]
+filenum: 000010
+props:
+  rocksdb.num.entries: 1
+  rocksdb.raw.key.size: 4
+  rocksdb.raw.value.size: 1
+  rocksdb.num.data.blocks: 1
+  rocksdb.compression: Snappy
+  rocksdb.compression_options: window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0;

table_stats.go

@@ -440,7 +440,7 @@ func (d *DB) loadTableRangeDelStats(
 		// the size of the range key block relative to the overall size of the
 		// table is expected to be small.
 		if level == numLevels-1 && meta.SmallestSeqNum < maxRangeDeleteSeqNum {
-			size, err := r.EstimateDiskUsage(start, end, env)
+			size, err := r.EstimateDiskUsage(start, end, env, meta.IterTransforms())
 			if err != nil {
 				return nil, err
 			}
@@ -649,7 +649,7 @@ func (d *DB) estimateReclaimedSizeBeneath(
 				var size uint64
 				err := d.fileCache.withReader(ctx, block.NoReadEnv, file,
 					func(r *sstable.Reader, env sstable.ReadEnv) (err error) {
-						size, err = r.EstimateDiskUsage(start, end, env)
+						size, err = r.EstimateDiskUsage(start, end, env, file.IterTransforms())
 						return err
 					})
 				if err != nil {

tool/sstable.go

@@ -542,7 +542,7 @@ func (s *sstableT) runScan(cmd *cobra.Command, args []string) {
 func (s *sstableT) runSpace(cmd *cobra.Command, args []string) {
 	stdout, stderr := cmd.OutOrStdout(), cmd.OutOrStderr()
 	s.foreachSstable(stderr, args, func(path string, r *sstable.Reader, props sstable.Properties) {
-		bytes, err := r.EstimateDiskUsage(s.start, s.end, sstable.NoReadEnv)
+		bytes, err := r.EstimateDiskUsage(s.start, s.end, sstable.NoReadEnv, sstable.NoTransforms)
 		if err != nil {
 			fmt.Fprintf(stderr, "%s\n", err)
 			return