sstable: remove VirtualReader and CommonReader

Refactor VirtualReaderParams into block.ReadEnv to
get rid of VirtualReader and CommonReader.

Author: EdwardX29

checkpoint.go

@@ -283,7 +283,7 @@ func (d *DB) Checkpoint(
 			}
 
 			fileBacking := f.FileBacking
-			if f.Virtual {
+			if f.Virtual != nil {
 				if _, ok := requiredVirtualBackingFiles[fileBacking.DiskFileNum]; ok {
 					continue
 				}

cockroachkvs/cockroachkvs_bench_test.go

@@ -132,7 +132,7 @@ func benchmarkRandSeekInSST(
 	rp := sstable.MakeTrivialReaderProvider(reader)
 	iter, err := reader.NewPointIter(ctx, sstable.IterOptions{
 		FilterBlockSizeLimit: sstable.NeverUseFilterBlock,
-		Env:                  block.ReadEnv{Stats: &stats, IterStats: nil},
+		Env:                  sstable.ReadEnv{Block: block.ReadEnv{Stats: &stats, IterStats: nil}},
 		ReaderProvider:       rp,
 	})
 	require.NoError(b, err)
@@ -150,7 +150,7 @@ func benchmarkRandSeekInSST(
 		key := queryKeys[i%numQueryKeys]
 		iter, err := reader.NewPointIter(ctx, sstable.IterOptions{
 			FilterBlockSizeLimit: sstable.NeverUseFilterBlock,
-			Env:                  block.ReadEnv{Stats: &stats, IterStats: nil},
+			Env:                  sstable.ReadEnv{Block: block.ReadEnv{Stats: &stats, IterStats: nil}},
 			ReaderProvider:       rp,
 		})
 		if err != nil {

compaction.go

@@ -424,7 +424,7 @@ func newCompaction(
 		if mustCopy {
 			// If the source is virtual, it's best to just rewrite the file as all
 			// conditions in the above comment are met.
-			if !meta.Virtual {
+			if meta.Virtual == nil {
 				c.kind = compactionKindCopy
 			}
 		} else {
@@ -966,7 +966,7 @@ func (c *compaction) newRangeDelIter(
 	iterSet, err := newIters(context.Background(), f.TableMetadata, &opts,
 		internalIterOpts{
 			compaction: true,
-			readEnv:    block.ReadEnv{BufferPool: &c.bufferPool},
+			readEnv:    sstable.ReadEnv{Block: block.ReadEnv{BufferPool: &c.bufferPool}},
 		}, iterRangeDeletions)
 	if err != nil {
 		return nil, err
@@ -1310,8 +1310,8 @@ func (d *DB) runIngestFlush(c *compaction) (*manifest.VersionEdit, error) {
 
 		// This file fits perfectly within the excise span, so we can slot it at L6.
 		if ingestFlushable.exciseSpan.Valid() &&
-			ingestFlushable.exciseSpan.Contains(d.cmp, file.TableMetadata.Smallest) &&
-			ingestFlushable.exciseSpan.Contains(d.cmp, file.TableMetadata.Largest) {
+			ingestFlushable.exciseSpan.Contains(d.cmp, file.Smallest) &&
+			ingestFlushable.exciseSpan.Contains(d.cmp, file.Largest) {
 			level = 6
 		} else {
 			// TODO(radu): this can perform I/O; we should not do this while holding DB.mu.
@@ -1320,18 +1320,18 @@ func (d *DB) runIngestFlush(c *compaction) (*manifest.VersionEdit, error) {
 				return nil, err
 			}
 			level, fileToSplit, err = ingestTargetLevel(
-				ctx, d.cmp, lsmOverlap, baseLevel, d.mu.compact.inProgress, file.TableMetadata, suggestSplit,
+				ctx, d.cmp, lsmOverlap, baseLevel, d.mu.compact.inProgress, file, suggestSplit,
 			)
 			if err != nil {
 				return nil, err
 			}
 		}
 
 		// Add the current flushableIngest file to the version.
-		ve.NewTables = append(ve.NewTables, newTableEntry{Level: level, Meta: file.TableMetadata})
+		ve.NewTables = append(ve.NewTables, newTableEntry{Level: level, Meta: file})
 		if fileToSplit != nil {
 			ingestSplitFiles = append(ingestSplitFiles, ingestSplitFile{
-				ingestFile: file.TableMetadata,
+				ingestFile: file,
 				splitFile:  fileToSplit,
 				level:      level,
 			})
@@ -2401,7 +2401,7 @@ func (d *DB) cleanupVersionEdit(ve *versionEdit) {
 		deletedFiles[key.FileNum] = struct{}{}
 	}
 	for i := range ve.NewTables {
-		if ve.NewTables[i].Meta.Virtual {
+		if ve.NewTables[i].Meta.Virtual != nil {
 			// We handle backing files separately.
 			continue
 		}
@@ -2561,8 +2561,8 @@ func (d *DB) runCopyCompaction(
 			panic("pebble: scheduled a copy compaction that is not actually moving files to shared storage")
 		}
 		// Note that based on logic in the compaction picker, we're guaranteed
-		// inputMeta.Virtual is false.
-		if inputMeta.Virtual {
+		// inputMeta.Virtual is nil.
+		if inputMeta.Virtual != nil {
 			panic(errors.AssertionFailedf("cannot do a copy compaction of a virtual sstable across local/remote storage"))
 		}
 	}
@@ -2660,12 +2660,12 @@ func (d *DB) runCopyCompaction(
 
 		// NB: external files are always virtual.
 		var wrote uint64
-		err = d.fileCache.withVirtualReader(ctx, block.NoReadEnv, inputMeta.VirtualMeta(), func(r sstable.VirtualReader, _ block.ReadEnv) error {
+		err = d.fileCache.withReader(ctx, block.NoReadEnv, inputMeta.VirtualMeta(), func(r *sstable.Reader, env sstable.ReadEnv) error {
 			var err error
 			// TODO(radu): plumb a ReadEnv to CopySpan (it could use the buffer pool
 			// or update category stats).
 			wrote, err = sstable.CopySpan(ctx,
-				src, r.UnsafeReader(), d.opts.MakeReaderOptions(),
+				src, r, d.opts.MakeReaderOptions(),
 				w, d.opts.MakeWriterOptions(c.outputLevel.level, d.TableFormat()),
 				start, end,
 			)
@@ -2700,7 +2700,7 @@ func (d *DB) runCopyCompaction(
 		Level: c.outputLevel.level,
 		Meta:  newMeta,
 	}}
-	if newMeta.Virtual {
+	if newMeta.Virtual != nil {
 		ve.CreatedBackingTables = []*fileBacking{newMeta.FileBacking}
 	}
 	c.metrics[c.outputLevel.level] = &LevelMetrics{
@@ -2904,7 +2904,7 @@ func (d *DB) runDeleteOnlyCompaction(
 	// NewFiles.
 	usedBackingFiles := make(map[base.DiskFileNum]struct{})
 	for _, e := range ve.NewTables {
-		if e.Meta.Virtual {
+		if e.Meta.Virtual != nil {
 			usedBackingFiles[e.Meta.FileBacking.DiskFileNum] = struct{}{}
 		}
 	}
@@ -3078,18 +3078,18 @@ func (d *DB) compactAndWrite(
 	// translate to 3 MiB per compaction.
 	c.bufferPool.Init(12)
 	defer c.bufferPool.Release()
-	env := block.ReadEnv{
+	blockReadEnv := block.ReadEnv{
 		BufferPool: &c.bufferPool,
 		Stats:      &c.stats,
 		IterStats: d.fileCache.SSTStatsCollector().Accumulator(
 			uint64(uintptr(unsafe.Pointer(c))),
 			categoryCompaction,
 		),
 	}
-	c.valueFetcher.Init(d.fileCache, env)
+	c.valueFetcher.Init(d.fileCache, blockReadEnv)
 	iiopts := internalIterOpts{
 		compaction:       true,
-		readEnv:          env,
+		readEnv:          sstable.ReadEnv{Block: blockReadEnv},
 		blobValueFetcher: &c.valueFetcher,
 	}
 

compaction_picker.go

@@ -1186,7 +1186,7 @@ func pickCompactionSeedFile(
 func responsibleForGarbageBytes(
 	virtualBackings *manifest.VirtualBackings, m *tableMetadata,
 ) uint64 {
-	if !m.Virtual {
+	if m.Virtual == nil {
 		return 0
 	}
 	useCount, virtualizedSize := virtualBackings.Usage(m.FileBacking.DiskFileNum)

compaction_test.go

@@ -600,7 +600,7 @@ func TestCompaction(t *testing.T) {
 		defer provider.Close()
 		for _, levelMetadata := range v.Levels {
 			for meta := range levelMetadata.All() {
-				if meta.Virtual {
+				if meta.Virtual != nil {
 					continue
 				}
 				f, err := provider.OpenForReading(context.Background(), base.FileTypeTable, meta.FileBacking.DiskFileNum, objstorage.OpenOptions{})
@@ -3001,7 +3001,7 @@ func hasExternalFiles(d *DB) bool {
 	for level := 0; level < manifest.NumLevels; level++ {
 		iter := v.Levels[level].Iter()
 		for m := iter.First(); m != nil; m = iter.Next() {
-			if m.Virtual {
+			if m.Virtual != nil {
 				meta, err := d.objProvider.Lookup(base.FileTypeTable, m.FileBacking.DiskFileNum)
 				if err != nil {
 					panic(err)

data_test.go

@@ -1286,11 +1286,13 @@ func runSSTablePropertiesCmd(t *testing.T, td *datadriven.TestData, d *DB) strin
 	}
 	defer r.Close()
 
-	var v sstable.VirtualReader
 	props := r.Properties.String()
-	if m != nil && m.Virtual {
-		v = sstable.MakeVirtualReader(r, m.VirtualMeta().VirtualReaderParams(false /* isShared */))
-		props = v.Properties.String()
+	env := sstable.ReadEnv{}
+	if m != nil && m.Virtual != nil {
+		m.InitVirtual(false /* isShared */)
+		env.Virtual = m.Virtual
+		scaledProps := r.Properties.GetScaledProperties(env.Virtual.BackingSize, env.Virtual.Size)
+		props = scaledProps.String()
 	}
 	if len(td.Input) == 0 {
 		return props

db.go

@@ -1353,7 +1353,7 @@ type internalIterOpts struct {
 	// NewCompactionIter; these iterators have a more constrained interface
 	// and are optimized for the sequential scan of a compaction.
 	compaction         bool
-	readEnv            block.ReadEnv
+	readEnv            sstable.ReadEnv
 	boundLimitedFilter sstable.BoundLimitedBlockPropertyFilter
 	// blobValueFetcher is the base.ValueFetcher to use when constructing
 	// internal values to represent values stored externally in blob files.
@@ -1423,7 +1423,7 @@ func (i *Iterator) constructPointIter(
 	}
 	i.blobValueFetcher.Init(i.fc, readEnv)
 	internalOpts := internalIterOpts{
-		readEnv:          readEnv,
+		readEnv:          sstable.ReadEnv{Block: readEnv},
 		blobValueFetcher: &i.blobValueFetcher,
 	}
 	if i.opts.RangeKeyMasking.Filter != nil {
@@ -2248,7 +2248,7 @@ func (d *DB) SSTables(opts ...SSTablesOption) ([][]SSTableInfo, error) {
 				}
 				destTables[j].Properties = p
 			}
-			destTables[j].Virtual = m.Virtual
+			destTables[j].Virtual = m.Virtual != nil
 			destTables[j].BackingSSTNum = m.FileBacking.DiskFileNum
 			objMeta, err := d.objProvider.Lookup(base.FileTypeTable, m.FileBacking.DiskFileNum)
 			if err != nil {

download.go

@@ -403,7 +403,7 @@ func firstExternalFileInLevelIter(
 		f = it.Next()
 	}
 	for ; f != nil && endBound.IsUpperBoundFor(cmp, f.Smallest.UserKey); f = it.Next() {
-		if f.Virtual && objstorage.IsExternalTable(objProvider, f.FileBacking.DiskFileNum) {
+		if f.Virtual != nil && objstorage.IsExternalTable(objProvider, f.FileBacking.DiskFileNum) {
 			return f
 		}
 	}

download_test.go

@@ -181,7 +181,7 @@ func TestDownloadTask(t *testing.T) {
 					ch <- ErrCancelledCompaction
 				} else {
 					fmt.Fprintf(&buf, "downloading %s\n", f.FileNum)
-					f.Virtual = false
+					f.Virtual = nil
 					f.FileBacking.DiskFileNum = base.DiskFileNum(f.FileNum)
 					ch <- nil
 				}

excise.go

@@ -12,6 +12,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/manifest"
+	"github.com/cockroachdb/pebble/sstable/virtual"
 )
 
 // Excise atomically deletes all data overlapping with the provided span. All
@@ -102,7 +103,7 @@ func (d *DB) exciseTable(
 	// https://github.com/cockroachdb/pebble/issues/2112 .
 	if d.cmp(m.Smallest.UserKey, exciseSpan.Start) < 0 {
 		leftTable = &tableMetadata{
-			Virtual:     true,
+			Virtual:     &virtual.VirtualReaderParams{},
 			FileBacking: m.FileBacking,
 			FileNum:     d.mu.versions.getNextFileNum(),
 			// Note that these are loose bounds for smallest/largest seqnums, but they're
@@ -135,7 +136,7 @@ func (d *DB) exciseTable(
 		// See comment before the definition of leftFile for the motivation behind
 		// calculating tight user-key bounds.
 		rightTable = &tableMetadata{
-			Virtual:     true,
+			Virtual:     &virtual.VirtualReaderParams{},
 			FileBacking: m.FileBacking,
 			FileNum:     d.mu.versions.getNextFileNum(),
 			// Note that these are loose bounds for smallest/largest seqnums, but they're
@@ -351,7 +352,7 @@ func applyExciseToVersionEdit(
 	if leftTable == nil && rightTable == nil {
 		return
 	}
-	if !originalTable.Virtual {
+	if originalTable.Virtual == nil {
 		// If the original table was virtual, then its file backing is already known
 		// to the manifest; we don't need to create another file backing. Note that
 		// there must be only one CreatedBackingTables entry per backing sstable.