db: clear a subset of Iterator fields on Close

Adjust the logic that zeroes an Iterator to only clear fields after a marker
`clearForReuseBoundary` field, and use this to delegate clearing of the
blob.ValueFetcher to the blob.ValueFetcher's Close method. When initializing
the Iterator, only set the fields that need to be set (instead relying on the
allocated Iterator's other fields having already been zeroed.  This in
preparation for recycling a buffer within the blob.ValueFetcher across reuses
of the containing Iterator.

```
goos: darwin
goarch: arm64
pkg: github.com/cockroachdb/pebble
cpu: Apple M1 Pro
                    │   old.txt    │             fetcher.txt             │
                    │    sec/op    │   sec/op     vs base                │
NewIterClose/1-10     1014.0n ± 1%   849.2n ± 2%  -16.25% (p=0.000 n=25)
NewIterClose/10-10     5.246µ ± 0%   5.184µ ± 1%   -1.18% (p=0.000 n=25)
NewIterClose/100-10    78.68µ ± 1%   79.25µ ± 2%        ~ (p=0.151 n=25)
geomean                7.480µ        7.040µ        -5.89%
```

Author: jbowens

db.go

@@ -1059,22 +1059,20 @@ func (d *DB) newIter(
 	// them together.
 	buf := newIterAlloc()
 	dbi := &buf.dbi
-	*dbi = Iterator{
-		ctx:                 ctx,
-		alloc:               buf,
-		merge:               d.merge,
-		comparer:            d.opts.Comparer,
-		readState:           readState,
-		version:             newIterOpts.snapshot.vers,
-		keyBuf:              buf.keyBuf,
-		prefixOrFullSeekKey: buf.prefixOrFullSeekKey,
-		boundsBuf:           buf.boundsBuf,
-		fc:                  d.fileCache,
-		newIters:            newIters,
-		newIterRangeKey:     newIterRangeKey,
-		seqNum:              seqNum,
-		batchOnlyIter:       newIterOpts.batch.batchOnly,
-	}
+	dbi.ctx = ctx
+	dbi.alloc = buf
+	dbi.merge = d.merge
+	dbi.comparer = d.opts.Comparer
+	dbi.readState = readState
+	dbi.version = newIterOpts.snapshot.vers
+	dbi.keyBuf = buf.keyBuf
+	dbi.prefixOrFullSeekKey = buf.prefixOrFullSeekKey
+	dbi.boundsBuf = buf.boundsBuf
+	dbi.fc = d.fileCache
+	dbi.newIters = newIters
+	dbi.newIterRangeKey = newIterRangeKey
+	dbi.seqNum = seqNum
+	dbi.batchOnlyIter = newIterOpts.batch.batchOnly
 	if o != nil {
 		dbi.opts = *o
 		dbi.processBounds(o.LowerBound, o.UpperBound)

iterator.go

@@ -194,6 +194,17 @@ type LazyValue = base.LazyValue
 // Next, Prev) return without advancing if the iterator has an accumulated
 // error.
 type Iterator struct {
+	// blobValueFetcher is the ValueFetcher to use when retrieving values stored
+	// externally in blob files.
+	blobValueFetcher blob.ValueFetcher
+
+	// All fields below this field are cleared during Iterator.Close before
+	// returning the Iterator to the pool. Any fields above this field must also
+	// be cleared, but may be cleared as a part of the body of Iterator.Close
+	// (eg, if these types have their own Close method that zeroes their
+	// fields).
+	clearForReuseBoundary struct{}
+
 	// The context is stored here since (a) Iterators are expected to be
 	// short-lived (since they pin memtables and sstables), (b) plumbing a
 	// context into every method is very painful, (c) they do not (yet) respect
@@ -228,9 +239,6 @@ type Iterator struct {
 	// For use in LazyValue.Value.
 	lazyValueBuf []byte
 	valueCloser  io.Closer
-	// blobValueFetcher is the ValueFetcher to use when retrieving values stored
-	// externally in blob files.
-	blobValueFetcher blob.ValueFetcher
 	// boundsBuf holds two buffers used to store the lower and upper bounds.
 	// Whenever the Iterator's bounds change, the new bounds are copied into
 	// boundsBuf[boundsBufIdx]. The two bounds share a slice to reduce
@@ -315,6 +323,13 @@ type Iterator struct {
 	nextPrefixNotPermittedByUpperBound bool
 }
 
+const clearOff = unsafe.Offsetof(Iterator{}.clearForReuseBoundary)
+const clearLen = unsafe.Sizeof(Iterator{}) - clearOff
+
+func (i *Iterator) clearForReuse() {
+	*(*[clearLen]byte)(unsafe.Add(unsafe.Pointer(i), clearOff)) = [clearLen]byte{}
+}
+
 // cmp is a convenience shorthand for the i.comparer.Compare function.
 func (i *Iterator) cmp(a, b []byte) int {
 	return i.comparer.Compare(a, b)
@@ -2452,7 +2467,7 @@ func (i *Iterator) Close() error {
 	if i.batch != nil {
 		alloc.batchState = iteratorBatchState{}
 	}
-	alloc.dbi = Iterator{}
+	alloc.dbi.clearForReuse()
 	alloc.merging = mergingIter{}
 	clear(mergingIterHeapItems[:cap(mergingIterHeapItems)])
 	alloc.merging.heap.items = mergingIterHeapItems

sstable/blob/fetcher.go

@@ -61,8 +61,8 @@ type ValueFetcher struct {
 	readerProvider ReaderProvider
 	env            block.ReadEnv
 	fetchCount     int
-	readers        [maxCachedReaders]cachedReader
 	bufMangler     invariants.BufMangler
+	readers        [maxCachedReaders]cachedReader
 }
 
 // TODO(jackson): Support setting up a read handle for compaction when relevant.
@@ -184,6 +184,11 @@ func (r *ValueFetcher) Close() error {
 			err = errors.CombineErrors(err, r.readers[i].Close())
 		}
 	}
+	r.fileMapping = nil
+	r.readerProvider = nil
+	r.env = block.ReadEnv{}
+	r.fetchCount = 0
+	r.bufMangler = invariants.BufMangler{}
 	return err
 }