db: pull more recovery logic into recoverState

Additionally, remove the call to wal.Scan in Open, instead relying on the WALs
scanned in recoverState. This was accidentally overlooked during the refactor
that began scanning wals in recoverState.

Author: jbowens

open.go

@@ -306,11 +306,6 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 			Buckets: FsyncLatencyBuckets,
 		})
 	}
-	walDirs := d.dirs.WALDirs()
-	wals, err := wal.Scan(walDirs...)
-	if err != nil {
-		return nil, err
-	}
 
 	walManager, err := wal.Init(walOpts, rs.walsReplay)
 	if err != nil {
@@ -339,44 +334,17 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 	d.blobFileDiskUsageAnnotator = d.makeBlobFileDiskSpaceUsageAnnotator()
 
 	d.mu.versions.markFileNumUsed(rs.maxFilenumUsed)
-	if n := len(wals); n > 0 {
-		// Don't reuse any obsolete file numbers to avoid modifying an
-		// ingested sstable's original external file.
-		d.mu.versions.markFileNumUsed(base.DiskFileNum(wals[n-1].Num))
-	}
-
-	// Ratchet d.mu.versions.nextFileNum ahead of all known objects in the
-	// objProvider. This avoids FileNum collisions with obsolete sstables.
-	objects := d.objProvider.List()
-	for _, obj := range objects {
-		d.mu.versions.markFileNumUsed(obj.DiskFileNum)
-	}
-
-	// Validate the most-recent OPTIONS file, if there is one.
-	if rs.previousOptionsFilename != "" {
-		path := opts.FS.PathJoin(dirname, rs.previousOptionsFilename)
-		previousOptions, err := readOptionsFile(opts, path)
-		if err != nil {
-			return nil, err
-		}
-		if err := opts.CheckCompatibility(dirname, previousOptions); err != nil {
-			return nil, err
-		}
-	}
 
 	// Replay any newer log files than the ones named in the manifest.
 	var flushableIngests []*ingestedFlushable
-	for i, w := range wals {
-		if base.DiskFileNum(w.Num) < d.mu.versions.minUnflushedLogNum {
-			continue
-		}
+	for i, w := range rs.walsReplay {
 		// WALs other than the last one would have been closed cleanly.
 		//
 		// Note: we used to never require strict WAL tails when reading from older
 		// versions: RocksDB 6.2.1 and the version of Pebble included in CockroachDB
 		// 20.1 do not guarantee that closed WALs end cleanly. But the earliest
 		// compatible Pebble format is newer and guarantees a clean EOF.
-		strictWALTail := i < len(wals)-1
+		strictWALTail := i < len(rs.walsReplay)-1
 		fi, maxSeqNum, err := d.replayWAL(jobID, w, strictWALTail)
 		if err != nil {
 			return nil, err

recovery.go

@@ -108,12 +108,34 @@ func (rs *recoveredState) init(opts *Options, dirname string) error {
 		}
 	}
 
+	// Validate the most-recent OPTIONS file, if there is one.
+	if rs.previousOptionsFilename != "" {
+		path := opts.FS.PathJoin(dirname, rs.previousOptionsFilename)
+		previousOptions, err := readOptionsFile(opts, path)
+		if err != nil {
+			return err
+		}
+		if err := opts.CheckCompatibility(dirname, previousOptions); err != nil {
+			return err
+		}
+	}
+
+	// Ratchet rs.maxFilenumUsed ahead of all known objects in the objProvider.
+	// This avoids FileNum collisions with obsolete sstables.
+	objects := rs.objProvider.List()
+	for _, obj := range objects {
+		rs.maxFilenumUsed = max(rs.maxFilenumUsed, obj.DiskFileNum)
+	}
+
 	// Find all the WAL files across the various WAL directories.
 	wals, err := wal.Scan(rs.dirs.WALDirs()...)
 	if err != nil {
 		return err
 	}
 	for _, w := range wals {
+		// Don't reuse any obsolete file numbers to avoid modifying an ingested
+		// sstable's original external file.
+		rs.maxFilenumUsed = max(rs.maxFilenumUsed, base.DiskFileNum(w.Num))
 		if rs.recoveredVersion == nil || base.DiskFileNum(w.Num) >= rs.recoveredVersion.minUnflushedLogNum {
 			rs.walsReplay = append(rs.walsReplay, w)
 		} else {

testdata/checkpoint

@@ -358,9 +358,9 @@ open-dir: checkpoints/checkpoint1
 open-dir: checkpoints/checkpoint1
 open: checkpoints/checkpoint1/MANIFEST-000001
 close: checkpoints/checkpoint1/MANIFEST-000001
-open-dir: checkpoints/checkpoint1
 open: checkpoints/checkpoint1/OPTIONS-000002
 close: checkpoints/checkpoint1/OPTIONS-000002
+open-dir: checkpoints/checkpoint1
 open: checkpoints/checkpoint1/000006.log (options: *vfs.sequentialReadsOption)
 close: checkpoints/checkpoint1/000006.log
 get-disk-usage: checkpoints/checkpoint1
@@ -426,9 +426,9 @@ open-dir: checkpoints/checkpoint2
 open-dir: checkpoints/checkpoint2
 open: checkpoints/checkpoint2/MANIFEST-000001
 close: checkpoints/checkpoint2/MANIFEST-000001
-open-dir: checkpoints/checkpoint2
 open: checkpoints/checkpoint2/OPTIONS-000002
 close: checkpoints/checkpoint2/OPTIONS-000002
+open-dir: checkpoints/checkpoint2
 open: checkpoints/checkpoint2/000006.log (options: *vfs.sequentialReadsOption)
 close: checkpoints/checkpoint2/000006.log
 get-disk-usage: checkpoints/checkpoint2
@@ -469,9 +469,9 @@ open-dir: checkpoints/checkpoint3
 open-dir: checkpoints/checkpoint3
 open: checkpoints/checkpoint3/MANIFEST-000001
 close: checkpoints/checkpoint3/MANIFEST-000001
-open-dir: checkpoints/checkpoint3
 open: checkpoints/checkpoint3/OPTIONS-000002
 close: checkpoints/checkpoint3/OPTIONS-000002
+open-dir: checkpoints/checkpoint3
 open: checkpoints/checkpoint3/000006.log (options: *vfs.sequentialReadsOption)
 close: checkpoints/checkpoint3/000006.log
 get-disk-usage: checkpoints/checkpoint3
@@ -613,9 +613,9 @@ open-dir: checkpoints/checkpoint4
 open-dir: checkpoints/checkpoint4
 open: checkpoints/checkpoint4/MANIFEST-000001
 close: checkpoints/checkpoint4/MANIFEST-000001
-open-dir: checkpoints/checkpoint4
 open: checkpoints/checkpoint4/OPTIONS-000002
 close: checkpoints/checkpoint4/OPTIONS-000002
+open-dir: checkpoints/checkpoint4
 open: checkpoints/checkpoint4/000008.log (options: *vfs.sequentialReadsOption)
 close: checkpoints/checkpoint4/000008.log
 get-disk-usage: checkpoints/checkpoint4
@@ -729,9 +729,9 @@ open-dir: checkpoints/checkpoint5
 open-dir: checkpoints/checkpoint5
 open: checkpoints/checkpoint5/MANIFEST-000001
 close: checkpoints/checkpoint5/MANIFEST-000001
-open-dir: checkpoints/checkpoint5
 open: checkpoints/checkpoint5/OPTIONS-000002
 close: checkpoints/checkpoint5/OPTIONS-000002
+open-dir: checkpoints/checkpoint5
 open: checkpoints/checkpoint5/000008.log (options: *vfs.sequentialReadsOption)
 close: checkpoints/checkpoint5/000008.log
 create: checkpoints/checkpoint5/temporary.000018.dbtmp
@@ -833,9 +833,9 @@ open-dir: checkpoints/checkpoint6
 open-dir: checkpoints/checkpoint6
 open: checkpoints/checkpoint6/MANIFEST-000001
 close: checkpoints/checkpoint6/MANIFEST-000001
-open-dir: checkpoints/checkpoint6
 open: checkpoints/checkpoint6/OPTIONS-000002
 close: checkpoints/checkpoint6/OPTIONS-000002
+open-dir: checkpoints/checkpoint6
 open: checkpoints/checkpoint6/000008.log (options: *vfs.sequentialReadsOption)
 close: checkpoints/checkpoint6/000008.log
 create: checkpoints/checkpoint6/temporary.000018.dbtmp
@@ -1093,9 +1093,9 @@ open-dir: checkpoints/checkpoint8
 open-dir: checkpoints/checkpoint8
 open: checkpoints/checkpoint8/MANIFEST-000001
 close: checkpoints/checkpoint8/MANIFEST-000001
-open-dir: checkpoints/checkpoint8
 open: checkpoints/checkpoint8/OPTIONS-000002
 close: checkpoints/checkpoint8/OPTIONS-000002
+open-dir: checkpoints/checkpoint8
 open: checkpoints/checkpoint8/000004.log (options: *vfs.sequentialReadsOption)
 close: checkpoints/checkpoint8/000004.log
 get-disk-usage: checkpoints/checkpoint8
@@ -1207,9 +1207,9 @@ open-dir: checkpoints/checkpoint9
 open-dir: checkpoints/checkpoint9
 open: checkpoints/checkpoint9/MANIFEST-000001
 close: checkpoints/checkpoint9/MANIFEST-000001
-open-dir: checkpoints/checkpoint9
 open: checkpoints/checkpoint9/OPTIONS-000002
 close: checkpoints/checkpoint9/OPTIONS-000002
+open-dir: checkpoints/checkpoint9
 open: checkpoints/checkpoint9/000007.log (options: *vfs.sequentialReadsOption)
 close: checkpoints/checkpoint9/000007.log
 get-disk-usage: checkpoints/checkpoint9

testdata/checkpoint_shared

@@ -339,9 +339,9 @@ close: checkpoints/checkpoint1/REMOTE-OBJ-CATALOG-000001
 open-dir: checkpoints/checkpoint1
 open: checkpoints/checkpoint1/MANIFEST-000001
 close: checkpoints/checkpoint1/MANIFEST-000001
-open-dir: checkpoints/checkpoint1
 open: checkpoints/checkpoint1/OPTIONS-000002
 close: checkpoints/checkpoint1/OPTIONS-000002
+open-dir: checkpoints/checkpoint1
 open: checkpoints/checkpoint1/000006.log (options: *vfs.sequentialReadsOption)
 close: checkpoints/checkpoint1/000006.log
 get-disk-usage: checkpoints/checkpoint1
@@ -392,9 +392,9 @@ close: checkpoints/checkpoint2/REMOTE-OBJ-CATALOG-000001
 open-dir: checkpoints/checkpoint2
 open: checkpoints/checkpoint2/MANIFEST-000001
 close: checkpoints/checkpoint2/MANIFEST-000001
-open-dir: checkpoints/checkpoint2
 open: checkpoints/checkpoint2/OPTIONS-000002
 close: checkpoints/checkpoint2/OPTIONS-000002
+open-dir: checkpoints/checkpoint2
 open: checkpoints/checkpoint2/000006.log (options: *vfs.sequentialReadsOption)
 close: checkpoints/checkpoint2/000006.log
 get-disk-usage: checkpoints/checkpoint2

testdata/cleaner

@@ -254,10 +254,10 @@ open-dir: db1
 open-dir: db1
 open: db1/MANIFEST-000001
 close: db1/MANIFEST-000001
-remove: db1_wal/000003.log
-open-dir: db1_wal
 open: db1/OPTIONS-000002
 close: db1/OPTIONS-000002
+remove: db1_wal/000003.log
+open-dir: db1_wal
 open: db1_wal/000004.log (options: *vfs.sequentialReadsOption)
 close: db1_wal/000004.log
 create: db1/temporary.000458.dbtmp