db: remove obsolete WALs before proceeding with Init

Fixes: #5354

Author: annrpom

open.go

@@ -33,7 +33,6 @@ import (
 	"github.com/cockroachdb/pebble/record"
 	"github.com/cockroachdb/pebble/vfs"
 	"github.com/cockroachdb/pebble/wal"
-	"github.com/cockroachdb/redact"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -413,11 +412,40 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 	if err != nil {
 		return nil, err
 	}
-	d.opts.Logger.Infof("Found %d WALs", redact.Safe(len(wals)))
-	for i := range wals {
-		d.opts.Logger.Infof("  - %s", wals[i])
+
+	// Remove obsolete WAL files now (as opposed to relying on asynchronous cleanup)
+	// to prevent crash loops due to no disk space (ENOSPC).
+	var retainedWALs wal.Logs
+	for _, w := range wals {
+		if base.DiskFileNum(w.Num) < d.mu.versions.minUnflushedLogNum {
+			// Log obsolete WALs that will be removed.
+			for i := range w.NumSegments() {
+				fs, path := w.SegmentLocation(i)
+				if err := fs.Remove(path); err != nil {
+					// It's not a big deal if we can't delete the file now.
+					// We'll try to remove it later in the cleanup process.
+					d.opts.EventListener.WALDeleted(WALDeleteInfo{
+						JobID:   0,
+						Path:    path,
+						FileNum: base.DiskFileNum(w.Num),
+						Err:     err,
+					})
+					retainedWALs = append(retainedWALs, w)
+				} else {
+					d.opts.EventListener.WALDeleted(WALDeleteInfo{
+						JobID:   0,
+						Path:    path,
+						FileNum: base.DiskFileNum(w.Num),
+						Err:     nil,
+					})
+				}
+			}
+		} else {
+			retainedWALs = append(retainedWALs, w)
+		}
 	}
-	walManager, err := wal.Init(walOpts, wals)
+
+	walManager, err := wal.Init(walOpts, retainedWALs)
 	if err != nil {
 		return nil, err
 	}

testdata/cleaner

@@ -244,6 +244,7 @@ open-dir: db1
 open: db1/MANIFEST-000001
 close: db1/MANIFEST-000001
 lock: db1_wal/LOCK
+remove: db1_wal/000002.log
 open-dir: db1_wal
 open: db1/OPTIONS-000003
 close: db1/OPTIONS-000003
@@ -255,7 +256,6 @@ create: db1/marker.manifest.000002.MANIFEST-000458
 close: db1/marker.manifest.000002.MANIFEST-000458
 remove: db1/marker.manifest.000001.MANIFEST-000001
 sync: db1
-remove: db1_wal/000002.log
 remove: db1_wal/000004.log
 create: db1_wal/000459.log
 sync: db1_wal

wal/wal.go

@@ -156,7 +156,10 @@ type Options struct {
 }
 
 // Init constructs and initializes a WAL manager from the provided options and
-// the set of initial logs.
+// the set of initial logs. The initial parameter must contain all WAL files
+// that have not been deleted. It is only used to queue these up for deletion
+// at the appropriate time (i.e. when their numbers are less than minUnflushedNum)
+// and to ensure that they won't be recycled.
 func Init(o Options, initial Logs) (Manager, error) {
 	var m Manager
 	if o.Secondary == (Dir{}) {