server: Add EngineTypeDefault, resolve it to Pebble if last used

Adds a new engine type called EngineTypeDefault, that resolves
at start time to RocksDB, or Pebble if that was last used
to write to the first store. This effectively makes the
--storage-engine flag sticky for future runs where no
engine is specified. If the flag is passed in again, that
specified engine type takes priority over the one used last.

Fixes #42445 .

Release note: None.

pkg/cli/cliflags/flags.go

@@ -716,7 +716,11 @@ Also, if you use equal signs in the file path to a store, you must use the
 		Name: "storage-engine",
 		Description: `
 Storage engine to use for all stores on this cockroach node. Options are rocksdb,
-or pebble.`,
+or pebble.
+
+If left unspecified, the storage engine last used to write to the
+first store directory is used (see --store). If unspecified and store directory
+is uninitialized, rocksdb is used as the default storage engine.`,
 	}
 
 	Size = FlagInfo{

pkg/cli/debug.go

@@ -58,6 +58,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/uuid"
 	"github.com/cockroachdb/pebble"
 	"github.com/cockroachdb/pebble/tool"
+	"github.com/cockroachdb/pebble/vfs"
 	"github.com/gogo/protobuf/jsonpb"
 	"github.com/kr/pretty"
 	"github.com/pkg/errors"
@@ -140,8 +141,19 @@ func OpenEngine(dir string, stopper *stop.Stopper, opts OpenEngineOptions) (engi
 	}
 
 	var db engine.Engine
+	storageEngine := engine.DefaultStorageEngine
+	if storageEngine == enginepb.EngineTypeDefault {
+		storageEngine = enginepb.EngineTypeRocksDB
+		// Check if this storage directory was last written to by pebble. In that
+		// case, default to opening a Pebble engine.
+		if version, err := pebble.GetVersion(storageConfig.Dir, vfs.Default); err == nil {
+			if version != "" && !strings.HasPrefix(version, "rocksdb") {
+				storageEngine = enginepb.EngineTypePebble
+			}
+		}
+	}
 
-	switch engine.DefaultStorageEngine {
+	switch storageEngine {
 	case enginepb.EngineTypePebble:
 		cfg := engine.PebbleConfig{
 			StorageConfig: storageConfig,

pkg/cli/start.go

@@ -39,6 +39,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/server/status"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/storage/engine"
+	"github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb"
 	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/grpcutil"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
@@ -51,6 +52,8 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/sysutil"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/cockroachdb/cockroach/pkg/util/tracing"
+	"github.com/cockroachdb/pebble"
+	"github.com/cockroachdb/pebble/vfs"
 	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -530,6 +533,19 @@ func runStart(cmd *cobra.Command, args []string, disableReplication bool) error
 	if serverCfg.Settings.ExternalIODir, err = initExternalIODir(ctx, serverCfg.Stores.Specs[0]); err != nil {
 		return err
 	}
+	// If the storage engine is set to "default", check the engine type used in
+	// this store directory in a past run. If this check fails for any reason,
+	// use RocksDB as the default engine type.
+	if serverCfg.StorageEngine == enginepb.EngineTypeDefault {
+		serverCfg.StorageEngine = enginepb.EngineTypeRocksDB
+		// Only check if the first store is written by pebble. If that's the case,
+		// set engine type to pebble.
+		if version, err := pebble.GetVersion(serverCfg.Stores.Specs[0].Path, vfs.Default); err == nil {
+			if version != "" && !strings.HasPrefix(version, "rocksdb") {
+				serverCfg.StorageEngine = enginepb.EngineTypePebble
+			}
+		}
+	}
 	// Find a StoreSpec that has encryption at rest turned on. If can't find
 	// one, use the first StoreSpec in the list.
 	var specIdx = 0

pkg/server/config.go

@@ -437,10 +437,13 @@ func (cfg *Config) CreateEngines(ctx context.Context) (Engines, error) {
 		details = append(details, fmt.Sprintf("Pebble cache size: %s", humanizeutil.IBytes(cfg.CacheSize)))
 		pebbleCache = pebble.NewCache(cfg.CacheSize)
 		defer pebbleCache.Unref()
-	} else if cfg.StorageEngine == enginepb.EngineTypeRocksDB || cfg.StorageEngine == enginepb.EngineTypeTeePebbleRocksDB {
+	}
+	if cfg.StorageEngine == enginepb.EngineTypeRocksDB || cfg.StorageEngine == enginepb.EngineTypeTeePebbleRocksDB {
 		details = append(details, fmt.Sprintf("RocksDB cache size: %s", humanizeutil.IBytes(cfg.CacheSize)))
 		cache = engine.NewRocksDBCache(cfg.CacheSize)
 		defer cache.Release()
+	} else if cfg.StorageEngine == enginepb.EngineTypeDefault {
+		return nil, errors.New("engine type 'default' left unresolved")
 	}
 
 	var physicalStores int

pkg/server/server.go

@@ -1750,6 +1750,8 @@ func (s *Server) Start(ctx context.Context) error {
 	switch s.cfg.StorageEngine {
 	case enginepb.EngineTypePebble:
 		nodeStartCounter += "pebble."
+	case enginepb.EngineTypeDefault:
+		nodeStartCounter += "default."
 	case enginepb.EngineTypeRocksDB:
 		nodeStartCounter += "rocksdb."
 	case enginepb.EngineTypeTeePebbleRocksDB:

pkg/storage/engine/engine.go

@@ -34,7 +34,7 @@ import (
 var DefaultStorageEngine enginepb.EngineType
 
 func init() {
-	_ = DefaultStorageEngine.Set(envutil.EnvOrDefaultString("COCKROACH_STORAGE_ENGINE", "rocksdb"))
+	_ = DefaultStorageEngine.Set(envutil.EnvOrDefaultString("COCKROACH_STORAGE_ENGINE", "default"))
 }
 
 // SimpleIterator is an interface for iterating over key/value pairs in an

pkg/storage/engine/enginepb/engine.go

@@ -18,6 +18,8 @@ func (e *EngineType) Type() string { return "string" }
 // String implements the pflag.Value interface.
 func (e *EngineType) String() string {
 	switch *e {
+	case EngineTypeDefault:
+		return "default"
 	case EngineTypeRocksDB:
 		return "rocksdb"
 	case EngineTypePebble:
@@ -31,6 +33,8 @@ func (e *EngineType) String() string {
 // Set implements the pflag.Value interface.
 func (e *EngineType) Set(s string) error {
 	switch s {
+	case "default":
+		*e = EngineTypeDefault
 	case "rocksdb":
 		*e = EngineTypeRocksDB
 	case "pebble":

pkg/storage/engine/enginepb/engine.proto

@@ -19,11 +19,15 @@ enum EngineType {
   option (gogoproto.goproto_enum_prefix) = false;
   option (gogoproto.goproto_enum_stringer) = false;
 
+  // Denotes the default engine as the underlying storage engine type. Resolves
+  // during start to the engine type last used. If left unresolved, it's treated
+  // the same as EngineTypeRocksDB.
+  EngineTypeDefault = 0;
   // Denotes RocksDB as the underlying storage engine type.
-  EngineTypeRocksDB = 0;
+  EngineTypeRocksDB = 1;
   // Denotes Pebble as the underlying storage engine type.
-  EngineTypePebble = 1;
+  EngineTypePebble = 2;
   // Denotes TeePebbleRocksDB as the underlying storage engine type. Only use
   // for testing purposes.
-  EngineTypeTeePebbleRocksDB = 2;
+  EngineTypeTeePebbleRocksDB = 3;
 }

pkg/storage/engine/in_mem.go

@@ -30,6 +30,8 @@ func NewInMem(
 		return newTeeInMem(ctx, attrs, cacheSize)
 	case enginepb.EngineTypePebble:
 		return newPebbleInMem(ctx, attrs, cacheSize)
+	case enginepb.EngineTypeDefault:
+		fallthrough
 	case enginepb.EngineTypeRocksDB:
 		return newRocksDBInMem(attrs, cacheSize)
 	}

pkg/storage/engine/temp_engine.go

@@ -37,6 +37,8 @@ func NewTempEngine(
 		fallthrough
 	case enginepb.EngineTypePebble:
 		return NewPebbleTempEngine(ctx, tempStorage, storeSpec)
+	case enginepb.EngineTypeDefault:
+		fallthrough
 	case enginepb.EngineTypeRocksDB:
 		return NewRocksDBTempEngine(tempStorage, storeSpec)
 	}