metamorphic: Add TestMetaTwoInstance

This change adds a two-instance metamorphic test as well
as a new metamorphic test command, db.Replicate, that under
some testOptions does a shared ingestion (i.e. ScanInternal
with skip-shared iteration to only get keys above files in
shared levels) along with an excise, and in other testOptions
it does a normal ingestion with range deletes/rangekeydels to clear out
the "replicated" range

Fixes #2711.

internal/metamorphic/meta_test.go

@@ -70,3 +70,24 @@ func TestMeta(t *testing.T) {
 		metamorphic.RunAndCompare(t, runFlags.Dir, opts...)
 	}
 }
+
+func TestMetaTwoInstance(t *testing.T) {
+	switch {
+	case runOnceFlags.Compare != "":
+		runDirs := strings.Split(runOnceFlags.Compare, ",")
+		onceOpts := runOnceFlags.MakeRunOnceOptions()
+		metamorphic.Compare(t, runOnceFlags.Dir, runOnceFlags.Seed, runDirs, onceOpts...)
+
+	case runOnceFlags.RunDir != "":
+		// The --run-dir flag is specified either in the child process (see
+		// runOptions() below) or the user specified it manually in order to re-run
+		// a test.
+		onceOpts := runOnceFlags.MakeRunOnceOptions()
+		metamorphic.RunOnce(t, runOnceFlags.RunDir, runOnceFlags.Seed, filepath.Join(runOnceFlags.RunDir, "history"), onceOpts...)
+
+	default:
+		opts := runFlags.MakeRunOptions()
+		opts = append(opts, metamorphic.MultiInstance(2))
+		metamorphic.RunAndCompare(t, runFlags.Dir, opts...)
+	}
+}

internal/metamorphic/metaflags/meta_flags.go

@@ -35,6 +35,9 @@ type CommonFlags struct {
 	Keep bool
 	// MaxThreads used by a single run. See "max-threads" flag below.
 	MaxThreads int
+	// NumInstances is the number of Pebble instances to create in one run. See
+	// "num-instances" flag below.
+	NumInstances int
 }
 
 func initCommonFlags() *CommonFlags {
@@ -65,6 +68,8 @@ func initCommonFlags() *CommonFlags {
 	flag.IntVar(&c.MaxThreads, "max-threads", math.MaxInt,
 		"limit execution of a single run to the provided number of threads; must be ≥ 1")
 
+	flag.IntVar(&c.NumInstances, "num-instances", 1, "number of pebble instances to create (default: 1)")
+
 	return c
 }
 
@@ -179,6 +184,9 @@ func (ro *RunOnceFlags) MakeRunOnceOptions() []metamorphic.RunOnceOption {
 	if ro.ErrorRate > 0 {
 		onceOpts = append(onceOpts, metamorphic.InjectErrorsRate(ro.ErrorRate))
 	}
+	if ro.NumInstances > 1 {
+		onceOpts = append(onceOpts, metamorphic.MultiInstance(ro.NumInstances))
+	}
 	return onceOpts
 }
 
@@ -204,6 +212,9 @@ func (r *RunFlags) MakeRunOptions() []metamorphic.RunOption {
 	if r.PreviousOps != "" {
 		opts = append(opts, metamorphic.ExtendPreviousRun(r.PreviousOps, r.InitialStatePath, r.InitialStateDesc))
 	}
+	if r.NumInstances > 1 {
+		opts = append(opts, metamorphic.MultiInstance(r.NumInstances))
+	}
 
 	// If the filesystem type was forced, all tests will use that value.
 	switch r.FS {

metamorphic/config.go

@@ -38,6 +38,7 @@ const (
 	newIterUsingClone
 	newSnapshot
 	readerGet
+	replicate
 	snapshotClose
 	writerApply
 	writerDelete
@@ -65,6 +66,10 @@ type config struct {
 	// skewing towards most recent timestamps.
 	writeSuffixDist randvar.Dynamic
 
+	// numInstances defines the number of pebble instances created for this
+	// metamorphic test run.
+	numInstances int
+
 	// TODO(peter): unimplemented
 	// keyDist        randvar.Dynamic
 	// keySizeDist    randvar.Static
@@ -107,6 +112,8 @@ var presetConfigs = []config{
 		withOpWeight(writerMerge, 0),
 }
 
+var multiInstancePresetConfig = multiInstanceConfig()
+
 func defaultConfig() config {
 	return config{
 		// dbClose is not in this list since it is deterministically generated once, at the end of the test.
@@ -139,6 +146,7 @@ func defaultConfig() config {
 			newIterUsingClone:           5,
 			newSnapshot:                 10,
 			readerGet:                   100,
+			replicate:                   0,
 			snapshotClose:               10,
 			writerApply:                 10,
 			writerDelete:                100,
@@ -161,6 +169,19 @@ func defaultConfig() config {
 	}
 }
 
+func multiInstanceConfig() config {
+	cfg := defaultConfig()
+	cfg.ops[replicate] = 5
+	cfg.ops[writerSingleDelete] = 0
+	cfg.ops[writerMerge] = 0
+	// TODO(bilal): The disabled operations below should also be supported
+	// in the two-instance test, once they're updated to work in multi-instance
+	// mode.
+	cfg.ops[newSnapshot] = 0
+	cfg.ops[snapshotClose] = 0
+	return cfg
+}
+
 func mustDynamic(dyn randvar.Dynamic, err error) randvar.Dynamic {
 	if err != nil {
 		panic(err)