metamorphic: support --initial-state with --try-to-reduce

Author: RaduBerinde

internal/metamorphic/meta_test.go

@@ -76,7 +76,7 @@ func runTestMeta(t *testing.T, addtlOptions ...option) {
 		}
 		testRootDir, runSubdirs := runOnceFlags.ParseCompare()
 		if runOnceFlags.TryToReduce {
-			tryToReduceCompare(t, runOnceFlags.Dir, testRootDir, runSubdirs, runOnceFlags.ReduceAttempts)
+			tryToReduceCompare(t, runOnceFlags.Dir, testRootDir, runSubdirs, runOnceFlags.InitialStatePath, runOnceFlags.ReduceAttempts)
 			return
 		}
 		metamorphic.Compare(t, testRootDir, runOnceFlags.Seed, runSubdirs, onceOpts...)
@@ -90,7 +90,7 @@ func runTestMeta(t *testing.T, addtlOptions ...option) {
 			onceOpts = append(onceOpts, opt)
 		}
 		if runOnceFlags.TryToReduce {
-			tryToReduce(t, runOnceFlags.Dir, runOnceFlags.RunDir, runOnceFlags.ReduceAttempts)
+			tryToReduce(t, runOnceFlags.Dir, runOnceFlags.RunDir, runOnceFlags.InitialStatePath, runOnceFlags.ReduceAttempts)
 			return
 		}
 		metamorphic.RunOnce(t, runOnceFlags.RunDir, runOnceFlags.Seed,

internal/metamorphic/reduce_test.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -32,10 +33,12 @@ import (
 //
 // The test will save the smallest reproduction found and print out the relevant
 // information.
-func tryToReduce(t *testing.T, testStateDir string, runDir string, reduceAttempts int) {
+func tryToReduce(
+	t *testing.T, testStateDir string, runDir string, initialStatePath string, reduceAttempts int,
+) {
 	testRootDir := filepath.Dir(runDir)
 	runSubdir := filepath.Base(runDir)
-	r := makeReducer(t, testStateDir, testRootDir, []string{runSubdir}, reduceAttempts)
+	r := makeReducer(t, testStateDir, testRootDir, []string{runSubdir}, initialStatePath, reduceAttempts)
 	r.Run(t)
 }
 
@@ -52,19 +55,26 @@ func tryToReduce(t *testing.T, testStateDir string, runDir string, reduceAttempt
 // The test will save the smallest reproduction found and print out the relevant
 // information.
 func tryToReduceCompare(
-	t *testing.T, testStateDir string, testRootDir string, runSubdirs []string, reduceAttempts int,
+	t *testing.T,
+	testStateDir string,
+	testRootDir string,
+	runSubdirs []string,
+	initialStatePath string,
+	reduceAttempts int,
 ) {
-	r := makeReducer(t, testStateDir, testRootDir, runSubdirs, reduceAttempts)
+	r := makeReducer(t, testStateDir, testRootDir, runSubdirs, initialStatePath, reduceAttempts)
 	r.Run(t)
 }
 
 // reducer is a helper that starts with a reproduction of a RunOnce failure and
 // tries to reduce the number of operations.
 type reducer struct {
 	// testRootDir is the directory of the test, which contains the "ops" file.
-	testRootDir    string
-	configs        []testConfig
-	reduceAttempts int
+	testRootDir string
+	configs     []testConfig
+	// initialStatePath stores the --initial-state value, if set.
+	initialStatePath string
+	reduceAttempts   int
 
 	ops []string
 
@@ -84,7 +94,12 @@ type testConfig struct {
 }
 
 func makeReducer(
-	t *testing.T, testStateDir string, testRootDir string, runSubdirs []string, reduceAttempts int,
+	t *testing.T,
+	testStateDir string,
+	testRootDir string,
+	runSubdirs []string,
+	initialStatePath string,
+	reduceAttempts int,
 ) *reducer {
 	// All run dirs should have the same parent path.
 	opsData, err := os.ReadFile(filepath.Join(testRootDir, "ops"))
@@ -105,11 +120,12 @@ func makeReducer(
 	t.Logf("Starting with %d operations", len(ops))
 
 	return &reducer{
-		testRootDir:    testRootDir,
-		configs:        tc,
-		ops:            ops,
-		reduceAttempts: reduceAttempts,
-		testStateDir:   testStateDir,
+		testRootDir:      testRootDir,
+		configs:          tc,
+		initialStatePath: initialStatePath,
+		ops:              ops,
+		reduceAttempts:   reduceAttempts,
+		testStateDir:     testStateDir,
 	}
 }
 
@@ -144,15 +160,16 @@ func (r *reducer) try(t *testing.T, ops []string) bool {
 		"--keep",
 	}
 
-	var runFlags []string
 	if len(runSubdirs) == 1 {
 		// RunOnce mode.
-		runFlags = []string{"--run-dir", filepath.Join(testRootDir, runSubdirs[0])}
+		args = append(args, "--run-dir", filepath.Join(testRootDir, runSubdirs[0]))
 	} else {
 		// Compare mode.
-		runFlags = []string{"--compare", filepath.Join(testRootDir, fmt.Sprintf("{%s}", strings.Join(runSubdirs, ",")))}
+		args = append(args, "--compare", filepath.Join(testRootDir, fmt.Sprintf("{%s}", strings.Join(runSubdirs, ","))))
+	}
+	if r.initialStatePath != "" {
+		args = append(args, "--initial-state", r.initialStatePath)
 	}
-	args = append(args, runFlags...)
 
 	var output bytes.Buffer
 	cmd := exec.CommandContext(context.Background(), os.Args[0], args...)
@@ -188,7 +205,7 @@ func (r *reducer) try(t *testing.T, ops []string) bool {
 		t.Logf("  Diagram: %s", diagramPath)
 	}
 
-	t.Logf(`  go test ./internal/metamorphic -tags invariants -run "%s$" -v %s %q`, t.Name(), runFlags[0], runFlags[1])
+	t.Logf("  go test ./internal/metamorphic -tags invariants %s", shellJoin(cmd.Args[1:]))
 	if r.lastSavedDir != "" {
 		require.NoError(t, os.RemoveAll(r.lastSavedDir))
 	}
@@ -238,3 +255,16 @@ func randomSubset(t *testing.T, ops []string, removeProbability float64) []strin
 	}
 	return res
 }
+
+func shellJoin(args []string) string {
+	quoted := make([]string, len(args))
+	for i, a := range args {
+		// Quote if the word has bytes that the shell would interpret.
+		if strings.ContainsAny(a, " \t\n\"'\\$`*?[]{}<>|&;()") {
+			quoted[i] = strconv.Quote(a)
+		} else {
+			quoted[i] = a
+		}
+	}
+	return strings.Join(quoted, " ")
+}

metamorphic/history.go

@@ -183,13 +183,7 @@ func CompareHistories(t TestingT, paths []string) (i int, diff string) {
 	for i := 1; i < len(paths); i++ {
 		lines := readHistory(t, paths[i])
 		lines = reorderHistory(lines)
-		diff := difflib.UnifiedDiff{
-			A:       base,
-			B:       lines,
-			Context: 5,
-		}
-		text, err := difflib.GetUnifiedDiffString(diff)
-		require.NoError(t, err)
+		text := lineByLineDiff(base, lines)
 		if text != "" {
 			return i, text
 		}