Add shrinking limits (#297)

* Add shrinking limits, improve "call removal" logic, fixed a bug where corpus results could re-record if replayed/reshrunk differently

* Removed comment indicating shrinkLimit must be greater than zero (untrue)

* Re-order unexecuted sequences so test result corpus items are replayed first

* change where shrinkIncrement is updated

---------

Co-authored-by: Anish Naik <anish.naik@trailofbits.com>

fuzzing/config/config.go

@@ -38,14 +38,17 @@ type FuzzingConfig struct {
 	// so that memory from its underlying chain is freed.
 	WorkerResetLimit int `json:"workerResetLimit"`
 
-	// Timeout describes a time in seconds for which the fuzzing operation should run. Providing negative or zero value
-	// will result in no timeout.
+	// Timeout describes a time threshold in seconds for which the fuzzing operation should run. Providing negative or
+	// zero value will result in no timeout.
 	Timeout int `json:"timeout"`
 
 	// TestLimit describes a threshold for the number of transactions to test, after which it will exit. This number
 	// must be non-negative. A zero value indicates the test limit should not be enforced.
 	TestLimit uint64 `json:"testLimit"`
 
+	// ShrinkLimit describes a threshold for the iterations (call sequence tests) which shrinking should perform.
+	ShrinkLimit uint64 `json:"shrinkLimit"`
+
 	// CallSequenceLength describes the maximum length a transaction sequence can be generated as.
 	CallSequenceLength int `json:"callSequenceLength"`
 

fuzzing/config/config_defaults.go

@@ -37,6 +37,7 @@ func GetDefaultProjectConfig(platform string) (*ProjectConfig, error) {
 			WorkerResetLimit:        50,
 			Timeout:                 0,
 			TestLimit:               0,
+			ShrinkLimit:             5_000,
 			CallSequenceLength:      100,
 			TargetContracts:         []string{},
 			TargetContractsBalances: []*big.Int{},

fuzzing/corpus/corpus.go

@@ -293,15 +293,21 @@ func (c *Corpus) Initialize(baseTestChain *chain.TestChain, contractDefinitions
 
 	// Next we replay every call sequence, checking its validity on this chain and measuring coverage. Valid sequences
 	// are added to the corpus for mutations, re-execution, etc.
-	err = c.initializeSequences(c.mutableSequenceFiles, testChain, deployedContracts, true)
+	//
+	// The order of initializations here is important, as it determines the order of "unexecuted sequences" to replay
+	// when the fuzzer's worker starts up. We want to replay test results first, so that other corpus items
+	// do not trigger the same test failures instead.
+	err = c.initializeSequences(c.testResultSequenceFiles, testChain, deployedContracts, false)
 	if err != nil {
 		return 0, 0, err
 	}
-	err = c.initializeSequences(c.immutableSequenceFiles, testChain, deployedContracts, false)
+
+	err = c.initializeSequences(c.mutableSequenceFiles, testChain, deployedContracts, true)
 	if err != nil {
 		return 0, 0, err
 	}
-	err = c.initializeSequences(c.testResultSequenceFiles, testChain, deployedContracts, false)
+
+	err = c.initializeSequences(c.immutableSequenceFiles, testChain, deployedContracts, false)
 	if err != nil {
 		return 0, 0, err
 	}

fuzzing/fuzzer.go

@@ -749,6 +749,7 @@ func (f *Fuzzer) printMetricsLoop() {
 		callsTested := f.metrics.CallsTested()
 		sequencesTested := f.metrics.SequencesTested()
 		workerStartupCount := f.metrics.WorkerStartupCount()
+		workersShrinking := f.metrics.WorkersShrinkingCount()
 
 		// Calculate time elapsed since the last update
 		secondsSinceLastUpdate := time.Since(lastPrintedTime).Seconds()
@@ -767,6 +768,7 @@ func (f *Fuzzer) printMetricsLoop() {
 		logBuffer.Append(", seq/s: ", colors.Bold, fmt.Sprintf("%d", uint64(float64(new(big.Int).Sub(sequencesTested, lastSequencesTested).Uint64())/secondsSinceLastUpdate)), colors.Reset)
 		logBuffer.Append(", coverage: ", colors.Bold, fmt.Sprintf("%d", f.corpus.ActiveMutableSequenceCount()), colors.Reset)
 		if f.logger.Level() <= zerolog.DebugLevel {
+			logBuffer.Append(", shrinking: ", colors.Bold, fmt.Sprintf("%v", workersShrinking), colors.Reset)
 			logBuffer.Append(", mem: ", colors.Bold, fmt.Sprintf("%v/%v MB", memoryUsedMB, memoryTotalMB), colors.Reset)
 			logBuffer.Append(", resets/s: ", colors.Bold, fmt.Sprintf("%d", uint64(float64(new(big.Int).Sub(workerStartupCount, lastWorkerStartupCount).Uint64())/secondsSinceLastUpdate)), colors.Reset)
 		}

fuzzing/fuzzer_metrics.go

@@ -19,6 +19,9 @@ type fuzzerWorkerMetrics struct {
 
 	// workerStartupCount describes the amount of times the worker was generated, or re-generated for this index.
 	workerStartupCount *big.Int
+
+	// shrinking indicates whether the fuzzer worker is currently shrinking.
+	shrinking bool
 }
 
 // newFuzzerMetrics obtains a new FuzzerMetrics struct for a given number of workers specified by workerCount.
@@ -63,3 +66,14 @@ func (m *FuzzerMetrics) WorkerStartupCount() *big.Int {
 	}
 	return workerStartupCount
 }
+
+// WorkersShrinkingCount returns the amount of workers currently performing shrinking operations.
+func (m *FuzzerMetrics) WorkersShrinkingCount() uint64 {
+	shrinkingCount := uint64(0)
+	for _, workerMetrics := range m.workerMetrics {
+		if workerMetrics.shrinking {
+			shrinkingCount++
+		}
+	}
+	return shrinkingCount
+}

fuzzing/fuzzer_worker.go

@@ -315,8 +315,8 @@ func (fw *FuzzerWorker) testNextCallSequence() (calls.CallSequence, []ShrinkCall
 
 	// If this was not a new call sequence, indicate not to save the shrunken result to the corpus again.
 	if !isNewSequence {
-		for _, shrinkRequest := range shrinkCallSequenceRequests {
-			shrinkRequest.RecordResultInCorpus = false
+		for i := 0; i < len(shrinkCallSequenceRequests); i++ {
+			shrinkCallSequenceRequests[i].RecordResultInCorpus = false
 		}
 	}
 
@@ -391,73 +391,95 @@ func (fw *FuzzerWorker) testShrunkenCallSequence(possibleShrunkSequence calls.Ca
 // shrinkCallSequence takes a provided call sequence and attempts to shrink it by looking for redundant
 // calls which can be removed, and values which can be minimized, while continuing to satisfy the provided shrink
 // verifier.
+//
+// This function should *always* be called if there are shrink requests, and should always report a result,
+// even if it is the original sequence provided.
+//
 // Returns a call sequence that was optimized to include as little calls as possible to trigger the
 // expected conditions, or an error if one occurred.
 func (fw *FuzzerWorker) shrinkCallSequence(callSequence calls.CallSequence, shrinkRequest ShrinkCallSequenceRequest) (calls.CallSequence, error) {
 	// Define a variable to track our most optimized sequence across all optimization iterations.
 	optimizedSequence := callSequence
 
-	// First try to remove any calls we can. We go from start to end to avoid index shifting.
-	for i := 0; i < len(optimizedSequence); {
-		// If our fuzzer context is done, exit out immediately without results.
-		if utils.CheckContextDone(fw.fuzzer.ctx) {
-			return nil, nil
-		}
-
-		// Recreate our current optimized sequence without the item at this index
-		possibleShrunkSequence, err := optimizedSequence.Clone()
-		if err != nil {
-			return nil, err
-		}
-		possibleShrunkSequence = append(possibleShrunkSequence[:i], possibleShrunkSequence[i+1:]...)
-
-		// Test the shrunken sequence.
-		validShrunkSequence, err := fw.testShrunkenCallSequence(possibleShrunkSequence, shrinkRequest)
-		if err != nil {
-			return nil, err
-		}
-
-		// If this current sequence satisfied our conditions, set it as our optimized sequence.
-		if validShrunkSequence {
-			optimizedSequence = possibleShrunkSequence
-		} else {
-			// We didn't remove an item at this index, so we'll iterate to the next one.
-			i++
-		}
+	// Obtain our shrink limits and begin shrinking.
+	shrinkIteration := uint64(0)
+	shrinkLimit := fw.fuzzer.config.Fuzzing.ShrinkLimit
+	shrinkingEnded := func() bool {
+		return shrinkIteration >= shrinkLimit || utils.CheckContextDone(fw.fuzzer.ctx)
 	}
+	if shrinkLimit > 0 {
+		// The first pass of shrinking is greedy towards trying to remove any unnecessary calls.
+		// For each call in the sequence, the following removal strategies are used:
+		// 1) Plain removal (lower block/time gap between surrounding blocks, maintain properties of max delay)
+		// 2) Add block/time delay to previous call (retain original block/time, possibly exceed max delays)
+		// At worst, this costs `2 * len(callSequence)` shrink iterations.
+		fw.workerMetrics().shrinking = true
+		for removalStrategy := 0; removalStrategy < 2 && !shrinkingEnded(); removalStrategy++ {
+			for i := len(optimizedSequence) - 1; i >= 0 && !shrinkingEnded(); i-- {
+				// Recreate our current optimized sequence without the item at this index
+				possibleShrunkSequence, err := optimizedSequence.Clone()
+				removedCall := possibleShrunkSequence[i]
+				if err != nil {
+					return nil, err
+				}
+				possibleShrunkSequence = append(possibleShrunkSequence[:i], possibleShrunkSequence[i+1:]...)
+
+				// Exercise the next removal strategy for this call.
+				if removalStrategy == 0 {
+					// Case 1: Plain removal.
+				} else if removalStrategy == 1 {
+					// Case 2: Add block/time delay to previous call.
+					if i > 0 {
+						possibleShrunkSequence[i-1].BlockNumberDelay += removedCall.BlockNumberDelay
+						possibleShrunkSequence[i-1].BlockTimestampDelay += removedCall.BlockTimestampDelay
+					}
+				}
 
-	// Next try to shrink our values of every transaction a given number of rounds.
-	for i := 0; i < len(optimizedSequence); i++ {
-		for optimizationRound := 0; optimizationRound < 200; optimizationRound++ {
-			// If our fuzzer context is done, exit out immediately without results.
-			if utils.CheckContextDone(fw.fuzzer.ctx) {
-				return nil, nil
+				// Test the shrunken sequence.
+				validShrunkSequence, err := fw.testShrunkenCallSequence(possibleShrunkSequence, shrinkRequest)
+				shrinkIteration++
+				if err != nil {
+					return nil, err
+				}
+
+				// If the current sequence satisfied our conditions, set it as our optimized sequence.
+				if validShrunkSequence {
+					optimizedSequence = possibleShrunkSequence
+				}
 			}
+		}
 
-			// Clone the optimized sequence.
-			possibleShrunkSequence, _ := optimizedSequence.Clone()
+		// The second pass of shrinking attempts to shrink values for each call in our call sequence.
+		// This is performed exhaustively in a round-robin fashion for each call, until the shrink limit is hit.
+		for !shrinkingEnded() {
+			for i := len(optimizedSequence) - 1; i >= 0 && !shrinkingEnded(); i-- {
+				// Clone the optimized sequence.
+				possibleShrunkSequence, _ := optimizedSequence.Clone()
+
+				// Loop for each argument in the currently indexed call to mutate it.
+				abiValuesMsgData := possibleShrunkSequence[i].Call.DataAbiValues
+				for j := 0; j < len(abiValuesMsgData.InputValues); j++ {
+					mutatedInput, err := valuegeneration.MutateAbiValue(fw.sequenceGenerator.config.ValueGenerator, fw.shrinkingValueMutator, &abiValuesMsgData.Method.Inputs[j].Type, abiValuesMsgData.InputValues[j])
+					if err != nil {
+						return nil, fmt.Errorf("error when shrinking call sequence input argument: %v", err)
+					}
+					abiValuesMsgData.InputValues[j] = mutatedInput
+				}
 
-			// Loop for each argument in the currently indexed call to mutate it.
-			abiValuesMsgData := possibleShrunkSequence[i].Call.DataAbiValues
-			for j := 0; j < len(abiValuesMsgData.InputValues); j++ {
-				mutatedInput, err := valuegeneration.MutateAbiValue(fw.sequenceGenerator.config.ValueGenerator, fw.shrinkingValueMutator, &abiValuesMsgData.Method.Inputs[j].Type, abiValuesMsgData.InputValues[j])
+				// Test the shrunken sequence.
+				validShrunkSequence, err := fw.testShrunkenCallSequence(possibleShrunkSequence, shrinkRequest)
+				shrinkIteration++
 				if err != nil {
-					return nil, fmt.Errorf("error when shrinking call sequence input argument: %v", err)
+					return nil, err
 				}
-				abiValuesMsgData.InputValues[j] = mutatedInput
-			}
 
-			// Test the shrunken sequence.
-			validShrunkSequence, err := fw.testShrunkenCallSequence(possibleShrunkSequence, shrinkRequest)
-			if err != nil {
-				return nil, err
-			}
-
-			// If this current sequence satisfied our conditions, set it as our optimized sequence.
-			if validShrunkSequence {
-				optimizedSequence = possibleShrunkSequence
+				// If this current sequence satisfied our conditions, set it as our optimized sequence.
+				if validShrunkSequence {
+					optimizedSequence = possibleShrunkSequence
+				}
 			}
 		}
+		fw.workerMetrics().shrinking = false
 	}
 
 	// If the shrink request wanted the sequence recorded in the corpus, do so now.

fuzzing/fuzzer_worker_sequence_generator.go

@@ -192,7 +192,7 @@ func (g *CallSequenceGenerator) InitializeNextSequence() (bool, error) {
 	g.fetchIndex = 0
 	g.prefetchModifyCallFunc = nil
 
-	// Check if there are any previously une-xecuted corpus call sequences. If there are, the fuzzer should execute
+	// Check if there are any previously un-executed corpus call sequences. If there are, the fuzzer should execute
 	// those first.
 	unexecutedSequence := g.worker.fuzzer.corpus.UnexecutedCallSequence()
 	if unexecutedSequence != nil {