[libFuzzer] Mutation tracking and logging implemented.

Summary:
Code now exists to track number of mutations that are used in fuzzing in total
and ones that produce new coverage. The stats are currently being dumped to the
command line.

Patch by Kodé Williams (@kodewilliams).

Reviewers: metzman, Dor1s, morehouse, kcc

Reviewed By: Dor1s, morehouse, kcc

Subscribers: delcypher, kubamracek, kcc, morehouse, llvm-commits, #sanitizers, mgorny

Differential Revision: https://reviews.llvm.org/D48054

llvm-svn: 337324

compiler-rt/lib/fuzzer/FuzzerDriver.cpp

@@ -615,6 +615,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   Options.PrintNewCovPcs = Flags.print_pcs;
   Options.PrintNewCovFuncs = Flags.print_funcs;
   Options.PrintFinalStats = Flags.print_final_stats;
+  Options.PrintMutationStats = Flags.print_mutation_stats;
   Options.PrintCorpusStats = Flags.print_corpus_stats;
   Options.PrintCoverage = Flags.print_coverage;
   Options.PrintUnstableStats = Flags.print_unstable_stats;

compiler-rt/lib/fuzzer/FuzzerFlags.def

@@ -155,3 +155,4 @@ FUZZER_DEPRECATED_FLAG(use_equivalence_server)
 FUZZER_FLAG_INT(analyze_dict, 0, "Experimental")
 FUZZER_DEPRECATED_FLAG(use_clang_coverage)
 FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace")
+FUZZER_FLAG_INT(print_mutation_stats, 0, "Experimental")

compiler-rt/lib/fuzzer/FuzzerLoop.cpp

@@ -360,6 +360,7 @@ void Fuzzer::PrintFinalStats() {
     TPC.DumpCoverage();
   if (Options.PrintCorpusStats)
     Corpus.PrintStats();
+  if (Options.PrintMutationStats) MD.PrintMutationStats();
   if (!Options.PrintFinalStats)
     return;
   size_t ExecPerSec = execPerSec();

compiler-rt/lib/fuzzer/FuzzerMutate.cpp

@@ -30,34 +30,36 @@ MutationDispatcher::MutationDispatcher(Random &Rand,
   DefaultMutators.insert(
       DefaultMutators.begin(),
       {
-          {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes"},
-          {&MutationDispatcher::Mutate_InsertByte, "InsertByte"},
+          {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes", 0, 0},
+          {&MutationDispatcher::Mutate_InsertByte, "InsertByte", 0, 0},
           {&MutationDispatcher::Mutate_InsertRepeatedBytes,
-           "InsertRepeatedBytes"},
-          {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"},
-          {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"},
-          {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"},
-          {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"},
-          {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"},
-          {&MutationDispatcher::Mutate_CopyPart, "CopyPart"},
-          {&MutationDispatcher::Mutate_CrossOver, "CrossOver"},
+           "InsertRepeatedBytes", 0, 0},
+          {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte", 0, 0},
+          {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit", 0, 0},
+          {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes", 0, 0},
+          {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt", 0,
+           0},
+          {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt", 0,
+           0},
+          {&MutationDispatcher::Mutate_CopyPart, "CopyPart", 0, 0},
+          {&MutationDispatcher::Mutate_CrossOver, "CrossOver", 0, 0},
           {&MutationDispatcher::Mutate_AddWordFromManualDictionary,
-           "ManualDict"},
+           "ManualDict", 0, 0},
           {&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary,
-           "PersAutoDict"},
+           "PersAutoDict", 0, 0},
       });
   if(Options.UseCmp)
     DefaultMutators.push_back(
-        {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"});
+        {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP", 0, 0});
 
   if (EF->LLVMFuzzerCustomMutator)
-    Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"});
+    Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom", 0, 0});
   else
     Mutators = DefaultMutators;
 
   if (EF->LLVMFuzzerCustomCrossOver)
     Mutators.push_back(
-        {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"});
+        {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver", 0, 0});
 }
 
 static char RandCh(Random &Rand) {
@@ -464,6 +466,7 @@ void MutationDispatcher::RecordSuccessfulMutationSequence() {
     if (!PersistentAutoDictionary.ContainsWord(DE->GetW()))
       PersistentAutoDictionary.push_back({DE->GetW(), 1});
   }
+  RecordUsefulMutations();
 }
 
 void MutationDispatcher::PrintRecommendedDictionary() {
@@ -484,8 +487,7 @@ void MutationDispatcher::PrintRecommendedDictionary() {
 
 void MutationDispatcher::PrintMutationSequence() {
   Printf("MS: %zd ", CurrentMutatorSequence.size());
-  for (auto M : CurrentMutatorSequence)
-    Printf("%s-", M.Name);
+  for (auto M : CurrentMutatorSequence) Printf("%s-", M->Name);
   if (!CurrentDictionaryEntrySequence.empty()) {
     Printf(" DE: ");
     for (auto DE : CurrentDictionaryEntrySequence) {
@@ -513,12 +515,13 @@ size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size,
   // in which case they will return 0.
   // Try several times before returning un-mutated data.
   for (int Iter = 0; Iter < 100; Iter++) {
-    auto M = Mutators[Rand(Mutators.size())];
-    size_t NewSize = (this->*(M.Fn))(Data, Size, MaxSize);
+    auto M = &Mutators[Rand(Mutators.size())];
+    size_t NewSize = (this->*(M->Fn))(Data, Size, MaxSize);
     if (NewSize && NewSize <= MaxSize) {
       if (Options.OnlyASCII)
         ToASCII(Data, NewSize);
       CurrentMutatorSequence.push_back(M);
+      M->TotalCount++;
       return NewSize;
     }
   }
@@ -531,4 +534,21 @@ void MutationDispatcher::AddWordToManualDictionary(const Word &W) {
       {W, std::numeric_limits<size_t>::max()});
 }
 
+void MutationDispatcher::RecordUsefulMutations() {
+  for (auto M : CurrentMutatorSequence) M->UsefulCount++;
+}
+
+void MutationDispatcher::PrintMutationStats() {
+  Printf("\nstat::mutation_usefulness:      ");
+  for (size_t i = 0; i < Mutators.size(); i++) {
+    double UsefulPercentage =
+        Mutators[i].TotalCount
+            ? (100.0 * Mutators[i].UsefulCount) / Mutators[i].TotalCount
+            : 0;
+    Printf("%.3f", UsefulPercentage);
+    if (i < Mutators.size() - 1) Printf(",");
+  }
+  Printf("\n");
+}
+
 }  // namespace fuzzer

compiler-rt/lib/fuzzer/FuzzerMutate.h

@@ -86,11 +86,16 @@ class MutationDispatcher {
 
   Random &GetRand() { return Rand; }
 
-private:
+  void PrintMutationStats();
 
+  void RecordUsefulMutations();
+
+ private:
   struct Mutator {
     size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max);
     const char *Name;
+    uint64_t UsefulCount;
+    uint64_t TotalCount;
   };
 
   size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size,
@@ -128,8 +133,8 @@ class MutationDispatcher {
   // entries that led to successful discoveries in the past mutations.
   Dictionary PersistentAutoDictionary;
 
-  Vector<Mutator> CurrentMutatorSequence;
   Vector<DictionaryEntry *> CurrentDictionaryEntrySequence;
+  Vector<Mutator *> CurrentMutatorSequence;
 
   static const size_t kCmpDictionaryEntriesDequeSize = 16;
   DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize];

compiler-rt/lib/fuzzer/FuzzerOptions.h

@@ -52,6 +52,7 @@ struct FuzzingOptions {
   bool PrintNewCovPcs = false;
   int PrintNewCovFuncs = 0;
   bool PrintFinalStats = false;
+  bool PrintMutationStats = false;
   bool PrintCorpusStats = false;
   bool PrintCoverage = false;
   bool PrintUnstableStats = false;

compiler-rt/test/fuzzer/fuzzer-mutationstats.test

@@ -0,0 +1,5 @@
+RUN: %cpp_compiler %S/SimpleTest.cpp -o %t-MutationStatsTest
+RUN: not %run %t-MutationStatsTest -print_mutation_stats=1 2>&1 | FileCheck %s
+
+# Ensures there are some non-zero values in the usefulness percentages printed.
+CHECK: stat::mutation_usefulness:   {{[0-9]+\.[0-9]+}}