From 061b4af998fc4a211566505555403806c373886a Mon Sep 17 00:00:00 2001 From: Max Moroz Date: Tue, 17 Jul 2018 20:37:40 +0000 Subject: [PATCH] [libFuzzer] Mutation tracking and logging implemented. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Code now exists to track number of mutations that are used in fuzzing in total and ones that produce new coverage. The stats are currently being dumped to the command line. Patch by Kodé Williams (@kodewilliams). Reviewers: metzman, Dor1s, morehouse, kcc Reviewed By: Dor1s, morehouse, kcc Subscribers: delcypher, kubamracek, kcc, morehouse, llvm-commits, #sanitizers, mgorny Differential Revision: https://reviews.llvm.org/D48054 llvm-svn: 337324 --- compiler-rt/lib/fuzzer/FuzzerDriver.cpp | 1 + compiler-rt/lib/fuzzer/FuzzerFlags.def | 1 + compiler-rt/lib/fuzzer/FuzzerLoop.cpp | 1 + compiler-rt/lib/fuzzer/FuzzerMutate.cpp | 58 +++++++++++++------ compiler-rt/lib/fuzzer/FuzzerMutate.h | 9 ++- compiler-rt/lib/fuzzer/FuzzerOptions.h | 1 + .../test/fuzzer/fuzzer-mutationstats.test | 5 ++ 7 files changed, 55 insertions(+), 21 deletions(-) create mode 100644 compiler-rt/test/fuzzer/fuzzer-mutationstats.test diff --git a/compiler-rt/lib/fuzzer/FuzzerDriver.cpp b/compiler-rt/lib/fuzzer/FuzzerDriver.cpp index 9375925b2c94e..eb849fcd07fe3 100644 --- a/compiler-rt/lib/fuzzer/FuzzerDriver.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerDriver.cpp @@ -615,6 +615,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.PrintNewCovPcs = Flags.print_pcs; Options.PrintNewCovFuncs = Flags.print_funcs; Options.PrintFinalStats = Flags.print_final_stats; + Options.PrintMutationStats = Flags.print_mutation_stats; Options.PrintCorpusStats = Flags.print_corpus_stats; Options.PrintCoverage = Flags.print_coverage; Options.PrintUnstableStats = Flags.print_unstable_stats; diff --git a/compiler-rt/lib/fuzzer/FuzzerFlags.def b/compiler-rt/lib/fuzzer/FuzzerFlags.def index e50b82ab70a5e..dc92758e4b7c9 100644 --- a/compiler-rt/lib/fuzzer/FuzzerFlags.def +++ b/compiler-rt/lib/fuzzer/FuzzerFlags.def @@ -155,3 +155,4 @@ FUZZER_DEPRECATED_FLAG(use_equivalence_server) FUZZER_FLAG_INT(analyze_dict, 0, "Experimental") FUZZER_DEPRECATED_FLAG(use_clang_coverage) FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace") +FUZZER_FLAG_INT(print_mutation_stats, 0, "Experimental") diff --git a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp index a2d53ee48dbce..1ba0765fab987 100644 --- a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp @@ -360,6 +360,7 @@ void Fuzzer::PrintFinalStats() { TPC.DumpCoverage(); if (Options.PrintCorpusStats) Corpus.PrintStats(); + if (Options.PrintMutationStats) MD.PrintMutationStats(); if (!Options.PrintFinalStats) return; size_t ExecPerSec = execPerSec(); diff --git a/compiler-rt/lib/fuzzer/FuzzerMutate.cpp b/compiler-rt/lib/fuzzer/FuzzerMutate.cpp index 6f6ce075ab9ad..e89e1a475a110 100644 --- a/compiler-rt/lib/fuzzer/FuzzerMutate.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerMutate.cpp @@ -30,34 +30,36 @@ MutationDispatcher::MutationDispatcher(Random &Rand, DefaultMutators.insert( DefaultMutators.begin(), { - {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes"}, - {&MutationDispatcher::Mutate_InsertByte, "InsertByte"}, + {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes", 0, 0}, + {&MutationDispatcher::Mutate_InsertByte, "InsertByte", 0, 0}, {&MutationDispatcher::Mutate_InsertRepeatedBytes, - "InsertRepeatedBytes"}, - {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"}, - {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"}, - {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"}, - {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"}, - {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"}, - {&MutationDispatcher::Mutate_CopyPart, "CopyPart"}, - {&MutationDispatcher::Mutate_CrossOver, "CrossOver"}, + "InsertRepeatedBytes", 0, 0}, + {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte", 0, 0}, + {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit", 0, 0}, + {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes", 0, 0}, + {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt", 0, + 0}, + {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt", 0, + 0}, + {&MutationDispatcher::Mutate_CopyPart, "CopyPart", 0, 0}, + {&MutationDispatcher::Mutate_CrossOver, "CrossOver", 0, 0}, {&MutationDispatcher::Mutate_AddWordFromManualDictionary, - "ManualDict"}, + "ManualDict", 0, 0}, {&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary, - "PersAutoDict"}, + "PersAutoDict", 0, 0}, }); if(Options.UseCmp) DefaultMutators.push_back( - {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"}); + {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP", 0, 0}); if (EF->LLVMFuzzerCustomMutator) - Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"}); + Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom", 0, 0}); else Mutators = DefaultMutators; if (EF->LLVMFuzzerCustomCrossOver) Mutators.push_back( - {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"}); + {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver", 0, 0}); } static char RandCh(Random &Rand) { @@ -464,6 +466,7 @@ void MutationDispatcher::RecordSuccessfulMutationSequence() { if (!PersistentAutoDictionary.ContainsWord(DE->GetW())) PersistentAutoDictionary.push_back({DE->GetW(), 1}); } + RecordUsefulMutations(); } void MutationDispatcher::PrintRecommendedDictionary() { @@ -484,8 +487,7 @@ void MutationDispatcher::PrintRecommendedDictionary() { void MutationDispatcher::PrintMutationSequence() { Printf("MS: %zd ", CurrentMutatorSequence.size()); - for (auto M : CurrentMutatorSequence) - Printf("%s-", M.Name); + for (auto M : CurrentMutatorSequence) Printf("%s-", M->Name); if (!CurrentDictionaryEntrySequence.empty()) { Printf(" DE: "); for (auto DE : CurrentDictionaryEntrySequence) { @@ -513,12 +515,13 @@ size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size, // in which case they will return 0. // Try several times before returning un-mutated data. for (int Iter = 0; Iter < 100; Iter++) { - auto M = Mutators[Rand(Mutators.size())]; - size_t NewSize = (this->*(M.Fn))(Data, Size, MaxSize); + auto M = &Mutators[Rand(Mutators.size())]; + size_t NewSize = (this->*(M->Fn))(Data, Size, MaxSize); if (NewSize && NewSize <= MaxSize) { if (Options.OnlyASCII) ToASCII(Data, NewSize); CurrentMutatorSequence.push_back(M); + M->TotalCount++; return NewSize; } } @@ -531,4 +534,21 @@ void MutationDispatcher::AddWordToManualDictionary(const Word &W) { {W, std::numeric_limits::max()}); } +void MutationDispatcher::RecordUsefulMutations() { + for (auto M : CurrentMutatorSequence) M->UsefulCount++; +} + +void MutationDispatcher::PrintMutationStats() { + Printf("\nstat::mutation_usefulness: "); + for (size_t i = 0; i < Mutators.size(); i++) { + double UsefulPercentage = + Mutators[i].TotalCount + ? (100.0 * Mutators[i].UsefulCount) / Mutators[i].TotalCount + : 0; + Printf("%.3f", UsefulPercentage); + if (i < Mutators.size() - 1) Printf(","); + } + Printf("\n"); +} + } // namespace fuzzer diff --git a/compiler-rt/lib/fuzzer/FuzzerMutate.h b/compiler-rt/lib/fuzzer/FuzzerMutate.h index 996d756cd5988..d8c1ddc5268cf 100644 --- a/compiler-rt/lib/fuzzer/FuzzerMutate.h +++ b/compiler-rt/lib/fuzzer/FuzzerMutate.h @@ -86,11 +86,16 @@ class MutationDispatcher { Random &GetRand() { return Rand; } -private: + void PrintMutationStats(); + void RecordUsefulMutations(); + + private: struct Mutator { size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max); const char *Name; + uint64_t UsefulCount; + uint64_t TotalCount; }; size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size, @@ -128,8 +133,8 @@ class MutationDispatcher { // entries that led to successful discoveries in the past mutations. Dictionary PersistentAutoDictionary; - Vector CurrentMutatorSequence; Vector CurrentDictionaryEntrySequence; + Vector CurrentMutatorSequence; static const size_t kCmpDictionaryEntriesDequeSize = 16; DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize]; diff --git a/compiler-rt/lib/fuzzer/FuzzerOptions.h b/compiler-rt/lib/fuzzer/FuzzerOptions.h index e32b7d59b853d..daa9104f5c591 100644 --- a/compiler-rt/lib/fuzzer/FuzzerOptions.h +++ b/compiler-rt/lib/fuzzer/FuzzerOptions.h @@ -52,6 +52,7 @@ struct FuzzingOptions { bool PrintNewCovPcs = false; int PrintNewCovFuncs = 0; bool PrintFinalStats = false; + bool PrintMutationStats = false; bool PrintCorpusStats = false; bool PrintCoverage = false; bool PrintUnstableStats = false; diff --git a/compiler-rt/test/fuzzer/fuzzer-mutationstats.test b/compiler-rt/test/fuzzer/fuzzer-mutationstats.test new file mode 100644 index 0000000000000..95743a818d1f4 --- /dev/null +++ b/compiler-rt/test/fuzzer/fuzzer-mutationstats.test @@ -0,0 +1,5 @@ +RUN: %cpp_compiler %S/SimpleTest.cpp -o %t-MutationStatsTest +RUN: not %run %t-MutationStatsTest -print_mutation_stats=1 2>&1 | FileCheck %s + +# Ensures there are some non-zero values in the usefulness percentages printed. +CHECK: stat::mutation_usefulness: {{[0-9]+\.[0-9]+}}