Skip to content

Commit

Permalink
Revert "Refactor mutation strategies into a standalone library"
Browse files Browse the repository at this point in the history
This reverts commit c4a41cd due to
buildbot failure.
  • Loading branch information
morehouse committed May 26, 2021
1 parent 97f15ed commit fd0a2f7
Show file tree
Hide file tree
Showing 32 changed files with 1,366 additions and 2,561 deletions.
45 changes: 19 additions & 26 deletions compiler-rt/lib/fuzzer/CMakeLists.txt
@@ -1,4 +1,5 @@
set(LIBFUZZER_SOURCES
FuzzerCrossOver.cpp
FuzzerDataFlowTrace.cpp
FuzzerDriver.cpp
FuzzerExtFunctionsDlsym.cpp
Expand Down Expand Up @@ -28,6 +29,7 @@ set(LIBFUZZER_HEADERS
FuzzerCorpus.h
FuzzerDataFlowTrace.h
FuzzerDefs.h
FuzzerDictionary.h
FuzzerExtFunctions.def
FuzzerExtFunctions.h
FuzzerFlags.def
Expand Down Expand Up @@ -82,32 +84,6 @@ else()
endif()
endif()

macro(partially_link_libcxx name dir arch)
if(${arch} MATCHES "i386")
set(EMULATION_ARGUMENT "-m" "elf_i386")
else()
set(EMULATION_ARGUMENT "")
endif()
set(cxx_${arch}_merge_dir "${CMAKE_CURRENT_BINARY_DIR}/cxx_${arch}_merge.dir")
file(MAKE_DIRECTORY ${cxx_${arch}_merge_dir})
add_custom_command(TARGET clang_rt.${name}-${arch} POST_BUILD
COMMAND ${CMAKE_LINKER} ${EMULATION_ARGUMENT} --whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" --no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o
COMMAND ${CMAKE_OBJCOPY} --localize-hidden ${name}.o
COMMAND ${CMAKE_COMMAND} -E remove "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>"
COMMAND ${CMAKE_AR} qcs "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" ${name}.o
WORKING_DIRECTORY ${cxx_${arch}_merge_dir}
)
endmacro()

add_subdirectory(mutagen)
foreach(X IN LISTS LIBFUZZER_MUTAGEN_SOURCES)
list(APPEND LIBFUZZER_SOURCES "mutagen/${X}")
endforeach()
foreach(X IN LISTS LIBFUZZER_MUTAGEN_HEADERS)
list(APPEND LIBFUZZER_HEADERS "mutagen/${X}")
endforeach()
include_directories(.)

add_compiler_rt_component(fuzzer)

add_compiler_rt_object_libraries(RTfuzzer
Expand Down Expand Up @@ -159,6 +135,23 @@ add_compiler_rt_runtime(clang_rt.fuzzer_interceptors
if(OS_NAME MATCHES "Linux|Fuchsia" AND
COMPILER_RT_LIBCXX_PATH AND
COMPILER_RT_LIBCXXABI_PATH)
macro(partially_link_libcxx name dir arch)
if(${arch} MATCHES "i386")
set(EMULATION_ARGUMENT "-m" "elf_i386")
else()
set(EMULATION_ARGUMENT "")
endif()
set(cxx_${arch}_merge_dir "${CMAKE_CURRENT_BINARY_DIR}/cxx_${arch}_merge.dir")
file(MAKE_DIRECTORY ${cxx_${arch}_merge_dir})
add_custom_command(TARGET clang_rt.${name}-${arch} POST_BUILD
COMMAND ${CMAKE_LINKER} ${EMULATION_ARGUMENT} --whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" --no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o
COMMAND ${CMAKE_OBJCOPY} --localize-hidden ${name}.o
COMMAND ${CMAKE_COMMAND} -E remove "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>"
COMMAND ${CMAKE_AR} qcs "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" ${name}.o
WORKING_DIRECTORY ${cxx_${arch}_merge_dir}
)
endmacro()

foreach(arch ${FUZZER_SUPPORTED_ARCH})
get_target_flags_for_arch(${arch} TARGET_CFLAGS)
set(LIBCXX_${arch}_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libcxx_fuzzer_${arch})
Expand Down
@@ -1,4 +1,4 @@
//===- MutagenCrossOver.cpp - Cross over two test inputs ------------------===//
//===- FuzzerCrossOver.cpp - Cross over two test inputs -------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
Expand All @@ -8,11 +8,12 @@
// Cross over test inputs.
//===----------------------------------------------------------------------===//

#include "FuzzerDefs.h"
#include "FuzzerMutate.h"
#include "FuzzerRandom.h"
#include "MutagenDispatcher.h"
#include <cstring>

namespace mutagen {
namespace fuzzer {

// Cross Data1 and Data2, store the result (up to MaxOutSize bytes) in Out.
size_t MutationDispatcher::CrossOver(const uint8_t *Data1, size_t Size1,
Expand All @@ -39,12 +40,12 @@ size_t MutationDispatcher::CrossOver(const uint8_t *Data1, size_t Size1,
(*InPos) += ExtraSize;
}
// Use the other input data on the next iteration.
InPos = CurrentlyUsingFirstData ? &Pos2 : &Pos1;
InPos = CurrentlyUsingFirstData ? &Pos2 : &Pos1;
InSize = CurrentlyUsingFirstData ? Size2 : Size1;
Data = CurrentlyUsingFirstData ? Data2 : Data1;
Data = CurrentlyUsingFirstData ? Data2 : Data1;
CurrentlyUsingFirstData = !CurrentlyUsingFirstData;
}
return OutPos;
}

} // namespace mutagen
} // namespace fuzzer
36 changes: 4 additions & 32 deletions compiler-rt/lib/fuzzer/FuzzerDefs.h
Expand Up @@ -15,18 +15,21 @@
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <limits>
#include <memory>
#include <set>
#include <string>
#include <vector>


namespace fuzzer {

template <class T> T Min(T a, T b) { return a < b ? a : b; }
template <class T> T Max(T a, T b) { return a > b ? a : b; }

class Random;
class Dictionary;
class DictionaryEntry;
class MutationDispatcher;
struct FuzzingOptions;
class InputCorpus;
struct InputInfo;
Expand Down Expand Up @@ -57,37 +60,6 @@ using Set = std::set<T, std::less<T>, fuzzer_allocator<T>>;

typedef Vector<uint8_t> Unit;
typedef Vector<Unit> UnitVector;

// A simple POD sized array of bytes.
template <size_t kMaxSizeT> class FixedWord {
public:
static const size_t kMaxSize = kMaxSizeT;
FixedWord() { memset(Data, 0, kMaxSize); }
FixedWord(const uint8_t *B, size_t S) { Set(B, S); }

void Set(const uint8_t *B, size_t S) {
static_assert(kMaxSizeT <= std::numeric_limits<uint8_t>::max(),
"FixedWord::kMaxSizeT cannot fit in a uint8_t.");
assert(S <= kMaxSize);
memcpy(Data, B, S);
Size = static_cast<uint8_t>(S);
}

bool operator==(const FixedWord<kMaxSize> &w) const {
return Size == w.Size && 0 == memcmp(Data, w.Data, Size);
}

static size_t GetMaxSize() { return kMaxSize; }
const uint8_t *data() const { return Data; }
uint8_t size() const { return Size; }

private:
uint8_t Size = 0;
uint8_t Data[kMaxSize];
};

typedef FixedWord<64> Word;

typedef int (*UserCallback)(const uint8_t *Data, size_t Size);

int FuzzerDriver(int *argc, char ***argv, UserCallback Callback);
Expand Down
120 changes: 120 additions & 0 deletions compiler-rt/lib/fuzzer/FuzzerDictionary.h
@@ -0,0 +1,120 @@
//===- FuzzerDictionary.h - Internal header for the Fuzzer ------*- C++ -* ===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
// fuzzer::Dictionary
//===----------------------------------------------------------------------===//

#ifndef LLVM_FUZZER_DICTIONARY_H
#define LLVM_FUZZER_DICTIONARY_H

#include "FuzzerDefs.h"
#include "FuzzerIO.h"
#include "FuzzerUtil.h"
#include <algorithm>
#include <limits>

namespace fuzzer {
// A simple POD sized array of bytes.
template <size_t kMaxSizeT> class FixedWord {
public:
static const size_t kMaxSize = kMaxSizeT;
FixedWord() {}
FixedWord(const uint8_t *B, size_t S) { Set(B, S); }

void Set(const uint8_t *B, size_t S) {
static_assert(kMaxSizeT <= std::numeric_limits<uint8_t>::max(),
"FixedWord::kMaxSizeT cannot fit in a uint8_t.");
assert(S <= kMaxSize);
memcpy(Data, B, S);
Size = static_cast<uint8_t>(S);
}

bool operator==(const FixedWord<kMaxSize> &w) const {
return Size == w.Size && 0 == memcmp(Data, w.Data, Size);
}

static size_t GetMaxSize() { return kMaxSize; }
const uint8_t *data() const { return Data; }
uint8_t size() const { return Size; }

private:
uint8_t Size = 0;
uint8_t Data[kMaxSize];
};

typedef FixedWord<64> Word;

class DictionaryEntry {
public:
DictionaryEntry() {}
DictionaryEntry(Word W) : W(W) {}
DictionaryEntry(Word W, size_t PositionHint) : W(W), PositionHint(PositionHint) {}
const Word &GetW() const { return W; }

bool HasPositionHint() const { return PositionHint != std::numeric_limits<size_t>::max(); }
size_t GetPositionHint() const {
assert(HasPositionHint());
return PositionHint;
}
void IncUseCount() { UseCount++; }
void IncSuccessCount() { SuccessCount++; }
size_t GetUseCount() const { return UseCount; }
size_t GetSuccessCount() const {return SuccessCount; }

void Print(const char *PrintAfter = "\n") {
PrintASCII(W.data(), W.size());
if (HasPositionHint())
Printf("@%zd", GetPositionHint());
Printf("%s", PrintAfter);
}

private:
Word W;
size_t PositionHint = std::numeric_limits<size_t>::max();
size_t UseCount = 0;
size_t SuccessCount = 0;
};

class Dictionary {
public:
static const size_t kMaxDictSize = 1 << 14;

bool ContainsWord(const Word &W) const {
return std::any_of(begin(), end(), [&](const DictionaryEntry &DE) {
return DE.GetW() == W;
});
}
const DictionaryEntry *begin() const { return &DE[0]; }
const DictionaryEntry *end() const { return begin() + Size; }
DictionaryEntry & operator[] (size_t Idx) {
assert(Idx < Size);
return DE[Idx];
}
void push_back(DictionaryEntry DE) {
if (Size < kMaxDictSize)
this->DE[Size++] = DE;
}
void clear() { Size = 0; }
bool empty() const { return Size == 0; }
size_t size() const { return Size; }

private:
DictionaryEntry DE[kMaxDictSize];
size_t Size = 0;
};

// Parses one dictionary entry.
// If successful, write the enty to Unit and returns true,
// otherwise returns false.
bool ParseOneDictionaryEntry(const std::string &Str, Unit *U);
// Parses the dictionary file, fills Units, returns true iff all lines
// were parsed successfully.
bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units);

} // namespace fuzzer

#endif // LLVM_FUZZER_DICTIONARY_H
8 changes: 3 additions & 5 deletions compiler-rt/lib/fuzzer/FuzzerDriver.cpp
Expand Up @@ -19,16 +19,15 @@
#include "FuzzerPlatform.h"
#include "FuzzerRandom.h"
#include "FuzzerTracePC.h"
#include "mutagen/MutagenDispatcher.h"
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <mutex>
#include <string>
#include <thread>
#include <fstream>

// This function should be present in the libFuzzer so that the client
// binary can test for its existence.
Expand Down Expand Up @@ -804,9 +803,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
ReadCorpora(*Inputs, {}));
}

LLVMMutagenConfiguration Config;
ConfigureMutagen(Seed, Options, &Config);
auto *MD = new MutationDispatcher(&Config);
Random Rand(Seed);
auto *MD = new MutationDispatcher(Rand, Options);
auto *Corpus = new InputCorpus(Options.OutputCorpus, Entropic);
auto *F = new Fuzzer(Callback, *Corpus, *MD, Options);

Expand Down
5 changes: 0 additions & 5 deletions compiler-rt/lib/fuzzer/FuzzerInternal.h
Expand Up @@ -18,7 +18,6 @@
#include "FuzzerOptions.h"
#include "FuzzerSHA1.h"
#include "FuzzerValueBitMap.h"
#include "mutagen/MutagenDispatcher.h"
#include <algorithm>
#include <atomic>
#include <chrono>
Expand All @@ -27,12 +26,8 @@
#include <string.h>

namespace fuzzer {
namespace {

using namespace std::chrono;
using mutagen::MutationDispatcher;

} // namespace

class Fuzzer {
public:
Expand Down
9 changes: 4 additions & 5 deletions compiler-rt/lib/fuzzer/FuzzerLoop.cpp
Expand Up @@ -177,7 +177,7 @@ void Fuzzer::DumpCurrentUnit(const char *Prefix) {
if (!CurrentUnitData)
return; // Happens when running individual inputs.
ScopedDisableMsanInterceptorChecks S;
PrintMutationSequence(MD);
MD.PrintMutationSequence();
Printf("; base unit: %s\n", Sha1ToString(BaseSha1).c_str());
size_t UnitSize = CurrentUnitSize;
if (UnitSize <= kMaxUnitSizeToPrint) {
Expand Down Expand Up @@ -539,9 +539,8 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
TimeOfUnit, UniqFeatureSetTmp, DFT, II);
WriteFeatureSetToFile(Options.FeaturesDir, Sha1ToString(NewII->Sha1),
NewII->UniqFeatureSet);
const auto &MS = MD.MutationSequence();
WriteEdgeToMutationGraphFile(Options.MutationGraphFile, NewII, II,
MS.GetString());
MD.MutationSequence());
return true;
}
if (II && FoundUniqFeaturesOfII &&
Expand Down Expand Up @@ -653,7 +652,7 @@ void Fuzzer::PrintStatusForNewUnit(const Unit &U, const char *Text) {
PrintStats(Text, "");
if (Options.Verbosity) {
Printf(" L: %zd/%zd ", U.size(), Corpus.MaxInputSize());
PrintMutationSequence(MD, Options.Verbosity >= 2);
MD.PrintMutationSequence(Options.Verbosity >= 2);
Printf("\n");
}
}
Expand Down Expand Up @@ -899,7 +898,7 @@ void Fuzzer::Loop(Vector<SizedFile> &CorporaFiles) {
}

PrintStats("DONE ", "\n");
PrintRecommendedDictionary(MD);
MD.PrintRecommendedDictionary();
}

void Fuzzer::MinimizeCrashLoop(const Unit &U) {
Expand Down

0 comments on commit fd0a2f7

Please sign in to comment.