Bochscpu: edge coverage

README.md

@@ -279,7 +279,7 @@ Check out the [CustomMutator_t](src/wtf/fuzzer_tlv_server.cc) class in the [fuzz
 In this section I briefly mention various differences between the execution backends.
 
 ### bochscpu
-- ✅ Full system code-coverage,
+- ✅ Full system code-coverage (edge coverage available via `--edges`),
 - ✅ Demand-paging,
 - ✅ Timeout is the number of instructions which is very precise,
 - ✅ Full execution traces are supported,
@@ -304,7 +304,7 @@ In this section I briefly mention various differences between the execution back
 
 ## Build
 
-The [CI](https://github.com/0vercl0k/wtf/actions/workflows/wtf.yml) builds **wtf** on Ubuntu 20.04 using [clang++-14](https://clang.llvm.org/) / [g++-11](https://gcc.gnu.org/gcc-11/) and on Windows using Microsoft's [Visual Studio 2019](https://visualstudio.microsoft.com/vs/community/).
+The [CI](https://github.com/0vercl0k/wtf/actions/workflows/wtf.yml) builds **wtf** on Ubuntu using both [clang++](https://clang.llvm.org/) / [g++](https://gcc.gnu.org/gcc-11/), on Windows using Microsoft's [Visual Studio](https://visualstudio.microsoft.com/vs/community/) and on OSX using [clang++](https://clang.llvm.org/).
 
 To build it yourself you need to start a *Visual Studio Developper Command Prompt* and either run [build-release.bat](src/build/build-release.bat) which uses the [Ninja](https://ninja-build.org/) generator or [build-release-msvc.bat](src/build/build-release-msvc.bat) to generate a Visual Studio solution file:
 

src/libs/bochscpu-bins/build-bochscpu.bat

@@ -18,7 +18,7 @@ nmake
 
 REM Remove old files in bochscpu.
 rmdir /s /q ..\..\..\bochscpu\bochs
-rmdir /s /q ..\..\..\bochscpu\libs
+rmdir /s /q ..\..\..\bochscpu\lib
 
 REM Create the libs directory where we stuff all the libs.
 mkdir ..\..\..\bochscpu\lib

src/libs/bochscpu-bins/include/bochscpu.hpp

@@ -6,6 +6,28 @@
 #include <ostream>
 #include <new>
 
+static const uint32_t BX_INSTR_IS_JMP = 10;
+
+static const uint32_t BOCHSCPU_INSTR_IS_JMP_INDIRECT = 11;
+
+static const uint32_t BOCHSCPU_INSTR_IS_CALL = 12;
+
+static const uint32_t BOCHSCPU_INSTR_IS_CALL_INDIRECT = 13;
+
+static const uint32_t BOCHSCPU_INSTR_IS_RET = 14;
+
+static const uint32_t BOCHSCPU_INSTR_IS_IRET = 15;
+
+static const uint32_t BOCHSCPU_INSTR_IS_INT = 16;
+
+static const uint32_t BOCHSCPU_INSTR_IS_SYSCALL = 17;
+
+static const uint32_t BOCHSCPU_INSTR_IS_SYSRET = 18;
+
+static const uint32_t BOCHSCPU_INSTR_IS_SYSENTER = 19;
+
+static const uint32_t BOCHSCPU_INSTR_IS_SYSEXIT = 20;
+
 static const uint32_t BOCHSCPU_HOOK_MEM_READ = 0;
 
 static const uint32_t BOCHSCPU_HOOK_MEM_WRITE = 1;
@@ -203,6 +225,8 @@ void bochscpu_cpu_state(bochscpu_cpu_t p, bochscpu_cpu_state_t *s);
 
 void bochscpu_cpu_set_state(bochscpu_cpu_t p, const bochscpu_cpu_state_t *s);
 
+void bochscpu_cpu_set_state_no_flush(bochscpu_cpu_t p, const bochscpu_cpu_state_t *s);
+
 void bochscpu_cpu_set_exception(bochscpu_cpu_t p, uint32_t vector, uint16_t error);
 
 uint64_t bochscpu_cpu_rax(bochscpu_cpu_t p);

src/wtf/bochscpu_backend.cc

@@ -195,6 +195,30 @@ void StaticHltHook(void *Context, uint32_t Cpu) {
   reinterpret_cast<BochscpuBackend_t *>(Context)->OpcodeHlt(Cpu);
 }
 
+void StaticUcNearBranchHook(void *Context, uint32_t Cpu, uint32_t What,
+                            uint64_t Rip, uint64_t NextRip) {
+  if ((What == BOCHSCPU_INSTR_IS_JMP_INDIRECT) ||
+      (What == BOCHSCPU_INSTR_IS_CALL_INDIRECT)) {
+
+    //
+    // Invoking the member function now.
+    //
+
+    reinterpret_cast<BochscpuBackend_t *>(Context)->RecordEdge(Cpu, Rip,
+                                                               NextRip);
+  }
+}
+
+void StaticCNearBranchHook(void *Context, uint32_t Cpu, uint64_t Rip,
+                           uint64_t NextRip) {
+
+  //
+  // Invoking the member function now.
+  //
+
+  reinterpret_cast<BochscpuBackend_t *>(Context)->RecordEdge(Cpu, Rip, NextRip);
+}
+
 BochscpuBackend_t::BochscpuBackend_t() {
 
   //
@@ -239,6 +263,17 @@ bool BochscpuBackend_t::Initialize(const Options_t &Opts,
   Hooks_.hlt = StaticHltHook;
   // Hooks_.opcode = StaticOpcodeHook;
 
+  //
+  // If edge coverage is enabled, configure hooks to be able to record
+  // edges from branches.
+  //
+
+  if (Opts.Edges) {
+    Hooks_.cnear_branch_taken = StaticCNearBranchHook;
+    Hooks_.cnear_branch_not_taken = StaticCNearBranchHook;
+    Hooks_.ucnear_branch = StaticUcNearBranchHook;
+  }
+
   //
   // Initialize the hook chain with only one set of hooks.
   //
@@ -624,6 +659,37 @@ void BochscpuBackend_t::OpcodeHlt(/*void *Context, */ uint32_t) {
   bochscpu_cpu_stop(Cpu_);
 }
 
+void BochscpuBackend_t::RecordEdge(/*void *Context, */ uint32_t Cpu,
+                                   uint64_t Rip, uint64_t NextRip) {
+
+  uint64_t Edge = Rip;
+
+  //
+  // splitmix64 Rip, might be overkill, a single shift is probably sufficient to
+  // avoid collisions?
+  //
+
+  Edge ^= Edge >> 30;
+  Edge *= 0xbf58476d1ce4e5b9U;
+  Edge ^= Edge >> 27;
+  Edge *= 0x94d049bb133111ebU;
+  Edge ^= Edge >> 31;
+
+  //
+  // XOR with NextRip.
+  //
+
+  Edge ^= NextRip;
+
+  const auto &[_, NewCoverage] = AggregatedCodeCoverage_.emplace(Edge);
+  if (NewCoverage) {
+    LastNewCoverage_.emplace(Edge);
+    RunStats_.NumberUniqueEdges++;
+  }
+
+  RunStats_.NumberEdges++;
+}
+
 bool BochscpuBackend_t::Restore(const CpuState_t &CpuState) {
 
   //

src/wtf/bochscpu_backend.h

@@ -19,6 +19,8 @@ struct BochscpuRunStats_t {
   uint64_t NumberMemoryAccesses = 0;
   uint64_t AggregatedCodeCoverage = 0;
   uint64_t DirtyGpas = 0;
+  uint64_t NumberEdges = 0;
+  uint64_t NumberUniqueEdges = 0;
 
   void Print() const {
     fmt::print("--------------------------------------------------\n");
@@ -30,11 +32,15 @@ struct BochscpuRunStats_t {
     fmt::print("          Dirty pages: {}\n", BytesToHuman(DirtyMemoryBytes));
     fmt::print("      Memory accesses: {}\n",
                BytesToHuman(NumberMemoryAccesses));
+    fmt::print("       Edges executed: {} ({} unique)\n",
+               NumberToHuman(NumberEdges), NumberToHuman(NumberUniqueEdges));
   }
 
   void Reset() {
     NumberInstructionsExecuted = 0;
     NumberMemoryAccesses = 0;
+    NumberEdges = 0;
+    NumberUniqueEdges = 0;
   }
 };
 
@@ -290,6 +296,9 @@ class BochscpuBackend_t : public Backend_t {
 
   void OpcodeHlt(/*void *Context, */ uint32_t Cpu);
 
+  void RecordEdge(/*void *Context, */ uint32_t Cpu, uint64_t Rip,
+                  uint64_t NextRip);
+
 private:
   //
   // Dirty every physical pages included in a virtual memory range.

src/wtf/globals.h

@@ -1188,6 +1188,7 @@ enum class TraceType_t {
 enum class BackendType_t { Bochscpu, Whv, Kvm };
 
 struct FuzzOptions_t {
+
   //
   // Path to the target folder.
   //
@@ -1358,6 +1359,12 @@ struct Options_t {
 
   fs::path CoveragePath;
 
+  //
+  // Use edge coverage (only with bxcpu).
+  //
+
+  bool Edges = false;
+
   //
   // Options for the subcommand 'run'.
   //

src/wtf/server.h

@@ -328,12 +328,6 @@ class Server_t {
 
   tsl::robin_set<Gva_t> Coverage_;
 
-  //
-  // File where the aggregated code-coverage is written.
-  //
-
-  FILE *FileCoverage_ = nullptr;
-
   //
   // Number of mutations we have performed.
   //
@@ -352,11 +346,6 @@ class Server_t {
   //
 
   ~Server_t() {
-    if (FileCoverage_) {
-      fclose(FileCoverage_);
-      FileCoverage_ = nullptr;
-    }
-
     for (const auto &[Fd, _] : Clients_) {
       CloseSocket(Fd);
     }
@@ -377,23 +366,6 @@ class Server_t {
 
     fmt::print("Seeded with {}\n", Opts_.Seed);
 
-    //
-    // Open the file where the aggregated coverage is written to.
-    //
-
-    constexpr std::string_view FileCoverageName("aggregate.cov");
-    if (fs::exists(FileCoverageName)) {
-      fmt::print("Please remove / save the {} file to continue\n",
-                 FileCoverageName);
-      return EXIT_FAILURE;
-    }
-
-    FileCoverage_ = fopen(FileCoverageName.data(), "a");
-    if (FileCoverage_ == nullptr) {
-      fmt::print("Failed to open {}\n", FileCoverageName);
-      return EXIT_FAILURE;
-    }
-
     //
     // Initialize our internal state.
     //
@@ -844,16 +816,11 @@ class Server_t {
     if (Coverage.size() > 0) {
 
       //
-      // Emplace the new coverage in our data and update the file on disk.
+      // Emplace the new coverage in our data.
       //
 
       const size_t SizeBefore = Coverage_.size();
-      for (const auto &Gva : Coverage) {
-        const auto &[_, NewCoverage] = Coverage_.emplace(Gva);
-        if (NewCoverage) {
-          fmt::print(FileCoverage_, "{:#x}\n", Gva.U64());
-        }
-      }
+      Coverage_.insert(Coverage.cbegin(), Coverage.cend());
 
       //
       // If the coverage size has changed, it means that this testcase
@@ -863,13 +830,6 @@ class Server_t {
       const bool NewCoverage = Coverage_.size() > SizeBefore;
       if (NewCoverage) {
 
-        //
-        // New coverage means that we added new content to the file, so let's
-        // flush it.
-        //
-
-        fflush(FileCoverage_);
-
         //
         // Allocate a test that will get moved into the corpus and maybe
         // saved on disk.

src/wtf/utils.cc

@@ -215,6 +215,18 @@ bool SanitizeCpuState(CpuState_t &CpuState) {
     }
   }
 
+  //
+  // If mxcsr_mask is equal to 0 it means something is wrong (old version of
+  // bdump, etc.) and will cause #GPs. In that case, let's use a default value
+  // that's been taken from the Linux kernel (src:
+  // https://github.com/yrp604/bdump/commit/5a86bdc45acaf65a32aa9149ea47b717d899c85e)
+  //
+
+  if (CpuState.MxcsrMask == 0) {
+    fmt::print("Setting mxcsr_mask to 0xffbf.\n");
+    CpuState.MxcsrMask = 0xff'bf;
+  }
+
   return true;
 }
 

src/wtf/wtf.cc

@@ -138,6 +138,17 @@ int main(int argc, const char *argv[]) {
           throw new CLI::ParseError("state", EXIT_FAILURE);
         }
 
+        //
+        // Ensure that if the 'edge' mode is turned on, bxcpu is used as the
+        // backend.
+        //
+
+        if (Opts.Edges && Opts.Backend != BackendType_t::Bochscpu) {
+          fmt::print(
+              "Edge coverage is only available with the bxcpu backend.\n");
+          throw new CLI::ParseError("edge", EXIT_FAILURE);
+        }
+
 #ifdef LINUX
         if (!std::filesystem::exists(Opts.SymbolFilePath)) {
           fmt::print(
@@ -228,6 +239,10 @@ int main(int argc, const char *argv[]) {
       ->check(CLI::ExistingDirectory)
       ->description("Directory where all the coverage files are stored in.");
 
+  RunCmd->add_flag("--edges", Opts.Edges, "Edge coverage")
+      ->default_val(false)
+      ->description("Turn on edge coverage (bxcpu only).");
+
   RunCmd->add_option("--runs", Opts.Run.Runs, "Runs")
       ->description("Number of mutations done.")
       ->default_val(1);
@@ -272,6 +287,17 @@ int main(int argc, const char *argv[]) {
           throw new CLI::ParseError("target", EXIT_FAILURE);
         }
 
+        //
+        // Ensure that if the 'edge' mode is turned on, bxcpu is used as the
+        // backend.
+        //
+
+        if (Opts.Edges && Opts.Backend != BackendType_t::Bochscpu) {
+          fmt::print(
+              "Edge coverage is only available with the bxcpu backend.\n");
+          throw new CLI::ParseError("edge", EXIT_FAILURE);
+        }
+
         if (Opts.Fuzz.Seed == 0) {
           std::random_device R;
           Opts.Fuzz.Seed = (uint64_t(R()) << 32) | R();
@@ -293,6 +319,10 @@ int main(int argc, const char *argv[]) {
       ->description("Execution backend.")
       ->required();
 
+  FuzzCmd->add_flag("--edges", Opts.Edges, "Edge coverage")
+      ->default_val(false)
+      ->description("Turn on edge coverage (bxcpu only).");
+
   FuzzCmd->add_option("--name", Opts.TargetName, "Target name")
       ->description("Name of the target fuzzer.")
       ->required();