Merge pull request #833 from mull-project/mutate-diffs

Incremental mutation testing using GitDiffFilter

docs/Features.rst

@@ -23,6 +23,9 @@ Features
 
 - Parallelized execution of tests
 
+- `Incremental mutation testing <IncrementalMutationTesting.html>`_.
+  Working with mutations found in Git Diff changesets.
+
 - Mull requires test programs to be compiled with Clang/LLVM. Mull supports
   all LLVM versions starting from LLVM 6.
 

docs/IncrementalMutationTesting.rst

@@ -0,0 +1,28 @@
+Incremental mutation testing
+============================
+
+Normally, Mull looks for mutations in all files of a project. Depending on a
+project's size, a number of mutations can be very large, so running Mull
+against all of them might be a rather slow process. Speed aside, an analysis of
+a large mutation data sets can be very time consuming work to be done by a
+user.
+
+Incremental mutation testing is a feature that enables running Mull only on the
+mutations found in Git Diff changesets. Instead of analysing all files and
+functions, Mull only finds mutations in the source lines that are covered by
+a particular Git Diff changeset.
+
+Example: if a Git diff is created from a project's Git tree and the diff is only
+one line, Mull will only find mutations in that line and will skip everything
+else.
+
+To enable incremental mutation testing, two arguments have to be provided to
+Mull: ``-git-diff-ref=<branch or commit>`` and ``-git-project-root=<path>``
+which is a path to a project's Git root path.
+
+An additional debug option ``-debug`` can be useful for a visualization of how
+exactly Mull whitelists or blacklists found source lines.
+
+**Note:** Incremental mutation testing is an experimental feature. Things might
+go wrong. If you encounter any issues, please report them on the
+`mull/issues <https://github.com/mull-project/mull/issues>`_ tracker.

docs/generated/CLIOptions.rst

@@ -50,6 +50,10 @@
 
 --exclude-path regex		File/directory paths to ignore (supports regex)
 
+--git-diff-ref git commit		Git branch to run diff against (enables incremental testing)
+
+--git-project-root git project root		Path to project's Git root (used together with -git-diff-ref)
+
 --mutators mutator		Choose mutators:
 
     Groups:

docs/index.rst

@@ -10,6 +10,7 @@ Welcome to Mull's documentation!
    Installation
    Tutorials
    SupportedMutations
+   IncrementalMutationTesting
    CommandLineReference
    HowMullWorks
    HackingOnMull

include/mull/Filters/GitDiffFilter.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "mull/Diagnostics/Diagnostics.h"
+#include "mull/Filters/Filter.h"
+#include "mull/Filters/GitDiffReader.h"
+#include "mull/Filters/InstructionFilter.h"
+
+namespace mull {
+struct SourceLocation;
+class GitDiffFilter : public InstructionFilter {
+public:
+  static GitDiffFilter *createFromGitDiff(Diagnostics &diagnostics,
+                                          const std::string &gitProjectRoot,
+                                          const std::string &gitDiffBranch);
+
+  GitDiffFilter(Diagnostics &diagnostics, GitDiffInfo gitDiffInfo);
+
+  std::string name() override;
+  bool shouldSkip(llvm::Instruction *instruction) const override;
+
+private:
+  Diagnostics &diagnostics;
+  const GitDiffInfo gitDiffInfo;
+};
+} // namespace mull

include/mull/Filters/GitDiffReader.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "mull/Diagnostics/Diagnostics.h"
+
+#include <map>
+#include <string>
+#include <vector>
+
+namespace mull {
+class Diagnostics;
+
+typedef std::pair<int, int> GitDiffSourceFileRange;
+typedef std::vector<GitDiffSourceFileRange> GitDiffSourceFileRanges;
+typedef std::map<std::string, GitDiffSourceFileRanges> GitDiffInfo;
+
+class GitDiffReader {
+public:
+  GitDiffReader(Diagnostics &diagnostics, const std::string gitRepoPath);
+  GitDiffInfo readGitDiff(const std::string &gitBranch);
+  GitDiffInfo parseDiffContent(const std::string &diffContent);
+
+private:
+  Diagnostics &diagnostics;
+  const std::string gitRepoPath;
+};
+} // namespace mull

include/mull/Toolchain/Runner.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "mull/ExecutionResult.h"
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -13,7 +14,8 @@ class Runner {
   explicit Runner(Diagnostics &diagnostics);
   ExecutionResult runProgram(const std::string &program, const std::vector<std::string> &arguments,
                              const std::vector<std::string> &environment, long long int timeout,
-                             bool captureOutput);
+                             bool captureOutput,
+                             std::optional<std::string> optionalWorkingDirectory);
 
 private:
   Diagnostics &diagnostics;

lib/CMakeLists.txt

@@ -84,7 +84,11 @@ set(mull_sources
   Filters/JunkMutationFilter.cpp
   Filters/NoDebugInfoFilter.cpp
   Filters/FilePathFilter.cpp
-  JunkDetection/CXX/Visitors/ScalarValueVisitor.cpp)
+  Filters/GitDiffReader.cpp
+  Filters/GitDiffFilter.cpp
+
+  JunkDetection/CXX/Visitors/ScalarValueVisitor.cpp
+)
 
 set(MULL_INCLUDE_DIR ${MULL_SOURCE_DIR}/include/mull)
 

lib/Driver.cpp

@@ -58,8 +58,8 @@ std::vector<MutationPoint *> Driver::findMutationPoints() {
   if (!config.skipSanityCheckRun) {
     Runner runner(diagnostics);
     singleTask.execute("Sanity check run", [&]() {
-      ExecutionResult result =
-          runner.runProgram(config.executable, {}, {}, config.timeout, config.captureTestOutput);
+      ExecutionResult result = runner.runProgram(
+          config.executable, {}, {}, config.timeout, config.captureTestOutput, std::nullopt);
       if (result.status != Passed) {
         std::stringstream failureMessage;
         failureMessage << "Original test failed\n";
@@ -238,12 +238,13 @@ Driver::normalRunMutations(const std::vector<MutationPoint *> &mutationPoints,
   /// As we take the execution time as a baseline for timeout it makes sense to have an additional
   /// warm up run so that the next runs will be a bit faster
   singleTask.execute("Warm up run", [&]() {
-    runner.runProgram(executable, {}, {}, config.timeout, config.captureMutantOutput);
+    runner.runProgram(executable, {}, {}, config.timeout, config.captureMutantOutput, std::nullopt);
   });
 
   ExecutionResult baseline;
   singleTask.execute("Baseline run", [&]() {
-    baseline = runner.runProgram(executable, {}, {}, config.timeout, config.captureMutantOutput);
+    baseline = runner.runProgram(
+        executable, {}, {}, config.timeout, config.captureMutantOutput, std::nullopt);
   });
 
   std::vector<std::unique_ptr<MutationResult>> mutationResults;

lib/Filters/GitDiffFilter.cpp

@@ -0,0 +1,82 @@
+#include "mull/Filters/GitDiffFilter.h"
+
+#include "mull/Diagnostics/Diagnostics.h"
+#include "mull/SourceLocation.h"
+#include "mull/Toolchain/Runner.h"
+
+#include "llvm/IR/InstIterator.h"
+#include <llvm/IR/DebugInfoMetadata.h>
+#include <llvm/IR/DebugLoc.h>
+#include <llvm/IR/Function.h>
+#include <llvm/Support/FileSystem.h>
+
+#include <iostream>
+#include <regex>
+#include <sstream>
+#include <unistd.h>
+
+using namespace mull;
+
+GitDiffFilter *GitDiffFilter::createFromGitDiff(Diagnostics &diagnostics,
+                                                const std::string &gitProjectRoot,
+                                                const std::string &gitDiffBranch) {
+  mull::GitDiffReader gitDiffReader(diagnostics, gitProjectRoot);
+  mull::GitDiffInfo gitDiffInfo = gitDiffReader.readGitDiff(gitDiffBranch);
+  return new GitDiffFilter(diagnostics, gitDiffInfo);
+}
+
+GitDiffFilter::GitDiffFilter(Diagnostics &diagnostics, GitDiffInfo gitDiffInfo)
+    : diagnostics(diagnostics), gitDiffInfo(gitDiffInfo) {}
+
+std::string GitDiffFilter::name() {
+  return "Git Diff";
+}
+
+bool GitDiffFilter::shouldSkip(llvm::Instruction *instruction) const {
+  SourceLocation sourceLocation = SourceLocation::locationFromInstruction(instruction);
+  if (sourceLocation.isNull()) {
+    return true;
+  }
+
+  /// If no diff, then filtering out.
+  if (gitDiffInfo.size() == 0) {
+    std::stringstream debugMessage;
+    debugMessage << "GitDiffFilter: git diff is empty. Skipping instruction: ";
+    debugMessage << sourceLocation.filePath << ":";
+    debugMessage << sourceLocation.line << ":" << sourceLocation.column;
+    diagnostics.debug(debugMessage.str());
+    return true;
+  }
+
+  /// If file is not in the diff, then filtering out.
+  if (gitDiffInfo.count(sourceLocation.filePath) == 0) {
+    std::stringstream debugMessage;
+    debugMessage << "GitDiffFilter: the file is not present in the git diff. ";
+    debugMessage << "Skipping instruction: ";
+    debugMessage << sourceLocation.filePath << ":";
+    debugMessage << sourceLocation.line << ":" << sourceLocation.column;
+    diagnostics.debug(debugMessage.str());
+    return true;
+  }
+
+  GitDiffSourceFileRanges ranges = gitDiffInfo.at(sourceLocation.filePath);
+  for (auto &range : ranges) {
+    int rangeEnd = range.first + range.second - 1;
+    if (range.first <= sourceLocation.line && sourceLocation.line <= rangeEnd) {
+      std::stringstream debugMessage;
+      debugMessage << "GitDiffFilter: whitelisting instruction: ";
+      debugMessage << sourceLocation.filePath << ":";
+      debugMessage << sourceLocation.line << ":" << sourceLocation.column;
+      diagnostics.debug(debugMessage.str());
+      return false;
+    }
+  }
+
+  std::stringstream debugMessage;
+  debugMessage << "GitDiffFilter: skipping instruction: ";
+  debugMessage << sourceLocation.filePath << ":";
+  debugMessage << sourceLocation.line << ":" << sourceLocation.column;
+  diagnostics.debug(debugMessage.str());
+
+  return true;
+}

lib/Filters/GitDiffReader.cpp

@@ -0,0 +1,76 @@
+#include "mull/Filters/GitDiffReader.h"
+
+#include "mull/Diagnostics/Diagnostics.h"
+#include <mull/Path.h>
+#include "mull/Toolchain/Runner.h"
+
+#include <iostream>
+#include <regex>
+#include <sstream>
+
+using namespace mull;
+
+GitDiffReader::GitDiffReader(Diagnostics &diagnostics, const std::string gitRepoPath)
+    : diagnostics(diagnostics), gitRepoPath(gitRepoPath) {}
+
+GitDiffInfo GitDiffReader::readGitDiff(const std::string &gitBranch) {
+  /// The implementation is borrowed from the git-clang-format Python tool.
+  /// https://opensource.apple.com/source/clang/clang-800.0.38/src/tools/clang/tools/clang-format/git-clang-format.auto.html
+  Runner runner(diagnostics);
+
+  std::vector<std::string> args = { "diff", "-U0", gitBranch };
+
+  ExecutionResult result =
+      runner.runProgram("git", args, {}, 5000, true, gitRepoPath);
+  if (result.exitStatus != 0) {
+    diagnostics.warning(
+        std::string("GitDiffReader: cannot get git diff information. Received output: ") +
+        result.stderrOutput);
+    return GitDiffInfo();
+  }
+
+  const std::string diffContent = result.stdoutOutput;
+  GitDiffInfo gitDiffInfo = parseDiffContent(diffContent);
+  return gitDiffInfo;
+}
+
+GitDiffInfo GitDiffReader::parseDiffContent(const std::string &diffContent) {
+  GitDiffInfo gitDiffInfo;
+  if (diffContent.empty()) {
+    return gitDiffInfo;
+  }
+
+  std::stringstream diffStream(diffContent);
+  std::string currentLine;
+
+  std::string currentFileName;
+
+  while (std::getline(diffStream, currentLine, '\n')) {
+    std::regex lineRegex("^\\+\\+\\+\\ [^/]+/(.*)");
+    std::regex rangeRegex("^@@ -[0-9,]+ \\+(\\d+)(,(\\d+))?");
+    std::smatch matches;
+
+    if (std::regex_search(currentLine, matches, lineRegex)) {
+      currentFileName = absoluteFilePath(this->gitRepoPath, matches[1].str());
+      continue;
+    }
+
+    if (std::regex_search(currentLine, matches, rangeRegex)) {
+      const std::string startLineStr = matches[1].str();
+      int startLine = std::stoi(startLineStr);
+
+      size_t lineCount = 1;
+      const std::string lineCountStr = matches[3].str();
+      if (lineCountStr.size() > 0) {
+        lineCount = std::stoi(lineCountStr);
+      }
+      if (lineCount > 0) {
+        if (gitDiffInfo.count(currentFileName) == 0) {
+          gitDiffInfo[currentFileName] = std::vector<GitDiffSourceFileRange>();
+        }
+        gitDiffInfo[currentFileName].push_back(std::pair<int, int>(startLine, lineCount));
+      }
+    }
+  }
+  return gitDiffInfo;
+}

lib/JunkDetection/CXX/ASTStorage.cpp

@@ -78,8 +78,12 @@ const clang::FileEntry *ThreadSafeASTUnit::findFileEntry(const std::string &file
   auto end = sourceManager.fileinfo_end();
   const clang::FileEntry *file = nullptr;
   for (auto it = begin; it != end; it++) {
-    StringRef name(it->first->getName());
-    if (name.equals(filePath)) {
+    llvm::StringRef currentSourceFilePath = it->first->getName();
+    /// In LLVM 6, it->first->getName() does not expand to full path for header files.
+    if (!llvm::sys::path::is_absolute(currentSourceFilePath)) {
+      currentSourceFilePath = it->first->tryGetRealPathName();
+    }
+    if (currentSourceFilePath.equals(filePath)) {
       file = it->first;
       break;
     }

lib/Parallelization/Tasks/MutantExecutionTask.cpp

@@ -25,7 +25,8 @@ void MutantExecutionTask::operator()(iterator begin, iterator end, Out &storage,
                                  {},
                                  { mutant->getIdentifier() },
                                  baseline.runningTime * 10,
-                                 configuration.captureMutantOutput);
+                                 configuration.captureMutantOutput,
+                                 std::nullopt);
     } else {
       result.status = NotCovered;
     }

lib/Toolchain/Linker.cpp

@@ -29,8 +29,8 @@ std::string Linker::linkObjectFiles(const std::vector<std::string> &objects) {
   std::copy(std::begin(objects), std::end(objects), std::back_inserter(arguments));
   arguments.emplace_back("-o");
   arguments.push_back(resultPath.str().str());
-  ExecutionResult result =
-      runner.runProgram(configuration.linker, arguments, {}, configuration.linkerTimeout, true);
+  ExecutionResult result = runner.runProgram(
+      configuration.linker, arguments, {}, configuration.linkerTimeout, true, std::nullopt);
   std::stringstream commandStream;
   commandStream << configuration.linker;
   for (std::string &argument : arguments) {

lib/Toolchain/Runner.cpp

@@ -30,7 +30,8 @@ Runner::Runner(Diagnostics &diagnostics) : diagnostics(diagnostics) {}
 ExecutionResult Runner::runProgram(const std::string &program,
                                    const std::vector<std::string> &arguments,
                                    const std::vector<std::string> &environment,
-                                   long long int timeout, bool captureOutput) {
+                                   long long int timeout, bool captureOutput,
+                                   std::optional<std::string> optionalWorkingDirectory) {
   std::vector<std::pair<std::string, std::string>> env;
   env.reserve(environment.size());
   for (auto &e : environment) {
@@ -40,6 +41,9 @@ ExecutionResult Runner::runProgram(const std::string &program,
   reproc::options options;
   options.env.extra = reproc::env(env);
   options.redirect.err.type = reproc::redirect::type::pipe;
+  if (auto &workingDirectory = optionalWorkingDirectory) {
+    options.working_directory = workingDirectory->c_str();
+  }
 
   std::vector<std::string> allArguments{ program };
   std::copy(std::begin(arguments), std::end(arguments), std::back_inserter(allArguments));

tests-lit/.gitignore

@@ -7,4 +7,5 @@
 *.tmp
 *.profdata
 *.profraw
+**/Output/**
 

tests-lit/CMakeLists.txt

@@ -41,6 +41,7 @@ set(LIT_COMMAND
 )
 
 add_custom_target(tests-lit
+  COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && make clean
   COMMAND ${LIT_COMMAND}
   DEPENDS mull-cxx
 )