Treat mutants surviving as an error

Surviving mutants would typically indicate insufficient test coverage. As such, it makes sense for Mull to treat it as an error and exit with a non-zero status. This changes the default behavior of mull-runner which up until this point has exited with zero regardless of how many mutants survived. This behavior can be "restored" by passing the --allow-surviving CLI argument to mull-runner. Since the mutation scores for both fmtlib and openssl are very low, the --allow-surviving flag is passed when running the integration tests This does not treat the absence of mutants (i.e. infinite mutation score) as an error
mull-project · Jan 17, 2023 · f4ccfb5 · f4ccfb5
1 parent 53045fe
commit f4ccfb5
Show file tree

Hide file tree

Showing 26 changed files with 65 additions and 33 deletions.
diff --git a/docs/command-line/generated/mull-runner-cli-options.rst b/docs/command-line/generated/mull-runner-cli-options.rst
@@ -28,6 +28,8 @@
 
 --strict		Enables Strict Mode: all warning messages are treated as fatal errors
 
+--allow-surviving	Do not treat mutants surviving as an error
+
 --no-test-output		Does not capture output from test runs
 
 --no-mutant-output		Does not capture output from mutant runs

diff --git a/infrastructure/helpers/integration-tests.yaml b/infrastructure/helpers/integration-tests.yaml
@@ -43,7 +43,7 @@
     chdir: "{{ working_dir }}/integration/fmt.build.dir"
 
 - name: Run Mull against fmt
-  command: mull-runner-{{ llvm_major }} {{ working_dir }}/integration/fmt.build.dir/bin/core-test
+  command: mull-runner-{{ llvm_major }} --allow-surviving {{ working_dir }}/integration/fmt.build.dir/bin/core-test
 
 - name: Clone OpenSSL
   git:
@@ -96,4 +96,4 @@
     LD_LIBRARY_PATH: "{{ working_dir }}/integration/openssl"
 
 - name: Run Mull against OpenSSL
-  command: mull-runner-{{ llvm_major }} {{ working_dir }}/integration/openssl/test/bio_enc_test
+  command: mull-runner-{{ llvm_major }} --allow-surviving {{ working_dir }}/integration/openssl/test/bio_enc_test
diff --git a/tests-lit/tests/filters/coverage/01/main.c b/tests-lit/tests/filters/coverage/01/main.c
@@ -11,7 +11,7 @@ int main(int argc, char **argv) {
 
 // clang-format off
 // RUN: %clang_cc %sysroot %s %pass_mull_ir_frontend -g -o %s.exe
-// RUN: unset TERM; %mull_runner %s.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=CHECK-NO-COVERAGE
+// RUN: unset TERM; %mull_runner --allow-surviving %s.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=CHECK-NO-COVERAGE
 // CHECK-NO-COVERAGE:[info] Survived mutants (1/1):
 // CHECK-NO-COVERAGE:{{^.*}}main.c:2:5: warning: Survived: Replaced + with - [cxx_add_to_sub]
 // CHECK-NO-COVERAGE:[info] Mutation score: 0%

diff --git a/tests-lit/tests/filters/coverage/partial-covered-functions/main.c b/tests-lit/tests/filters/coverage/partial-covered-functions/main.c
@@ -23,7 +23,7 @@ int main(){
 }
 // clang-format off
 // RUN: %clang_cc %sysroot %s %pass_mull_ir_frontend -g -o %s.exe
-// RUN: unset TERM; %mull_runner %s.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=CHECK-NO-COVERAGE
+// RUN: unset TERM; %mull_runner --allow-surviving %s.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=CHECK-NO-COVERAGE
 // CHECK-NO-COVERAGE:[info] Survived mutants (4/4):
 // CHECK-NO-COVERAGE:{{^.*}}main.c:5:16: warning: Survived: Replaced + with - [cxx_add_to_sub]
 // CHECK-NO-COVERAGE:{{^.*}}main.c:10:11: warning: Survived: Replaced + with - [cxx_add_to_sub]
@@ -32,10 +32,10 @@ int main(){
 // CHECK-NO-COVERAGE:[info] Mutation score: 0%
 
 // RUN: %clang_cc %sysroot %s %pass_mull_ir_frontend -g -fprofile-instr-generate -fcoverage-mapping -o %s-cov.exe
-// RUN: unset TERM; %mull_runner -include-not-covered %s-cov.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=CHECK-COVERAGE
+// RUN: unset TERM; %mull_runner --allow-surviving -include-not-covered %s-cov.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=CHECK-COVERAGE
 // RUN: env LLVM_PROFILE_FILE=%s.profraw %s-cov.exe
 // RUN: %llvm_profdata merge %s.profraw -o %s.profdata
-// RUN: unset TERM; %mull_runner -coverage-info %s.profdata -include-not-covered %s-cov.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=CHECK-COVERAGE
+// RUN: unset TERM; %mull_runner --allow-surviving -coverage-info %s.profdata -include-not-covered %s-cov.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=CHECK-COVERAGE
 // CHECK-COVERAGE:[info] Survived mutants (2/4):
 // CHECK-COVERAGE:{{^.*}}main.c:10:11: warning: Survived: Replaced + with - [cxx_add_to_sub]
 // CHECK-COVERAGE:{{^.*}}main.c:14:21: warning: Survived: Replaced + with - [cxx_add_to_sub]

diff --git a/...s/junk-detection/00_error_compilation_flags_database_does_not_match_real_flags/sample.cpp b/...s/junk-detection/00_error_compilation_flags_database_does_not_match_real_flags/sample.cpp
@@ -18,10 +18,10 @@ RUN: sed -e "s:%PWD:%S:g" %S/compile_commands.with_flag.json.template > %S/compi
 
 RUN: cd %CURRENT_DIR
 RUN: cd / && env MULL_CONFIG=%S/mull.no_junk.yml %clang_cxx -O0 %sysroot %pass_mull_ir_frontend -g -DFLAG=1 %s -o %s-ir-no-junk.exe
-RUN: %mull_runner -reporters=IDE %s-ir-no-junk.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-JUNK-DETECTION
+RUN: %mull_runner --allow-surviving -reporters=IDE %s-ir-no-junk.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-JUNK-DETECTION
 
 RUN: cd %S && env MULL_CONFIG=%S/mull.no_flag.yml %clang_cxx %sysroot -O0 %pass_mull_ir_frontend -g -DFLAG=1 %s -o %s-ir-no-flag.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITH-JUNK-DETECTION-NO-FLAG-MUTATE
-RUN: %mull_runner %s-ir-no-flag.exe -reporters=IDE -ide-reporter-show-killed 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITH-JUNK-DETECTION-NO-FLAG
+RUN: %mull_runner --allow-surviving %s-ir-no-flag.exe -reporters=IDE -ide-reporter-show-killed 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITH-JUNK-DETECTION-NO-FLAG
 
 RUN: cd %S && env MULL_CONFIG=%S/mull.with_flag.yml %clang_cxx %sysroot -O0 %pass_mull_ir_frontend -g -DFLAG=1 %s -o %s-ir-with-flag.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITH-JUNK-DETECTION-WITH-FLAG-MUTATE
 RUN: %mull_runner %s-ir-with-flag.exe -reporters=IDE -ide-reporter-show-killed 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITH-JUNK-DETECTION-WITH-FLAG

diff --git a/tests-lit/tests/filters/junk-detection/01_junk_detection/sample.cpp b/tests-lit/tests/filters/junk-detection/01_junk_detection/sample.cpp
@@ -14,7 +14,7 @@ RUN: sed -e "s:%PWD:%S:g" %S/compile_commands.json.template > %S/Output/compile_
 RUN: sed -e "s:%CC:%clang_cc:g" %S/Output/compile_commands.json.temp > %S/compile_commands.json
 RUN: cd %CURRENT_DIR
 RUN: env MULL_CONFIG=%S/mull.no_junk.yml %clang_cxx %sysroot -O0 %pass_mull_ir_frontend -g %s -o %s-ir-no-junk.exe
-RUN: %mull_runner -workers=1 -reporters=IDE %s-ir-no-junk.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-JUNK-DETECTION
+RUN: %mull_runner --allow-surviving -workers=1 -reporters=IDE %s-ir-no-junk.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-JUNK-DETECTION
 WITHOUT-JUNK-DETECTION:{{^.*}}[info] Running mutants (threads: 1){{$}}
 WITHOUT-JUNK-DETECTION:{{^.*}}sample.cpp:2:13: warning: Survived: Removed the call to the function [cxx_remove_void_call]{{$}}
 WITHOUT-JUNK-DETECTION-NOT:[info] Mutation score: 100%

diff --git a/tests-lit/tests/filters/junk-detection/02_junk_detection_using_extra_flags/sample.cpp b/tests-lit/tests/filters/junk-detection/02_junk_detection_using_extra_flags/sample.cpp
@@ -15,7 +15,7 @@ int main() {
 /**
 RUN: cd %CURRENT_DIR
 RUN: cd / && env MULL_CONFIG=%S/mull.no_junk.yml %clang_cxx %sysroot -O0 %pass_mull_ir_frontend -g -DFLAG=1 %s -o %s-ir-no-junk.exe
-RUN: %mull_runner -reporters=IDE %s-ir-no-junk.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-JUNK-DETECTION
+RUN: %mull_runner --allow-surviving -reporters=IDE %s-ir-no-junk.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-JUNK-DETECTION
 
 WITHOUT-JUNK-DETECTION:{{^.*}}sample.cpp:5:13: warning: Survived: Removed the call to the function [cxx_remove_void_call]{{$}}
 

diff --git a/...tests/filters/junk-detection/03_junk_detection_merging_comp_db_and_extra_flags/sample.cpp b/...tests/filters/junk-detection/03_junk_detection_merging_comp_db_and_extra_flags/sample.cpp
@@ -21,7 +21,7 @@ RUN: sed -e "s:%PWD:%S:g" %S/compile_commands.json.template > %S/compile_command
 RUN: cd %CURRENT_DIR
 
 RUN: cd / && env MULL_CONFIG=%S/mull.no_junk.yml %clang_cxx %sysroot -O0 %pass_mull_ir_frontend -g -DFLAG_VIA_COMP_DB=1 -DFLAG_VIA_EXTRA_FLAGS=1 %s -o %s-ir-no-junk.exe
-RUN: %mull_runner %s-ir-no-junk.exe -reporters=IDE | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-JUNK-DETECTION
+RUN: %mull_runner --allow-surviving %s-ir-no-junk.exe -reporters=IDE | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-JUNK-DETECTION
 
 WITHOUT-JUNK-DETECTION:{{^.*}}sample.cpp:9:13: warning: Survived: Removed the call to the function [cxx_remove_void_call]{{$}}
 

diff --git a/...04_junk_detection_using_bitcode_compilation_flags_via_record_command_line_flag/sample.cpp b/...04_junk_detection_using_bitcode_compilation_flags_via_record_command_line_flag/sample.cpp
@@ -16,7 +16,7 @@ int main() {
 RUN: cd %CURRENT_DIR
 
 RUN: cd / && env MULL_CONFIG=%S/mull.no_junk.yml %clang_cxx %sysroot %pass_mull_ir_frontend -g -DFLAG=1 %s -o %s-ir-no-junk.exe
-RUN: (unset TERM; %mull_runner -reporters=IDE -ide-reporter-show-killed %s-ir-no-junk.exe 2>&1; test $? = 0) | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-RECORD-COMMAND-LINE
+RUN: (unset TERM; %mull_runner --allow-surviving -reporters=IDE -ide-reporter-show-killed %s-ir-no-junk.exe 2>&1; test $? = 0) | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-RECORD-COMMAND-LINE
 WITHOUT-RECORD-COMMAND-LINE-MUTATE-NOT:Found compilation flags in the input bitcode
 WITHOUT-RECORD-COMMAND-LINE:{{^.*}}sample.cpp:5:13: warning: Survived: Removed the call to the function [cxx_remove_void_call]{{$}}
 

diff --git a/tests-lit/tests/mutations/logical/and_or_combined/01/main.c b/tests-lit/tests/mutations/logical/and_or_combined/01/main.c
@@ -86,7 +86,7 @@ int main() {
 // clang-format off
 
 // RUN: %clang_cc %sysroot -O0 %pass_mull_ir_frontend -g %s -o %s-ir.exe
-// RUN: unset TERM; %mull_runner -ide-reporter-show-killed -reporters=IDE %s-ir.exe | %filecheck %s --dump-input=fail
+// RUN: unset TERM; %mull_runner --allow-surviving -ide-reporter-show-killed -reporters=IDE %s-ir.exe | %filecheck %s --dump-input=fail
 // CHECK:[info] Killed mutants (4/8):
 // CHECK:{{.*}}main.c:6:23: warning: Killed: Replaced || with && [cxx_logical_or_to_and]
 // CHECK:  if (a < b && (b < c || a < c)) {

diff --git a/tests-lit/tests/mutations/logical/cxx_logical_and_to_or/01/main.c b/tests-lit/tests/mutations/logical/cxx_logical_and_to_or/01/main.c
@@ -92,7 +92,7 @@ int main() {
 // clang-format off
 
 // RUN: %clang_cc %sysroot -O0 %pass_mull_ir_frontend -g %s -o %s-ir.exe
-// RUN: unset TERM; %mull_runner -ide-reporter-show-killed -reporters=IDE %s-ir.exe | %filecheck %s --dump-input=fail
+// RUN: unset TERM; %mull_runner --allow-surviving -ide-reporter-show-killed -reporters=IDE %s-ir.exe | %filecheck %s --dump-input=fail
 // CHECK:[info] Killed mutants (2/5):
 // CHECK:{{.*}}main.c:8:13: warning: Killed: Replaced && with || [cxx_logical_and_to_or]
 // CHECK:  if (a < b && b < c) {

diff --git a/tests-lit/tests/mutations/logical/cxx_logical_or_to_and/01/main.c b/tests-lit/tests/mutations/logical/cxx_logical_or_to_and/01/main.c
@@ -92,7 +92,7 @@ int main() {
 // clang-format off
 
 // RUN: %clang_cc %sysroot -O0 %pass_mull_ir_frontend -g %s -o %s-ir.exe
-// RUN: unset TERM; %mull_runner -ide-reporter-show-killed -reporters=IDE %s-ir.exe | %filecheck %s --dump-input=fail
+// RUN: unset TERM; %mull_runner --allow-surviving -ide-reporter-show-killed -reporters=IDE %s-ir.exe | %filecheck %s --dump-input=fail
 // CHECK:[info] Killed mutants (2/5):
 // CHECK:{{.*}}8:13: warning: Killed: Replaced || with && [cxx_logical_or_to_and]
 // CHECK:  if (a < b || b < c) {

diff --git a/tests-lit/tests/mutations/logical/cxx_remove_negation/main.cpp b/tests-lit/tests/mutations/logical/cxx_remove_negation/main.cpp
@@ -15,7 +15,7 @@ int main() {
 
 /**
 RUN: %clang_cxx %sysroot -O0 %pass_mull_ir_frontend -g %s -o %s-ir.exe
-RUN: unset TERM; %mull_runner -workers=1 --ide-reporter-show-killed -reporters=IDE %s-ir.exe | %filecheck %s --dump-input=fail
+RUN: unset TERM; %mull_runner --allow-surviving -workers=1 --ide-reporter-show-killed -reporters=IDE %s-ir.exe | %filecheck %s --dump-input=fail
 CHECK:[info] Running mutants (threads: 1)
 CHECK:{{^       \[################################\] 2/2\. Finished .*}}
 CHECK:[info] Killed mutants (1/2):
@@ -24,5 +24,6 @@ CHECK:[info] Survived mutants (1/2):
 CHECK:{{^.*}}main.cpp:6:10: warning: Survived: Replaced !a with a [cxx_remove_negation]{{$}}
 CHECK:[info] Mutation score: 50%
 CHECK:[info] Total execution time: {{.*}}
+CHECK:[info] Surviving mutants: 1
 CHECK-EMPTY:
 **/
diff --git a/tests-lit/tests/mutations/logical/phi_nodes/main.c b/tests-lit/tests/mutations/logical/phi_nodes/main.c
@@ -9,7 +9,7 @@ int main() {
 }
 
 // RUN: %clang_cc %sysroot -O0 %pass_mull_ir_frontend -g %s -o %s.exe
-// RUN: unset TERM; %mull_runner -reporters=IDE %s.exe | %filecheck %s --dump-input=fail
+// RUN: unset TERM; %mull_runner --allow-surviving -reporters=IDE %s.exe | %filecheck %s --dump-input=fail
 // CHECK:[info] Survived mutants (3/3):
 // CHECK-NEXT:{{.*}}main.c:5:9: warning: Survived: Replaced && with || [cxx_logical_and_to_or]
 // CHECK-NEXT:  a = a && a && a;

diff --git a/tests-lit/tests/mutations/math/cxx_add_to_sub/03_no_ret/sample.cpp b/tests-lit/tests/mutations/math/cxx_add_to_sub/03_no_ret/sample.cpp
@@ -11,6 +11,6 @@ int main() {
 
 /**
 RUN: %clang_cxx %sysroot %pass_mull_ir_frontend -g -O0 %s -o %s-ir.exe
-RUN: unset TERM; %mull_runner -reporters=IDE %s-ir.exe | %filecheck %s --dump-input=fail
+RUN: unset TERM; %mull_runner --allow-surviving -reporters=IDE %s-ir.exe | %filecheck %s --dump-input=fail
 CHECK-NOT:[error]
 **/
diff --git a/tests-lit/tests/options/-ide-reporter-show-killed/02_one_survived/sample.cpp b/tests-lit/tests/options/-ide-reporter-show-killed/02_one_survived/sample.cpp
@@ -7,8 +7,8 @@ RUN: %clang_cxx %sysroot %pass_mull_ir_frontend -g %s -o %s.exe
 /// 1) when -reporters=IDE is not provided
 /// 2) when -reporters=IDE is provided
 
-RUN: (unset TERM; %mull_runner %s.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-OPTION)
-RUN: (unset TERM; %mull_runner -reporters=IDE %s.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-OPTION)
+RUN: (unset TERM; %mull_runner --allow-surviving %s.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-OPTION)
+RUN: (unset TERM; %mull_runner --allow-surviving -reporters=IDE %s.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-OPTION)
 WITHOUT-OPTION:[info] Running mutants (threads: 1)
 WITHOUT-OPTION:{{^       \[################################\] 1/1\. Finished .*}}
 WITHOUT-OPTION:[info] Survived mutants (1/1):
@@ -17,8 +17,8 @@ WITHOUT-OPTION-NEXT:  int result = a + b;
 WITHOUT-OPTION-NEXT:                 ^
 WITHOUT-OPTION-NEXT:[info] Mutation score: 0%
 
-RUN: (unset TERM; %mull_runner -ide-reporter-show-killed %s.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITH-OPTION)
-RUN: (unset TERM; %mull_runner -reporters=IDE -ide-reporter-show-killed %s.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITH-OPTION)
+RUN: (unset TERM; %mull_runner --allow-surviving -ide-reporter-show-killed %s.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITH-OPTION)
+RUN: (unset TERM; %mull_runner --allow-surviving -reporters=IDE -ide-reporter-show-killed %s.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITH-OPTION)
 WITH-OPTION:[info] Running mutants (threads: 1)
 WITH-OPTION:{{^       \[################################\] 1/1\. Finished .*}}
 WITH-OPTION:[info] Survived mutants (1/1):

diff --git a/tests-lit/tests/options/-ide-reporter-show-killed/03_one_surviving_one_killed/sample.cpp b/tests-lit/tests/options/-ide-reporter-show-killed/03_one_surviving_one_killed/sample.cpp
@@ -7,8 +7,8 @@ RUN: %clang_cxx %sysroot %pass_mull_ir_frontend -g %s -o %s.exe
 /// 1) when -reporters=IDE is not provided
 /// 2) when -reporters=IDE is provided
 
-RUN: (unset TERM; %mull_runner %s.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-OPTION)
-RUN: (unset TERM; %mull_runner -reporters=IDE %s.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-OPTION)
+RUN: (unset TERM; %mull_runner --allow-surviving %s.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-OPTION)
+RUN: (unset TERM; %mull_runner --allow-surviving -reporters=IDE %s.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITHOUT-OPTION)
 WITHOUT-OPTION:[info] Running mutants (threads: 2)
 WITHOUT-OPTION:{{^       \[################################\] 2/2\. Finished .*}}
 WITHOUT-OPTION:[info] Survived mutants (1/2):
@@ -17,8 +17,8 @@ WITHOUT-OPTION-NEXT:  result = result + 0;
 WITHOUT-OPTION-NEXT:                  ^
 WITHOUT-OPTION-NEXT:[info] Mutation score: 50%
 
-RUN: (unset TERM; %mull_runner -ide-reporter-show-killed %s.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITH-OPTION)
-RUN: (unset TERM; %mull_runner -reporters=IDE -ide-reporter-show-killed %s.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITH-OPTION)
+RUN: (unset TERM; %mull_runner --allow-surviving -ide-reporter-show-killed %s.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITH-OPTION)
+RUN: (unset TERM; %mull_runner --allow-surviving -reporters=IDE -ide-reporter-show-killed %s.exe | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=WITH-OPTION)
 WITH-OPTION:[info] Running mutants (threads: 2)
 WITH-OPTION:{{^       \[################################\] 2/2\. Finished .*}}
 WITH-OPTION:[info] Killed mutants (1/2):

diff --git a/tests-lit/tests/reporters/github-reporter/equality/main.cpp b/tests-lit/tests/reporters/github-reporter/equality/main.cpp
@@ -18,7 +18,7 @@ RUN: cd %S/Output/sandbox
 /// We cd to the the test directory and compile using relative paths.
 RUN: cd %S; %clang_cxx %sysroot -g -O0 %pass_mull_ir_frontend Output/sandbox/main.cpp -o Output/main.cpp-ir.exe
 
-RUN: cd %S/Output && echo $PATH; (unset TERM; %mull_runner main.cpp-ir.exe --report-name test --reporters GithubAnnotations --reporters IDE; test $? = 0) | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines
+RUN: cd %S/Output && echo $PATH; (unset TERM; %mull_runner --allow-surviving main.cpp-ir.exe --report-name test --reporters GithubAnnotations --reporters IDE; test $? = 0) | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines
 
 CHECK:[info] Github Annotations:
 CHECK:{{::}}warning file={{.*}}/main.cpp,line=2,col=12,endLine=2,endColumn=14::[cxx_eq_to_ne] Replaced == with !=

diff --git a/tests-lit/tests/reporters/github-reporter/shift/main.cpp b/tests-lit/tests/reporters/github-reporter/shift/main.cpp
@@ -18,7 +18,7 @@ RUN: cd %S/Output/sandbox
 /// We cd to the the test directory and compile using relative paths.
 RUN: cd %S; %clang_cxx %sysroot -O0 %pass_mull_ir_frontend -g Output/sandbox/main.cpp -o Output/main.cpp-ir.exe
 
-RUN: cd %S/Output && echo $PATH; (unset TERM; %mull_runner main.cpp-ir.exe --report-name test --reporters GithubAnnotations --reporters IDE; test $? = 0) | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines
+RUN: cd %S/Output && echo $PATH; (unset TERM; %mull_runner --allow-surviving main.cpp-ir.exe --report-name test --reporters GithubAnnotations --reporters IDE; test $? = 0) | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines
 
 CHECK:[info] Github Annotations:
 CHECK:{{::}}warning file={{.*}}/main.cpp,line=2,col=12,endLine=2,endColumn=14::[cxx_lshift_to_rshift] Replaced << with >>

diff --git a/tests-lit/tests/reporters/invalid-paths/main.c b/tests-lit/tests/reporters/invalid-paths/main.c
@@ -12,7 +12,7 @@ int main() {
 RUN: cp %s Output/main.c
 RUN: %clang_cc %sysroot -g -O0 %pass_mull_ir_frontend Output/main.c -o Output/main.exe
 RUN: rm Output/main.c
-RUN: %mull_runner Output/main.exe -reporters IDE -reporters Patches -reporters Elements | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines
+RUN: %mull_runner --allow-surviving Output/main.exe -reporters IDE -reporters Patches -reporters Elements | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines
 CHECK:[warning] IDEReporter: Cannot report 'cxx_add_to_sub:{{.*}}/Output/main.c:2:12': cannot read {{.*}}Output/main.c
 CHECK:[warning] PatchesReporter: Cannot report 'cxx_add_to_sub:{{.*}}/Output/main.c:2:12': cannot read {{.*}}Output/main.c
 CHECK:[warning] ElementsReporter: Cannot report 'cxx_add_to_sub:{{.*}}/Output/main.c:2:12': cannot read {{.*}}Output/main.c

diff --git a/tests-lit/tests/reporters/sqlite-reporter/equality/main.cpp b/tests-lit/tests/reporters/sqlite-reporter/equality/main.cpp
@@ -15,7 +15,7 @@ int main() {
 /**
 RUN: %clang_cxx %sysroot -g -O0 %pass_mull_ir_frontend %s -o Output/main.cpp.exe
 RUN: rm -f test.sqlite
-RUN: %mull_runner Output/main.cpp.exe --report-name test --reporters SQLite
+RUN: %mull_runner --allow-surviving Output/main.cpp.exe --report-name test --reporters SQLite
 RUN: sqlite3 ./test.sqlite -line "select * from information order by key" | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=CHECK-INFO
 CHECK-INFO:  key = Build Date
 CHECK-INFO:  key = Commit