[libFuzzer] Enable tests on Windows

Summary:
Enable tests on Windows and make check-fuzzer pass on it.  Make
check-fuzzer pass on Windows by fixing libFuzzer, fixing tests, and by
disabling tests on Windows. Most of these are disabled temporarily as
support for the tests and the features they test will be added
incrementally.  Other tests will not be enabled since they require
things that are not on Windows (eg: afl_driver tests).  Every test
that was explicitly disabled on Windows has a comment explaining why
(unless obvious like merge-posix.test).

The lit.cfg file was modified to support running tests on windows.
fuzzer-dirs.test was fixed by making the Windows implementation print
the same error message as the posix version.
merge-control-file.test was fixed by making the test binary end with
the ".exe" extension (on all platforms).

Patch By: metzman

Reviewers: morehouse

Reviewed By: morehouse

Subscribers: srhines, mgorny

Differential Revision: https://reviews.llvm.org/D51549

llvm-svn: 341385

compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp

@@ -111,7 +111,7 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
   {
     if (GetLastError() == ERROR_FILE_NOT_FOUND)
       return;
-    Printf("No such directory: %s; exiting\n", Dir.c_str());
+    Printf("No such file or directory: %s; exiting\n", Dir.c_str());
     exit(1);
   }
 

compiler-rt/test/CMakeLists.txt

@@ -62,10 +62,7 @@ if(COMPILER_RT_CAN_EXECUTE_TESTS)
     compiler_rt_test_runtime(sanitizer_common)
 
     # OpenBSD not supporting asan, cannot run the tests
-    # Tests are broken for now on Windows
-    if(COMPILER_RT_BUILD_LIBFUZZER
-      AND NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "OpenBSD"
-      AND NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "Windows" AND NOT ANDROID)
+    if(COMPILER_RT_BUILD_LIBFUZZER AND NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "OpenBSD" AND NOT ANDROID)
       compiler_rt_test_runtime(fuzzer)
     endif()
 

compiler-rt/test/fuzzer/afl-driver-extra-stats.test

@@ -1,3 +1,5 @@
+# AFL doesn't work on Windows. No reason to test the driver.
+UNSUPPORTED: windows
 XFAIL: ios
 RUN: %no_fuzzer_cpp_compiler %S/AFLDriverTest.cpp %libfuzzer_src/afl/afl_driver.cpp -o %t-AFLDriverTest
 

compiler-rt/test/fuzzer/afl-driver-stderr.test

@@ -1,5 +1,6 @@
+# AFL doesn't work on Windows. No reason to test the driver.
+UNSUPPORTED: freebsd, windows
 XFAIL: ios
-UNSUPPORTED: freebsd
 RUN: %no_fuzzer_cpp_compiler %S/AFLDriverTest.cpp %libfuzzer_src/afl/afl_driver.cpp -o %t-AFLDriverTest
 
 ; Test that not specifying a stderr file isn't broken.

compiler-rt/test/fuzzer/bogus-initialize.test

@@ -1,3 +1,5 @@
+# FIXME: Disabled on Windows since LLVMFuzzerInitialize does not yet work.
+UNSUPPORTED: windows
 RUN: %cpp_compiler %S/BogusInitializeTest.cpp -o %t-BogusInitializeTest
 
 RUN: not %run %t-BogusInitializeTest 2>&1 | FileCheck %s --check-prefix=BOGUS_INITIALIZE

compiler-rt/test/fuzzer/coverage.test

@@ -1,4 +1,5 @@
-UNSUPPORTED: aarch64
+# FIXME: Disabled on Windows because -fPIC cannot be used to compile for Windows.
+UNSUPPORTED: aarch64, windows
 RUN: %cpp_compiler -mllvm -use-unknown-locations=Disable  %S/NullDerefTest.cpp -o %t-NullDerefTest
 RUN: %cpp_compiler -mllvm -use-unknown-locations=Disable %S/DSO1.cpp -fPIC %ld_flags_rpath_so1 -shared -o %dynamiclib1
 RUN: %cpp_compiler -mllvm -use-unknown-locations=Disable %S/DSO2.cpp -fPIC %ld_flags_rpath_so2 -shared -o %dynamiclib2

compiler-rt/test/fuzzer/dso.test

@@ -1,3 +1,5 @@
+# FIXME: Disabled on Windows because -fPIC cannot be used to compile for Windows.
+UNSUPPORTED: windows
 RUN: %cpp_compiler %S/DSO1.cpp -fPIC %ld_flags_rpath_so1 -shared -o %dynamiclib1
 RUN: %cpp_compiler %S/DSO2.cpp -fPIC %ld_flags_rpath_so2 -shared -o %dynamiclib2
 RUN: %cpp_compiler %S/DSOTestMain.cpp %S/DSOTestExtra.cpp %ld_flags_rpath_exe1 %ld_flags_rpath_exe2 -o %t-DSOTest

compiler-rt/test/fuzzer/dump_coverage.test

@@ -1,4 +1,5 @@
-UNSUPPORTED: freebsd
+# FIXME: Disabled on Windows because -fPIC cannot be used to compile for Windows.
+UNSUPPORTED: freebsd, windows
 RUN: %cpp_compiler -fsanitize-coverage=0 -fsanitize-coverage=trace-pc-guard %S/DSO1.cpp -fPIC -shared -o %dynamiclib1 %ld_flags_rpath_so1
 RUN: %cpp_compiler -fsanitize-coverage=0 -fsanitize-coverage=trace-pc-guard %S/DSO2.cpp -fPIC -shared -o %dynamiclib2 %ld_flags_rpath_so2
 RUN: %cpp_compiler -fsanitize-coverage=0 -fsanitize-coverage=trace-pc-guard %S/DSOTestMain.cpp %S/DSOTestExtra.cpp %ld_flags_rpath_exe1 %ld_flags_rpath_exe2 -o %t-DSOTest

compiler-rt/test/fuzzer/exit_on_src_pos.test

@@ -1,3 +1,5 @@
+# FIXME: Disabled on Windows until symbolization works properly.
+UNSUPPORTED: windows
 # Temporary use -mllvm -use-unknown-locations=Disable so that
 # all instructions have debug info (file line numbers) attached.
 # TODO: Find out why test fails on Darwin with -O2.

compiler-rt/test/fuzzer/fuzzer-customcrossover.test

@@ -1,3 +1,5 @@
+# FIXME: Disabled on Windows since LLVMFuzzerCustomCrossOver does not yet work.
+UNSUPPORTED: windows
 RUN: %cpp_compiler %S/CustomCrossOverTest.cpp -o %t-CustomCrossOverTest
 
 RUN: not %run %t-CustomCrossOverTest -seed=1 -runs=1000000                2>&1 | FileCheck %s --check-prefix=CHECK_CO

compiler-rt/test/fuzzer/fuzzer-custommutator.test

@@ -1,3 +1,5 @@
+# FIXME: Disabled on Windows since LLVMFuzzerCustomMutator does not yet work.
+UNSUPPORTED: windows
 RUN: %cpp_compiler %S/CustomMutatorTest.cpp -o %t-CustomMutatorTest
 RUN: not %run %t-CustomMutatorTest 2>&1 | FileCheck %s --check-prefix=LLVMFuzzerCustomMutator
 LLVMFuzzerCustomMutator: In LLVMFuzzerCustomMutator

compiler-rt/test/fuzzer/fuzzer-flags.test

@@ -1,3 +1,5 @@
+# FIXME: Disabled on Windows since LLVMFuzzerInitialize does not yet work.
+UNSUPPORTED: windows
 RUN: %cpp_compiler %S/FlagsTest.cpp -o %t-FlagsTest
 RUN: %run %t-FlagsTest -runs=10 -foo_bar=1 2>&1 | FileCheck %s --check-prefix=FOO_BAR
 FOO_BAR: WARNING: unrecognized flag '-foo_bar=1'; use -help=1 to list all flags

compiler-rt/test/fuzzer/fuzzer-oom.test

@@ -1,4 +1,5 @@
-UNSUPPORTED: aarch64
+# FIXME: Disabled on Windows until symbolization works properly.
+UNSUPPORTED: aarch64, windows
 RUN: %cpp_compiler %S/OutOfMemoryTest.cpp -o %t-OutOfMemoryTest
 RUN: %cpp_compiler %S/OutOfMemorySingleLargeMallocTest.cpp -o %t-OutOfMemorySingleLargeMallocTest
 RUN: %cpp_compiler %S/AccumulateAllocationsTest.cpp -o %t-AccumulateAllocationsTest

compiler-rt/test/fuzzer/handle-unstable.test

@@ -1,5 +1,6 @@
 # Tests -handle_unstable
-UNSUPPORTED: aarch64
+# FIXME: Disabled on Windows until symbolization works properly.
+UNSUPPORTED: aarch64, windows
 
 RUN: %cpp_compiler %S/PrintUnstableStatsTest.cpp -o %t-HandleUnstableTest
 

compiler-rt/test/fuzzer/initialize.test

@@ -1,3 +1,5 @@
+# FIXME: Disabled on Windows because memmem is a GNU extension.
+UNSUPPORTED: windows
 CHECK: BINGO
 RUN: %cpp_compiler %S/InitializeTest.cpp -o %t-InitializeTest
 RUN: not %run %t-InitializeTest -use_value_profile=1 2>&1 | FileCheck %s

compiler-rt/test/fuzzer/lit.cfg

@@ -24,15 +24,17 @@ else:
 # the test runner updated.
 config.test_format = lit.formats.ShTest(execute_external)
 
-# LeakSanitizer is not supported on OSX right now.
-if sys.platform.startswith('darwin') or sys.platform.startswith('freebsd'):
+# LeakSanitizer is not supported on OSX or Windows right now.
+if (sys.platform.startswith('darwin') or
+    sys.platform.startswith('freebsd') or
+    sys.platform.startswith('win')):
   lit_config.note('lsan feature unavailable')
 else:
   lit_config.note('lsan feature available')
   config.available_features.add('lsan')
 
-# MemorySanitizer is not supported on OSX right now
-if sys.platform.startswith('darwin'):
+# MemorySanitizer is not supported on OSX or Windows right now
+if sys.platform.startswith('darwin') or sys.platform.startswith('win'):
   lit_config.note('msan feature unavailable')
   assert 'msan' not in config.available_features
 else:
@@ -70,7 +72,11 @@ def generate_compiler_cmd(is_cpp=True, fuzzer_enabled=True, msan_enabled=False):
   else:
     link_cmd = '-lstdc++'
 
-  std_cmd = '--driver-mode=g++ -std=c++11' if is_cpp else ''
+  if is_cpp and not sys.platform.startswith('win'):
+    std_cmd = '--driver-mode=g++ -std=c++11'
+  else:
+    std_cmd = ''
+
   if msan_enabled:
     sanitizers = ['memory']
   else:

compiler-rt/test/fuzzer/merge-control-file.test

@@ -1,6 +1,8 @@
 XFAIL: ios
 RUN: mkdir -p %t
-RUN: %cpp_compiler %S/FullCoverageSetTest.cpp -o %t/T
+# Use a ".exe" extension because it is needed on Windows to call system()
+# to execute itself again.
+RUN: %cpp_compiler %S/FullCoverageSetTest.cpp -o %t/T.exe
 
 RUN: rm -rf %t/T0 %t/T1 %t/T2
 RUN: mkdir -p %t/T0 %t/T1 %t/T2
@@ -11,9 +13,9 @@ RUN: echo ..Z... > %t/T0/3
 # Test what happens if the control file is junk.
 
 RUN: echo JUNK > %t/MCF
-RUN: not %run %t/T -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=JUNK
+RUN: not %run %t/T.exe -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=JUNK
 RUN: echo 3 > %t/MCF; echo 0 >> %t/MCF; echo %t/T1/1 >> %t/MCF
-RUN: not %run %t/T -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=JUNK
+RUN: not %run %t/T.exe -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=JUNK
 JUNK: MERGE-OUTER: non-empty control file provided: {{.*}}MCF
 JUNK: MERGE-OUTER: bad control file, will overwrite it
 
@@ -22,18 +24,18 @@ JUNK: MERGE-OUTER: bad control file, will overwrite it
 
 RUN: rm -f %t/T1/*; cp %t/T0/* %t/T1
 RUN: echo 3 > %t/MCF; echo 0 >> %t/MCF; echo %t/T1/1 >> %t/MCF; echo %t/T1/2 >> %t/MCF; echo %t/T1/3 >> %t/MCF
-RUN: %run %t/T -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=OK_0
+RUN: %run %t/T.exe -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=OK_0
 OK_0: MERGE-OUTER: control file ok, 3 files total, first not processed file 0
 OK_0: MERGE-OUTER: 3 new files with {{.*}} new features added
 
 RUN: rm -f %t/T1/*; cp %t/T0/* %t/T1
 RUN: echo 3 > %t/MCF; echo 0 >> %t/MCF; echo %t/T1/1 >> %t/MCF; echo %t/T1/2 >> %t/MCF; echo %t/T1/3 >> %t/MCF
-RUN: %run %t/T -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF -save_coverage_summary=%t/SUMMARY 2>&1 | FileCheck %s --check-prefix=SAVE_SUMMARY
+RUN: %run %t/T.exe -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF -save_coverage_summary=%t/SUMMARY 2>&1 | FileCheck %s --check-prefix=SAVE_SUMMARY
 SAVE_SUMMARY: MERGE-OUTER: writing coverage summary for 3 files to {{.*}}/SUMMARY
 
 RUN: rm -f %t/T1/*; cp %t/T0/* %t/T1
 RUN: echo 3 > %t/MCF; echo 0 >> %t/MCF; echo %t/T1/1 >> %t/MCF; echo %t/T1/2 >> %t/MCF; echo %t/T1/3 >> %t/MCF
-RUN: %run %t/T -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF -load_coverage_summary=%t/SUMMARY 2>&1 | FileCheck %s --check-prefix=LOAD_SUMMARY
+RUN: %run %t/T.exe -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF -load_coverage_summary=%t/SUMMARY 2>&1 | FileCheck %s --check-prefix=LOAD_SUMMARY
 LOAD_SUMMARY: MERGE-OUTER: coverage summary loaded from
 
 RUN: rm -f %t/T1/*; cp %t/T0/* %t/T1
@@ -42,7 +44,7 @@ RUN: echo STARTED 0 1 >> %t/MCF
 RUN: echo DONE 0 11 >> %t/MCF
 RUN: echo STARTED 1 2 >> %t/MCF
 RUN: echo DONE 1 12 >> %t/MCF
-RUN: %run %t/T -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=OK_2
+RUN: %run %t/T.exe -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=OK_2
 OK_2: MERGE-OUTER: control file ok, 3 files total, first not processed file 2
 OK_2: MERGE-OUTER: 3 new files with {{.*}} new features added
 
@@ -54,5 +56,5 @@ RUN: echo STARTED 1 2 >> %t/MCF
 RUN: echo DONE 1 12 >> %t/MCF
 RUN: echo STARTED 2 2 >> %t/MCF
 RUN: echo DONE 2 13 >> %t/MCF
-RUN: %run %t/T -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=OK_3
+RUN: %run %t/T.exe -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=OK_3
 OK_3: MERGE-OUTER: nothing to do, merge has been completed before

compiler-rt/test/fuzzer/merge-posix.test

@@ -1,4 +1,5 @@
 XFAIL: ios
+UNSUPPORTED: windows
 RUN: %cpp_compiler %S/FullCoverageSetTest.cpp -o %t-FullCoverageSetTest
 
 RUN: rm -rf  %tmp/T1 %tmp/T2

compiler-rt/test/fuzzer/merge-sigusr.test

@@ -1,5 +1,7 @@
 # Check that libFuzzer honors SIGUSR1/SIGUSR2
-UNSUPPORTED: darwin
+# FIXME: Disabled on Windows for now because of reliance on posix only features
+# (eg: export, "&", pkill).
+UNSUPPORTED: darwin, windows
 RUN: rm -rf %t
 RUN: mkdir -p %t
 RUN: %cpp_compiler %S/SleepOneSecondTest.cpp -o %t/LFSIGUSR

compiler-rt/test/fuzzer/print_unstable_stats.test

@@ -1,10 +1,11 @@
 # Tests -print_unstable_stats
-UNSUPPORTED: aarch64
+# Disabled on Windows because of differences symbolizing and flakiness.
+UNSUPPORTED: aarch64, windows
 
 RUN: %cpp_compiler %S/PrintUnstableStatsTest.cpp -o %t-PrintUnstableStatsTest
 
 RUN: %run %t-PrintUnstableStatsTest -print_unstable_stats=1 -runs=100000 2>&1 | FileCheck %s --check-prefix=LONG
-; We do not observe ini functions since we take the minimum hit counts, and minimum hit counts for ini is 0.
+# We do not observe ini functions since we take the minimum hit counts, and minimum hit counts for ini is 0.
 LONG: UNSTABLE_FUNCTIONS:
 LONG-NOT: det0()
 LONG-NOT: det1()

compiler-rt/test/fuzzer/sigusr.test

@@ -1,4 +1,6 @@
-UNSUPPORTED: darwin
+# FIXME: Disabled on Windows for now because of reliance on posix only features
+# (eg: export, "&", pkill).
+UNSUPPORTED: darwin, windows
 # Check that libFuzzer honors SIGUSR1/SIGUSR2
 RUN: rm -rf %t
 RUN: mkdir -p %t

compiler-rt/test/fuzzer/standalone.test

@@ -1,3 +1,5 @@
+# FIXME: Disabled on Windows because memmem is a GNU extension.
+UNSUPPORTED: windows
 RUN: %no_fuzzer_c_compiler %libfuzzer_src/standalone/StandaloneFuzzTargetMain.c -c -o %t_1.o
 RUN: %no_fuzzer_cpp_compiler %S/InitializeTest.cpp -c -o %t_2.o
 

compiler-rt/test/fuzzer/symbolize-deadlock.test

@@ -1,2 +1,4 @@
+# Disabled on Widows because <unistd.h> is a posix-only header.
+UNSUPPORTED: windows
 RUN: %cpp_compiler %S/SymbolizeDeadlock.cpp -o %t
 RUN: not %run %t -rss_limit_mb=20 2>&1

compiler-rt/test/fuzzer/trace-malloc-threaded.test

@@ -1,6 +1,7 @@
 // FIXME: This test infinite loops on darwin because it crashes
 // printing a stack trace repeatedly
-UNSUPPORTED: darwin, aarch64
+// FIXME: Disabled on Windows because of a crash (possibly related to above).
+UNSUPPORTED: darwin, aarch64, windows
 
 RUN: %cpp_compiler %S/TraceMallocThreadedTest.cpp -o \
 RUN:   %t-TraceMallocThreadedTest

compiler-rt/test/fuzzer/trace-malloc-unbalanced.test

@@ -1,6 +1,7 @@
 // FIXME: This test infinite loops on darwin because it crashes
 // printing a stack trace repeatedly
-UNSUPPORTED: darwin
+// FIXME: Disabled on Windows because of reliance on shebang line and failure.
+UNSUPPORTED: darwin, windows
 
 // Verifies lib/fuzzer/scripts/unbalanced_allocs.py script
 

compiler-rt/test/fuzzer/ulimit.test

@@ -1,3 +1,5 @@
+# FIXME: Disabled on Windows for now because Windows has no ulimit command.
+UNSUPPORTED: windows
 RUN: %cpp_compiler %S/SimpleTest.cpp -o %t-SimpleTest
 RUN: ulimit -s 1000
 RUN: not %run %t-SimpleTest

compiler-rt/test/fuzzer/value-profile-cmp.test

@@ -1,3 +1,5 @@
+# FIXME: Disabled on Windows because of hangs.
+UNSUPPORTED: windows
 CHECK: BINGO
 RUN: %cpp_compiler %S/SimpleCmpTest.cpp -o %t-SimpleCmpTest
 RUN: not %run %t-SimpleCmpTest -seed=1 -use_cmp=0 -use_value_profile=1 -runs=100000000 2>&1 | FileCheck %s

compiler-rt/test/fuzzer/value-profile-cmp4.test

@@ -1,3 +1,5 @@
+# FIXME: Disabled on Windows because of hangs.
+UNSUPPORTED: windows
 CHECK: BINGO
 RUN: %cpp_compiler %S/AbsNegAndConstant64Test.cpp -o %t-AbsNegAndConstant64Test
 RUN: not %run %t-AbsNegAndConstant64Test -seed=1 -use_cmp=0 -use_value_profile=1 -runs=100000000 2>&1 | FileCheck %s

compiler-rt/test/fuzzer/value-profile-load.test

@@ -1,3 +1,5 @@
+# FIXME: Disabled on Windows because of hangs.
+UNSUPPORTED: windows
 CHECK: AddressSanitizer: global-buffer-overflow
 RUN: %cpp_compiler %S/LoadTest.cpp -fsanitize-coverage=trace-gep -o %t-LoadTest
 RUN: not %run %t-LoadTest -seed=2 -use_cmp=0 -use_value_profile=1 -runs=20000000 2>&1 | FileCheck %s