[infra] Add support for dataflow builds to the helper script and build check (#1632).

.travis.yml

@@ -42,5 +42,10 @@ matrix:
         - TRAVIS_ENGINE=libfuzzer
         - TRAVIS_SANITIZER=address
         - TRAVIS_ARCHITECTURE=i386
+    - name: "dataflow dataflow x86_64"
+      env:
+        - TRAVIS_ENGINE=dataflow
+        - TRAVIS_SANITIZER=dataflow
+        - TRAVIS_ARCHITECTURE=x86_64
 
 script: ./infra/travis/travis_build.py

infra/base-images/base-builder/Dockerfile

@@ -35,7 +35,7 @@ ENV SANITIZER_FLAGS_undefined "-fsanitize=bool,array-bounds,float-divide-by-zero
 
 ENV SANITIZER_FLAGS_memory "-fsanitize=memory -fsanitize-memory-track-origins"
 
-ENV SANITIZER_FLAGS_dataflow "-fsanitize=dataflow -fsanitize-coverage=trace-pc-guard,pc-table,func,trace-cmp"
+ENV SANITIZER_FLAGS_dataflow "-fsanitize=dataflow"
 
 # Do not use any sanitizers in the coverage build.
 ENV SANITIZER_FLAGS_coverage ""
@@ -51,6 +51,9 @@ ENV COVERAGE_FLAGS="-fsanitize=fuzzer-no-link"
 # messages which are treated as errors by some projects.
 ENV COVERAGE_FLAGS_coverage "-fprofile-instr-generate -fcoverage-mapping -pthread -Wl,--no-as-needed -Wl,-ldl -Wl,-lm -Wno-unused-command-line-argument"
 
+# Coverage isntrumentation flags for dataflow builds.
+ENV COVERAGE_FLAGS_dataflow="-fsanitize-coverage=trace-pc-guard,pc-table,func,trace-cmp"
+
 # Default sanitizer, fuzzing engine and architecture to use.
 ENV SANITIZER="address"
 ENV FUZZING_ENGINE="libfuzzer"

infra/base-images/base-builder/compile

@@ -17,6 +17,11 @@
 
 echo "---------------------------------------------------------------"
 
+if [ "$SANITIZER" = "dataflow" ] && [ "$FUZZING_ENGINE" != "dataflow" ]; then
+  echo "ERROR: 'dataflow' sanitizer can be used with 'dataflow' engine only."
+  exit 1
+fi
+
 if [ -z "${SANITIZER_FLAGS-}" ]; then
   FLAGS_VAR="SANITIZER_FLAGS_${SANITIZER}"
   export SANITIZER_FLAGS=${!FLAGS_VAR-}
@@ -53,8 +58,8 @@ then
   export COVERAGE_FLAGS="${!COVERAGE_FLAGS_VAR}"
 fi
 
- # Don't need coverage instrumentation for engine-less or DFSan builds.
-if [ $FUZZING_ENGINE = "none" ] || [ $FUZZING_ENGINE = "dataflow" ]; then
+ # Don't need coverage instrumentation for engine-less builds.
+if [ $FUZZING_ENGINE = "none" ]; then
   export COVERAGE_FLAGS=
 fi
 

infra/base-images/base-builder/compile_dataflow

@@ -19,8 +19,7 @@ echo -n "Compiling DataFlow to $LIB_FUZZING_ENGINE... "
 mkdir -p $WORK/libfuzzer
 pushd $WORK/libfuzzer > /dev/null
 
-# Intentionally do not use $SANITIZER_FLAGS, we need -fsanitize=dataflow only.
-$CXX $CXXFLAGS -fsanitize=dataflow -std=c++11 -O2 -c \
+$CXX $CXXFLAGS $SANITIZER_FLAGS -std=c++11 -O2 -c \
     $SRC/libfuzzer/dataflow/*.cpp
 ar r $LIB_FUZZING_ENGINE $WORK/libfuzzer/*.o
 popd > /dev/null

infra/base-images/base-runner/bad_build_check

@@ -27,6 +27,11 @@ THRESHOLD_FOR_NUMBER_OF_EDGES=100
 ASAN_CALLS_THRESHOLD_FOR_ASAN_BUILD=1000
 ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD=0
 
+# The value below can definitely be higher (like 500-1000), but avoid being too
+# agressive here while still evaluating the DFT-based fuzzing approach.
+DFSAN_CALLS_THRESHOLD_FOR_DFSAN_BUILD=100
+DFSAN_CALLS_THRESHOLD_FOR_NON_DFSAN_BUILD=0
+
 MSAN_CALLS_THRESHOLD_FOR_MSAN_BUILD=1000
 MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD=0
 
@@ -89,6 +94,10 @@ function check_engine {
       cat $FUZZER_OUTPUT
       return 1
     fi
+  elif [[ "$FUZZING_ENGINE" == dataflow ]]; then
+    # TODO(https://github.com/google/oss-fuzz/issues/1632): add check for
+    # binaries compiled with dataflow engine when the interface becomes stable.
+    return 0
   fi
 
   # TODO: add checks for other fuzzing engines if possible.
@@ -113,6 +122,10 @@ function check_startup_crash {
     if [ $(egrep "target binary (crashed|terminated)" -c $FUZZER_OUTPUT) -eq 0 ]; then
       CHECK_PASSED=1
     fi
+  elif [[ "$FUZZING_ENGINE" = dataflow ]]; then
+    # TODO(https://github.com/google/oss-fuzz/issues/1632): add check for
+    # binaries compiled with dataflow engine when the interface becomes stable.
+    CHECK_PASSED=1
   else
     # TODO: add checks for another fuzzing engines if possible.
     CHECK_PASSED=1
@@ -131,15 +144,53 @@ function check_startup_crash {
 function check_asan_build {
   local FUZZER=$1
   local ASAN_CALLS=$2
-  local MSAN_CALLS=$3
-  local UBSAN_CALLS=$4
+  local DFSAN_CALLS=$3
+  local MSAN_CALLS=$4
+  local UBSAN_CALLS=$5
 
   # Perform all the checks for more detailed error message.
   if (( $ASAN_CALLS < $ASAN_CALLS_THRESHOLD_FOR_ASAN_BUILD )); then
     echo "BAD BUILD: $FUZZER does not seem to be compiled with ASan."
     return 1
   fi
 
+  if (( $DFSAN_CALLS > $DFSAN_CALLS_THRESHOLD_FOR_NON_DFSAN_BUILD )); then
+    echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with DFSan."
+    return 1
+  fi
+
+  if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then
+    echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with MSan."
+    return 1
+  fi
+
+  if (( $UBSAN_CALLS > $UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD )); then
+    echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with UBSan."
+    return 1
+  fi
+
+  return 0
+}
+
+# Mixed sanitizers check for DFSan build.
+function check_dfsan_build {
+  local FUZZER=$1
+  local ASAN_CALLS=$2
+  local DFSAN_CALLS=$3
+  local MSAN_CALLS=$4
+  local UBSAN_CALLS=$5
+
+  # Perform all the checks for more detailed error message.
+  if (( $ASAN_CALLS > $ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD )); then
+    echo "BAD BUILD: DFSan build of $FUZZER seems to be compiled with ASan."
+    return 1
+  fi
+
+  if (( $DFSAN_CALLS < $DFSAN_CALLS_THRESHOLD_FOR_DFSAN_BUILD )); then
+    echo "BAD BUILD: $FUZZER does not seem to be compiled with DFSan."
+    return 1
+  fi
+
   if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then
     echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with MSan."
     return 1
@@ -153,19 +204,26 @@ function check_asan_build {
   return 0
 }
 
+
 # Mixed sanitizers check for MSan build.
 function check_msan_build {
   local FUZZER=$1
   local ASAN_CALLS=$2
-  local MSAN_CALLS=$3
-  local UBSAN_CALLS=$4
+  local DFSAN_CALLS=$3
+  local MSAN_CALLS=$4
+  local UBSAN_CALLS=$5
 
   # Perform all the checks for more detailed error message.
   if (( $ASAN_CALLS > $ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD )); then
     echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with ASan."
     return 1
   fi
 
+  if (( $DFSAN_CALLS > $DFSAN_CALLS_THRESHOLD_FOR_NON_DFSAN_BUILD )); then
+    echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with DFSan."
+    return 1
+  fi
+
   if (( $MSAN_CALLS < $MSAN_CALLS_THRESHOLD_FOR_MSAN_BUILD )); then
     echo "BAD BUILD: $FUZZER does not seem to be compiled with MSan."
     return 1
@@ -183,8 +241,9 @@ function check_msan_build {
 function check_ubsan_build {
   local FUZZER=$1
   local ASAN_CALLS=$2
-  local MSAN_CALLS=$3
-  local UBSAN_CALLS=$4
+  local DFSAN_CALLS=$3
+  local MSAN_CALLS=$4
+  local UBSAN_CALLS=$5
 
   if [[ "$FUZZING_ENGINE" != libfuzzer ]]; then
     # Ignore UBSan checks for fuzzing engines other than libFuzzer because:
@@ -199,6 +258,11 @@ function check_ubsan_build {
     return 1
   fi
 
+  if (( $DFSAN_CALLS > $DFSAN_CALLS_THRESHOLD_FOR_NON_DFSAN_BUILD )); then
+    echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with DFSan."
+    return 1
+  fi
+
   if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then
     echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with MSan."
     return 1
@@ -233,17 +297,21 @@ function check_mixed_sanitizers {
     esac
   fi
   local ASAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__asan" -c)
+  local DFSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__dfsan" -c)
   local MSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__msan" -c)
   local UBSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__ubsan" -c)
 
   if [[ "$SANITIZER" = address ]]; then
-    check_asan_build $FUZZER $ASAN_CALLS $MSAN_CALLS $UBSAN_CALLS
+    check_asan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS
+    result=$?
+  elif [[ "$SANITIZER" = dataflow ]]; then
+    check_dfsan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS
     result=$?
   elif [[ "$SANITIZER" = memory ]]; then
-    check_msan_build $FUZZER $ASAN_CALLS $MSAN_CALLS $UBSAN_CALLS
+    check_msan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS
     result=$?
   elif [[ "$SANITIZER" = undefined ]]; then
-    check_ubsan_build $FUZZER $ASAN_CALLS $MSAN_CALLS $UBSAN_CALLS
+    check_ubsan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS
     result=$?
   fi
 
@@ -302,6 +370,7 @@ function check_architecture {
 }
 
 function main {
+  return 1
   local FUZZER=$1
   local checks_failed=0
   local result=0
@@ -330,8 +399,8 @@ function main {
 
 
 if [ $# -ne 1 ]; then
-    echo "Usage: $0 <fuzz_target_binary>"
-    exit 1
+  echo "Usage: $0 <fuzz_target_binary>"
+  exit 1
 fi
 
 # Fuzz target path.

infra/base-images/base-runner/test_all

@@ -55,7 +55,8 @@ for FUZZER_BINARY in $(find $OUT/ -maxdepth 1 -executable -type f); do
 
   LOG_PATH_FOR_BROKEN_TARGET="${BROKEN_TARGETS_DIR}/${FUZZER}"
 
-  # Launch bad build check process in the background.
+  # Launch bad build check process in the background. Ignore the exit codes, as
+  # we check the percentage of broken fuzz targets after running all the checks.
   bad_build_check $FUZZER_BINARY &> $LOG_PATH_FOR_BROKEN_TARGET &
 
   # Count total number of fuzz targets being tested.

infra/helper.py

@@ -97,9 +97,9 @@ def main():
   check_build_parser = subparsers.add_parser(
       'check_build', help='Checks that fuzzers execute without errors.')
   _add_architecture_args(check_build_parser)
-  _add_engine_args(check_build_parser, choices=['libfuzzer', 'afl'])
+  _add_engine_args(check_build_parser, choices=['libfuzzer', 'afl', 'dataflow'])
   _add_sanitizer_args(
-      check_build_parser, choices=['address', 'memory', 'undefined'])
+      check_build_parser, choices=['address', 'memory', 'undefined', 'dataflow'])
   _add_environment_args(check_build_parser)
   check_build_parser.add_argument('project_name', help='name of the project')
   check_build_parser.add_argument(
@@ -160,6 +160,15 @@ def main():
                                              help='Pull base images.')
 
   args = parser.parse_args()
+
+  # We have different default values for `sanitizer` depending on the `engine`.
+  # Some commands do not have `sanitizer` argument, so `hasattr` is necessary.
+  if hasattr(args, 'sanitizer') and not args.sanitizer:
+    if args.engine == 'dataflow':
+      args.sanitizer = 'dataflow'
+    else:
+      args.sanitizer = 'address'
+
   if args.command == 'generate':
     return generate(args)
   elif args.command == 'build_image':
@@ -268,7 +277,11 @@ def _add_sanitizer_args(
         parser,
         choices=('address', 'memory', 'undefined', 'coverage', 'dataflow')):
   """Add common sanitizer args."""
-  parser.add_argument('--sanitizer', default='address', choices=choices)
+  parser.add_argument(
+      '--sanitizer',
+      default=None,
+      choices=choices,
+      help='the default is "address"; "dataflow" for "dataflow" engine')
 
 
 def _add_environment_args(parser):