[infra] Remove sancov and rename "profile" to "coverage". (#1839)

* [infra] Remove sancov and rename "profile" to "coverage".

* Bring coverage flags back.

* Update projects files that rely on SANITIZER="profile".

infra/README.md

@@ -23,7 +23,6 @@ Continuous Integration infrastracture:
 | `build_image`   | Builds a docker image for a given project |
 | `build_fuzzers` | Builds fuzz targets for a given project |
 | `run_fuzzer`    | Runs a fuzz target in a docker container |
-| `coverage`      | Runs a fuzz target in a docker container and computes a coverage report |
-| `profile`       | Runs a fuzz target in a docker container and generates a code coverage report. See [Code Coverage doc](../docs/code_coverage.md) |
+| `coverage`      | Runs fuzz target(s) in a docker container and generates a code coverage report. See [Code Coverage doc](../docs/code_coverage.md) |
 | `reproduce`     | Runs a testcase to reproduce a crash |
 | `shell`         | Starts a shell inside the docker image for a project |

infra/base-images/base-builder/Dockerfile

@@ -25,22 +25,19 @@ ENV SANITIZER_FLAGS_address "-fsanitize=address -fsanitize-address-use-after-sco
 ENV SANITIZER_FLAGS_undefined "-fsanitize=bool,array-bounds,float-divide-by-zero,function,integer-divide-by-zero,return,shift,signed-integer-overflow,unsigned-integer-overflow,vla-bound,vptr -fno-sanitize-recover=bool,array-bounds,float-divide-by-zero,function,integer-divide-by-zero,return,shift,signed-integer-overflow,vla-bound,vptr"
 ENV SANITIZER_FLAGS_memory "-fsanitize=memory -fsanitize-memory-track-origins"
 
-# Use '-Wno-unused-command-line-argument' to suppress "warning: -ldl: 'linker' input unused"
-# messages which are treated as errors by some projects.
-ENV SANITIZER_FLAGS_profile "-fprofile-instr-generate -fcoverage-mapping -pthread -Wl,-ldl -Wno-unused-command-line-argument"
+# Do not use any sanitizers in the coverage build.
+ENV SANITIZER_FLAGS_coverage ""
 
 # We use unsigned-integer-overflow as an additional coverage signal and have to
 # suppress error messages. See https://github.com/google/oss-fuzz/issues/910.
 ENV UBSAN_OPTIONS="silence_unsigned_overflow=1"
 
-# Default build flags for coverage.
+# Default build flags for coverage feedback.
 ENV COVERAGE_FLAGS="-fsanitize=fuzzer-no-link"
 
-# Coverage flags for generating coverage reports.
-ENV COVERAGE_FLAGS_coverage="-fsanitize-coverage=bb,no-prune,trace-pc-guard -O0"
-
-# Don't mix sanitizer coverage with clang coverage.
-ENV COVERAGE_FLAGS_profile=""
+# Use '-Wno-unused-command-line-argument' to suppress "warning: -ldl: 'linker' input unused"
+# messages which are treated as errors by some projects.
+ENV COVERAGE_FLAGS_coverage "-fprofile-instr-generate -fcoverage-mapping -pthread -Wl,-ldl -Wno-unused-command-line-argument"
 
 # Default sanitizer and fuzzing engine to use.
 ENV SANITIZER="address"
@@ -66,6 +63,6 @@ RUN mkdir honggfuzz && \
     tar -xzv --strip-components=1 -f $SRC/oss-fuzz.tar.gz && \
     rm -rf $SRC/oss-fuzz.tar.gz
 
-COPY compile compile_afl compile_libfuzzer compile_honggfuzz coverage_report srcmap /usr/local/bin/
+COPY compile compile_afl compile_libfuzzer compile_honggfuzz /usr/local/bin/
 
 CMD ["compile"]

infra/base-images/base-builder/compile

@@ -49,7 +49,7 @@ then
   export COVERAGE_FLAGS="${!COVERAGE_FLAGS_VAR}"
 fi
 
-# Don't need coverage instrumentation for engine-less builds.
+ # Don't need coverage instrumentation for engine-less builds.
 if [[ $FUZZING_ENGINE = "none" ]]; then
   export COVERAGE_FLAGS=
 fi
@@ -74,13 +74,13 @@ if [ "${BUILD_UID-0}" -ne "0" ]; then
   adduser -u $BUILD_UID --disabled-password --gecos '' builder
   chown -R builder $SRC $OUT $WORK
   su -c "$BUILD_CMD" builder
-  if [ "$SANITIZER" = "profile" ]; then
+  if [ "$SANITIZER" = "coverage" ]; then
     # Some directories have broken symlinks (e.g. honggfuzz), ignore the errors.
     su -c "$COPY_SOURCES_CMD" builder || true
   fi
 else
   $BUILD_CMD
-  if [ "$SANITIZER" = "profile" ]; then
+  if [ "$SANITIZER" = "coverage" ]; then
     # Some directories have broken symlinks (e.g. honggfuzz), ignore the errors.
     $COPY_SOURCES_CMD || true
   fi

infra/base-images/base-runner/Dockerfile

@@ -43,7 +43,6 @@ COPY bad_build_check \
     reproduce \
     run_fuzzer \
     run_minijail \
-    sancov \
     targets_list \
     test_all \
     /usr/local/bin/

infra/gcb/build_and_run_coverage.py

@@ -13,7 +13,7 @@
 
 import build_project
 
-SANITIZER = 'profile'
+SANITIZER = 'coverage'
 CONFIGURATION = ['FUZZING_ENGINE=libfuzzer', 'SANITIZER=%s' % SANITIZER]
 PLATFORM = 'linux'
 

infra/gcb/build_project.py

@@ -29,7 +29,6 @@
     'sanitizer-address': ['SANITIZER=address'],
     'sanitizer-memory': ['SANITIZER=memory'],
     'sanitizer-undefined': ['SANITIZER=undefined'],
-    'sanitizer-coverage': ['SANITIZER=coverage'],
     'engine-libfuzzer': ['FUZZING_ENGINE=libfuzzer'],
     'engine-afl': ['FUZZING_ENGINE=afl'],
     'engine-honggfuzz': ['FUZZING_ENGINE=honggfuzz'],
@@ -43,8 +42,7 @@
     'libfuzzer':
         EngineInfo(
             upload_bucket='clusterfuzz-builds',
-            supported_sanitizers=['address', 'memory', 'undefined',
-                                  'coverage']),
+            supported_sanitizers=['address', 'memory', 'undefined']),
     'afl':
         EngineInfo(
             upload_bucket='clusterfuzz-builds-afl',
@@ -121,10 +119,6 @@ def get_sanitizers(project_yaml):
       for key in sanitizer.iterkeys():
         processed_sanitizers.append(key)
 
-  # Always make a coverage build.
-  if 'coverage' not in processed_sanitizers:
-    processed_sanitizers.append('coverage')
-
   return processed_sanitizers
 
 

infra/helper.py

@@ -112,33 +112,29 @@ def main():
                                  nargs=argparse.REMAINDER)
 
   coverage_parser = subparsers.add_parser(
-      'coverage', help='Run a fuzzer for a while and generate coverage.')
-  coverage_parser.add_argument('--run_time', default=60,
-                      help='time in seconds to run fuzzer')
+      'coverage', help='Generate code coverage report for the project.')
+  coverage_parser.add_argument('--no-corpus-download', action='store_true',
+                               help='do not download corpus backup from '
+                               'OSS-Fuzz; use corpus located in '
+                               'build/corpus/<project>/<fuzz_target>/')
+  coverage_parser.add_argument('--port', default='8008', help='specify port for'
+                               ' a local HTTP server rendering coverage report')
+  coverage_parser.add_argument('--fuzz-target', help='specify name of a fuzz '
+                               'target to be run for generating coverage '
+                               'report')
+  coverage_parser.add_argument('--corpus-dir', help='specify location of corpus'
+                               ' to be used (requires --fuzz-target argument)')
   coverage_parser.add_argument('project_name', help='name of the project')
-  coverage_parser.add_argument('fuzzer_name', help='name of the fuzzer')
-  coverage_parser.add_argument('fuzzer_args', help='arguments to pass to the fuzzer',
-                               nargs=argparse.REMAINDER)
+  coverage_parser.add_argument('extra_args', help='additional arguments to '
+                               'pass to llvm-cov utility.', nargs='*')
 
   profile_parser = subparsers.add_parser(
-      'profile', help='Generate code coverage report for the project.')
-  profile_parser.add_argument('--no-corpus-download', action='store_true',
-                              help='do not download corpus backup from OSS-Fuzz; '
-                              'use corpus located in build/corpus/<project>/<fuzz_target>/')
-  profile_parser.add_argument('--port', default='8008', help='specify port for '
-                              'a local HTTP server rendering coverage report')
-  profile_parser.add_argument('--fuzz-target', help='specify name of a fuzz '
-                              'target to be run for generating coverage report')
-  profile_parser.add_argument('--corpus-dir', help='specify location of corpus '
-                              'to be used (requires --fuzz-target argument)')
-  profile_parser.add_argument('project_name', help='name of the project')
-  profile_parser.add_argument('extra_args', help='additional arguments to '
-                              'pass to llvm-cov utility.', nargs='*')
+      'profile', help='"profile" command was renamed to "coverage".')
 
   reproduce_parser = subparsers.add_parser(
       'reproduce', help='Reproduce a crash.')
   reproduce_parser.add_argument('--valgrind', action='store_true',
-                       help='run with valgrind')
+                                help='run with valgrind')
   reproduce_parser.add_argument('project_name', help='name of the project')
   reproduce_parser.add_argument('fuzzer_name', help='name of the fuzzer')
   reproduce_parser.add_argument('testcase_path', help='path of local testcase')
@@ -170,7 +166,9 @@ def main():
   elif args.command == 'coverage':
     return coverage(args)
   elif args.command == 'profile':
-    return profile(args)
+    print(
+        'ERROR: "profile" command was renamed to "coverage".', file=sys.stderr)
+    return 1
   elif args.command == 'reproduce':
     return reproduce(args)
   elif args.command == 'shell':
@@ -258,7 +256,7 @@ def _add_engine_args(parser):
 def _add_sanitizer_args(parser):
   """Add common sanitizer args."""
   parser.add_argument('--sanitizer', default='address',
-                      choices=['address', 'memory', 'undefined', 'coverage', 'profile'])
+                      choices=['address', 'memory', 'undefined', 'coverage'])
 
 
 def _add_environment_args(parser):
@@ -617,7 +615,7 @@ def _download_for_single_target(fuzz_target):
   return all(thread_pool.map(_download_for_single_target, fuzz_targets))
 
 
-def profile(args):
+def coverage(args):
   """Generate code coverage using clang source based code coverage."""
   if args.corpus_dir and not args.fuzz_target:
     print('ERROR: --corpus-dir requires specifying a particular fuzz target '
@@ -635,7 +633,7 @@ def profile(args):
   env = [
       'FUZZING_ENGINE=libfuzzer',
       'PROJECT=%s' % args.project_name,
-      'SANITIZER=profile',
+      'SANITIZER=coverage',
       'HTTP_PORT=%s' % args.port,
       'COVERAGE_EXTRA_ARGS=%s' % ' '.join(args.extra_args),
   ]
@@ -693,47 +691,6 @@ def run_fuzzer(args):
   return docker_run(run_args)
 
 
-def coverage(args):
-  """Runs a fuzzer in the container."""
-  if not _check_project_exists(args.project_name):
-    return 1
-
-  if not _check_fuzzer_exists(args.project_name, args.fuzzer_name):
-    return 1
-
-  temp_dir = tempfile.mkdtemp()
-
-  run_args = [
-      '-e', 'FUZZING_ENGINE=libfuzzer',
-      '-e', 'ASAN_OPTIONS=coverage_dir=/cov',
-      '-e', 'MSAN_OPTIONS=coverage_dir=/cov',
-      '-e', 'UBSAN_OPTIONS=coverage_dir=/cov',
-      '-v', '%s:/out' % _get_output_dir(args.project_name),
-      '-v', '%s:/cov' % temp_dir,
-      '-w', '/cov',
-      '-t', 'gcr.io/oss-fuzz-base/base-runner',
-      'run_fuzzer',
-      args.fuzzer_name,
-      '-dump_coverage=1',
-      '-max_total_time=%s' % args.run_time
-  ] + args.fuzzer_args
-
-  print('This may take a while (running your fuzzer for %s seconds)...' %
-        args.run_time)
-  docker_run(run_args, print_output=False)
-
-  run_args = [
-      '-v', '%s:/out' % _get_output_dir(args.project_name),
-      '-v', '%s:/cov' % temp_dir,
-      '-w', '/cov',
-      '-p', '8001:8001',
-      '-t', 'gcr.io/oss-fuzz/%s' % args.project_name,
-      'coverage_report', '/out/%s' % args.fuzzer_name,
-  ]
-
-  docker_run(run_args)
-
-
 def reproduce(args):
   """Reproduces a testcase in the container."""
   if not _check_project_exists(args.project_name):

projects/bad_example/build.sh

@@ -23,7 +23,7 @@ $CXX $CXXFLAGS -std=c++11 -I. -DINTENTIONAL_STARTUP_CRASH \
 
 
 # The latest two examples won't for for coverage build, bail out.
-if [[ $SANITIZER = *coverage* ]] || [[ $SANITIZER = *profile* ]]; then
+if [[ $SANITIZER = *coverage* ]]; then
   exit 0
 fi
 

projects/envoy/build.sh

@@ -71,7 +71,7 @@ bazel build --verbose_failures --dynamic_mode=off --spawn_strategy=standalone \
 
 # Profiling with coverage requires that we resolve+copy all Bazel symlinks and
 # also remap everything under proc/self/cwd to correspond to Bazel build paths.
-if [ "$SANITIZER" = "profile" ]
+if [ "$SANITIZER" = "coverage" ]
 then
   # The build invoker looks for sources in $SRC, but it turns out that we need
   # to not be buried under src/, paths are expected at out/proc/self/cwd by

projects/firefox/build.sh

@@ -47,7 +47,7 @@ source $HOME/.cargo/env
 
 # Packages Firefox only to immediately extract the archive. Some files are
 # replaced with gtest-variants, which is required by the fuzzing interface.
-# Weighs in shy of 1GB afterwards. About double for profile builds.
+# Weighs in shy of 1GB afterwards. About double for coverage builds.
 ./mach package
 tar -xf $MOZ_OBJDIR/dist/firefox*bz2 -C $OUT
 cp -L $MOZ_OBJDIR/dist/bin/gtest/libxul.so $OUT/firefox

projects/grpc/build.sh

@@ -62,7 +62,7 @@ bazel build --dynamic_mode=off --spawn_strategy=standalone --genrule_strategy=st
 # Copied from projects/envoy/build.sh which also uses Bazel.
 # Profiling with coverage requires that we resolve+copy all Bazel symlinks and
 # also remap everything under proc/self/cwd to correspond to Bazel build paths.
-if [ "$SANITIZER" = "profile" ]
+if [ "$SANITIZER" = "coverage" ]
 then
   # The build invoker looks for sources in $SRC, but it turns out that we need
   # to not be buried under src/, paths are expected at out/proc/self/cwd by

projects/skia/build.sh

@@ -22,11 +22,8 @@ rm -rf build
 mkdir build
 
 cd build
-if [ $SANITIZER == "profile" ]; then
+if [ $SANITIZER == "coverage" ]; then
   cmake ..
-elif [ $SANITIZER == "coverage" ]; then
-  # TODO(metzman): Remove this once "coverage" builds are removed from OSS-Fuzz.
-  CFLAGS= CXXFLAGS="-stdlib=libc++" cmake ..
 else
   if [ $SANITIZER == "address" ]; then
     CMAKE_SANITIZER="ASAN"