From 86fd2f4ff8be8d6313ed537f14e448ef2c517e78 Mon Sep 17 00:00:00 2001 From: Kenneth Giusti Date: Mon, 20 Nov 2017 11:39:12 -0500 Subject: [PATCH] DISPATCH-892: add test suite code coverage analysis This feature was copied unabashedly from the coverage work done on the Proton project by Andrew Stitcher, Jiri Danek, Chuck Rolke, et al. --- CMakeLists.txt | 32 ++++++++++++++++++---- README | 19 +++++++++++++ bin/record-coverage.sh | 60 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 5 deletions(-) create mode 100755 bin/record-coverage.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c9aa2a676..f2dc844306 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,11 +18,14 @@ ## # Set default build type. Must come before project() which sets default to "" -set (CMAKE_BUILD_TYPE RelWithDebInfo CACHE string - "Build type: Debug, Release, RelWithDebInfo or MinSizeRel (default RelWithDebInfo)") -if (CMAKE_BUILD_TYPE MATCHES "Deb") +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE RelWithDebInfo CACHE string + "Build type: Debug, Release, RelWithDebInfo, MinSizeRel or Coverage (default RelWithDebInfo)" FORCE) +endif(NOT CMAKE_BUILD_TYPE) +if (CMAKE_BUILD_TYPE MATCHES "Deb|Cover") set (has_debug_symbols " (has debug symbols)") -endif (CMAKE_BUILD_TYPE MATCHES "Deb") +endif (CMAKE_BUILD_TYPE MATCHES "Deb|Cover") +message(STATUS "Build type is \"${CMAKE_BUILD_TYPE}\"${has_debug_symbols}") project(qpid-dispatch C) @@ -115,7 +118,11 @@ find_package_handle_standard_args(VALGRIND DEFAULT_MSG VALGRIND_EXECUTABLE) option(USE_VALGRIND "Use valgrind when running tests" OFF) if (USE_VALGRIND) - set(TEST_RUNNER "${VALGRIND_EXECUTABLE} --leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite --error-exitcode=42 --suppressions=${CMAKE_SOURCE_DIR}/tests/valgrind.supp") + if (CMAKE_BUILD_TYPE MATCHES "Coverage") + message(WARNING "Building for coverage analysis; disabling valgrind run-time error detection") + else () + set(TEST_RUNNER "${VALGRIND_EXECUTABLE} --leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite --error-exitcode=42 --suppressions=${CMAKE_SOURCE_DIR}/tests/valgrind.supp") + endif () endif() option(USE_SANITIZERS "Compile with sanitizers (ASan, UBSan, TSan); incompatible with Valgrind" OFF) @@ -153,6 +160,21 @@ if (NOT CMAKE_SYSTEM_NAME STREQUAL SunOS) endif(HAS_PEDANTIC_FLAG) add_compile_options(-std=gnu99) add_compile_options(-pthread) + if (CMAKE_BUILD_TYPE MATCHES "Coverage") + set (CMAKE_C_FLAGS_COVERAGE "-g -O0 --coverage") + set (CMAKE_EXE_LINKER_FLAGS_COVERAGE "--coverage") + set (CMAKE_MODULE_LINKER_FLAGS_COVERAGE "--coverage") + set (CMAKE_SHARED_LINKER_FLAGS_COVERAGE "--coverage") + mark_as_advanced( + CMAKE_C_FLAGS_COVERAGE + CMAKE_EXE_LINKER_FLAGS_COVERAGE + CMAKE_MODULE_LINKER_FLAGS_COVERAGE + CMAKE_SHARED_LINKER_FLAGS_COVERAGE) + make_directory(${CMAKE_BINARY_DIR}/coverage_results) + add_custom_target(coverage + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/coverage_results + COMMAND ${CMAKE_SOURCE_DIR}/bin/record-coverage.sh ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) + endif(CMAKE_BUILD_TYPE MATCHES "Coverage") else (NOT CMAKE_SYSTEM_NAME STREQUAL SunOS) add_compile_options(-xc99) add_compile_options(-errwarn=%all) diff --git a/README b/README index 641ee64581..bbb8de2315 100644 --- a/README +++ b/README @@ -52,6 +52,25 @@ Run it without arguments to get a summary of how it can be used: $ ./run.py +Test Suite Code Coverage (GNU tools only) +========================================= + +Use coverage analysis to ensure that all code paths are exercised by +the test suite. To run the tests and perform code coverage analysis: + +# install the lcov package +$ yum install lcov + +# configure and build for the Coverage build type (from the directory): +$ cmake -DCMAKE_BUILD_TYPE=Coverage .. && make + +# run the test suite and generate the coverage html output +$ ctest && make coverage + +# Then point your browser at: + /coverage_results/html/index.html + + Clean build, install and test ============================= diff --git a/bin/record-coverage.sh b/bin/record-coverage.sh new file mode 100755 index 0000000000..69bfc340d9 --- /dev/null +++ b/bin/record-coverage.sh @@ -0,0 +1,60 @@ +#! /usr/bin/env bash + +# This script collates coverage data already present from running instrumented code. +# +# It requires the lcov tool to be installed (this provides the lcov and genhtml commands) +# +# It will produce a coverage analysis for gcc or clang compiled builds and currently for +# C and C++ parts of the build tree. +# +# It takes two command line arguments: +# - The first is the dispatch router source tree: this is mandatory. +# - The second is the build tree: this is optional and if not specified is assumed to be the +# current directory. +# +# The output is in the form of an html report which will be found in the generated html direectory. +# - There will also be a number of intermediate files left in the current directory. +# +# The typical way to use it would be to use the "Coverage" build type to get instrumented +# code, then to run the tests then to extract the coverage information from running the +# tests. +# Something like: +# cmake -DCMAKE_BUILD_TYPE=Coverage .. +# make +# make test +# make coverage + +# set -x + +# get full path +function getpath { + pushd -n $1 > /dev/null + echo $(dirs -0 -l) + popd -n > /dev/null +} + +SRC=${1?} +BLD=${2:-.} + +BLDPATH=$(getpath $BLD) +SRCPATH=$(getpath $SRC) + +# Get base profile +# - this initialises 0 counts for every profiled file +# without this step any file with no counts at all wouldn't +# show up on the final output. +lcov -c -i -d $BLDPATH -o dispatch-base.info + +# Get actual coverage data +lcov -c -d $BLDPATH -o dispatch-ctest.info + +# Total them up +lcov --add dispatch-base.info --add dispatch-ctest.info > dispatch-total-raw.info + +# Snip out stuff in /usr (we don't care about coverage in system code) +lcov --remove dispatch-total-raw.info "/usr/include*" "/usr/share*" "${SRCPATH}/tests/*" > dispatch-total.info + +# Generate report +rm -rf html +genhtml -p $SRCPATH -p $BLDPATH dispatch-total.info --title "Dispatch Router Test Coverage" --demangle-cpp -o html +