Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[cpp] Add unit test coverage report for cpp/ dir
Extract tool into test/coverage.sh

Had to disable pyos_test, hm
  • Loading branch information
Andy C committed Jul 10, 2022
1 parent 61d8882 commit f1851a3
Show file tree
Hide file tree
Showing 7 changed files with 195 additions and 163 deletions.
10 changes: 8 additions & 2 deletions cpp/NINJA-steps.sh
Expand Up @@ -80,11 +80,17 @@ setglobal_compile_flags() {

(ubsan)
# faster build with -O0
flags+="$flags -O0 -g -fsanitize=undefined"
flags="$flags -O0 -g -fsanitize=undefined"
;;
(testgc)
# TODO: GC_REPORT and GC_VERBOSE instead?
flags+="$flags -g -D GC_PROTECT -D GC_DEBUG -D GC_EVERY_ALLOC"
flags="$flags -g -D GC_PROTECT -D GC_DEBUG -D GC_EVERY_ALLOC"
;;

(leaky)
# Could this be ASAN?
# For cpp/gc_binding_test
flags="$flags -O0 -g -D LEAKY_BINDINGS -D LEAKY_TEST_MODE"
;;

(opt)
Expand Down
41 changes: 23 additions & 18 deletions cpp/leaky_binding_test.cc
Expand Up @@ -233,23 +233,6 @@ TEST bool_stat_test() {
PASS();
}

TEST pyos_test() {
// This test isn't hermetic but it should work in most places, including in a
// container
int err_num = pyos::Chdir(new Str("/"));
ASSERT(err_num == 0);

err_num = pyos::Chdir(new Str("/nonexistent__"));
ASSERT(err_num != 0);

Dict<Str*, Str*>* env = pyos::Environ();
Str* p = env->get(new Str("PATH"));
ASSERT(p != nullptr);
log("PATH = %s", p->data_);

PASS();
}

TEST pyos_readbyte_test() {
// Write 2 bytes to this file
const char* tmp_name = "_tmp/pyos_ReadByte";
Expand Down Expand Up @@ -349,12 +332,30 @@ TEST putenv_test() {
PASS();
}

TEST pyos_test() {
// This test isn't hermetic but it should work in most places, including in a
// container
int err_num = pyos::Chdir(new Str("/"));
ASSERT(err_num == 0);

err_num = pyos::Chdir(new Str("/nonexistent__"));
ASSERT(err_num != 0);

Dict<Str*, Str*>* env = pyos::Environ();
Str* p = env->get(new Str("PATH"));
ASSERT(p != nullptr);
log("PATH = %s", p->data_);

PASS();
}

GREATEST_MAIN_DEFS();

int main(int argc, char** argv) {
// TODO: use garbage collection in this test?

GREATEST_MAIN_BEGIN();

RUN_TEST(show_sizeof);
RUN_TEST(match_test);
RUN_TEST(util_test);
Expand All @@ -368,8 +369,12 @@ int main(int argc, char** argv) {
RUN_TEST(os_path_test);
RUN_TEST(putenv_test);

// Disabled because changing the dir somehow prevents the
// leaky_binding_test.profraw file from being created
//
// Must come last because it does chdir()
RUN_TEST(pyos_test);
//
// RUN_TEST(pyos_test);

GREATEST_MAIN_END(); /* display results */
return 0;
Expand Down
140 changes: 79 additions & 61 deletions cpp/test.sh
Expand Up @@ -14,17 +14,15 @@ source cpp/NINJA-steps.sh # for compile_and_link function
# https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer
export ASAN_OPTIONS='detect_leaks=0'

readonly LEAKY_TEST_SRC=(
cpp/leaky_binding_test.cc \
cpp/leaky_core_pyos.cc \
cpp/leaky_core_pyutil.cc \
cpp/leaky_frontend_match.cc \
cpp/leaky_libc.cc \
cpp/leaky_osh_bool_stat.cc \
cpp/leaky_posix.cc \
cpp/leaky_pylib_os_path.cc \
mycpp/mylib_leaky.cc
)
pre-build() {
# TODO: sort out these deps
# This is part of build/dev.sh oil-cpp

build/codegen.sh ast-id-lex # id.h, osh-types.h, osh-lex.h
build/codegen.sh flag-gen-cpp # _build/cpp/arg_types.h
build/dev.sh oil-asdl-to-cpp # unit tests depend on id_kind_asdl.h, etc.
build/dev.sh cpp-codegen
}

readonly LEAKY_FLAG_SPEC_SRC=(
cpp/leaky_flag_spec_test.cc \
Expand All @@ -36,60 +34,49 @@ readonly LEAKY_FLAG_SPEC_SRC=(
leaky-flag-spec-test() {
### Test generated code

local dir=_bin/cxx-dbg/cpp
mkdir -p $dir
local compiler=${1:-cxx}
local variant=${2:-dbg}

local dir=_bin/$compiler-$variant/cpp
mkdir -p $dir
local bin=$dir/leaky_flag_spec_test

local more_cxx_flags='-D LEAKY_BINDINGS -D CPP_UNIT_TEST -D DUMB_ALLOC'
compile_and_link cxx dbg "$more_cxx_flags" $bin \
compile_and_link $compiler $variant "$more_cxx_flags" $bin \
"${LEAKY_FLAG_SPEC_SRC[@]}" cpp/leaky_dumb_alloc.cc

$bin "$@"
run-test $bin $compiler $variant
}

# TODO:
#
# - Fold variants of leaky_binding_test and gc_binding_test into Ninja
# - And add cxx-ubsan and clang-coverage -- with HTML output
# - Problem: We don't have fine-grained dependencies for ASDL
#
# Possible solution is to factor into:
#
# build/cpp.sh gen-cpp
# build/cpp.sh all
#
# And then any time a Python source file changes (_build/app-deps), you do the
# whole build/cpp.sh gen-cpp?


leaky-binding-test-asan() {
local dir=_bin/cxx-asan/cpp
mkdir -p $dir

local bin=$dir/leaky_binding_test

compile_and_link cxx asan '-D LEAKY_BINDINGS -D CPP_UNIT_TEST' $bin \
"${LEAKY_TEST_SRC[@]}"

$bin "$@"
}
readonly LEAKY_TEST_SRC=(
cpp/leaky_binding_test.cc \
cpp/leaky_core_pyos.cc \
cpp/leaky_core_pyutil.cc \
cpp/leaky_frontend_match.cc \
cpp/leaky_libc.cc \
cpp/leaky_osh_bool_stat.cc \
cpp/leaky_posix.cc \
cpp/leaky_pylib_os_path.cc \
mycpp/mylib_leaky.cc
)

leaky-binding-test() {
### Test hand-written code

local dir=_bin/cxx-dbg/cpp
mkdir -p $dir
local compiler=${1:-cxx}
local variant=${2:-dbg}

local dir=_bin/$compiler-$variant/cpp
mkdir -p $dir
local bin=$dir/leaky_binding_test

# leaky_dumb_alloc.cc exposes allocator alignment issues?

local more_cxx_flags='-D LEAKY_BINDINGS -D CPP_UNIT_TEST -D DUMB_ALLOC'
compile_and_link cxx dbg "$more_cxx_flags" $bin \
compile_and_link $compiler $variant "$more_cxx_flags" $bin \
"${LEAKY_TEST_SRC[@]}" cpp/leaky_dumb_alloc.cc

$bin "$@"
run-test $bin $compiler $variant
}

readonly GC_TEST_SRC=(
Expand All @@ -100,37 +87,68 @@ readonly GC_TEST_SRC=(
gc-binding-test() {
local leaky_mode=${1:-}

local out_dir=_bin/cxx-testgc/cpp
local compiler=${1:-cxx}
local variant=${2:-dbg}

local out_dir=_bin/$compiler-$variant/cpp
mkdir -p $out_dir

local more_cxx_flags='-D DUMB_ALLOC' # do we need this?
if test -n "$leaky_mode"; then
# LEAKY_BINDINGS is in the qsn_qsn.h header; LEAKY_TEST_MODE is gc_binding_test.cc
more_cxx_flags+=' -D LEAKY_BINDINGS -D LEAKY_TEST_MODE'
fi
local bin=$out_dir/gc_binding_test

local bin=$out_dir/gc_binding_test${leaky_mode}
compile_and_link cxx testgc "$more_cxx_flags" $bin \
compile_and_link $compiler $variant "$more_cxx_flags" $bin \
"${GC_TEST_SRC[@]}" cpp/leaky_dumb_alloc.cc

$bin "$@"
run-test $bin $compiler $variant
}

all-gc-binding-test() {
gc-binding-test # normal GC mode
gc-binding-test '.LEAKY' # test in leaky mode too
run-test() {
local bin=$1
local compiler=$2
local variant=$3

local dir=_test/$compiler-$variant/cpp

mkdir -p $dir

local name=$(basename $bin)
export LLVM_PROFILE_FILE=$dir/$name.profraw

local log=$dir/$name.log
echo "RUN $bin > $log"
$bin > $log
}

# TODO:
#
# - These tests can use Ninja dependencies with -M
# - separate all the files
# - Put logs in _test/
# - Make HTML links to all the logs
# - Add coverage report

unit() {
### Run by test/cpp-unit.sh

# Generated code
leaky-flag-spec-test
gc-binding-test '' testgc
gc-binding-test '' leaky

# Has generated code
leaky-flag-spec-test '' ''
leaky-flag-spec-test '' asan

leaky-binding-test '' ''
leaky-binding-test '' asan
}

coverage() {
pre-build

leaky-binding-test
leaky-binding-test-asan
gc-binding-test clang coverage
leaky-flag-spec-test clang coverage
leaky-binding-test clang coverage

all-gc-binding-test
test/coverage.sh html-report cpp
}

"$@"
78 changes: 2 additions & 76 deletions mycpp/test.sh
Expand Up @@ -9,9 +9,6 @@ set -o nounset
set -o pipefail
set -o errexit

REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
source build/common.sh # $CLANG_DIR

# in case binaries weren't built
shopt -s failglob

Expand Down Expand Up @@ -201,91 +198,20 @@ soil-run() {
unit '' ubsan
}

#
# Coverage
#

coverage-report() {
local suite=${1:-mycpp-unit}

local prof_dir="_test/clang-coverage/$suite"
local bin_dir="_bin/clang-coverage/$suite"

local merged=$prof_dir/ALL.profdata
$CLANG_DIR/bin/llvm-profdata merge -sparse $prof_dir/*.profraw \
-o $merged

# https://llvm.org/docs/CommandGuide/llvm-cov.html

local -a args=()
for b in $bin_dir/*; do
args+=(--object $b)
done

# Text report
# $CLANG_DIR/bin/llvm-cov show --instr-profile $dir/ALL.profdata "${args[@]}"

local html_dir=$prof_dir/html
mkdir -p $html_dir

$CLANG_DIR/bin/llvm-cov show \
--format html --output-dir $html_dir \
--project-title "$suite" \
--ignore-filename-regex '_test.cc$' \
--ignore-filename-regex 'greatest.h$' \
--ignore-filename-regex 'mycpp/demo' \
--ignore-filename-regex '_test/' \
--ignore-filename-regex '_build/' \
--show-instantiation-summary \
--instr-profile $merged \
"${args[@]}"

#echo "Wrote $html"
#ls -l --si -h $html # 2.2 MB of HTML

# Clang quirk: permissions of this tree aren't right. Without this, the Soil
# host won't be able to zip and publish them.

# make sure files are readable
echo 'fix FILES'
chmod --changes -R o+r $html_dir
echo

# make sure dirs can be listed
echo 'fix DIRS'
find $html_dir -type d | xargs -- chmod --changes o+x
echo

# 2.4 MB of HTML
du --si -s $html_dir
echo

$CLANG_DIR/bin/llvm-cov report --instr-profile $merged "${args[@]}"

# Also TODO: leaky_bindings_test, etc.
}

llvm-cov-help() {
# many options for filtering
# --name-allowlist

$CLANG_DIR/bin/llvm-cov show --help
}

unit-test-coverage() {
### Invoked by Soil

unit clang coverage

coverage-report
test/coverage.sh html-report mycpp-unit
}

examples-coverage() {
### Invoked by Soil

examples-variant clang coverage

coverage-report mycpp-examples
test/coverage.sh html-report mycpp-examples
}

"$@"
1 change: 1 addition & 0 deletions soil/worker.sh
Expand Up @@ -193,6 +193,7 @@ extract-clang soil/deps-binary.sh extract-clang-in-container -
mycpp-unit-coverage mycpp/test.sh unit-test-coverage _test/clang-coverage/mycpp-unit/html/index.html
HACK-asdl build/dev.sh oil-asdl-to-cpp -
mycpp-examples-coverage mycpp/test.sh examples-coverage _test/clang-coverage/mycpp-examples/html/index.html
cpp-coverage cpp/test.sh coverage _test/clang-coverage/cpp/html/index.html
EOF
}

Expand Down

0 comments on commit f1851a3

Please sign in to comment.