diff --git a/cpp/NINJA-steps.sh b/cpp/NINJA-steps.sh index e006420099..d1790eaba9 100755 --- a/cpp/NINJA-steps.sh +++ b/cpp/NINJA-steps.sh @@ -80,11 +80,17 @@ setglobal_compile_flags() { (ubsan) # faster build with -O0 - flags+="$flags -O0 -g -fsanitize=undefined" + flags="$flags -O0 -g -fsanitize=undefined" ;; (testgc) # TODO: GC_REPORT and GC_VERBOSE instead? - flags+="$flags -g -D GC_PROTECT -D GC_DEBUG -D GC_EVERY_ALLOC" + flags="$flags -g -D GC_PROTECT -D GC_DEBUG -D GC_EVERY_ALLOC" + ;; + + (leaky) + # Could this be ASAN? + # For cpp/gc_binding_test + flags="$flags -O0 -g -D LEAKY_BINDINGS -D LEAKY_TEST_MODE" ;; (opt) diff --git a/cpp/leaky_binding_test.cc b/cpp/leaky_binding_test.cc index 1067e1dc76..4e0c47ffb7 100644 --- a/cpp/leaky_binding_test.cc +++ b/cpp/leaky_binding_test.cc @@ -233,23 +233,6 @@ TEST bool_stat_test() { PASS(); } -TEST pyos_test() { - // This test isn't hermetic but it should work in most places, including in a - // container - int err_num = pyos::Chdir(new Str("/")); - ASSERT(err_num == 0); - - err_num = pyos::Chdir(new Str("/nonexistent__")); - ASSERT(err_num != 0); - - Dict* env = pyos::Environ(); - Str* p = env->get(new Str("PATH")); - ASSERT(p != nullptr); - log("PATH = %s", p->data_); - - PASS(); -} - TEST pyos_readbyte_test() { // Write 2 bytes to this file const char* tmp_name = "_tmp/pyos_ReadByte"; @@ -349,12 +332,30 @@ TEST putenv_test() { PASS(); } +TEST pyos_test() { + // This test isn't hermetic but it should work in most places, including in a + // container + int err_num = pyos::Chdir(new Str("/")); + ASSERT(err_num == 0); + + err_num = pyos::Chdir(new Str("/nonexistent__")); + ASSERT(err_num != 0); + + Dict* env = pyos::Environ(); + Str* p = env->get(new Str("PATH")); + ASSERT(p != nullptr); + log("PATH = %s", p->data_); + + PASS(); +} + GREATEST_MAIN_DEFS(); int main(int argc, char** argv) { // TODO: use garbage collection in this test? GREATEST_MAIN_BEGIN(); + RUN_TEST(show_sizeof); RUN_TEST(match_test); RUN_TEST(util_test); @@ -368,8 +369,12 @@ int main(int argc, char** argv) { RUN_TEST(os_path_test); RUN_TEST(putenv_test); + // Disabled because changing the dir somehow prevents the + // leaky_binding_test.profraw file from being created + // // Must come last because it does chdir() - RUN_TEST(pyos_test); + // + // RUN_TEST(pyos_test); GREATEST_MAIN_END(); /* display results */ return 0; diff --git a/cpp/test.sh b/cpp/test.sh index 845daf30fb..32f18c5390 100755 --- a/cpp/test.sh +++ b/cpp/test.sh @@ -14,17 +14,15 @@ source cpp/NINJA-steps.sh # for compile_and_link function # https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer export ASAN_OPTIONS='detect_leaks=0' -readonly LEAKY_TEST_SRC=( - cpp/leaky_binding_test.cc \ - cpp/leaky_core_pyos.cc \ - cpp/leaky_core_pyutil.cc \ - cpp/leaky_frontend_match.cc \ - cpp/leaky_libc.cc \ - cpp/leaky_osh_bool_stat.cc \ - cpp/leaky_posix.cc \ - cpp/leaky_pylib_os_path.cc \ - mycpp/mylib_leaky.cc -) +pre-build() { + # TODO: sort out these deps + # This is part of build/dev.sh oil-cpp + + build/codegen.sh ast-id-lex # id.h, osh-types.h, osh-lex.h + build/codegen.sh flag-gen-cpp # _build/cpp/arg_types.h + build/dev.sh oil-asdl-to-cpp # unit tests depend on id_kind_asdl.h, etc. + build/dev.sh cpp-codegen +} readonly LEAKY_FLAG_SPEC_SRC=( cpp/leaky_flag_spec_test.cc \ @@ -36,60 +34,49 @@ readonly LEAKY_FLAG_SPEC_SRC=( leaky-flag-spec-test() { ### Test generated code - local dir=_bin/cxx-dbg/cpp - mkdir -p $dir + local compiler=${1:-cxx} + local variant=${2:-dbg} + local dir=_bin/$compiler-$variant/cpp + mkdir -p $dir local bin=$dir/leaky_flag_spec_test local more_cxx_flags='-D LEAKY_BINDINGS -D CPP_UNIT_TEST -D DUMB_ALLOC' - compile_and_link cxx dbg "$more_cxx_flags" $bin \ + compile_and_link $compiler $variant "$more_cxx_flags" $bin \ "${LEAKY_FLAG_SPEC_SRC[@]}" cpp/leaky_dumb_alloc.cc - $bin "$@" + run-test $bin $compiler $variant } -# TODO: -# -# - Fold variants of leaky_binding_test and gc_binding_test into Ninja -# - And add cxx-ubsan and clang-coverage -- with HTML output -# - Problem: We don't have fine-grained dependencies for ASDL -# -# Possible solution is to factor into: -# -# build/cpp.sh gen-cpp -# build/cpp.sh all -# -# And then any time a Python source file changes (_build/app-deps), you do the -# whole build/cpp.sh gen-cpp? - - -leaky-binding-test-asan() { - local dir=_bin/cxx-asan/cpp - mkdir -p $dir - - local bin=$dir/leaky_binding_test - - compile_and_link cxx asan '-D LEAKY_BINDINGS -D CPP_UNIT_TEST' $bin \ - "${LEAKY_TEST_SRC[@]}" - - $bin "$@" -} +readonly LEAKY_TEST_SRC=( + cpp/leaky_binding_test.cc \ + cpp/leaky_core_pyos.cc \ + cpp/leaky_core_pyutil.cc \ + cpp/leaky_frontend_match.cc \ + cpp/leaky_libc.cc \ + cpp/leaky_osh_bool_stat.cc \ + cpp/leaky_posix.cc \ + cpp/leaky_pylib_os_path.cc \ + mycpp/mylib_leaky.cc +) leaky-binding-test() { ### Test hand-written code - local dir=_bin/cxx-dbg/cpp - mkdir -p $dir + local compiler=${1:-cxx} + local variant=${2:-dbg} + local dir=_bin/$compiler-$variant/cpp + mkdir -p $dir local bin=$dir/leaky_binding_test # leaky_dumb_alloc.cc exposes allocator alignment issues? local more_cxx_flags='-D LEAKY_BINDINGS -D CPP_UNIT_TEST -D DUMB_ALLOC' - compile_and_link cxx dbg "$more_cxx_flags" $bin \ + compile_and_link $compiler $variant "$more_cxx_flags" $bin \ "${LEAKY_TEST_SRC[@]}" cpp/leaky_dumb_alloc.cc - $bin "$@" + run-test $bin $compiler $variant } readonly GC_TEST_SRC=( @@ -100,37 +87,68 @@ readonly GC_TEST_SRC=( gc-binding-test() { local leaky_mode=${1:-} - local out_dir=_bin/cxx-testgc/cpp + local compiler=${1:-cxx} + local variant=${2:-dbg} + + local out_dir=_bin/$compiler-$variant/cpp mkdir -p $out_dir local more_cxx_flags='-D DUMB_ALLOC' # do we need this? - if test -n "$leaky_mode"; then - # LEAKY_BINDINGS is in the qsn_qsn.h header; LEAKY_TEST_MODE is gc_binding_test.cc - more_cxx_flags+=' -D LEAKY_BINDINGS -D LEAKY_TEST_MODE' - fi + local bin=$out_dir/gc_binding_test - local bin=$out_dir/gc_binding_test${leaky_mode} - compile_and_link cxx testgc "$more_cxx_flags" $bin \ + compile_and_link $compiler $variant "$more_cxx_flags" $bin \ "${GC_TEST_SRC[@]}" cpp/leaky_dumb_alloc.cc - $bin "$@" + run-test $bin $compiler $variant } -all-gc-binding-test() { - gc-binding-test # normal GC mode - gc-binding-test '.LEAKY' # test in leaky mode too +run-test() { + local bin=$1 + local compiler=$2 + local variant=$3 + + local dir=_test/$compiler-$variant/cpp + + mkdir -p $dir + + local name=$(basename $bin) + export LLVM_PROFILE_FILE=$dir/$name.profraw + + local log=$dir/$name.log + echo "RUN $bin > $log" + $bin > $log } +# TODO: +# +# - These tests can use Ninja dependencies with -M +# - separate all the files +# - Put logs in _test/ +# - Make HTML links to all the logs +# - Add coverage report + unit() { ### Run by test/cpp-unit.sh - # Generated code - leaky-flag-spec-test + gc-binding-test '' testgc + gc-binding-test '' leaky + + # Has generated code + leaky-flag-spec-test '' '' + leaky-flag-spec-test '' asan + + leaky-binding-test '' '' + leaky-binding-test '' asan +} + +coverage() { + pre-build - leaky-binding-test - leaky-binding-test-asan + gc-binding-test clang coverage + leaky-flag-spec-test clang coverage + leaky-binding-test clang coverage - all-gc-binding-test + test/coverage.sh html-report cpp } "$@" diff --git a/mycpp/test.sh b/mycpp/test.sh index 3686199f74..0ee493b688 100755 --- a/mycpp/test.sh +++ b/mycpp/test.sh @@ -9,9 +9,6 @@ set -o nounset set -o pipefail set -o errexit -REPO_ROOT=$(cd "$(dirname $0)/.."; pwd) -source build/common.sh # $CLANG_DIR - # in case binaries weren't built shopt -s failglob @@ -201,83 +198,12 @@ soil-run() { unit '' ubsan } -# -# Coverage -# - -coverage-report() { - local suite=${1:-mycpp-unit} - - local prof_dir="_test/clang-coverage/$suite" - local bin_dir="_bin/clang-coverage/$suite" - - local merged=$prof_dir/ALL.profdata - $CLANG_DIR/bin/llvm-profdata merge -sparse $prof_dir/*.profraw \ - -o $merged - - # https://llvm.org/docs/CommandGuide/llvm-cov.html - - local -a args=() - for b in $bin_dir/*; do - args+=(--object $b) - done - - # Text report - # $CLANG_DIR/bin/llvm-cov show --instr-profile $dir/ALL.profdata "${args[@]}" - - local html_dir=$prof_dir/html - mkdir -p $html_dir - - $CLANG_DIR/bin/llvm-cov show \ - --format html --output-dir $html_dir \ - --project-title "$suite" \ - --ignore-filename-regex '_test.cc$' \ - --ignore-filename-regex 'greatest.h$' \ - --ignore-filename-regex 'mycpp/demo' \ - --ignore-filename-regex '_test/' \ - --ignore-filename-regex '_build/' \ - --show-instantiation-summary \ - --instr-profile $merged \ - "${args[@]}" - - #echo "Wrote $html" - #ls -l --si -h $html # 2.2 MB of HTML - - # Clang quirk: permissions of this tree aren't right. Without this, the Soil - # host won't be able to zip and publish them. - - # make sure files are readable - echo 'fix FILES' - chmod --changes -R o+r $html_dir - echo - - # make sure dirs can be listed - echo 'fix DIRS' - find $html_dir -type d | xargs -- chmod --changes o+x - echo - - # 2.4 MB of HTML - du --si -s $html_dir - echo - - $CLANG_DIR/bin/llvm-cov report --instr-profile $merged "${args[@]}" - - # Also TODO: leaky_bindings_test, etc. -} - -llvm-cov-help() { - # many options for filtering - # --name-allowlist - - $CLANG_DIR/bin/llvm-cov show --help -} - unit-test-coverage() { ### Invoked by Soil unit clang coverage - coverage-report + test/coverage.sh html-report mycpp-unit } examples-coverage() { @@ -285,7 +211,7 @@ examples-coverage() { examples-variant clang coverage - coverage-report mycpp-examples + test/coverage.sh html-report mycpp-examples } "$@" diff --git a/soil/worker.sh b/soil/worker.sh index 064596ebab..87bf0656b3 100755 --- a/soil/worker.sh +++ b/soil/worker.sh @@ -193,6 +193,7 @@ extract-clang soil/deps-binary.sh extract-clang-in-container - mycpp-unit-coverage mycpp/test.sh unit-test-coverage _test/clang-coverage/mycpp-unit/html/index.html HACK-asdl build/dev.sh oil-asdl-to-cpp - mycpp-examples-coverage mycpp/test.sh examples-coverage _test/clang-coverage/mycpp-examples/html/index.html +cpp-coverage cpp/test.sh coverage _test/clang-coverage/cpp/html/index.html EOF } diff --git a/test/coverage.sh b/test/coverage.sh new file mode 100755 index 0000000000..33362e209e --- /dev/null +++ b/test/coverage.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# +# Usage: +# test/coverage.sh + +set -o nounset +set -o pipefail +set -o errexit + +REPO_ROOT=$(cd "$(dirname $0)/.."; pwd) +source build/common.sh # $CLANG_DIR + +html-report() { + local suite=${1:-mycpp-unit} + + local prof_dir="_test/clang-coverage/$suite" + local bin_dir="_bin/clang-coverage/$suite" + + local merged=$prof_dir/ALL.profdata + $CLANG_DIR/bin/llvm-profdata merge -sparse $prof_dir/*.profraw \ + -o $merged + + # https://llvm.org/docs/CommandGuide/llvm-cov.html + + local -a args=() + for b in $bin_dir/*; do + args+=(--object $b) + done + + # Text report + # $CLANG_DIR/bin/llvm-cov show --instr-profile $dir/ALL.profdata "${args[@]}" + + local html_dir=$prof_dir/html + mkdir -p $html_dir + + $CLANG_DIR/bin/llvm-cov show \ + --format html --output-dir $html_dir \ + --project-title "$suite" \ + --ignore-filename-regex '_test.cc$' \ + --ignore-filename-regex 'greatest.h$' \ + --ignore-filename-regex 'mycpp/demo' \ + --ignore-filename-regex '_test/' \ + --ignore-filename-regex '_build/' \ + --show-instantiation-summary \ + --instr-profile $merged \ + "${args[@]}" + + #echo "Wrote $html" + #ls -l --si -h $html # 2.2 MB of HTML + + # Clang quirk: permissions of this tree aren't right. Without this, the Soil + # host won't be able to zip and publish them. + + # make sure files are readable + echo 'fix FILES' + chmod --changes -R o+r $html_dir + echo + + # make sure dirs can be listed + echo 'fix DIRS' + find $html_dir -type d | xargs -- chmod --changes o+x + echo + + # 2.4 MB of HTML + du --si -s $html_dir + echo + + $CLANG_DIR/bin/llvm-cov report --instr-profile $merged "${args[@]}" + + # Also TODO: leaky_bindings_test, etc. +} + +llvm-cov-help() { + # many options for filtering + # --name-allowlist + + $CLANG_DIR/bin/llvm-cov show --help +} + +"$@" + diff --git a/test/cpp-unit.sh b/test/cpp-unit.sh index 2d915ff4fa..517129cda3 100755 --- a/test/cpp-unit.sh +++ b/test/cpp-unit.sh @@ -16,12 +16,7 @@ soil-run() { # which finds more bugs. mycpp/test.sh soil-run - # This is part of build/dev.sh oil-cpp - build/codegen.sh ast-id-lex # id.h, osh-types.h, osh-lex.h - build/codegen.sh flag-gen-cpp # _build/cpp/arg_types.h - build/dev.sh oil-asdl-to-cpp # unit tests depend on id_kind_asdl.h, etc. - build/dev.sh cpp-codegen - + cpp/test.sh pre-build cpp/test.sh unit asdl/test.sh unit