Skip to content

Commit

Permalink
feat(bb): wasmtime and remote benchmarking (AztecProtocol#4204)
Browse files Browse the repository at this point in the history
- provides a script that runs threaded goblin_bench in WASM with 16
cores
- point to https://github.com/AztecProtocol/google-benchmark with
changes for our custom WASI environment. This included removing CMake's
thread library heuristics (which were unneeded, anyway) and adding
checks for a wasm environment when measuring time. See
AztecProtocol/barretenberg#837 for followups.
- remove conditional BENCHMARKS in CMakeLists.txt. There's no reason to
conditionally provide benchmark targets now that WASM can process the
google benchmark cmake
- add some WASMTIME_ENV_HACK guards around fake SRS and a fixed core
count.
- helpers for running benchmarks on remote nodes

---------

Co-authored-by: ludamad <adam@aztecprotocol.com>
  • Loading branch information
ludamad and ludamad0 committed Jan 30, 2024
1 parent 14e8c5c commit fd27808
Show file tree
Hide file tree
Showing 56 changed files with 356 additions and 379 deletions.
5 changes: 4 additions & 1 deletion .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -153,5 +153,8 @@
"C_Cpp.default.enableConfigurationSquiggles": false,
"C_Cpp.formatting": "disabled",
"C_Cpp.vcpkg.enabled": false,
"C_Cpp.default.includePath": ["barretenberg/cpp/src"],
"C_Cpp.default.includePath": [
"barretenberg/cpp/src"
],
"cmake.sourceDirectory": "/mnt/user-data/adam/aztec-packages/barretenberg/cpp",
}
22 changes: 3 additions & 19 deletions barretenberg/cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,11 @@ configure_file(
@ONLY
)


option(DISABLE_ASM "Disable custom assembly" OFF)
option(DISABLE_ADX "Disable ADX assembly variant" OFF)
option(MULTITHREADING "Enable multi-threading" ON)
option(OMP_MULTITHREADING "Enable OMP multi-threading" OFF)
option(TESTING "Build tests" ON)
option(BENCHMARKS "Build benchmarks" ON)
option(FUZZING "Build fuzzing harnesses" OFF)
option(FUZZING "Build ONLY fuzzing harnesses" OFF)
option(DISABLE_TBB "Intel Thread Building Blocks" ON)
option(COVERAGE "Enable collecting coverage from tests" OFF)
option(ENABLE_ASAN "Address sanitizer for debugging tricky memory corruption" OFF)
Expand Down Expand Up @@ -52,30 +49,17 @@ if(FUZZING)
add_definitions(-DDISABLE_CUSTOM_MUTATORS=1)
endif()

set(SANITIZER_OPTIONS "")

if(ADDRESS_SANITIZER)
set(SANITIZER_OPTIONS ${SANITIZER_OPTIONS} -fsanitize=address)
endif()

if(UNDEFINED_BEHAVIOUR_SANITIZER)
set(SANITIZER_OPTIONS ${SANITIZER_OPTIONS} -fsanitize=undefined -fno-sanitize=alignment)
endif()

add_compile_options(-fsanitize=fuzzer-no-link ${SANITIZER_OPTIONS})
add_compile_options(-fsanitize=fuzzer)
add_link_options(-fsanitize=fuzzer)

set(WASM OFF)
set(BENCHMARKS OFF)
set(MULTITHREADING OFF)
set(TESTING OFF)
endif()

if(CMAKE_SYSTEM_PROCESSOR MATCHES "wasm32")
message(STATUS "Compiling for WebAssembly.")
set(WASM ON)
set(DISABLE_ASM ON)
set(OMP_MULTITHREADING OFF)
set(BENCHMARKS OFF)
set(DISABLE_TBB 1)
add_compile_definitions(_WASI_EMULATED_PROCESS_CLOCKS=1)
endif()
Expand Down
38 changes: 34 additions & 4 deletions barretenberg/cpp/CMakePresets.json
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,19 @@
"LDFLAGS": "-fsanitize=thread"
}
},
{
"name": "ubsan",
"displayName": "Debugging build with undefined behaviour sanitizer on Clang-16",
"description": "Build with undefined behaviour sanitizer on clang16 with debugging information",
"inherits": "clang16-dbg",
"binaryDir": "build-ubsan",
"generator": "Unix Makefiles",
"environment": {
"CFLAGS": "-fsanitize=undefined",
"CXXFLAGS": "-fsanitize=undefined",
"LDFLAGS": "-fsanitize=undefined"
}
},
{
"name": "msan",
"displayName": "Debugging build with memory sanitizer on Clang-16",
Expand All @@ -151,10 +164,6 @@
"CFLAGS": "-fsanitize=memory",
"CXXFLAGS": "-fsanitize=memory",
"LDFLAGS": "-fsanitize=memory"
},
"cacheVariables": {
"BENCHMARK": "OFF",
"TESTING": "OFF"
}
},
{
Expand Down Expand Up @@ -228,6 +237,16 @@
"MULTITHREADING": "ON"
}
},
{
"name": "wasm-bench",
"displayName": "WASM benchmarking.",
"description": "WASM benchmarking.",
"inherits": "wasm-threads",
"binaryDir": "build-wasm-bench",
"environment": {
"CXXFLAGS": "-DWASMTIME_ENV_HACK"
}
},
{
"name": "xray",
"displayName": "Build with multi-threaded XRay Profiling",
Expand Down Expand Up @@ -336,6 +355,11 @@
"inherits": "default",
"configurePreset": "tsan"
},
{
"name": "ubsan",
"inherits": "default",
"configurePreset": "ubsan"
},
{
"name": "coverage",
"inherits": "default",
Expand Down Expand Up @@ -368,6 +392,12 @@
"jobs": 0,
"targets": ["barretenberg.wasm"]
},
{
"name": "wasm-bench",
"configurePreset": "wasm-bench",
"inheritConfigureEnvironment": true,
"jobs": 0
},
{
"name": "xray",
"configurePreset": "xray",
Expand Down
36 changes: 15 additions & 21 deletions barretenberg/cpp/cmake/benchmark.cmake
Original file line number Diff line number Diff line change
@@ -1,25 +1,19 @@
if(NOT TESTING)
set(BENCHMARKS OFF)
endif()

if(BENCHMARKS)
include(FetchContent)
include(FetchContent)

FetchContent_Declare(
benchmark
GIT_REPOSITORY https://github.com/google/benchmark
GIT_TAG v1.7.1
FIND_PACKAGE_ARGS
)
FetchContent_Declare(
benchmark
GIT_REPOSITORY https://github.com/AztecProtocol/google-benchmark
GIT_TAG 7638387d2727853d970fc9420dcf95cf3e9bd112
FIND_PACKAGE_ARGS
)

set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Benchmark tests off")
set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "Benchmark installation off")
set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Benchmark tests off")
set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "Benchmark installation off")

FetchContent_MakeAvailable(benchmark)
if(NOT benchmark_FOUND)
# FetchContent_MakeAvailable calls FetchContent_Populate if `find_package` is unsuccessful
# so these variables will be available if we reach this case
set_property(DIRECTORY ${benchmark_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL)
set_property(DIRECTORY ${benchmark_BINARY_DIR} PROPERTY EXCLUDE_FROM_ALL)
endif()
FetchContent_MakeAvailable(benchmark)
if(NOT benchmark_FOUND)
# FetchContent_MakeAvailable calls FetchContent_Populate if `find_package` is unsuccessful
# so these variables will be available if we reach this case
set_property(DIRECTORY ${benchmark_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL)
set_property(DIRECTORY ${benchmark_BINARY_DIR} PROPERTY EXCLUDE_FROM_ALL)
endif()
84 changes: 41 additions & 43 deletions barretenberg/cpp/cmake/gtest.cmake
Original file line number Diff line number Diff line change
@@ -1,51 +1,49 @@
if(TESTING)
include(GoogleTest)
include(FetchContent)

FetchContent_Declare(
GTest
GIT_REPOSITORY https://github.com/google/googletest.git
# Version 1.12.1 is not compatible with WASI-SDK 12
GIT_TAG release-1.10.0
FIND_PACKAGE_ARGS
include(GoogleTest)
include(FetchContent)

FetchContent_Declare(
GTest
GIT_REPOSITORY https://github.com/google/googletest.git
# Version 1.12.1 is not compatible with WASI-SDK 12
GIT_TAG release-1.10.0
FIND_PACKAGE_ARGS
)

set(BUILD_GMOCK OFF CACHE BOOL "Build with gMock disabled")
set(INSTALL_GTEST OFF CACHE BOOL "gTest installation disabled")

FetchContent_MakeAvailable(GTest)

if (NOT GTest_FOUND)
# FetchContent_MakeAvailable calls FetchContent_Populate if `find_package` is unsuccessful
# so these variables will be available if we reach this case
set_property(DIRECTORY ${gtest_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL)
set_property(DIRECTORY ${gtest_BINARY_DIR} PROPERTY EXCLUDE_FROM_ALL)

# Disable all warning when compiling gtest
target_compile_options(
gtest
PRIVATE
-w
)

set(BUILD_GMOCK OFF CACHE BOOL "Build with gMock disabled")
set(INSTALL_GTEST OFF CACHE BOOL "gTest installation disabled")

FetchContent_MakeAvailable(GTest)

if (NOT GTest_FOUND)
# FetchContent_MakeAvailable calls FetchContent_Populate if `find_package` is unsuccessful
# so these variables will be available if we reach this case
set_property(DIRECTORY ${gtest_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL)
set_property(DIRECTORY ${gtest_BINARY_DIR} PROPERTY EXCLUDE_FROM_ALL)

# Disable all warning when compiling gtest
target_compile_options(
if(WASM)
target_compile_definitions(
gtest
PRIVATE
-w
)

if(WASM)
target_compile_definitions(
gtest
PRIVATE
-DGTEST_HAS_EXCEPTIONS=0
-DGTEST_HAS_STREAM_REDIRECTION=0
)
endif()

mark_as_advanced(
BUILD_GMOCK BUILD_GTEST BUILD_SHARED_LIBS
gmock_build_tests gtest_build_samples gtest_build_tests
gtest_disable_pthreads gtest_force_shared_crt gtest_hide_internal_symbols
-DGTEST_HAS_EXCEPTIONS=0
-DGTEST_HAS_STREAM_REDIRECTION=0
)

add_library(GTest::gtest ALIAS gtest)
add_library(GTest::gtest_main ALIAS gtest_main)
endif()

enable_testing()
mark_as_advanced(
BUILD_GMOCK BUILD_GTEST BUILD_SHARED_LIBS
gmock_build_tests gtest_build_samples gtest_build_tests
gtest_disable_pthreads gtest_force_shared_crt gtest_hide_internal_symbols
)

add_library(GTest::gtest ALIAS gtest)
add_library(GTest::gtest_main ALIAS gtest_main)
endif()

enable_testing()
74 changes: 38 additions & 36 deletions barretenberg/cpp/cmake/module.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ function(barretenberg_module MODULE_NAME)
endif()

file(GLOB_RECURSE TEST_SOURCE_FILES *.test.cpp)
if(TESTING AND TEST_SOURCE_FILES)
if(TEST_SOURCE_FILES AND NOT FUZZING)
add_library(
${MODULE_NAME}_test_objects
OBJECT
Expand Down Expand Up @@ -184,7 +184,6 @@ function(barretenberg_module MODULE_NAME)
${MODULE_NAME}_${FUZZER_NAME_STEM}_fuzzer
PRIVATE
"-fsanitize=fuzzer"
${SANITIZER_OPTIONS}
)

target_link_libraries(
Expand All @@ -196,44 +195,47 @@ function(barretenberg_module MODULE_NAME)
endif()

file(GLOB_RECURSE BENCH_SOURCE_FILES *.bench.cpp)
if(BENCHMARKS AND BENCH_SOURCE_FILES)
add_library(
${MODULE_NAME}_bench_objects
OBJECT
${BENCH_SOURCE_FILES}
)
list(APPEND lib_targets ${MODULE_NAME}_bench_objects)
if(BENCH_SOURCE_FILES AND NOT FUZZING)
foreach(BENCHMARK_SOURCE ${BENCH_SOURCE_FILES})
get_filename_component(BENCHMARK_NAME ${BENCHMARK_SOURCE} NAME_WE) # extract name without extension
add_library(
${BENCHMARK_NAME}_bench_objects
OBJECT
${BENCHMARK_SOURCE}
)
list(APPEND lib_targets ${BENCHMARK_NAME}_bench_objects)

target_link_libraries(
${MODULE_NAME}_bench_objects
PRIVATE
benchmark::benchmark
${TBB_IMPORTED_TARGETS}
)
target_link_libraries(
${BENCHMARK_NAME}_bench_objects
PRIVATE
benchmark::benchmark
${TBB_IMPORTED_TARGETS}
)

add_executable(
${MODULE_NAME}_bench
$<TARGET_OBJECTS:${MODULE_NAME}_bench_objects>
)
list(APPEND exe_targets ${MODULE_NAME}_bench)
add_executable(
${BENCHMARK_NAME}_bench
$<TARGET_OBJECTS:${BENCHMARK_NAME}_bench_objects>
)
list(APPEND exe_targets ${MODULE_NAME}_bench)

target_link_libraries(
${MODULE_NAME}_bench
PRIVATE
${MODULE_LINK_NAME}
${ARGN}
benchmark::benchmark
${TBB_IMPORTED_TARGETS}
)
target_link_libraries(
${BENCHMARK_NAME}_bench
PRIVATE
${MODULE_LINK_NAME}
${ARGN}
benchmark::benchmark
${TBB_IMPORTED_TARGETS}
)

# enable msgpack downloading via dependency (solves race condition)
add_dependencies(${MODULE_NAME}_bench_objects msgpack-c)
add_dependencies(${MODULE_NAME}_bench msgpack-c)
add_custom_target(
run_${MODULE_NAME}_bench
COMMAND ${MODULE_NAME}_bench
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
)
# enable msgpack downloading via dependency (solves race condition)
add_dependencies(${BENCHMARK_NAME}_bench_objects msgpack-c)
add_dependencies(${BENCHMARK_NAME}_bench msgpack-c)
add_custom_target(
run_${BENCHMARK_NAME}_bench
COMMAND ${BENCHMARK_NAME}_bench
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
)
endforeach()
endif()

set(${MODULE_NAME}_lib_targets ${lib_targets} PARENT_SCOPE)
Expand Down
8 changes: 2 additions & 6 deletions barretenberg/cpp/docs/Fuzzing.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,11 @@ cmake --build --preset fuzzing

Fuzzing build turns off building tests and benchmarks, since they are incompatible with libfuzzer interface.

To turn on address sanitizer add `-DADDRESS_SANITIZER=ON`. Note that address sanitizer can be used to explore crashes.
To turn on address sanitizer instead run `cmake --preset asan -DFUZZING=ON`. Note that address sanitizer can be used to explore crashes.
Sometimes you might have to specify the address of llvm-symbolizer. You have to do it with `export ASAN_SYMBOLIZER_PATH=<PATH_TO_SYMBOLIZER>`.
For undefined behavior sanitizer `-DUNDEFINED_BEHAVIOUR_SANITIZER=ON`.
For undefined behavior sanitizer use `cmake --preset ubsan -DFUZZING=ON`.
Note that the fuzzer can be orders of magnitude slower with ASan (2-3x slower) or UBSan on, so it is best to run a non-sanitized build first, minimize the testcase and then run it for a bit of time with sanitizers.

Building with clang 13 or later is recommended, since libfuzzer contains and by default utilizes the entropic power schedule, which is considered more efficient
than the standard one present in previous versions.
You can downloadload the latest clang+llvm release here: https://github.com/llvm/llvm-project/releases

To set up cmake with another version of clang and fuzzing on:

```bash
Expand Down

0 comments on commit fd27808

Please sign in to comment.