From f1ea7ee8efb74e56a5b44c5544046f5fed20f9e5 Mon Sep 17 00:00:00 2001 From: Harvey Tuch Date: Thu, 5 Mar 2020 15:12:10 -0500 Subject: [PATCH] fuzz: H1 capture fuzz test performance improvements. The main contribution in this patch is a "persistent" mode for h1_capture_[direct_response_]fuzz_test. Based on profiling observations, we were spending 30-40% of time rebuilding the Envoy server on each run. This is avoided by having fuzzer variants that makes the integration test proxy static. There is a downside of this approach, since different fuzz invocations may interfere with each other. Ideally we would snapshot/fork for each fuzz invocation, but Envoy doesn't like forking once events/dispatchers are up. So, for now we have two builds of the fuzzer, where we trade fuzz engine efficacy for fuzz target performance. Some form of VM snapshotting would be ideal. The persistent mode takes the H1 replay tests to O(10 exec/s) from O(1 exec/s). This is still not great. Doing some perf analysis, it seems that we're spending the bulk of time in ASAN. Running the fuzzers without ASAN gives O(100 exec/s), which seems reasonable for a LPM-style integration test. It's future work why ASAN is so expensive, ASAN advertises itself as generally a 2x slowdown. There is also some secondary effect from the cost of mocks used in the integration test TCP client (mock watermark buffer), this speaks to our general mocking performance problem in fuzzing. In addition to the above, this patch has an optimization for the direct response fuzzer (don't initiate upstream connections) and a --config=plain-fuzz mode for peformance work without confounding ASAN. Risk level: Low Testing: Manual bazel runs of the fuzzers, observing exec/s. Signed-off-by: Harvey Tuch --- .bazelrc | 6 +++ test/fuzz/fuzz_runner.h | 6 +++ test/integration/BUILD | 47 +++++++++++++++---- .../h1_capture_direct_response_fuzz_test.cc | 4 +- test/integration/h1_capture_fuzz_test.cc | 4 +- test/integration/h1_fuzz.cc | 14 ++++-- test/integration/h1_fuzz.h | 3 +- 7 files changed, 67 insertions(+), 17 deletions(-) diff --git a/.bazelrc b/.bazelrc index f1c01584c182..1224c91fd8bc 100644 --- a/.bazelrc +++ b/.bazelrc @@ -190,8 +190,14 @@ build:asan-fuzzer --config=clang-asan build:asan-fuzzer --define=FUZZING_ENGINE=libfuzzer build:asan-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION build:asan-fuzzer --copt=-fsanitize=fuzzer-no-link +build:asan-fuzzer --copt=-fno-omit-frame-pointer # Remove UBSAN halt_on_error to avoid crashing on protobuf errors. build:asan-fuzzer --test_env=UBSAN_OPTIONS=print_stacktrace=1 +# Fuzzing without ASAN. This is useful for profiling fuzzers without any ASAN artifacts. +build:plain-fuzzer --define=FUZZING_ENGINE=libfuzzer +build:plain-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +build:plain-fuzzer --copt=-fsanitize=fuzzer-no-link + try-import %workspace%/clang.bazelrc try-import %workspace%/user.bazelrc diff --git a/test/fuzz/fuzz_runner.h b/test/fuzz/fuzz_runner.h index d6a7c4a2f808..e2279365cab6 100644 --- a/test/fuzz/fuzz_runner.h +++ b/test/fuzz/fuzz_runner.h @@ -62,6 +62,12 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv); // See https://llvm.org/docs/LibFuzzer.html#fuzz-target. extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size); +#ifdef PERSISTENT_FUZZER +#define PERSISTENT_FUZZ_VAR static +#else +#define PERSISTENT_FUZZ_VAR +#endif + #define DEFINE_TEST_ONE_INPUT_IMPL \ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { \ EnvoyTestOneInput(data, size); \ diff --git a/test/integration/BUILD b/test/integration/BUILD index 57e5487d87b5..b9a57a7f0a21 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -975,19 +975,29 @@ envoy_cc_test( ], ) +H1_FUZZ_LIB_DEPS = [ + ":capture_fuzz_proto_cc_proto", + ":http_integration_lib", + "//source/common/common:assert_lib", + "//source/common/common:logger_lib", + "//test/fuzz:fuzz_runner_lib", + "//test/integration:integration_lib", + "//test/test_common:environment_lib", +] + envoy_cc_test_library( name = "h1_fuzz_lib", srcs = ["h1_fuzz.cc"], hdrs = ["h1_fuzz.h"], - deps = [ - ":capture_fuzz_proto_cc_proto", - ":http_integration_lib", - "//source/common/common:assert_lib", - "//source/common/common:logger_lib", - "//test/fuzz:fuzz_runner_lib", - "//test/integration:integration_lib", - "//test/test_common:environment_lib", - ], + deps = H1_FUZZ_LIB_DEPS, +) + +envoy_cc_test_library( + name = "h1_fuzz_persistent_lib", + srcs = ["h1_fuzz.cc"], + hdrs = ["h1_fuzz.h"], + copts = ["-DPERSISTENT_FUZZER"], + deps = H1_FUZZ_LIB_DEPS, ) envoy_cc_fuzz_test( @@ -997,6 +1007,14 @@ envoy_cc_fuzz_test( deps = [":h1_fuzz_lib"], ) +envoy_cc_fuzz_test( + name = "h1_capture_persistent_fuzz_test", + srcs = ["h1_capture_fuzz_test.cc"], + copts = ["-DPERSISTENT_FUZZER"], + corpus = "h1_corpus", + deps = [":h1_fuzz_persistent_lib"], +) + envoy_cc_fuzz_test( name = "h1_capture_direct_response_fuzz_test", srcs = ["h1_capture_direct_response_fuzz_test.cc"], @@ -1007,6 +1025,17 @@ envoy_cc_fuzz_test( ], ) +envoy_cc_fuzz_test( + name = "h1_capture_direct_response_persistent_fuzz_test", + srcs = ["h1_capture_direct_response_fuzz_test.cc"], + copts = ["-DPERSISTENT_FUZZER"], + corpus = "h1_corpus", + deps = [ + ":h1_fuzz_persistent_lib", + "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", + ], +) + envoy_cc_test( name = "scoped_rds_integration_test", srcs = [ diff --git a/test/integration/h1_capture_direct_response_fuzz_test.cc b/test/integration/h1_capture_direct_response_fuzz_test.cc index 1e797261d408..621179a18354 100644 --- a/test/integration/h1_capture_direct_response_fuzz_test.cc +++ b/test/integration/h1_capture_direct_response_fuzz_test.cc @@ -31,8 +31,8 @@ void H1FuzzIntegrationTest::initialize() { DEFINE_PROTO_FUZZER(const test::integration::CaptureFuzzTestCase& input) { RELEASE_ASSERT(!TestEnvironment::getIpVersionsForTest().empty(), ""); const auto ip_version = TestEnvironment::getIpVersionsForTest()[0]; - H1FuzzIntegrationTest h1_fuzz_integration_test(ip_version); - h1_fuzz_integration_test.replay(input); + PERSISTENT_FUZZ_VAR H1FuzzIntegrationTest h1_fuzz_integration_test(ip_version); + h1_fuzz_integration_test.replay(input, true); } } // namespace Envoy diff --git a/test/integration/h1_capture_fuzz_test.cc b/test/integration/h1_capture_fuzz_test.cc index d6b161a5ea17..426e74abb391 100644 --- a/test/integration/h1_capture_fuzz_test.cc +++ b/test/integration/h1_capture_fuzz_test.cc @@ -7,8 +7,8 @@ DEFINE_PROTO_FUZZER(const test::integration::CaptureFuzzTestCase& input) { // Pick an IP version to use for loopback, it doesn't matter which. RELEASE_ASSERT(!TestEnvironment::getIpVersionsForTest().empty(), ""); const auto ip_version = TestEnvironment::getIpVersionsForTest()[0]; - H1FuzzIntegrationTest h1_fuzz_integration_test(ip_version); - h1_fuzz_integration_test.replay(input); + PERSISTENT_FUZZ_VAR H1FuzzIntegrationTest h1_fuzz_integration_test(ip_version); + h1_fuzz_integration_test.replay(input, false); } } // namespace Envoy diff --git a/test/integration/h1_fuzz.cc b/test/integration/h1_fuzz.cc index cc53743fbd6f..f1abeb9481a1 100644 --- a/test/integration/h1_fuzz.cc +++ b/test/integration/h1_fuzz.cc @@ -10,9 +10,14 @@ namespace Envoy { -void H1FuzzIntegrationTest::replay(const test::integration::CaptureFuzzTestCase& input) { - initialize(); - fake_upstreams_[0]->set_allow_unexpected_disconnects(true); +void H1FuzzIntegrationTest::replay(const test::integration::CaptureFuzzTestCase& input, + bool ignore_response) { + PERSISTENT_FUZZ_VAR bool initialized = [this]() -> bool { + initialize(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + return true; + }(); + UNREFERENCED_PARAMETER(initialized); IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("http")); FakeRawConnectionPtr fake_upstream_connection; for (int i = 0; i < input.events().size(); ++i) { @@ -31,6 +36,9 @@ void H1FuzzIntegrationTest::replay(const test::integration::CaptureFuzzTestCase& // TODO(htuch): Should we wait for some data? break; case test::integration::Event::kUpstreamSendBytes: + if (ignore_response) { + break; + } if (fake_upstream_connection == nullptr) { if (!fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection, max_wait_ms_)) { // If we timed out, we fail out. diff --git a/test/integration/h1_fuzz.h b/test/integration/h1_fuzz.h index 5fec0263a3f3..e4ba7edccb0a 100644 --- a/test/integration/h1_fuzz.h +++ b/test/integration/h1_fuzz.h @@ -15,7 +15,8 @@ class H1FuzzIntegrationTest : public HttpIntegrationTest { : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, version) {} void initialize() override; - void replay(const test::integration::CaptureFuzzTestCase&); + void replay(const test::integration::CaptureFuzzTestCase&, bool ignore_response); const std::chrono::milliseconds max_wait_ms_{10}; }; + } // namespace Envoy