From 611b77af364ea096c3bfdde819e004fa88eadaa4 Mon Sep 17 00:00:00 2001 From: Tyler Rockwood Date: Mon, 26 Feb 2024 21:36:24 -0600 Subject: [PATCH] wasm/parser: bump limits We've seen our launch demo run into these limits, so make the containers non-contiguous and allow up to 1MiB of memory usage during parsing. This fits the compiled ML framework's function count (~17k), as now we allow ~21k functions. Signed-off-by: Tyler Rockwood (cherry picked from commit d07f77527ee2cf408ea9aa3fc1dfb90eb09c8317) --- src/v/wasm/parser/parser.cc | 16 +-- src/v/wasm/parser/parser.h | 5 +- src/v/wasm/parser/tests/parser_test.cc | 176 ++++++++++++------------- 3 files changed, 92 insertions(+), 105 deletions(-) diff --git a/src/v/wasm/parser/parser.cc b/src/v/wasm/parser/parser.cc index 1ae5649978a6..4f650c16917a 100644 --- a/src/v/wasm/parser/parser.cc +++ b/src/v/wasm/parser/parser.cc @@ -123,7 +123,7 @@ std::ostream& operator<<(std::ostream& os, const module_declarations& m_decls) { } namespace { -constexpr size_t max_vector_bytes = 128_KiB; +constexpr size_t max_vector_bytes = 1_MiB; constexpr size_t max_functions = max_vector_bytes / std::max(sizeof(function_signature), sizeof(declaration::function)); @@ -537,13 +537,13 @@ class module_extractor { // NOTE these vectors are all explicitly bounded as to prevent large // allocations. - std::vector _func_signatures; - std::vector _imports; - std::vector _functions; - std::vector _tables; - std::vector _memories; - std::vector _globals; - std::vector _exports; + chunked_vector _func_signatures; + chunked_vector _imports; + chunked_vector _functions; + chunked_vector _tables; + chunked_vector _memories; + chunked_vector _globals; + chunked_vector _exports; iobuf_parser_base* _parser; }; diff --git a/src/v/wasm/parser/parser.h b/src/v/wasm/parser/parser.h index 4acae232adce..b1fd457354bf 100644 --- a/src/v/wasm/parser/parser.h +++ b/src/v/wasm/parser/parser.h @@ -11,6 +11,7 @@ #include "bytes/iobuf.h" #include "seastarx.h" +#include "utils/fragmented_vector.h" #include @@ -152,8 +153,8 @@ struct module_export { * The declarations of a WebAssembly module. */ struct module_declarations { - std::vector exports; - std::vector imports; + chunked_vector exports; + chunked_vector imports; bool operator==(const module_declarations&) const = default; friend std::ostream& operator<<(std::ostream&, const module_declarations&); diff --git a/src/v/wasm/parser/tests/parser_test.cc b/src/v/wasm/parser/tests/parser_test.cc index e5d4b10ffff7..2b93a58cd043 100644 --- a/src/v/wasm/parser/tests/parser_test.cc +++ b/src/v/wasm/parser/tests/parser_test.cc @@ -12,6 +12,7 @@ #include "bytes/bytes.h" #include "bytes/iobuf_parser.h" #include "gtest/gtest.h" +#include "utils/fragmented_vector.h" #include "wasm/parser/parser.h" #include @@ -43,7 +44,8 @@ bytes wat2wasm(std::string_view wat) { struct test_data { std::string wat; - module_declarations expected; + std::vector exports; + std::vector imports; }; void PrintTo(const test_data& d, std::ostream* os) { *os << d.wat; } @@ -54,7 +56,13 @@ TEST_P(ParserTest, ExtractDeclarations) { const auto& testcase = GetParam(); bytes wasm = wat2wasm(testcase.wat); auto decls = extract_declarations(bytes_to_iobuf(wasm)).get(); - EXPECT_EQ(decls, testcase.expected); + module_declarations expected = { + .exports = chunked_vector( + testcase.exports.begin(), testcase.exports.end()), + .imports = chunked_vector( + testcase.imports.begin(), testcase.imports.end()), + }; + EXPECT_EQ(decls, std::ref(expected)); } module_export func( @@ -134,10 +142,8 @@ INSTANTIATE_TEST_SUITE_P( { { .wat = R"WAT((module))WAT", - .expected = { - .exports = {}, - .imports = {}, - } + .exports = {}, + .imports = {}, }, { .wat = R"WAT( @@ -154,15 +160,13 @@ INSTANTIATE_TEST_SUITE_P( (export "_start" (func $main)) ) )WAT", - .expected = { - .exports = { - func("_start", {}, {}) - }, - .imports = { - func("console", "log", {val_type::i32}, {}), - global("env", "from_js", val_type::i32), - }, - } + .exports = { + func("_start", {}, {}) + }, + .imports = { + func("console", "log", {val_type::i32}, {}), + global("env", "from_js", val_type::i32), + }, }, { .wat = R"WAT( @@ -186,26 +190,24 @@ INSTANTIATE_TEST_SUITE_P( (func (export "get-6") (result f64) (global.get 6)) ) )WAT", - .expected = { - .exports = { - func("get-0", {}, {val_type::i32}), - func("get-1", {}, {val_type::i32}), - func("get-x", {}, {val_type::i32}), - func("get-y", {}, {val_type::i32}), - func("get-4", {}, {val_type::i64}), - func("get-5", {}, {val_type::f32}), - func("get-6", {}, {val_type::f64}), - }, - .imports = { - global("spectest", "global_i32", val_type::i32), - global("spectest", "global_i32", val_type::i32), - global("spectest", "global_i32", val_type::i32), - global("spectest", "global_i32", val_type::i32), - global("spectest", "global_i64", val_type::i64), - global("spectest", "global_f32", val_type::f32), - global("spectest", "global_f64", val_type::f64), - }, - } + .exports = { + func("get-0", {}, {val_type::i32}), + func("get-1", {}, {val_type::i32}), + func("get-x", {}, {val_type::i32}), + func("get-y", {}, {val_type::i32}), + func("get-4", {}, {val_type::i64}), + func("get-5", {}, {val_type::f32}), + func("get-6", {}, {val_type::f64}), + }, + .imports = { + global("spectest", "global_i32", val_type::i32), + global("spectest", "global_i32", val_type::i32), + global("spectest", "global_i32", val_type::i32), + global("spectest", "global_i32", val_type::i32), + global("spectest", "global_i64", val_type::i64), + global("spectest", "global_f32", val_type::f32), + global("spectest", "global_f64", val_type::f64), + }, }, { .wat = R"WAT( @@ -221,10 +223,8 @@ INSTANTIATE_TEST_SUITE_P( (func $g (result i32) (i32.const 22)) ) )WAT", - .expected = { - .exports = {func("call", {val_type::i32}, {val_type::i32})}, - .imports = {table("spectest", "table", val_type::funcref, {.min = 10, .max = 20})}, - } + .exports = {func("call", {val_type::i32}, {val_type::i32})}, + .imports = {table("spectest", "table", val_type::funcref, {.min = 10, .max = 20})}, }, { .wat = R"WAT( @@ -235,46 +235,36 @@ INSTANTIATE_TEST_SUITE_P( (func (export "load") (param i32) (result i32) (i32.load (local.get 0))) ) )WAT", - .expected = { - .exports = {func("load", {val_type::i32}, {val_type::i32})}, - .imports = {memory("spectest", "memory", {.min = 1, .max = 2})}, - } + .exports = {func("load", {val_type::i32}, {val_type::i32})}, + .imports = {memory("spectest", "memory", {.min = 1, .max = 2})}, }, { .wat = R"WAT( (module (memory (export "a") 0)) )WAT", - .expected = { - .exports = {memory("a", {})}, - .imports = {}, - } + .exports = {memory("a", {})}, + .imports = {}, }, { .wat = R"WAT( (module (table 0 funcref) (export "a" (table 0))) )WAT", - .expected = { - .exports = {table("a", val_type::funcref, {})}, - .imports = {}, - } + .exports = {table("a", val_type::funcref, {})}, + .imports = {}, }, { .wat = R"WAT( (module (global i32 (i32.const 0)) (export "a" (global 0))) )WAT", - .expected = { - .exports = {global("a", val_type::i32)}, - .imports = {}, - } + .exports = {global("a", val_type::i32)}, + .imports = {}, }, { .wat = R"WAT( (module (func) (export "a" (func 0))) )WAT", - .expected = { - .exports = {func("a", {}, {})}, - .imports = {}, - } + .exports = {func("a", {}, {})}, + .imports = {}, }, { .wat = R"WAT( @@ -337,38 +327,36 @@ INSTANTIATE_TEST_SUITE_P( ) ) )WAT", - .expected = { - .exports = { - func("p1", {val_type::i32}, {}), - func("p2", {val_type::i32}, {}), - func("p3", {val_type::i32}, {}), - func("p4", {val_type::i32}, {}), - func("p5", {val_type::i32}, {}), - func("p6", {val_type::i32}, {}), - func("print32", {val_type::i32}, {}), - func("print64", {val_type::i64}, {}), - }, - .imports = { - func("spectest", "print_i32", {val_type::i32}, {}), - func("spectest", "print_i64", {val_type::i64}, {}), - func("spectest", "print_i32", {val_type::i32}, {}), - func("spectest", "print_i64", {val_type::i64}, {}), - func("spectest", "print_f32", {val_type::f32}, {}), - func("spectest", "print_f64", {val_type::f64}, {}), - func("spectest", "print_i32_f32", {val_type::i32, val_type::f32}, {}), - func("spectest", "print_f64_f64", {val_type::f64, val_type::f64}, {}), - func("spectest", "print_i32", {val_type::i32}, {}), - func("spectest", "print_f64", {val_type::f64}, {}), - func("test", "func-i64->i64", {val_type::i64}, {val_type::i64}), - func("spectest", "print_i32", {val_type::i32}, {}), - func("spectest", "print_i32", {val_type::i32}, {}), - func("spectest", "print_i32", {val_type::i32}, {}), - func("spectest", "print_i32", {val_type::i32}, {}), - func("spectest", "print_i32", {val_type::i32}, {}), - func("spectest", "print_i32", {val_type::i32}, {}), - func("spectest", "print_i32", {val_type::i32}, {}), - }, - } + .exports = { + func("p1", {val_type::i32}, {}), + func("p2", {val_type::i32}, {}), + func("p3", {val_type::i32}, {}), + func("p4", {val_type::i32}, {}), + func("p5", {val_type::i32}, {}), + func("p6", {val_type::i32}, {}), + func("print32", {val_type::i32}, {}), + func("print64", {val_type::i64}, {}), + }, + .imports = { + func("spectest", "print_i32", {val_type::i32}, {}), + func("spectest", "print_i64", {val_type::i64}, {}), + func("spectest", "print_i32", {val_type::i32}, {}), + func("spectest", "print_i64", {val_type::i64}, {}), + func("spectest", "print_f32", {val_type::f32}, {}), + func("spectest", "print_f64", {val_type::f64}, {}), + func("spectest", "print_i32_f32", {val_type::i32, val_type::f32}, {}), + func("spectest", "print_f64_f64", {val_type::f64, val_type::f64}, {}), + func("spectest", "print_i32", {val_type::i32}, {}), + func("spectest", "print_f64", {val_type::f64}, {}), + func("test", "func-i64->i64", {val_type::i64}, {val_type::i64}), + func("spectest", "print_i32", {val_type::i32}, {}), + func("spectest", "print_i32", {val_type::i32}, {}), + func("spectest", "print_i32", {val_type::i32}, {}), + func("spectest", "print_i32", {val_type::i32}, {}), + func("spectest", "print_i32", {val_type::i32}, {}), + func("spectest", "print_i32", {val_type::i32}, {}), + func("spectest", "print_i32", {val_type::i32}, {}), + }, }, { .wat = R"WAT( @@ -377,10 +365,8 @@ INSTANTIATE_TEST_SUITE_P( (data (i32.const 0) "a") ) )WAT", - .expected = { - .exports = {}, - .imports = {}, - } + .exports = {}, + .imports = {}, }, }));