Skip to content

Commit

Permalink
wasm/parser: bump limits
Browse files Browse the repository at this point in the history 
We've seen our launch demo run into these limits, so make the containers
non-contiguous and allow up to 1MiB of memory usage during parsing. This
fits the compiled ML framework's function count (~17k), as now we allow
~21k functions.

Signed-off-by: Tyler Rockwood <rockwood@redpanda.com>
(cherry picked from commit d07f775)
  • Loading branch information
@rockwotj
rockwotj committed Feb 28, 2024
1 parent 7dc0d73 commit 611b77a
Show file tree
Hide file tree
Showing 3 changed files with 92 additions and 105 deletions.
16 changes: 8 additions & 8 deletions src/v/wasm/parser/parser.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ std::ostream& operator<<(std::ostream& os, const module_declarations& m_decls) {
}

namespace {
constexpr size_t max_vector_bytes = 128_KiB;
constexpr size_t max_vector_bytes = 1_MiB;
constexpr size_t max_functions
= max_vector_bytes
/ std::max(sizeof(function_signature), sizeof(declaration::function));
Expand Down Expand Up @@ -537,13 +537,13 @@ class module_extractor {
// NOTE these vectors are all explicitly bounded as to prevent large
// allocations.

std::vector<function_signature> _func_signatures;
std::vector<module_import> _imports;
std::vector<declaration::function> _functions;
std::vector<declaration::table> _tables;
std::vector<declaration::memory> _memories;
std::vector<declaration::global> _globals;
std::vector<module_export> _exports;
chunked_vector<function_signature> _func_signatures;
chunked_vector<module_import> _imports;
chunked_vector<declaration::function> _functions;
chunked_vector<declaration::table> _tables;
chunked_vector<declaration::memory> _memories;
chunked_vector<declaration::global> _globals;
chunked_vector<module_export> _exports;

iobuf_parser_base* _parser;
};
Expand Down
5 changes: 3 additions & 2 deletions src/v/wasm/parser/parser.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

#include "bytes/iobuf.h"
#include "seastarx.h"
#include "utils/fragmented_vector.h"

#include <seastar/core/sstring.hh>

Expand Down Expand Up @@ -152,8 +153,8 @@ struct module_export {
* The declarations of a WebAssembly module.
*/
struct module_declarations {
std::vector<module_export> exports;
std::vector<module_import> imports;
chunked_vector<module_export> exports;
chunked_vector<module_import> imports;

bool operator==(const module_declarations&) const = default;
friend std::ostream& operator<<(std::ostream&, const module_declarations&);
Expand Down
176 changes: 81 additions & 95 deletions src/v/wasm/parser/tests/parser_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include "bytes/bytes.h"
#include "bytes/iobuf_parser.h"
#include "gtest/gtest.h"
#include "utils/fragmented_vector.h"
#include "wasm/parser/parser.h"

#include <gtest/gtest.h>
Expand Down Expand Up @@ -43,7 +44,8 @@ bytes wat2wasm(std::string_view wat) {

struct test_data {
std::string wat;
module_declarations expected;
std::vector<module_export> exports;
std::vector<module_import> imports;
};

void PrintTo(const test_data& d, std::ostream* os) { *os << d.wat; }
Expand All @@ -54,7 +56,13 @@ TEST_P(ParserTest, ExtractDeclarations) {
const auto& testcase = GetParam();
bytes wasm = wat2wasm(testcase.wat);
auto decls = extract_declarations(bytes_to_iobuf(wasm)).get();
EXPECT_EQ(decls, testcase.expected);
module_declarations expected = {
.exports = chunked_vector<module_export>(
testcase.exports.begin(), testcase.exports.end()),
.imports = chunked_vector<module_import>(
testcase.imports.begin(), testcase.imports.end()),
};
EXPECT_EQ(decls, std::ref(expected));
}

module_export func(
Expand Down Expand Up @@ -134,10 +142,8 @@ INSTANTIATE_TEST_SUITE_P(
{
{
.wat = R"WAT((module))WAT",
.expected = {
.exports = {},
.imports = {},
}
.exports = {},
.imports = {},
},
{
.wat = R"WAT(
Expand All @@ -154,15 +160,13 @@ INSTANTIATE_TEST_SUITE_P(
(export "_start" (func $main))
)
)WAT",
.expected = {
.exports = {
func("_start", {}, {})
},
.imports = {
func("console", "log", {val_type::i32}, {}),
global("env", "from_js", val_type::i32),
},
}
.exports = {
func("_start", {}, {})
},
.imports = {
func("console", "log", {val_type::i32}, {}),
global("env", "from_js", val_type::i32),
},
},
{
.wat = R"WAT(
Expand All @@ -186,26 +190,24 @@ INSTANTIATE_TEST_SUITE_P(
(func (export "get-6") (result f64) (global.get 6))
)
)WAT",
.expected = {
.exports = {
func("get-0", {}, {val_type::i32}),
func("get-1", {}, {val_type::i32}),
func("get-x", {}, {val_type::i32}),
func("get-y", {}, {val_type::i32}),
func("get-4", {}, {val_type::i64}),
func("get-5", {}, {val_type::f32}),
func("get-6", {}, {val_type::f64}),
},
.imports = {
global("spectest", "global_i32", val_type::i32),
global("spectest", "global_i32", val_type::i32),
global("spectest", "global_i32", val_type::i32),
global("spectest", "global_i32", val_type::i32),
global("spectest", "global_i64", val_type::i64),
global("spectest", "global_f32", val_type::f32),
global("spectest", "global_f64", val_type::f64),
},
}
.exports = {
func("get-0", {}, {val_type::i32}),
func("get-1", {}, {val_type::i32}),
func("get-x", {}, {val_type::i32}),
func("get-y", {}, {val_type::i32}),
func("get-4", {}, {val_type::i64}),
func("get-5", {}, {val_type::f32}),
func("get-6", {}, {val_type::f64}),
},
.imports = {
global("spectest", "global_i32", val_type::i32),
global("spectest", "global_i32", val_type::i32),
global("spectest", "global_i32", val_type::i32),
global("spectest", "global_i32", val_type::i32),
global("spectest", "global_i64", val_type::i64),
global("spectest", "global_f32", val_type::f32),
global("spectest", "global_f64", val_type::f64),
},
},
{
.wat = R"WAT(
Expand All @@ -221,10 +223,8 @@ INSTANTIATE_TEST_SUITE_P(
(func $g (result i32) (i32.const 22))
)
)WAT",
.expected = {
.exports = {func("call", {val_type::i32}, {val_type::i32})},
.imports = {table("spectest", "table", val_type::funcref, {.min = 10, .max = 20})},
}
.exports = {func("call", {val_type::i32}, {val_type::i32})},
.imports = {table("spectest", "table", val_type::funcref, {.min = 10, .max = 20})},
},
{
.wat = R"WAT(
Expand All @@ -235,46 +235,36 @@ INSTANTIATE_TEST_SUITE_P(
(func (export "load") (param i32) (result i32) (i32.load (local.get 0)))
)
)WAT",
.expected = {
.exports = {func("load", {val_type::i32}, {val_type::i32})},
.imports = {memory("spectest", "memory", {.min = 1, .max = 2})},
}
.exports = {func("load", {val_type::i32}, {val_type::i32})},
.imports = {memory("spectest", "memory", {.min = 1, .max = 2})},
},
{
.wat = R"WAT(
(module (memory (export "a") 0))
)WAT",
.expected = {
.exports = {memory("a", {})},
.imports = {},
}
.exports = {memory("a", {})},
.imports = {},
},
{
.wat = R"WAT(
(module (table 0 funcref) (export "a" (table 0)))
)WAT",
.expected = {
.exports = {table("a", val_type::funcref, {})},
.imports = {},
}
.exports = {table("a", val_type::funcref, {})},
.imports = {},
},
{
.wat = R"WAT(
(module (global i32 (i32.const 0)) (export "a" (global 0)))
)WAT",
.expected = {
.exports = {global("a", val_type::i32)},
.imports = {},
}
.exports = {global("a", val_type::i32)},
.imports = {},
},
{
.wat = R"WAT(
(module (func) (export "a" (func 0)))
)WAT",
.expected = {
.exports = {func("a", {}, {})},
.imports = {},
}
.exports = {func("a", {}, {})},
.imports = {},
},
{
.wat = R"WAT(
Expand Down Expand Up @@ -337,38 +327,36 @@ INSTANTIATE_TEST_SUITE_P(
)
)
)WAT",
.expected = {
.exports = {
func("p1", {val_type::i32}, {}),
func("p2", {val_type::i32}, {}),
func("p3", {val_type::i32}, {}),
func("p4", {val_type::i32}, {}),
func("p5", {val_type::i32}, {}),
func("p6", {val_type::i32}, {}),
func("print32", {val_type::i32}, {}),
func("print64", {val_type::i64}, {}),
},
.imports = {
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_i64", {val_type::i64}, {}),
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_i64", {val_type::i64}, {}),
func("spectest", "print_f32", {val_type::f32}, {}),
func("spectest", "print_f64", {val_type::f64}, {}),
func("spectest", "print_i32_f32", {val_type::i32, val_type::f32}, {}),
func("spectest", "print_f64_f64", {val_type::f64, val_type::f64}, {}),
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_f64", {val_type::f64}, {}),
func("test", "func-i64->i64", {val_type::i64}, {val_type::i64}),
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_i32", {val_type::i32}, {}),
},
}
.exports = {
func("p1", {val_type::i32}, {}),
func("p2", {val_type::i32}, {}),
func("p3", {val_type::i32}, {}),
func("p4", {val_type::i32}, {}),
func("p5", {val_type::i32}, {}),
func("p6", {val_type::i32}, {}),
func("print32", {val_type::i32}, {}),
func("print64", {val_type::i64}, {}),
},
.imports = {
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_i64", {val_type::i64}, {}),
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_i64", {val_type::i64}, {}),
func("spectest", "print_f32", {val_type::f32}, {}),
func("spectest", "print_f64", {val_type::f64}, {}),
func("spectest", "print_i32_f32", {val_type::i32, val_type::f32}, {}),
func("spectest", "print_f64_f64", {val_type::f64, val_type::f64}, {}),
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_f64", {val_type::f64}, {}),
func("test", "func-i64->i64", {val_type::i64}, {val_type::i64}),
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_i32", {val_type::i32}, {}),
func("spectest", "print_i32", {val_type::i32}, {}),
},
},
{
.wat = R"WAT(
Expand All @@ -377,10 +365,8 @@ INSTANTIATE_TEST_SUITE_P(
(data (i32.const 0) "a")
)
)WAT",
.expected = {
.exports = {},
.imports = {},
}
.exports = {},
.imports = {},
},
}));

Expand Down

0 comments on commit 611b77a

Please sign in to comment.