diff --git a/scripts/clusterfuzz/run.py b/scripts/clusterfuzz/run.py index b98c5658c50..fe814a2b048 100755 --- a/scripts/clusterfuzz/run.py +++ b/scripts/clusterfuzz/run.py @@ -33,7 +33,7 @@ # The V8 flags we put in the "fuzzer flags" files, which tell ClusterFuzz how to # run V8. By default we apply all staging flags. -FUZZER_FLAGS = '--wasm-staging --experimental-wasm-custom-descriptors --experimental-wasm-js-interop --experimental-wasm-acquire-release --experimental-wasm-wide-arithmetic' +FUZZER_FLAGS = '--wasm-staging --experimental-wasm-custom-descriptors --experimental-wasm-js-interop --experimental-wasm-acquire-release --experimental-wasm-wide-arithmetic --wasm-compact-imports' # Optional V8 flags to add to FUZZER_FLAGS, some of the time. OPTIONAL_FUZZER_FLAGS = [ diff --git a/scripts/test/shared.py b/scripts/test/shared.py index 5e3cd208270..25ab3271c9c 100644 --- a/scripts/test/shared.py +++ b/scripts/test/shared.py @@ -268,6 +268,7 @@ def has_shell_timeout(): '--experimental-wasm-js-interop', '--experimental-wasm-acquire-release', '--experimental-wasm-wide-arithmetic', + '--wasm-compact-imports', ] # external tools diff --git a/src/ir/memory-utils.h b/src/ir/memory-utils.h index db9ff2bcba8..a586109e809 100644 --- a/src/ir/memory-utils.h +++ b/src/ir/memory-utils.h @@ -30,6 +30,10 @@ namespace wasm::MemoryUtils { bool isSubType(const Memory& a, const Memory& b); +inline bool sameType(const Memory& a, const Memory& b) { + return isSubType(a, b) && isSubType(b, a); +} + // Flattens memory into a single data segment, or no segment. If there is // a segment, it starts at 0. // Returns true if successful (e.g. relocatable segments cannot be flattened). diff --git a/src/ir/table-utils.h b/src/ir/table-utils.h index f5473a00012..bf1e7d663f0 100644 --- a/src/ir/table-utils.h +++ b/src/ir/table-utils.h @@ -25,6 +25,11 @@ namespace wasm::TableUtils { +inline bool sameType(const Table& a, const Table& b) { + return a.type == b.type && a.initial == b.initial && a.max == b.max && + a.addressType == b.addressType; +} + struct FlatTable { std::vector names; bool valid; diff --git a/src/wasm/wasm-binary.cpp b/src/wasm/wasm-binary.cpp index 737c7a9656b..e03274d8428 100644 --- a/src/wasm/wasm-binary.cpp +++ b/src/wasm/wasm-binary.cpp @@ -18,6 +18,7 @@ #include #include +#include "ir/memory-utils.h" #include "ir/module-utils.h" #include "ir/names.h" #include "ir/table-utils.h" @@ -26,6 +27,7 @@ #include "support/bits.h" #include "support/stdckdint.h" #include "support/string.h" +#include "support/utilities.h" #include "wasm-annotations.h" #include "wasm-binary.h" #include "wasm-debug.h" @@ -337,51 +339,160 @@ void WasmBinaryWriter::writeImports() { return; } auto start = startSection(BinaryConsts::Section::Import); - o << U32LEB(num); - auto writeImportHeader = [&](Importable* import) { - writeInlineString(import->module.view()); - writeInlineString(import->base.view()); + + using ImportItem = std::variant; + std::vector imports; + imports.reserve(num); + + ModuleUtils::iterImports(*wasm, + [&](ImportItem item) { imports.push_back(item); }); + + auto getModule = [](const ImportItem& item) -> Name { + return std::visit([](auto* i) { return i->module; }, item); + }; + auto getBase = [](const ImportItem& item) -> Name { + return std::visit([](auto* i) { return i->base; }, item); + }; + + auto shareImportType = [&](const ImportItem& a, const ImportItem& b) -> bool { + return std::visit( + overloaded{ + [](Function* a, Function* b) { return a->type == b->type; }, + [](Global* a, Global* b) { + return a->type == b->type && a->mutable_ == b->mutable_; + }, + [](Tag* a, Tag* b) { return a->type == b->type; }, + [](Memory* a, Memory* b) { return MemoryUtils::sameType(*a, *b); }, + [](Table* a, Table* b) { return TableUtils::sameType(*a, *b); }, + [](const auto& a, const auto& b) { return false; }}, + a, + b); }; - ModuleUtils::iterImportedFunctions(*wasm, [&](Function* func) { - writeImportHeader(func); - uint32_t kind = ExternalKind::Function; - if (func->type.isExact()) { - kind |= BinaryConsts::ExactImport; + + struct ImportGroup { + enum Kind { Single, SharedAll, SharedModule } kind; + size_t start; + size_t count; + }; + + std::vector groups; + if (wasm->features.hasCompactImports()) { + size_t i = 0; + size_t numImports = imports.size(); + while (i < numImports) { + // If the next import shares the module and type, then greedily collect + // the following imports as long as they share both the module and type. + size_t run = 1; + while (i + run < numImports && + getModule(imports[i]) == getModule(imports[i + run]) && + shareImportType(imports[i], imports[i + run])) { + ++run; + } + if (run > 1) { + groups.push_back({ImportGroup::SharedAll, i, run}); + i += run; + continue; + } + // Otherwise, try greedily collecting imports that share just the module. + while (i + run < numImports && + getModule(imports[i]) == getModule(imports[i + run])) { + ++run; + } + if (run > 1) { + groups.push_back({ImportGroup::SharedModule, i, run}); + i += run; + continue; + } + // Otherwise, just use a normal import. + groups.push_back({ImportGroup::Single, i, 1}); + ++i; } - o << U32LEB(kind) << U32LEB(getTypeIndex(func->type.getHeapType())); - }); - ModuleUtils::iterImportedGlobals(*wasm, [&](Global* global) { - writeImportHeader(global); - o << U32LEB(int32_t(ExternalKind::Global)); - writeType(global->type); - o << U32LEB(global->mutable_); - }); - ModuleUtils::iterImportedTags(*wasm, [&](Tag* tag) { - writeImportHeader(tag); - o << U32LEB(int32_t(ExternalKind::Tag)); - o << uint8_t(0); // Reserved 'attribute' field. Always 0. - o << U32LEB(getTypeIndex(tag->type)); - }); - ModuleUtils::iterImportedMemories(*wasm, [&](Memory* memory) { - writeImportHeader(memory); - o << U32LEB(int32_t(ExternalKind::Memory)); - writeResizableLimits(memory->initial, - memory->max, - memory->hasMax(), - memory->shared, - memory->is64(), - memory->pageSizeLog2); - }); - ModuleUtils::iterImportedTables(*wasm, [&](Table* table) { - writeImportHeader(table); - o << U32LEB(int32_t(ExternalKind::Table)); - writeType(table->type); - writeResizableLimits(table->initial, - table->max, - table->hasMax(), - /*shared=*/false, - table->is64()); - }); + } else { + for (size_t i = 0; i < imports.size(); ++i) { + groups.push_back({ImportGroup::Single, i, 1}); + } + } + + o << U32LEB(groups.size()); + + auto writeImportDesc = [&](const ImportItem& item) { + std::visit(overloaded{[&](Function* func) { + uint32_t kind = ExternalKind::Function; + if (func->type.isExact()) { + kind |= BinaryConsts::ExactImport; + } + o << U32LEB(kind) + << U32LEB(getTypeIndex(func->type.getHeapType())); + }, + [&](Global* global) { + o << U32LEB(int32_t(ExternalKind::Global)); + writeType(global->type); + o << U32LEB(global->mutable_); + }, + [&](Tag* tag) { + o << U32LEB(int32_t(ExternalKind::Tag)); + // Reserved 'attribute' field. Always 0. + o << uint8_t(0); + o << U32LEB(getTypeIndex(tag->type)); + }, + [&](Memory* memory) { + o << U32LEB(int32_t(ExternalKind::Memory)); + writeResizableLimits(memory->initial, + memory->max, + memory->hasMax(), + memory->shared, + memory->is64(), + memory->pageSizeLog2); + }, + [&](Table* table) { + o << U32LEB(int32_t(ExternalKind::Table)); + writeType(table->type); + writeResizableLimits(table->initial, + table->max, + table->hasMax(), + /*shared=*/false, + table->is64()); + }}, + item); + }; + + for (const auto& group : groups) { + switch (group.kind) { + case ImportGroup::Single: { + const auto& item = imports[group.start]; + writeInlineString(getModule(item).view()); + writeInlineString(getBase(item).view()); + writeImportDesc(item); + break; + } + case ImportGroup::SharedAll: { + const auto& first = imports[group.start]; + writeInlineString(getModule(first).view()); + writeInlineString(""); + o << uint8_t(BinaryConsts::CompactImportsSharedAll); + writeImportDesc(first); + o << U32LEB(group.count); + for (size_t i = 0; i < group.count; ++i) { + writeInlineString(getBase(imports[group.start + i]).view()); + } + break; + } + case ImportGroup::SharedModule: { + const auto& first = imports[group.start]; + writeInlineString(getModule(first).view()); + writeInlineString(""); + o << uint8_t(BinaryConsts::CompactImportsSharedModule); + o << U32LEB(group.count); + for (size_t i = 0; i < group.count; ++i) { + const auto& item = imports[group.start + i]; + writeInlineString(getBase(item).view()); + writeImportDesc(item); + } + break; + } + } + } + finishSection(start); } diff --git a/test/lit/d8/fuzz_shell_exceptions.wast b/test/lit/d8/fuzz_shell_exceptions.wast index f4d5d082d85..b8d075c8e86 100644 --- a/test/lit/d8/fuzz_shell_exceptions.wast +++ b/test/lit/d8/fuzz_shell_exceptions.wast @@ -37,7 +37,7 @@ ;; Build to a binary wasm. ;; -;; RUN: wasm-opt %s -o %t.wasm -q -all +;; RUN: wasm-opt %s -o %t.wasm -q --enable-reference-types --enable-exception-handling ;; Run in node. ;; diff --git a/test/unit/input/reference_types_target_feature.wasm b/test/unit/input/reference_types_target_feature.wasm index b388b11bb57..95338c95b3f 100644 Binary files a/test/unit/input/reference_types_target_feature.wasm and b/test/unit/input/reference_types_target_feature.wasm differ diff --git a/test/unit/test_compact_imports.py b/test/unit/test_compact_imports.py new file mode 100644 index 00000000000..c759c0fc114 --- /dev/null +++ b/test/unit/test_compact_imports.py @@ -0,0 +1,73 @@ +from scripts.test import shared + +from . import utils + + +class CompactImportsTest(utils.BinaryenTestCase): + def get_binary(self, wat_str, flags): + cmd = shared.WASM_OPT + ['-o', '-'] + flags + p = shared.run_process( + cmd, input=wat_str, check=True, capture_output=True, decode_output=False, + ) + return p.stdout + + def test_shared_all_encoding(self): + wat = '''(module + (type $sig (func (param i32) (result i32))) + (import "env" "f1" (func (type $sig))) + (import "env" "f2" (func (type $sig))) + )''' + wasm_bytes = self.get_binary(wat, ['--enable-compact-imports']) + # 0x7E is CompactImportsSharedAll + self.assertIn(b'\x03env\x00\x7e', wasm_bytes) + + def test_shared_module_encoding(self): + wat = '''(module + (type $sig1 (func (param i32) (result i32))) + (type $sig2 (func (param f64) (result f64))) + (import "env" "f1" (func (type $sig1))) + (import "env" "f2" (func (type $sig2))) + )''' + wasm_bytes = self.get_binary(wat, ['--enable-compact-imports']) + # 0x7F is CompactImportsSharedModule + self.assertIn(b'\x03env\x00\x7f', wasm_bytes) + + def test_disabled_compact_imports(self): + wat = '''(module + (type $sig (func (param i32) (result i32))) + (import "env" "f1" (func (type $sig))) + (import "env" "f2" (func (type $sig))) + )''' + wasm_bytes = self.get_binary(wat, ['--disable-compact-imports']) + self.assertNotIn(b'\x03env\x00\x7e', wasm_bytes) + self.assertNotIn(b'\x03env\x00\x7f', wasm_bytes) + self.assertIn(b'\x03env\x02f1', wasm_bytes) + self.assertIn(b'\x03env\x02f2', wasm_bytes) + + def test_mixed_import_patterns(self): + wat = '''(module + (type $sig1 (func (param i32) (result i32))) + (type $sig2 (func (param f64) (result f64))) + ;; Run 1: SharedAll for "env" + (import "env" "f1" (func (type $sig1))) + (import "env" "f2" (func (type $sig1))) + ;; Run 2: SharedModule for "math" (different types) + (import "math" "sin" (func (type $sig1))) + (import "math" "sqrt" (func (type $sig2))) + ;; Run 3: Single import for "single" + (import "single" "m1" (memory 1 2)) + )''' + wasm_bytes = self.get_binary(wat, ['--enable-compact-imports']) + self.assertIn(b'\x03env\x00\x7e', wasm_bytes) + self.assertIn(b'\x04math\x00\x7f', wasm_bytes) + self.assertIn(b'\x06single\x02m1', wasm_bytes) + + def test_identical_imports_size_reduction(self): + imports = '\n'.join(['(import "env" "f" (func (type $sig)))'] * 1000) + wat = f'''(module + (type $sig (func (param i32) (result i32))) + {imports} + )''' + with_compact = self.get_binary(wat, ['--enable-compact-imports']) + without_compact = self.get_binary(wat, ['--disable-compact-imports']) + self.assertTrue(len(without_compact) / len(with_compact) > 3.9)