Opportunisticly emit compact imports

scripts/clusterfuzz/run.py

@@ -33,7 +33,7 @@
 
 # The V8 flags we put in the "fuzzer flags" files, which tell ClusterFuzz how to
 # run V8. By default we apply all staging flags.
-FUZZER_FLAGS = '--wasm-staging --experimental-wasm-custom-descriptors --experimental-wasm-js-interop --experimental-wasm-acquire-release --experimental-wasm-wide-arithmetic'
+FUZZER_FLAGS = '--wasm-staging --experimental-wasm-custom-descriptors --experimental-wasm-js-interop --experimental-wasm-acquire-release --experimental-wasm-wide-arithmetic --wasm-compact-imports'
 
 # Optional V8 flags to add to FUZZER_FLAGS, some of the time.
 OPTIONAL_FUZZER_FLAGS = [

scripts/test/shared.py

@@ -268,6 +268,7 @@ def has_shell_timeout():
     '--experimental-wasm-js-interop',
     '--experimental-wasm-acquire-release',
     '--experimental-wasm-wide-arithmetic',
+    '--wasm-compact-imports',
 ]
 
 # external tools

src/ir/memory-utils.h

@@ -30,6 +30,10 @@ namespace wasm::MemoryUtils {
 
 bool isSubType(const Memory& a, const Memory& b);
 
+inline bool sameType(const Memory& a, const Memory& b) {
+  return isSubType(a, b) && isSubType(b, a);
+}
+
 // Flattens memory into a single data segment, or no segment. If there is
 // a segment, it starts at 0.
 // Returns true if successful (e.g. relocatable segments cannot be flattened).

src/ir/table-utils.h

@@ -25,6 +25,11 @@
 
 namespace wasm::TableUtils {
 
+inline bool sameType(const Table& a, const Table& b) {
+  return a.type == b.type && a.initial == b.initial && a.max == b.max &&
+         a.addressType == b.addressType;
+}
+
 struct FlatTable {
   std::vector<Name> names;
   bool valid;

src/wasm/wasm-binary.cpp

@@ -18,6 +18,7 @@
 #include <fstream>
 #include <optional>
 
+#include "ir/memory-utils.h"
 #include "ir/module-utils.h"
 #include "ir/names.h"
 #include "ir/table-utils.h"
@@ -26,6 +27,7 @@
 #include "support/bits.h"
 #include "support/stdckdint.h"
 #include "support/string.h"
+#include "support/utilities.h"
 #include "wasm-annotations.h"
 #include "wasm-binary.h"
 #include "wasm-debug.h"
@@ -337,51 +339,160 @@ void WasmBinaryWriter::writeImports() {
     return;
   }
   auto start = startSection(BinaryConsts::Section::Import);
-  o << U32LEB(num);
-  auto writeImportHeader = [&](Importable* import) {
-    writeInlineString(import->module.view());
-    writeInlineString(import->base.view());
+
+  using ImportItem = std::variant<Function*, Global*, Tag*, Memory*, Table*>;
+  std::vector<ImportItem> imports;
+  imports.reserve(num);
+
+  ModuleUtils::iterImports(*wasm,
+                           [&](ImportItem item) { imports.push_back(item); });
+
+  auto getModule = [](const ImportItem& item) -> Name {
+    return std::visit([](auto* i) { return i->module; }, item);
+  };
+  auto getBase = [](const ImportItem& item) -> Name {
+    return std::visit([](auto* i) { return i->base; }, item);
+  };
+
+  auto shareImportType = [&](const ImportItem& a, const ImportItem& b) -> bool {
+    return std::visit(
+      overloaded{
+        [](Function* a, Function* b) { return a->type == b->type; },
+        [](Global* a, Global* b) {
+          return a->type == b->type && a->mutable_ == b->mutable_;
+        },
+        [](Tag* a, Tag* b) { return a->type == b->type; },
+        [](Memory* a, Memory* b) { return MemoryUtils::sameType(*a, *b); },
+        [](Table* a, Table* b) { return TableUtils::sameType(*a, *b); },
+        [](const auto& a, const auto& b) { return false; }},
+      a,
+      b);
   };
-  ModuleUtils::iterImportedFunctions(*wasm, [&](Function* func) {
-    writeImportHeader(func);
-    uint32_t kind = ExternalKind::Function;
-    if (func->type.isExact()) {
-      kind |= BinaryConsts::ExactImport;
+
+  struct ImportGroup {
+    enum Kind { Single, SharedAll, SharedModule } kind;
+    size_t start;
+    size_t count;
+  };
+
+  std::vector<ImportGroup> groups;
+  if (wasm->features.hasCompactImports()) {
+    size_t i = 0;
+    size_t numImports = imports.size();
+    while (i < numImports) {
+      // If the next import shares the module and type, then greedily collect
+      // the following imports as long as they share both the module and type.
+      size_t run = 1;
+      while (i + run < numImports &&
+             getModule(imports[i]) == getModule(imports[i + run]) &&
+             shareImportType(imports[i], imports[i + run])) {
+        ++run;
+      }
+      if (run > 1) {
+        groups.push_back({ImportGroup::SharedAll, i, run});
+        i += run;
+        continue;
+      }
+      // Otherwise, try greedily collecting imports that share just the module.
+      while (i + run < numImports &&
+             getModule(imports[i]) == getModule(imports[i + run])) {
+        ++run;
+      }
+      if (run > 1) {
+        groups.push_back({ImportGroup::SharedModule, i, run});
+        i += run;
+        continue;
+      }
+      // Otherwise, just use a normal import.
+      groups.push_back({ImportGroup::Single, i, 1});
+      ++i;
     }
-    o << U32LEB(kind) << U32LEB(getTypeIndex(func->type.getHeapType()));
-  });
-  ModuleUtils::iterImportedGlobals(*wasm, [&](Global* global) {
-    writeImportHeader(global);
-    o << U32LEB(int32_t(ExternalKind::Global));
-    writeType(global->type);
-    o << U32LEB(global->mutable_);
-  });
-  ModuleUtils::iterImportedTags(*wasm, [&](Tag* tag) {
-    writeImportHeader(tag);
-    o << U32LEB(int32_t(ExternalKind::Tag));
-    o << uint8_t(0); // Reserved 'attribute' field. Always 0.
-    o << U32LEB(getTypeIndex(tag->type));
-  });
-  ModuleUtils::iterImportedMemories(*wasm, [&](Memory* memory) {
-    writeImportHeader(memory);
-    o << U32LEB(int32_t(ExternalKind::Memory));
-    writeResizableLimits(memory->initial,
-                         memory->max,
-                         memory->hasMax(),
-                         memory->shared,
-                         memory->is64(),
-                         memory->pageSizeLog2);
-  });
-  ModuleUtils::iterImportedTables(*wasm, [&](Table* table) {
-    writeImportHeader(table);
-    o << U32LEB(int32_t(ExternalKind::Table));
-    writeType(table->type);
-    writeResizableLimits(table->initial,
-                         table->max,
-                         table->hasMax(),
-                         /*shared=*/false,
-                         table->is64());
-  });
+  } else {
+    for (size_t i = 0; i < imports.size(); ++i) {
+      groups.push_back({ImportGroup::Single, i, 1});
+    }
+  }
+
+  o << U32LEB(groups.size());
+
+  auto writeImportDesc = [&](const ImportItem& item) {
+    std::visit(overloaded{[&](Function* func) {
+                            uint32_t kind = ExternalKind::Function;
+                            if (func->type.isExact()) {
+                              kind |= BinaryConsts::ExactImport;
+                            }
+                            o << U32LEB(kind)
+                              << U32LEB(getTypeIndex(func->type.getHeapType()));
+                          },
+                          [&](Global* global) {
+                            o << U32LEB(int32_t(ExternalKind::Global));
+                            writeType(global->type);
+                            o << U32LEB(global->mutable_);
+                          },
+                          [&](Tag* tag) {
+                            o << U32LEB(int32_t(ExternalKind::Tag));
+                            // Reserved 'attribute' field. Always 0.
+                            o << uint8_t(0);
+                            o << U32LEB(getTypeIndex(tag->type));
+                          },
+                          [&](Memory* memory) {
+                            o << U32LEB(int32_t(ExternalKind::Memory));
+                            writeResizableLimits(memory->initial,
+                                                 memory->max,
+                                                 memory->hasMax(),
+                                                 memory->shared,
+                                                 memory->is64(),
+                                                 memory->pageSizeLog2);
+                          },
+                          [&](Table* table) {
+                            o << U32LEB(int32_t(ExternalKind::Table));
+                            writeType(table->type);
+                            writeResizableLimits(table->initial,
+                                                 table->max,
+                                                 table->hasMax(),
+                                                 /*shared=*/false,
+                                                 table->is64());
+                          }},
+               item);
+  };
+
+  for (const auto& group : groups) {
+    switch (group.kind) {
+      case ImportGroup::Single: {
+        const auto& item = imports[group.start];
+        writeInlineString(getModule(item).view());
+        writeInlineString(getBase(item).view());
+        writeImportDesc(item);
+        break;
+      }
+      case ImportGroup::SharedAll: {
+        const auto& first = imports[group.start];
+        writeInlineString(getModule(first).view());
+        writeInlineString("");
+        o << uint8_t(BinaryConsts::CompactImportsSharedAll);
+        writeImportDesc(first);
+        o << U32LEB(group.count);
+        for (size_t i = 0; i < group.count; ++i) {
+          writeInlineString(getBase(imports[group.start + i]).view());
+        }
+        break;
+      }
+      case ImportGroup::SharedModule: {
+        const auto& first = imports[group.start];
+        writeInlineString(getModule(first).view());
+        writeInlineString("");
+        o << uint8_t(BinaryConsts::CompactImportsSharedModule);
+        o << U32LEB(group.count);
+        for (size_t i = 0; i < group.count; ++i) {
+          const auto& item = imports[group.start + i];
+          writeInlineString(getBase(item).view());
+          writeImportDesc(item);
+        }
+        break;
+      }
+    }
+  }
+
   finishSection(start);
 }
 

test/lit/d8/fuzz_shell_exceptions.wast

@@ -37,7 +37,7 @@
 
 ;; Build to a binary wasm.
 ;;
-;; RUN: wasm-opt %s -o %t.wasm -q -all
+;; RUN: wasm-opt %s -o %t.wasm -q --enable-reference-types --enable-exception-handling
 
 ;; Run in node.
 ;;

test/unit/test_compact_imports.py

@@ -0,0 +1,73 @@
+from scripts.test import shared
+
+from . import utils
+
+
+class CompactImportsTest(utils.BinaryenTestCase):
+    def get_binary(self, wat_str, flags):
+        cmd = shared.WASM_OPT + ['-o', '-'] + flags
+        p = shared.run_process(
+            cmd, input=wat_str, check=True, capture_output=True, decode_output=False,
+        )
+        return p.stdout
+
+    def test_shared_all_encoding(self):
+        wat = '''(module
+            (type $sig (func (param i32) (result i32)))
+            (import "env" "f1" (func (type $sig)))
+            (import "env" "f2" (func (type $sig)))
+        )'''
+        wasm_bytes = self.get_binary(wat, ['--enable-compact-imports'])
+        # 0x7E is CompactImportsSharedAll
+        self.assertIn(b'\x03env\x00\x7e', wasm_bytes)
+
+    def test_shared_module_encoding(self):
+        wat = '''(module
+            (type $sig1 (func (param i32) (result i32)))
+            (type $sig2 (func (param f64) (result f64)))
+            (import "env" "f1" (func (type $sig1)))
+            (import "env" "f2" (func (type $sig2)))
+        )'''
+        wasm_bytes = self.get_binary(wat, ['--enable-compact-imports'])
+        # 0x7F is CompactImportsSharedModule
+        self.assertIn(b'\x03env\x00\x7f', wasm_bytes)
+
+    def test_disabled_compact_imports(self):
+        wat = '''(module
+            (type $sig (func (param i32) (result i32)))
+            (import "env" "f1" (func (type $sig)))
+            (import "env" "f2" (func (type $sig)))
+        )'''
+        wasm_bytes = self.get_binary(wat, ['--disable-compact-imports'])
+        self.assertNotIn(b'\x03env\x00\x7e', wasm_bytes)
+        self.assertNotIn(b'\x03env\x00\x7f', wasm_bytes)
+        self.assertIn(b'\x03env\x02f1', wasm_bytes)
+        self.assertIn(b'\x03env\x02f2', wasm_bytes)
+
+    def test_mixed_import_patterns(self):
+        wat = '''(module
+            (type $sig1 (func (param i32) (result i32)))
+            (type $sig2 (func (param f64) (result f64)))
+            ;; Run 1: SharedAll for "env"
+            (import "env" "f1" (func (type $sig1)))
+            (import "env" "f2" (func (type $sig1)))
+            ;; Run 2: SharedModule for "math" (different types)
+            (import "math" "sin" (func (type $sig1)))
+            (import "math" "sqrt" (func (type $sig2)))
+            ;; Run 3: Single import for "single"
+            (import "single" "m1" (memory 1 2))
+        )'''
+        wasm_bytes = self.get_binary(wat, ['--enable-compact-imports'])
+        self.assertIn(b'\x03env\x00\x7e', wasm_bytes)
+        self.assertIn(b'\x04math\x00\x7f', wasm_bytes)
+        self.assertIn(b'\x06single\x02m1', wasm_bytes)
+
+    def test_identical_imports_size_reduction(self):
+        imports = '\n'.join(['(import "env" "f" (func (type $sig)))'] * 1000)
+        wat = f'''(module
+            (type $sig (func (param i32) (result i32)))
+            {imports}
+        )'''
+        with_compact = self.get_binary(wat, ['--enable-compact-imports'])
+        without_compact = self.get_binary(wat, ['--disable-compact-imports'])
+        self.assertTrue(len(without_compact) / len(with_compact) > 3.9)