Initial new INITIAL_HEAP setting

ChangeLog.md

@@ -49,7 +49,10 @@ See docs/process.md for more on how version tagging works.
   loaded on demand using the syntax `--use-port=/path/to/my_port.py` (#21316)
 - Allow comments in response files. Any line starting with `#` is now ignored.
   This is useful when listing exported symbols. (#21330)
-
+- `INITIAL_HEAP` setting is introduced to control the amount of initial
+  memory available for dynamic allocation without capping it. If you are
+  using `INITIAL_MEMORY`, consider switching to `INITIAL_HEAP`. Note that
+  it is currently not supported in all configurations (#21071).
 
 3.1.53 - 01/29/24
 -----------------

site/source/docs/tools_reference/settings_reference.rst

@@ -151,6 +151,19 @@ Note that this setting does not affect the behavior of operator new in C++.
 This function will always abort on allocation failure if exceptions are disabled.
 If you want new to return 0 on failure, use it with std::nothrow.
 
+.. _initial_heap:
+
+INITIAL_HEAP
+============
+
+The initial amount of heap memory available to the program.  This is the
+memory region available for dynamic allocations via `sbrk`, `malloc` and `new`.
+
+Unlike INITIAL_MEMORY, this setting allows the static and dynamic regions of
+your programs memory to independently grow. In most cases we recommend using
+this setting rather than `INITIAL_MEMORY`. However, this setting does not work
+for imported memories (e.g. when dynamic linking is used).
+
 .. _initial_memory:
 
 INITIAL_MEMORY
@@ -162,6 +175,9 @@ we need to copy the old heap into a new one in that case.
 If ALLOW_MEMORY_GROWTH is set, this initial amount of memory can increase
 later; if not, then it is the final and total amount of memory.
 
+By default, this value is calculated based on INITIAL_HEAP, STACK_SIZE,
+as well the size of static data in input modules.
+
 (This option was formerly called TOTAL_MEMORY.)
 
 .. _maximum_memory:

src/postamble_minimal.js

@@ -216,7 +216,6 @@ WebAssembly.instantiate(Module['wasm'], imports).then((output) => {
   wasmMemory = wasmExports['memory'];
 #if ASSERTIONS
   assert(wasmMemory);
-  assert(wasmMemory.buffer.byteLength === {{{ INITIAL_MEMORY }}});
 #endif
   updateMemoryViews();
 #endif

src/preamble.js

@@ -983,10 +983,6 @@ function createWasm() {
     {{{ receivedSymbol('wasmMemory') }}}
 #if ASSERTIONS
     assert(wasmMemory, 'memory not found in wasm exports');
-    // This assertion doesn't hold when emscripten is run in --post-link
-    // mode.
-    // TODO(sbc): Read INITIAL_MEMORY out of the wasm file in post-link mode.
-    //assert(wasmMemory.buffer.byteLength === {{{ INITIAL_MEMORY }}});
 #endif
     updateMemoryViews();
 #endif

src/settings.js

@@ -154,15 +154,29 @@ var MALLOC = "dlmalloc";
 // [link]
 var ABORTING_MALLOC = true;
 
+// The initial amount of heap memory available to the program.  This is the
+// memory region available for dynamic allocations via `sbrk`, `malloc` and `new`.
+//
+// Unlike INITIAL_MEMORY, this setting allows the static and dynamic regions of
+// your programs memory to independently grow. In most cases we recommend using
+// this setting rather than `INITIAL_MEMORY`. However, this setting does not work
+// for imported memories (e.g. when dynamic linking is used).
+//
+// [link]
+var INITIAL_HEAP = 16777216;
+
 // The initial amount of memory to use. Using more memory than this will
 // cause us to expand the heap, which can be costly with typed arrays:
 // we need to copy the old heap into a new one in that case.
 // If ALLOW_MEMORY_GROWTH is set, this initial amount of memory can increase
 // later; if not, then it is the final and total amount of memory.
 //
+// By default, this value is calculated based on INITIAL_HEAP, STACK_SIZE,
+// as well the size of static data in input modules.
+//
 // (This option was formerly called TOTAL_MEMORY.)
 // [link]
-var INITIAL_MEMORY = 16777216;
+var INITIAL_MEMORY = -1;
 
 // Set the maximum size of memory in the wasm module (in bytes). This is only
 // relevant when ALLOW_MEMORY_GROWTH is set, as without growth, the size of

test/code_size/embind_val_wasm.json

@@ -3,8 +3,8 @@
   "a.html.gz": 431,
   "a.js": 7080,
   "a.js.gz": 2999,
-  "a.wasm": 11433,
-  "a.wasm.gz": 5725,
-  "total": 19186,
-  "total_gz": 9155
+  "a.wasm": 11434,
+  "a.wasm.gz": 5726,
+  "total": 19187,
+  "total_gz": 9156
 }

test/code_size/hello_webgl2_wasm.json

@@ -3,8 +3,8 @@
   "a.html.gz": 379,
   "a.js": 4589,
   "a.js.gz": 2341,
-  "a.wasm": 10451,
-  "a.wasm.gz": 6724,
-  "total": 15609,
-  "total_gz": 9444
+  "a.wasm": 10452,
+  "a.wasm.gz": 6725,
+  "total": 15610,
+  "total_gz": 9445
 }

test/code_size/hello_webgl_wasm.json

@@ -3,8 +3,8 @@
   "a.html.gz": 379,
   "a.js": 4075,
   "a.js.gz": 2170,
-  "a.wasm": 10451,
-  "a.wasm.gz": 6724,
-  "total": 15095,
-  "total_gz": 9273
+  "a.wasm": 10452,
+  "a.wasm.gz": 6725,
+  "total": 15096,
+  "total_gz": 9274
 }

test/code_size/hello_world_wasm.json

@@ -3,8 +3,8 @@
   "a.html.gz": 431,
   "a.js": 291,
   "a.js.gz": 249,
-  "a.wasm": 103,
-  "a.wasm.gz": 112,
-  "total": 1067,
-  "total_gz": 792
+  "a.wasm": 104,
+  "a.wasm.gz": 114,
+  "total": 1068,
+  "total_gz": 794
 }

test/code_size/math_wasm.json

@@ -3,8 +3,8 @@
   "a.html.gz": 431,
   "a.js": 110,
   "a.js.gz": 125,
-  "a.wasm": 2724,
-  "a.wasm.gz": 1667,
-  "total": 3507,
-  "total_gz": 2223
+  "a.wasm": 2725,
+  "a.wasm.gz": 1669,
+  "total": 3508,
+  "total_gz": 2225
 }

test/test_core.py

@@ -2109,7 +2109,7 @@ def test_memorygrowth_geometric_step(self):
     if not self.is_wasm():
       self.skipTest('wasm memory specific test')
 
-    self.emcc_args += ['-sALLOW_MEMORY_GROWTH', '-sMEMORY_GROWTH_GEOMETRIC_STEP=8.5', '-sMEMORY_GROWTH_GEOMETRIC_CAP=32MB']
+    self.emcc_args += ['-sINITIAL_MEMORY=16MB', '-sALLOW_MEMORY_GROWTH', '-sMEMORY_GROWTH_GEOMETRIC_STEP=8.5', '-sMEMORY_GROWTH_GEOMETRIC_CAP=32MB']
     self.do_core_test('test_memorygrowth_geometric_step.c')
 
   def test_memorygrowth_3_force_fail_reallocBuffer(self):
@@ -6007,6 +6007,7 @@ def test_unistd_sysconf_phys_pages(self):
         assert self.get_setting('INITIAL_MEMORY') == '2200mb'
         expected = (2200 * 1024 * 1024) // webassembly.WASM_PAGE_SIZE
     else:
+      self.set_setting('INITIAL_MEMORY', '16mb')
       expected = 16 * 1024 * 1024 // webassembly.WASM_PAGE_SIZE
     self.do_runf(filename, str(expected) + ', errno: 0')
 

test/test_other.py

@@ -6538,7 +6538,7 @@ def test_massive_alloc(self, wasm):
   return x == 0; // can't alloc it, but don't fail catastrophically, expect null
 }
     ''')
-    cmd = [EMCC, 'main.c', '-sALLOW_MEMORY_GROWTH']
+    cmd = [EMCC, 'main.c', '-sALLOW_MEMORY_GROWTH', '-sINITIAL_MEMORY=16MB']
     if not wasm:
       cmd += ['-sWASM=0']
     self.run_process(cmd)
@@ -6597,7 +6597,7 @@ def test_failing_alloc(self):
   printf("managed another malloc!\n");
 }
 ''' % (pre_fail, post_fail))
-          args = [EMXX, 'main.cpp', '-sEXPORTED_FUNCTIONS=_main,_sbrk'] + opts + aborting_args
+          args = [EMXX, 'main.cpp', '-sEXPORTED_FUNCTIONS=_main,_sbrk', '-sINITIAL_MEMORY=16MB'] + opts + aborting_args
           args += ['-sTEST_MEMORY_GROWTH_FAILS'] # In this test, force memory growing to fail
           if growth:
             args += ['-sALLOW_MEMORY_GROWTH']
@@ -8132,6 +8132,30 @@ def test_binaryen_warn_mem(self):
     self.run_process([EMCC, test_file('hello_world.c'), '-sINITIAL_MEMORY=' + str(16 * 1024 * 1024), '--pre-js', 'pre.js', '-sALLOW_MEMORY_GROWTH', '-sWASM_ASYNC_COMPILATION=0', '-sIMPORTED_MEMORY'])
     self.assertContained('hello, world!', self.run_js('a.out.js'))
 
+  @parameterized({
+    '': ([], 16 * 1024 * 1024), # Default behavior: 16MB initial heap
+    'with_initial_memory': (['-sINITIAL_MEMORY=40MB'], 0), # Backwards compatibility: no initial heap (we can't tell if it'll fit)
+    'with_maximum_memory': (['-sMAXIMUM_MEMORY=40MB'], 0), # Backwards compatibility: no initial heap (we can't tell if it'll fit)
+    'with_initial_heap': (['-sINITIAL_HEAP=64KB'], 64 * 1024), # Explicitly set initial heap is passed
+    'with_all': (['-sINITIAL_HEAP=128KB', '-sINITIAL_MEMORY=20MB', '-sMAXIMUM_MEMORY=40MB'], 128 * 1024),
+    'limited_by_initial_memory': (['-sINITIAL_HEAP=10MB', '-sINITIAL_MEMORY=10MB'], -1), # Not enough space for stack
+    'limited_by_maximum_memory': (['-sINITIAL_HEAP=5MB', '-sMAXIMUM_MEMORY=5MB'], -1), # Not enough space for stack
+  })
+  def test_initial_heap(self, args, expected_initial_heap):
+    cmd = [EMCC, test_file('hello_world.c'), '-v'] + args
+    print(' '.join(cmd))
+
+    if expected_initial_heap < 0:
+      out = self.expect_fail(cmd)
+      self.assertContained('wasm-ld: error:', out)
+      return
+
+    out = self.run_process(cmd, stderr=PIPE)
+    if expected_initial_heap != 0:
+      self.assertContained('--initial-heap=' + str(expected_initial_heap), out.stderr)
+    else:
+      self.assertNotContained('--initial-heap=', out.stderr)
+
   def test_memory_size(self):
     for args, expect_initial, expect_max in [
         ([], 320, 320),
@@ -8161,6 +8185,9 @@ def test_invalid_mem(self):
     self.assertContained('hello, world!', self.run_js('a.out.js'))
 
     # Must be a multiple of 64KB
+    ret = self.expect_fail([EMCC, test_file('hello_world.c'), '-sINITIAL_HEAP=32505857', '-sALLOW_MEMORY_GROWTH']) # 31MB + 1 byte
+    self.assertContained('INITIAL_HEAP must be a multiple of WebAssembly page size (64KiB)', ret)
+
     ret = self.expect_fail([EMCC, test_file('hello_world.c'), '-sINITIAL_MEMORY=33554433']) # 32MB + 1 byte
     self.assertContained('INITIAL_MEMORY must be a multiple of WebAssembly page size (64KiB)', ret)
 
@@ -8611,7 +8638,7 @@ def run(args, expected):
       result = self.run_js('a.out.js').strip()
       self.assertEqual(result, f'{expected}, errno: 0')
 
-    run([], 256)
+    run([], 258)
     run(['-sINITIAL_MEMORY=32MB'], 512)
     run(['-sINITIAL_MEMORY=32MB', '-sALLOW_MEMORY_GROWTH'], (2 * 1024 * 1024 * 1024) // webassembly.WASM_PAGE_SIZE)
     run(['-sINITIAL_MEMORY=32MB', '-sALLOW_MEMORY_GROWTH', '-sWASM=0'], (2 * 1024 * 1024 * 1024) // webassembly.WASM_PAGE_SIZE)

tools/building.py

@@ -220,13 +220,13 @@ def lld_flags_for_executable(external_symbols):
       cmd.append('--growable-table')
 
   if not settings.SIDE_MODULE:
-    # Export these two section start symbols so that we can extract the string
-    # data that they contain.
-    cmd += [
-      '-z', 'stack-size=%s' % settings.STACK_SIZE,
-      '--initial-memory=%d' % settings.INITIAL_MEMORY,
-      '--max-memory=%d' % settings.MAXIMUM_MEMORY,
-    ]
+    cmd += ['-z', 'stack-size=%s' % settings.STACK_SIZE]
+    cmd += ['--max-memory=%d' % settings.MAXIMUM_MEMORY]
+
+    if settings.INITIAL_HEAP != -1:
+      cmd += ['--initial-heap=%d' % settings.INITIAL_HEAP]
+    if settings.INITIAL_MEMORY != -1:
+      cmd += ['--initial-memory=%d' % settings.INITIAL_MEMORY]
 
     if settings.STANDALONE_WASM:
       # when settings.EXPECT_MAIN is set we fall back to wasm-ld default of _start