Respect the WASM_MEM_MAX limit in memory growth (#6937)

If the user specified a limit, it is set in the wasm binary, and we can't (and shouldn't) grow past it.

Also the docs in settings.js about this were wrong.

See #6042

src/preamble.js

@@ -1027,6 +1027,25 @@ function enlargeMemory() {
     }
   }
 
+#if WASM_MEM_MAX != -1
+  // A limit was set for how much we can grow. We should not exceed that
+  // (the wasm binary specifies it, so if we tried, we'd fail anyhow). That is,
+  // if we are at say 64MB, and the max is 100MB, then we should *not* try to
+  // grow 64->128MB which is the default behavior (doubling), as 128MB will
+  // fail because of the max limit. Instead, we should only try to grow
+  // 64->100MB in this example, which has a chance of succeeding (but may
+  // still fail for another reason, of actually running out of memory).
+  TOTAL_MEMORY = Math.min(TOTAL_MEMORY, {{{ WASM_MEM_MAX }}});
+  if (TOTAL_MEMORY == OLD_TOTAL_MEMORY) {
+#if ASSERTIONS
+    err('Failed to grow the heap from ' + OLD_TOTAL_MEMORY + ', as we reached the WASM_MEM_MAX limit (' + {{{ WASM_MEM_MAX }}} + ') set during compilation');
+#endif
+    // restore the state to before this call, we failed
+    TOTAL_MEMORY = OLD_TOTAL_MEMORY;
+    return false;
+  }
+#endif
+
 #if ASSERTIONS
   var start = Date.now();
 #endif

src/settings.js

@@ -766,7 +766,9 @@ var BINARYEN_PASSES = ""; // A comma-separated list of passes to run in the bina
                           // When set, this overrides the default passes we would normally run.
 var WASM_MEM_MAX = -1; // Set the maximum size of memory in the wasm module (in bytes).
                        // Without this, TOTAL_MEMORY is used (as it is used for the initial value),
-                       // or if memory growth is enabled, no limit is set. This overrides both of those.
+                       // or if memory growth is enabled, the default value here (-1) is to have
+                       // no limit, but you can set this to set a maximum size that growth will
+                       // stop at.
                        // (This option was formerly called BINARYEN_MEM_MAX)
 var BINARYEN_ASYNC_COMPILATION = 1; // Whether to compile the wasm asynchronously, which is more
                                     // efficient and does not block the main thread. This is currently

tests/core/test_memorygrowth_wasm_mem_max.c

@@ -0,0 +1,22 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+
+int main() {
+  const int MB = 1024 * 1024;
+  // TOTAL_MEMORY starts at 64MB, and max is 100. allocate enough
+  // to prove we can grow. 70 is enough to prove we can grow,
+  // higher can prove we stop at the right point.
+  for (int i = 0; 1; i++) {
+    printf("%d\n", i);
+    volatile int sink = (int)malloc(MB);
+    if (!sink) {
+      printf("failed at %d\n", i);
+      assert(i > 70);
+      break;
+    }
+    assert(i <= 100); // the wasm mem max limit, we must not get there
+  }
+  printf("grew memory ok.\n");
+}
+

tests/core/test_memorygrowth_wasm_mem_max.txt

@@ -0,0 +1 @@
+grew memory ok.

tests/test_core.py

@@ -1828,6 +1828,13 @@ def test_memorygrowth_3(self):
     self.emcc_args += ['-s', 'ALLOW_MEMORY_GROWTH=0', '-s', 'ABORTING_MALLOC=0', '-s', 'SAFE_HEAP=1']
     self.do_run_in_out_file_test('tests', 'core', 'test_memorygrowth_3')
 
+  def test_memorygrowth_wasm_mem_max(self):
+    if not self.is_wasm():
+      self.skipTest('wasm memory specific test')
+    # check that memory growth does not exceed the wasm mem max limit
+    self.emcc_args += ['-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'TOTAL_MEMORY=64Mb', '-s', 'WASM_MEM_MAX=100Mb']
+    self.do_run_in_out_file_test('tests', 'core', 'test_memorygrowth_wasm_mem_max')
+
   def test_memorygrowth_3_force_fail_reallocBuffer(self):
     self.emcc_args += ['-s', 'ALLOW_MEMORY_GROWTH=1', '-DFAIL_REALLOC_BUFFER']
     self.do_run_in_out_file_test('tests', 'core', 'test_memorygrowth_3')