Stop including malloc/free by default (#12213)

Before this we would always include malloc/free, and rely on metadce to remove them
when not needed. But this has downsides:

1. Slower compile times.
2. Larger builds when not using metadce.
3. It prevented us from writing a nice sbrk using __heap_base, as
if malloc is included first and removed by metadce that would still
leave some data in the memory initialization, increasing code size
without the workaround we had in sbrk to avoid it.

Much of the work here is to get deps_info.json to handle things properly and
include malloc/free when needed. I did this twice to double-check, but it is still
someone error-prone, as we need each code path that reached a malloc from
JS to be identified properly.

Some places use malloc/free in locations that are not part of the JS deps
system, and those now use makeMallocAbort. In those places we know
we should not reach them if malloc is not included, so we just have an
abort there. That would catch errors with deps_info.json.

In assertions builds, also help users by adding malloc/free in JS that show
a clear error if called, if malloc/free was not included but the user called
them anyhow.

With this PR we can remove the hack for sbrk, but I wanted to keep that
for a later PR as this is already not small.

ChangeLog.md

@@ -17,6 +17,9 @@ See docs/process.md for how version tagging works.
 
 Current Trunk
 -------------
+- Stop including `malloc` and `free` by default. If you need access to them from
+  JS, you must export them manually using
+  `-s EXPORTED_FUNCTIONS=['_malloc', ..]`.
 - Stop running Binaryen optimizations in `-O1`. This makes `-O1` builds a little
   larger but they compile a lot faster, which makes more sense in a "compromise"
   build (in between `-O0` and higher optimization levels suitable for release

emcc.py

@@ -1411,12 +1411,6 @@ def check(input_file):
     if shared.Settings.EMBIND:
       forced_stdlibs.append('libembind')
 
-    if not shared.Settings.MINIMAL_RUNTIME and not shared.Settings.STANDALONE_WASM:
-      # The normal JS runtime depends on malloc and free so always keep them alive.
-      # MINIMAL_RUNTIME avoids this dependency as does STANDALONE_WASM mode (since it has no
-      # JS runtime at all).
-      shared.Settings.EXPORTED_FUNCTIONS += ['_malloc', '_free']
-
     shared.Settings.EXPORTED_FUNCTIONS += ['_stackSave', '_stackRestore', '_stackAlloc']
     # We need to preserve the __data_end symbol so that wasm-emscripten-finalize can determine
     # where static data ends (and correspondingly where the stack begins).
@@ -1545,7 +1539,7 @@ def check(input_file):
 
     if shared.Settings.USE_PTHREADS:
       # memalign is used to ensure allocated thread stacks are aligned.
-      shared.Settings.EXPORTED_FUNCTIONS += ['_memalign', '_malloc']
+      shared.Settings.EXPORTED_FUNCTIONS += ['_memalign']
 
       # dynCall is used to call pthread entry points in worker.js (as
       # metadce does not consider worker.js, which is external, we must
@@ -1767,6 +1761,26 @@ def include_and_export(name):
     if sanitize and '-g4' in args:
       shared.Settings.LOAD_SOURCE_MAP = 1
 
+    # various settings require malloc/free support from JS
+    if shared.Settings.RELOCATABLE or \
+       shared.Settings.BUILD_AS_WORKER or \
+       shared.Settings.USE_WEBGPU or \
+       shared.Settings.USE_PTHREADS or \
+       shared.Settings.OFFSCREENCANVAS_SUPPORT or \
+       shared.Settings.LEGACY_GL_EMULATION or \
+       shared.Settings.DISABLE_EXCEPTION_CATCHING != 1 or \
+       shared.Settings.ASYNCIFY or \
+       shared.Settings.ASMFS or \
+       shared.Settings.DEMANGLE_SUPPORT or \
+       shared.Settings.FORCE_FILESYSTEM or \
+       shared.Settings.STB_IMAGE or \
+       shared.Settings.EMBIND or \
+       shared.Settings.FETCH or \
+       shared.Settings.PROXY_POSIX_SOCKETS or \
+       options.memory_profiler or \
+       sanitize:
+      shared.Settings.EXPORTED_FUNCTIONS += ['_malloc', '_free']
+
     options.binaryen_passes += backend_binaryen_passes()
 
     if shared.Settings.WASM2JS and use_source_map(options):

emscripten.py

@@ -249,10 +249,6 @@ def report_missing_symbols(all_implemented, pre):
   for requested in missing:
     if ('function ' + asstr(requested)) in pre:
       continue
-    # special-case malloc, EXPORTED by default for internal use, but we bake in a
-    # trivial allocator and warn at runtime if used in ASSERTIONS
-    if missing == '_malloc':
-      continue
     diagnostics.warning('undefined', 'undefined exported function: "%s"', requested)
 
   # Special hanlding for the `_main` symbol

src/deps_info.json

@@ -4,40 +4,50 @@
   "getenv": ["malloc", "free"],
   "SDL_getenv": ["malloc", "free"],
   "dlerror": ["malloc", "free"],
-  "readdir": ["malloc"],
-  "ttyname": ["malloc"],
-  "calloc": ["malloc"],
+  "dladdr": ["malloc", "free"],
+  "readdir": ["malloc", "free"],
+  "ttyname": ["malloc", "free"],
+  "calloc": ["malloc", "free"],
   "realloc": ["malloc", "free"],
-  "getlogin": ["malloc"],
-  "tmpnam": ["malloc"],
-  "mmap": ["memalign", "memset"],
-  "realpath": ["malloc"],
-  "strerror": ["malloc"],
-  "__ctype_b_loc": ["malloc"],
-  "__ctype_tolower_loc": ["malloc"],
-  "__ctype_toupper_loc": ["malloc"],
-  "newlocale": ["malloc"],
+  "getlogin": ["malloc", "free"],
+  "tmpnam": ["malloc", "free"],
+  "mmap": ["memalign", "memset", "malloc", "free"],
+  "munmap": ["memalign", "memset", "malloc", "free"],
+  "realpath": ["malloc", "free"],
+  "strerror": ["malloc", "free"],
+  "__ctype_b_loc": ["malloc", "free"],
+  "__ctype_tolower_loc": ["malloc", "free"],
+  "__ctype_toupper_loc": ["malloc", "free"],
+  "newlocale": ["malloc", "free"],
   "freelocale": ["free"],
-  "nl_langinfo": ["malloc"],
-  "inet_ntoa": ["malloc"],
+  "nl_langinfo": ["malloc", "free"],
+  "inet_ntoa": ["malloc", "free"],
   "gethostbyname": ["malloc", "free", "htons", "ntohs", "memcpy"],
   "gethostbyname_r": ["malloc", "free", "htons", "ntohs", "memcpy"],
-  "getaddrinfo": ["malloc", "htonl", "htons", "ntohs"],
+  "getaddrinfo": ["malloc", "free", "htonl", "htons", "ntohs"],
   "getnameinfo": ["htons", "ntohs"],
   "getpeername": ["htons", "ntohs"],
   "_inet_ntop6_raw": ["ntohs"],
   "_read_sockaddr": ["ntohs"],
   "freeaddrinfo": ["free"],
-  "gai_strerror": ["malloc"],
-  "setprotoent": ["malloc"],
+  "gai_strerror": ["malloc", "free"],
+  "setprotoent": ["malloc", "free"],
   "emscripten_run_script_string": ["malloc", "free"],
   "emscripten_log": ["strlen"],
   "uuid_clear": ["memset"],
   "uuid_compare": ["memcmp", "memcpy", "memset"],
   "uuid_copy": ["memcpy"],
   "SDL_Init": ["malloc", "free", "memset", "memcpy"],
+  "SDL_PushEvent": ["malloc", "free"],
+  "SDL_OpenAudio": ["malloc", "free"],
+  "SDL_LockSurface": ["malloc", "free"],
   "SDL_GL_GetProcAddress": ["emscripten_GetProcAddress"],
+  "SDL_CreateRGBSurface": ["malloc", "free"],
+  "SDL_malloc": ["malloc", "free"],
+  "SDL_free": ["free"],
+  "emscripten_SDL_SetEventHandler": ["malloc", "free"],
   "Mix_LoadWAV_RW": ["fileno"],
+  "eglQueryString": ["malloc", "free"],
   "eglGetProcAddress": ["emscripten_GetProcAddress"],
   "glfwGetProcAddress": ["emscripten_GetProcAddress"],
   "emscripten_GetProcAddress": ["strstr"],
@@ -46,6 +56,9 @@
   "emscripten_GetAlcProcAddress": ["strcmp"],
   "emscripten_GetAlProcAddress": ["strcmp"],
   "emscripten_get_preloaded_image_data_from_FILE": ["fileno"],
+  "alGetString": ["malloc", "free"],
+  "alcGetString": ["malloc", "free"],
+  "emscripten_alcGetStringiSOFT": ["malloc", "free"],
   "__gxx_personality_v0": ["_ZSt18uncaught_exceptionv", "__cxa_find_matching_catch"],
   "__cxa_find_matching_catch": ["__cxa_is_pointer_type", "__cxa_can_catch"],
   "__cxa_find_matching_catch_0": ["__cxa_is_pointer_type", "__cxa_can_catch"],
@@ -60,36 +73,43 @@
   "__cxa_find_matching_catch_9": ["__cxa_is_pointer_type", "__cxa_can_catch"],
   "__cxa_begin_catch": ["_ZSt18uncaught_exceptionv", "setThrew"],
   "__cxa_end_catch": ["free"],
-  "__cxa_allocate_exception": ["malloc", "setThrew"],
+  "__cxa_allocate_exception": ["malloc", "free", "setThrew"],
   "__cxa_free_exception": ["free"],
   "__cxa_throw": ["setThrew"],
   "formatString": ["strlen"],
+  "glfwInit": ["malloc", "free"],
   "glfwSleep": ["sleep"],
   "glBegin": ["malloc", "free"],
+  "glewInit": ["malloc", "free"],
   "bind": ["htonl", "htons", "ntohs"],
   "connect": ["htonl", "htons", "ntohs"],
   "socket": ["htonl", "htons", "ntohs"],
   "socketpair": ["htons", "ntohs"],
   "sleep": ["usleep"],
   "recv": ["htons", "ntohs"],
   "send": ["htons", "ntohs"],
-  "ctime": ["_get_tzname", "_get_daylight", "_get_timezone", "malloc"],
-  "ctime_r": ["_get_tzname", "_get_daylight", "_get_timezone", "malloc"],
-  "localtime": ["_get_tzname", "_get_daylight", "_get_timezone", "malloc"],
-  "localtime_r": ["_get_tzname", "_get_daylight", "_get_timezone", "malloc"],
-  "mktime": ["_get_tzname", "_get_daylight", "_get_timezone", "malloc"],
-  "timegm": ["_get_tzname", "_get_daylight", "_get_timezone", "malloc"],
-  "tzset": ["_get_tzname", "_get_daylight", "_get_timezone", "malloc"],
+  "ctime": ["_get_tzname", "_get_daylight", "_get_timezone", "malloc", "free"],
+  "ctime_r": ["_get_tzname", "_get_daylight", "_get_timezone", "malloc", "free"],
+  "gmtime":  ["malloc", "free"],
+  "gmtime_r":  ["malloc", "free"],
+  "localtime": ["_get_tzname", "_get_daylight", "_get_timezone", "malloc", "free"],
+  "localtime_r": ["_get_tzname", "_get_daylight", "_get_timezone", "malloc", "free"],
+  "mktime": ["_get_tzname", "_get_daylight", "_get_timezone", "malloc", "free"],
+  "timegm": ["_get_tzname", "_get_daylight", "_get_timezone", "malloc", "free"],
+  "tzset": ["_get_tzname", "_get_daylight", "_get_timezone", "malloc", "free"],
   "times": ["memset"],
+  "emscripten_pc_get_function": ["malloc", "free"],
+  "emscripten_pc_get_file": ["malloc", "free"],
   "emscripten_set_canvas_element_size_calling_thread": ["_emscripten_call_on_thread"],
-  "emscripten_set_offscreencanvas_size_on_target_thread": ["_emscripten_call_on_thread"],
-  "emscripten_websocket_new": ["malloc"],
-  "emscripten_wget_data": ["malloc"],
+  "emscripten_set_offscreencanvas_size_on_target_thread": ["_emscripten_call_on_thread", "malloc", "free"],
+  "emscripten_set_offscreencanvas_size_on_target_thread_js": ["malloc", "free"],
+  "emscripten_websocket_new": ["malloc", "free"],
+  "emscripten_wget_data": ["malloc", "free"],
   "emscripten_webgl_destroy_context": ["emscripten_webgl_make_context_current", "emscripten_webgl_get_current_context"],
   "emscripten_idb_async_load": ["malloc", "free"],
   "emscripten_idb_load": ["malloc", "free"],
   "wgpuDeviceCreateBuffer": ["malloc", "free"],
-  "stringToNewUTF8": ["malloc"],
+  "stringToNewUTF8": ["malloc", "free"],
   "_embind_register_std_string": ["malloc", "free"],
   "_embind_register_std_wstring": ["malloc", "free"],
   "pthread_create": ["malloc", "free", "emscripten_main_thread_process_queued_calls"],
@@ -103,5 +123,64 @@
   "setjmp": ["setThrew", "realloc", "testSetjmp", "saveSetjmp"],
   "longjmp": ["setThrew", "realloc", "testSetjmp", "saveSetjmp"],
   "siglongjmp": ["setThrew", "realloc", "testSetjmp", "saveSetjmp"],
-  "emscripten_longjmp_jmpbuf": ["setThrew", "realloc", "testSetjmp", "saveSetjmp"]
+  "emscripten_longjmp_jmpbuf": ["setThrew", "realloc", "testSetjmp", "saveSetjmp"],
+  "emscripten_websocket_new": ["malloc", "free"],
+  "emscripten_websocket_set_onmessage_callback_on_thread": ["malloc", "free"],
+  "emscripten_websocket_set_onclose_callback_on_thread": ["malloc", "free"],
+  "emscripten_websocket_set_onerror_callback_on_thread": ["malloc", "free"],
+  "emscripten_websocket_set_onopen_callback_on_thread": ["malloc", "free"],
+  "emscripten_init_websocket_to_posix_socket_bridge": ["malloc", "free"],
+  "emscripten_set_keypress_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_keyup_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_keydown_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_click_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_mousedown_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_mouseup_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_dblclick_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_mousemove_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_mousemove_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_mouseenter_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_mouseleave_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_mouseover_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_mouseout_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_wheel_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_resize_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_scroll_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_blur_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_focus_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_focusin_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_focusout_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_deviceorientation_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_orientationchange_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_fullscreenchange_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_pointerlockchange_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_pointerlockerror_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_visibilitychange_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_touchstart_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_touchend_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_touchmove_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_touchcancel_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_gamepadconnected_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_gamepaddisconnected_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_beforeunload_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_batterychargingchange_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_batterylevelchange_callback_on_thread": ["malloc", "free"],
+  "emscripten_set_devicemotion_callback_on_thread": ["malloc", "free"],
+  "emscripten_run_preload_plugins_data": ["malloc", "free"],
+  "emscripten_async_wget_data": ["malloc", "free"],
+  "emscripten_async_wget2_data": ["malloc", "free"],
+  "emscripten_get_window_title": ["malloc", "free"],
+  "emscripten_get_compiler_setting": ["malloc", "free"],
+  "emscripten_create_worker": ["malloc", "free"],
+  "emscripten_get_preloaded_image_data": ["malloc", "free"],
+  "emscripten_webgl_get_supported_extensions": ["malloc", "free"],
+  "emscripten_webgl_get_program_info_log_utf8": ["malloc", "free"],
+  "emscripten_webgl_get_shader_info_log_utf8": ["malloc", "free"],
+  "emscripten_webgl_get_shader_source_utf8": ["malloc", "free"],
+  "emscripten_webgl_get_parameter_utf8": ["malloc", "free"],
+  "emscripten_webgl_create_context": ["malloc", "free"],
+  "glMapBufferRange": ["malloc", "free"],
+  "glGetString": ["malloc", "free"],
+  "glGetStringi": ["malloc", "free"],
+  "syslog": ["malloc", "free"]
 }

src/library_fs.js

@@ -2026,7 +2026,7 @@ FS.staticInit();` +
     // page-aligned size, and clears the padding.
     mmapAlloc: function(size) {
       var alignedSize = alignMemory(size, {{{ POSIX_PAGE_SIZE }}});
-      var ptr = _malloc(alignedSize);
+      var ptr = {{{ makeMalloc('mmapAlloc', 'alignedSize') }}};
       while (size < alignedSize) HEAP8[ptr + size++] = 0;
       return ptr;
     }

src/library_sdl.js

@@ -1846,10 +1846,14 @@ var LibrarySDL = {
   SDL_SetError: function() {},
 
   SDL_malloc__sig: 'ii',
-  SDL_malloc: 'malloc',
+  SDL_malloc: function(size) {
+    return _malloc(size);
+  },
 
   SDL_free__sig: 'vi',
-  SDL_free: 'free',
+  SDL_free: function(ptr) {
+    _free(ptr);
+  },
 
   SDL_CreateRGBSurface__proxy: 'sync',
   SDL_CreateRGBSurface__sig: 'iiiiiiiii',
@@ -2653,7 +2657,7 @@ var LibrarySDL = {
     SDL.audio.paused = pauseOn;
   },
 
-  SDL_CloseAudio__deps: ['SDL_PauseAudio', 'free'],
+  SDL_CloseAudio__deps: ['SDL_PauseAudio'],
   SDL_CloseAudio__proxy: 'sync',
   SDL_CloseAudio__sig: 'v',
   SDL_CloseAudio: function() {

src/parseTools.js

@@ -1707,3 +1707,18 @@ function addReadyPromiseAssertions(promise) {
     `;
   }).join('\n');
 }
+
+function makeMalloc(source, param) {
+  if ('_malloc' in IMPLEMENTED_FUNCTIONS) {
+    return '_malloc(' + param + ')';
+  }
+  // It should be impossible to call some functions without malloc being
+  // included, unless we have a deps_info.json bug. To let closure not error
+  // on `_malloc` not being present, they don't call malloc and instead abort
+  // with an error at runtime.
+  // TODO: A more comprehensive deps system could catch this at compile time.
+  if (!ASSERTIONS) {
+    return "abort();";
+  }
+  return `abort('malloc was not included, but is needed in ${source}. Adding "_malloc" to EXPORTED_FUNCTIONS should fix that. This may be a bug in the compiler, please file an issue.');`
+}

src/preamble.js

@@ -187,6 +187,22 @@ function cwrap(ident, returnType, argTypes, opts) {
   }
 }
 
+#if ASSERTIONS
+// We used to include malloc/free by default in the past. Show a helpful error in
+// builds with assertions.
+#if !('_malloc' in IMPLEMENTED_FUNCTIONS)
+function _malloc() {
+  abort("malloc() called but not included in the build - add '_malloc' to EXPORTED_FUNCTIONS");
+}
+#endif // malloc
+#if !('_free' in IMPLEMENTED_FUNCTIONS)
+function _free() {
+  // Show a helpful error since we used to include free by default in the past.
+  abort("free() called but not included in the build - add '_free' to EXPORTED_FUNCTIONS");
+}
+#endif // free
+#endif // ASSERTIONS
+
 var ALLOC_NORMAL = 0; // Tries to use _malloc()
 var ALLOC_STACK = 1; // Lives for the duration of the current function call
 var ALLOC_NONE = 2; // Do not allocate
@@ -221,7 +237,7 @@ function allocate(slab, types, allocator, ptr) {
   if (allocator == ALLOC_NONE) {
     ret = ptr;
   } else {
-    ret = [_malloc,
+    ret = [{{{ ('_malloc' in IMPLEMENTED_FUNCTIONS) ? '_malloc' : 'null' }}},
 #if DECLARE_ASM_MODULE_EXPORTS
     stackAlloc,
 #else

src/runtime_strings_extra.js

@@ -209,7 +209,7 @@ function lengthBytesUTF32(str) {
 // It is the responsibility of the caller to free() that memory.
 function allocateUTF8(str) {
   var size = lengthBytesUTF8(str) + 1;
-  var ret = _malloc(size);
+  var ret = {{{ makeMalloc('allocateUTF8', 'size') }}};
   if (ret) stringToUTF8Array(str, HEAP8, ret, size);
   return ret;
 }

tests/code_size/hello_webgl2_wasm.json

@@ -4,7 +4,7 @@
   "a.js": 5035,
   "a.js.gz": 2395,
   "a.wasm": 10910,
-  "a.wasm.gz": 6930,
+  "a.wasm.gz": 6928,
   "total": 16508,
-  "total_gz": 9702
+  "total_gz": 9700
 }

tests/code_size/hello_webgl_wasm.json

@@ -4,7 +4,7 @@
   "a.js": 4519,
   "a.js.gz": 2219,
   "a.wasm": 10910,
-  "a.wasm.gz": 6930,
+  "a.wasm.gz": 6928,
   "total": 15992,
-  "total_gz": 9526
+  "total_gz": 9524
 }

tests/code_size/hello_webgl_wasm2js.json

@@ -2,9 +2,9 @@
   "a.html": 588,
   "a.html.gz": 386,
   "a.js": 21617,
-  "a.js.gz": 8296,
+  "a.js.gz": 8297,
   "a.mem": 3168,
   "a.mem.gz": 2711,
   "total": 25373,
-  "total_gz": 11393
+  "total_gz": 11394
 }