Link twice with EXPORTED_FUNCTIONS='@exported_list.txt'

.github/workflows/main.yml

@@ -494,6 +494,10 @@ jobs:
                   submodules: 'recursive'
                   fetch-depth: 0
 
+            - name: Install dependencies
+              run: |
+                  sudo apt install wabt
+
             - name: Prepare repository
               run: |
                   make apply_patches
@@ -514,7 +518,7 @@ jobs:
 
             - name: Build Wasm module
               run: |
-                  DUCKDB_PLATFORM="wasm_mvp" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize mvp
+                  USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_mvp" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize mvp
 
             - name: Upload artifact
               uses: actions/upload-artifact@v4
@@ -537,6 +541,10 @@ jobs:
                   submodules: 'recursive'
                   fetch-depth: 0
 
+            - name: Install dependencies
+              run: |
+                  sudo apt install wabt
+
             - name: Prepare repository
               run: |
                   make apply_patches
@@ -557,7 +565,7 @@ jobs:
 
             - name: Build Wasm module
               run: |
-                  DUCKDB_PLATFORM="wasm_eh" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize eh
+                  USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_eh" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize eh
 
             - name: Upload artifact
               uses: actions/upload-artifact@v4
@@ -580,6 +588,10 @@ jobs:
                   submodules: 'recursive'
                   fetch-depth: 0
 
+            - name: Install dependencies
+              run: |
+                  sudo apt install wabt
+
             - name: Prepare repository
               run: |
                   make apply_patches
@@ -600,7 +612,7 @@ jobs:
 
             - name: Build Wasm module
               run: |
-                  DUCKDB_PLATFORM="wasm_threads" DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize coi
+                  USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_threads" DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize coi
 
             - name: Upload artifact
               uses: actions/upload-artifact@v4

Makefile

@@ -357,7 +357,7 @@ app: wasm wasmpack shell docs js_tests_release
 	yarn workspace @duckdb/duckdb-wasm-app build:release
 
 build_loadable:
-	DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET}
+	USE_GENERATED_EXPORTED_LIST=yes DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET}
 
 build_loadable_unsigned: build_loadable
         # need to propagate the unsigned flag
@@ -447,3 +447,15 @@ submodules:
 build/bootstrap: submodules yarn_install
 	mkdir -p build
 	touch build/bootstrap
+
+update_exported_list:
+	cd build/relsize/${TARGET} && wasm2wat duckdb_wasm.wasm --enable-all -o duckdb-wasm.wat
+	cd build/relsize/${TARGET} && grep export duckdb-wasm.wat | cut -d \" -f2 | sed '$d' | grep -v "^orig" | grep -v "^dynCall_" > export_list.txt
+        ## filter list of c++ symbols
+	cd build/relsize/${TARGET} && cat export_list.txt | grep "^_" | grep -v "_Unwind_" | grep -v "__syscall_shutdown" | grep -v "0\\00\\0" | grep -v "^_ZZN5arrow" | grep -v "^_ZGVZN5arrow" | grep -v "^_ZN5arrow" | sort > cpp_list
+	cd build/relsize/${TARGET} && sed 's/^/_/g' cpp_list > exported_list.txt
+        ## filter list of c symbols
+	cd build/relsize/${TARGET} && cat export_list.txt | grep -v "^_" | grep -v "getTempRet" | grep -v "^sched_yield" | grep -v "emscripten_wget" | grep -v "0\\00\\0" | sort > c_exported_list
+	# prepend '_'
+	cd build/relsize/${TARGET} && sed 's/^/_/g' c_exported_list >> exported_list.txt
+	cd build/relsize/${TARGET} && echo '__ZNSt3__26chrono12system_clock9to_time_tERKNS0_10time_pointIS1_NS0_8durationIxNS_5ratioILx1ELx1000000EEEEEEE' >> exported_list.txt

lib/CMakeLists.txt

@@ -150,9 +150,15 @@ else()
 endif()
 
 if (DUCKDB_WASM_LOADABLE_EXTENSIONS)
-  set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1 -s FILESYSTEM=1 -s ENVIRONMENT='web,node,worker' -s ALLOW_TABLE_GROWTH -lembind")
+  set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=1 -s ENVIRONMENT='web,node,worker' -s ALLOW_TABLE_GROWTH -lembind")
+
+  if (USE_GENERATED_EXPORTED_LIST)
+    set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=2 -s EXPORTED_FUNCTIONS='@exported_list.txt'")
+  else()
+    set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1")
+  endif()
 else()
-  set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker'")
+  set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker' -s EXPORTED_FUNCTIONS='@../../../lib/base_exported_list.txt'")
 endif()
 
 # ---------------------------------------------------------------------------
@@ -287,56 +293,7 @@ if(EMSCRIPTEN)
       -s MAXIMUM_MEMORY=4GB \
       -s MODULARIZE=1 \
       -s EXPORT_NAME='DuckDB' \
-      -s EXPORTED_FUNCTIONS='[ \
-          _main, \
-          _malloc, \
-          _calloc, \
-          _free, \
-          _duckdb_web_clear_response, \
-          _duckdb_web_collect_file_stats, \
-          _duckdb_web_connect, \
-          _duckdb_web_copy_file_to_buffer, \
-          _duckdb_web_copy_file_to_path, \
-          _duckdb_web_disconnect, \
-          _duckdb_web_export_file_stats, \
-          _duckdb_web_fail_with, \
-          _duckdb_web_flush_file, \
-          _duckdb_web_flush_files, \
-          _duckdb_web_fs_drop_file, \
-          _duckdb_web_fs_drop_files, \
-          _duckdb_web_fs_get_file_info_by_id, \
-          _duckdb_web_fs_get_file_info_by_name, \
-          _duckdb_web_fs_glob_add_path, \
-          _duckdb_web_fs_glob_file_infos, \
-          _duckdb_web_fs_register_file_buffer, \
-          _duckdb_web_fs_register_file_url, \
-          _duckdb_web_get_feature_flags, \
-          _duckdb_web_get_global_file_info, \
-          _duckdb_web_get_tablenames, \
-          _duckdb_web_get_tablenames_buffer, \
-          _duckdb_web_get_version, \
-          _duckdb_web_insert_arrow_from_ipc_stream, \
-          _duckdb_web_insert_csv_from_path, \
-          _duckdb_web_insert_json_from_path, \
-          _duckdb_web_open, \
-          _duckdb_web_pending_query_cancel, \
-          _duckdb_web_pending_query_poll, \
-          _duckdb_web_pending_query_start, \
-          _duckdb_web_pending_query_start_buffer, \
-          _duckdb_web_prepared_close, \
-          _duckdb_web_prepared_create, \
-          _duckdb_web_prepared_create_buffer, \
-          _duckdb_web_prepared_run, \
-          _duckdb_web_prepared_send, \
-          _duckdb_web_query_fetch_results, \
-          _duckdb_web_query_run, \
-          _duckdb_web_query_run_buffer, \
-          _duckdb_web_reset, \
-          _duckdb_web_tokenize, \
-          _duckdb_web_tokenize_buffer, \
-          _duckdb_web_udf_scalar_create \
-      ]' \
-      -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\"]' \
+      -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\", \"createDyncallWrapper\", \"getTempRet0\", \"setTempRet0\"]' \
       --js-library=${CMAKE_SOURCE_DIR}/js-stubs.js")
 
 endif()

lib/base_exported_list.txt

@@ -0,0 +1,47 @@
+_main
+_malloc
+_calloc
+_free
+_duckdb_web_clear_response
+_duckdb_web_collect_file_stats
+_duckdb_web_connect
+_duckdb_web_copy_file_to_buffer
+_duckdb_web_copy_file_to_path
+_duckdb_web_disconnect
+_duckdb_web_export_file_stats
+_duckdb_web_fail_with
+_duckdb_web_flush_file
+_duckdb_web_flush_files
+_duckdb_web_fs_drop_file
+_duckdb_web_fs_drop_files
+_duckdb_web_fs_get_file_info_by_id
+_duckdb_web_fs_get_file_info_by_name
+_duckdb_web_fs_glob_add_path
+_duckdb_web_fs_glob_file_infos
+_duckdb_web_fs_register_file_buffer
+_duckdb_web_fs_register_file_url
+_duckdb_web_get_feature_flags
+_duckdb_web_get_global_file_info
+_duckdb_web_get_tablenames
+_duckdb_web_get_tablenames_buffer
+_duckdb_web_get_version
+_duckdb_web_insert_arrow_from_ipc_stream
+_duckdb_web_insert_csv_from_path
+_duckdb_web_insert_json_from_path
+_duckdb_web_open
+_duckdb_web_pending_query_cancel
+_duckdb_web_pending_query_poll
+_duckdb_web_pending_query_start
+_duckdb_web_pending_query_start_buffer
+_duckdb_web_prepared_close
+_duckdb_web_prepared_create
+_duckdb_web_prepared_create_buffer
+_duckdb_web_prepared_run
+_duckdb_web_prepared_send
+_duckdb_web_query_fetch_results
+_duckdb_web_query_run
+_duckdb_web_query_run_buffer
+_duckdb_web_reset
+_duckdb_web_tokenize
+_duckdb_web_tokenize_buffer
+_duckdb_web_udf_scalar_create

lib/src/utils/parking_lot.cc

@@ -71,7 +71,7 @@ void ParkingLot::Park(const void* addr, std::function<bool()> check, std::chrono
             bucket.cv.wait(lock);
         }
     } else {
-        auto stop = std::chrono::system_clock::now() + timeout;
+        auto stop = std::chrono::high_resolution_clock::now() + timeout;
         while (!check()) {
             if (bucket.cv.wait_until(lock, stop) == std::cv_status::timeout) break;
         }

scripts/wasm_build_lib.sh

@@ -69,8 +69,32 @@ emmake make \
     -j${CORES} \
     duckdb_wasm
 
-npm install -g js-beautify
+if [ "${USE_GENERATED_EXPORTED_LIST:-no}" == "yes" ]; then
+make TARGET=${FEATURES} update_exported_list
+
+emcmake cmake \
+    -S${CPP_SOURCE_DIR} \
+    -B${BUILD_DIR} \
+    -DDUCKDB_WASM_VERSION=${DUCKDB_WASM_VERSION_NAME} \
+    -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+    -DDUCKDB_LOCATION=${DUCKDB_LOCATION} \
+    -DWASM_LINK_FLAGS_EXT="${LINK_FLAGS}" \
+    -DDUCKDB_EXTENSION_CONFIGS=extension_config_wasm.cmake \
+    -DUSE_GENERATED_EXPORTED_LIST=1 \
+    ${ADDITIONAL_FLAGS}
+
+emmake make \
+    -C${BUILD_DIR} \
+    -j${CORES} \
+    duckdb_wasm
+fi
+
+js-beautify -v || npm install -g js-beautify
 js-beautify ${BUILD_DIR}/duckdb_wasm.js > ${BUILD_DIR}/beauty.js
+sed 's/case \"__table_base\"/case \"getTempRet0\": return getTempRet0;   case \"__table_base\"/g' ${BUILD_DIR}/beauty.js > ${BUILD_DIR}/beauty_sed.js
+cp ${BUILD_DIR}/beauty_sed.js ${BUILD_DIR}/beauty.js
+cp ${BUILD_DIR}/beauty.js ${BUILD_DIR}/duckdb_wasm.js
 awk '{gsub(/get\(stubs, prop\) \{/,"get(stubs,prop) { if (prop.startsWith(\"invoke_\")) {return createDyncallWrapper(prop.substring(7));}"); print}' ${BUILD_DIR}/beauty.js > ${BUILD_DIR}/beauty2.js
 awk '!(/var .*wasmExports\[/ || /var [_a-z0-9A-Z]+ = Module\[\"[_a-z0-9A-Z]+\"\] = [0-9]+;/) || /var _duckdb_web/ || /var _main/ || /var _calloc/ || /var _malloc/ || /var _free/ || /var stack/ || /var ___dl_seterr/ || /var __em/ || /var _em/ || /var _pthread/' ${BUILD_DIR}/beauty2.js > ${BUILD_DIR}/duckdb_wasm.js