From 5c25f2b0ff657a6b30100cc31645a433c460547c Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 13:13:38 +0200 Subject: [PATCH 01/16] Pass EXPORTED_FUNCTIONS as external file --- lib/CMakeLists.txt | 59 +++++++--------------------------------------- 1 file changed, 8 insertions(+), 51 deletions(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index f7df08312..cc9450cbd 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -150,11 +150,17 @@ else() endif() if (DUCKDB_WASM_LOADABLE_EXTENSIONS) - set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1 -s FILESYSTEM=1 -s ENVIRONMENT='web,node,worker' -s ALLOW_TABLE_GROWTH -lembind") + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=1 -s ENVIRONMENT='web,node,worker' -s ALLOW_TABLE_GROWTH -lembind") else() set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker'") endif() +if (FINAL_BUILD) + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=2 -s EXPORTED_FUNCTIONS='@exported_list.txt'") +else() + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1") +endif() + # --------------------------------------------------------------------------- # Parallelism @@ -287,56 +293,7 @@ if(EMSCRIPTEN) -s MAXIMUM_MEMORY=4GB \ -s MODULARIZE=1 \ -s EXPORT_NAME='DuckDB' \ - -s EXPORTED_FUNCTIONS='[ \ - _main, \ - _malloc, \ - _calloc, \ - _free, \ - _duckdb_web_clear_response, \ - _duckdb_web_collect_file_stats, \ - _duckdb_web_connect, \ - _duckdb_web_copy_file_to_buffer, \ - _duckdb_web_copy_file_to_path, \ - _duckdb_web_disconnect, \ - _duckdb_web_export_file_stats, \ - _duckdb_web_fail_with, \ - _duckdb_web_flush_file, \ - _duckdb_web_flush_files, \ - _duckdb_web_fs_drop_file, \ - _duckdb_web_fs_drop_files, \ - _duckdb_web_fs_get_file_info_by_id, \ - _duckdb_web_fs_get_file_info_by_name, \ - _duckdb_web_fs_glob_add_path, \ - _duckdb_web_fs_glob_file_infos, \ - _duckdb_web_fs_register_file_buffer, \ - _duckdb_web_fs_register_file_url, \ - _duckdb_web_get_feature_flags, \ - _duckdb_web_get_global_file_info, \ - _duckdb_web_get_tablenames, \ - _duckdb_web_get_tablenames_buffer, \ - _duckdb_web_get_version, \ - _duckdb_web_insert_arrow_from_ipc_stream, \ - _duckdb_web_insert_csv_from_path, \ - _duckdb_web_insert_json_from_path, \ - _duckdb_web_open, \ - _duckdb_web_pending_query_cancel, \ - _duckdb_web_pending_query_poll, \ - _duckdb_web_pending_query_start, \ - _duckdb_web_pending_query_start_buffer, \ - _duckdb_web_prepared_close, \ - _duckdb_web_prepared_create, \ - _duckdb_web_prepared_create_buffer, \ - _duckdb_web_prepared_run, \ - _duckdb_web_prepared_send, \ - _duckdb_web_query_fetch_results, \ - _duckdb_web_query_run, \ - _duckdb_web_query_run_buffer, \ - _duckdb_web_reset, \ - _duckdb_web_tokenize, \ - _duckdb_web_tokenize_buffer, \ - _duckdb_web_udf_scalar_create \ - ]' \ - -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\"]' \ + -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\", \"createDyncallWrapper\"]' \ --js-library=${CMAKE_SOURCE_DIR}/js-stubs.js") endif() From 4ca45c7bc8d6437ed6473586ff4c695249a2746a Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 13:17:00 +0200 Subject: [PATCH 02/16] Add Makefile step to generate hacky exported list --- Makefile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Makefile b/Makefile index e4c96a669..5302ad61f 100644 --- a/Makefile +++ b/Makefile @@ -447,3 +447,14 @@ submodules: build/bootstrap: submodules yarn_install mkdir -p build touch build/bootstrap + +update_exported_list: + cd build/relsize/${TARGET} && wasm2wat duckdb_wasm.wasm --enable-all -o duckdb-wasm.wat + cd build/relsize/${TARGET} && grep export duckdb-wasm.wat | cut -d \" -f2 | sed '$d' | grep -v "^orig" | grep -v "^dynCall_" > export_list.txt + ## filter list of c++ symbols + cd build/relsize/${TARGET} && cat export_list.txt | grep "^_" | grep -v "_Unwind_" | grep -v "__syscall_shutdown" | grep -v "0\\00\\0" | grep -v "^_ZZN5arrow" | grep -v "^_ZGVZN5arrow" | grep -v "^_ZN5arrow" | sort > cpp_list + cd build/relsize/${TARGET} && sed 's/^/_/g' cpp_list > exported_list.txt + ## filter list of c symbols + cd build/relsize/${TARGET} && cat export_list.txt | grep -v "^_" | grep -v "getTempRet" | grep -v "^sched_yield" | grep -v "emscripten_wget" | grep -v "0\\00\\0" | sort > c_exported_list + # prepend '_' + cd build/relsize/${TARGET} && sed 's/^/_/g' c_exported_list >> exported_list.txt From 2329ba86c5addbf648654a257ceb8cf5c85244b2 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 13:19:59 +0200 Subject: [PATCH 03/16] Conditionally on USE_GENERATED_EXPORTED_LIST, use just generated list --- lib/CMakeLists.txt | 2 +- scripts/wasm_build_lib.sh | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index cc9450cbd..cbc3367bf 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -155,7 +155,7 @@ else() set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker'") endif() -if (FINAL_BUILD) +if (USE_GENERATED_EXPORTED_LIST) set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=2 -s EXPORTED_FUNCTIONS='@exported_list.txt'") else() set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1") diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index cf2b268ad..a3e4f43e6 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -69,6 +69,27 @@ emmake make \ -j${CORES} \ duckdb_wasm +make update_exported_list + +if [ "$USE_GENERATED_EXPORTED_LIST" == "yes" ]; then +emcmake cmake \ + -S${CPP_SOURCE_DIR} \ + -B${BUILD_DIR} \ + -DDUCKDB_WASM_VERSION=${DUCKDB_WASM_VERSION_NAME} \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DDUCKDB_LOCATION=${DUCKDB_LOCATION} \ + -DWASM_LINK_FLAGS_EXT="${LINK_FLAGS}" \ + -DDUCKDB_EXTENSION_CONFIGS=extension_config_wasm.cmake \ + -DUSE_GENERATED_EXPORTED_LIST=1 \ + ${ADDITIONAL_FLAGS} + +emmake make \ + -C${BUILD_DIR} \ + -j${CORES} \ + duckdb_wasm +fi + npm install -g js-beautify js-beautify ${BUILD_DIR}/duckdb_wasm.js > ${BUILD_DIR}/beauty.js awk '{gsub(/get\(stubs, prop\) \{/,"get(stubs,prop) { if (prop.startsWith(\"invoke_\")) {return createDyncallWrapper(prop.substring(7));}"); print}' ${BUILD_DIR}/beauty.js > ${BUILD_DIR}/beauty2.js From 1df1b1b139f8ae03fc48d00b8906c6d0220b15b5 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 13:22:17 +0200 Subject: [PATCH 04/16] Use generated export list by default --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5302ad61f..382fff328 100644 --- a/Makefile +++ b/Makefile @@ -357,7 +357,7 @@ app: wasm wasmpack shell docs js_tests_release yarn workspace @duckdb/duckdb-wasm-app build:release build_loadable: - DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET} + USE_GENERATED_EXPORTED_LIST=1 DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET} build_loadable_unsigned: build_loadable # need to propagate the unsigned flag From 5579f7c01a61b3f5256de305b0bdfdda624ff455 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 13:52:12 +0200 Subject: [PATCH 05/16] Fixup parameter passing --- .github/workflows/main.yml | 24 ++++++++++++++++++++++++ Makefile | 2 +- scripts/wasm_build_lib.sh | 4 ++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c9d216edd..550514dc8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -364,6 +364,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches @@ -407,6 +411,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches @@ -450,6 +458,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches @@ -494,6 +506,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches @@ -537,6 +553,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches @@ -580,6 +600,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches diff --git a/Makefile b/Makefile index 382fff328..d4934abc8 100644 --- a/Makefile +++ b/Makefile @@ -357,7 +357,7 @@ app: wasm wasmpack shell docs js_tests_release yarn workspace @duckdb/duckdb-wasm-app build:release build_loadable: - USE_GENERATED_EXPORTED_LIST=1 DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET} + USE_GENERATED_EXPORTED_LIST=yes DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET} build_loadable_unsigned: build_loadable # need to propagate the unsigned flag diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index a3e4f43e6..88b343b86 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -69,7 +69,7 @@ emmake make \ -j${CORES} \ duckdb_wasm -make update_exported_list +TARGET=${FEATURES} make update_exported_list if [ "$USE_GENERATED_EXPORTED_LIST" == "yes" ]; then emcmake cmake \ @@ -90,7 +90,7 @@ emmake make \ duckdb_wasm fi -npm install -g js-beautify +js-beautify -v || npm install -g js-beautify js-beautify ${BUILD_DIR}/duckdb_wasm.js > ${BUILD_DIR}/beauty.js awk '{gsub(/get\(stubs, prop\) \{/,"get(stubs,prop) { if (prop.startsWith(\"invoke_\")) {return createDyncallWrapper(prop.substring(7));}"); print}' ${BUILD_DIR}/beauty.js > ${BUILD_DIR}/beauty2.js awk '!(/var .*wasmExports\[/ || /var [_a-z0-9A-Z]+ = Module\[\"[_a-z0-9A-Z]+\"\] = [0-9]+;/) || /var _duckdb_web/ || /var _main/ || /var _calloc/ || /var _malloc/ || /var _free/ || /var stack/ || /var ___dl_seterr/ || /var __em/ || /var _em/ || /var _pthread/' ${BUILD_DIR}/beauty2.js > ${BUILD_DIR}/duckdb_wasm.js From 17217a39a9b4d6c990b4107a2527e580dc86ad77 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 14:17:21 +0200 Subject: [PATCH 06/16] try this --- scripts/wasm_build_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index 88b343b86..9a95b31a8 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -71,7 +71,7 @@ emmake make \ TARGET=${FEATURES} make update_exported_list -if [ "$USE_GENERATED_EXPORTED_LIST" == "yes" ]; then +if [ "${USE_GENERATED_EXPORTED_LIST}-no" == "yes" ]; then emcmake cmake \ -S${CPP_SOURCE_DIR} \ -B${BUILD_DIR} \ From bf50bc426632fb0cefcc4d1fee653fbe5014889b Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 15:04:09 +0200 Subject: [PATCH 07/16] fix --- .github/workflows/main.yml | 6 +++--- scripts/wasm_build_lib.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 550514dc8..27a39ed60 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -530,7 +530,7 @@ jobs: - name: Build Wasm module run: | - DUCKDB_PLATFORM="wasm_mvp" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize mvp + USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_mvp" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize mvp - name: Upload artifact uses: actions/upload-artifact@v4 @@ -577,7 +577,7 @@ jobs: - name: Build Wasm module run: | - DUCKDB_PLATFORM="wasm_eh" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize eh + USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_eh" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize eh - name: Upload artifact uses: actions/upload-artifact@v4 @@ -624,7 +624,7 @@ jobs: - name: Build Wasm module run: | - DUCKDB_PLATFORM="wasm_threads" DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize coi + USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_threads" DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize coi - name: Upload artifact uses: actions/upload-artifact@v4 diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index 9a95b31a8..fb26069f2 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -71,7 +71,7 @@ emmake make \ TARGET=${FEATURES} make update_exported_list -if [ "${USE_GENERATED_EXPORTED_LIST}-no" == "yes" ]; then +if [ "${USE_GENERATED_EXPORTED_LIST:-no}" == "yes" ]; then emcmake cmake \ -S${CPP_SOURCE_DIR} \ -B${BUILD_DIR} \ From 2047a9bea77ae86e7d82ed1c339cdad0c89079c5 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 15:35:24 +0200 Subject: [PATCH 08/16] invert --- scripts/wasm_build_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index fb26069f2..80a6eb5d4 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -69,7 +69,7 @@ emmake make \ -j${CORES} \ duckdb_wasm -TARGET=${FEATURES} make update_exported_list +make TARGET=${FEATURES} update_exported_list if [ "${USE_GENERATED_EXPORTED_LIST:-no}" == "yes" ]; then emcmake cmake \ From fd58926bf8b04aaa3abe1bb513d6c039dda7e45d Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 16:06:08 +0200 Subject: [PATCH 09/16] skip not loadable --- lib/CMakeLists.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index cbc3367bf..16dced808 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -151,14 +151,14 @@ endif() if (DUCKDB_WASM_LOADABLE_EXTENSIONS) set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=1 -s ENVIRONMENT='web,node,worker' -s ALLOW_TABLE_GROWTH -lembind") -else() - set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker'") -endif() -if (USE_GENERATED_EXPORTED_LIST) - set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=2 -s EXPORTED_FUNCTIONS='@exported_list.txt'") + if (USE_GENERATED_EXPORTED_LIST) + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=2 -s EXPORTED_FUNCTIONS='@exported_list.txt'") + else() + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1") + endif() else() - set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1") + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker'") endif() # --------------------------------------------------------------------------- From c631b6e0893eb40a7faafd29751afa881f9a9bad Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 17:49:08 +0200 Subject: [PATCH 10/16] Fix parking lot --- lib/src/utils/parking_lot.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/utils/parking_lot.cc b/lib/src/utils/parking_lot.cc index 888272734..fd9162d3d 100644 --- a/lib/src/utils/parking_lot.cc +++ b/lib/src/utils/parking_lot.cc @@ -71,7 +71,7 @@ void ParkingLot::Park(const void* addr, std::function check, std::chrono bucket.cv.wait(lock); } } else { - auto stop = std::chrono::system_clock::now() + timeout; + auto stop = std::chrono::high_resolution_clock::now() + timeout; while (!check()) { if (bucket.cv.wait_until(lock, stop) == std::cv_status::timeout) break; } From e4615a2bde8b356ac6053c7385fffe7e588e6a0c Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 13 May 2025 08:28:51 +0200 Subject: [PATCH 11/16] Temporarily add chrono --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index d4934abc8..ec31a55a2 100644 --- a/Makefile +++ b/Makefile @@ -458,3 +458,4 @@ update_exported_list: cd build/relsize/${TARGET} && cat export_list.txt | grep -v "^_" | grep -v "getTempRet" | grep -v "^sched_yield" | grep -v "emscripten_wget" | grep -v "0\\00\\0" | sort > c_exported_list # prepend '_' cd build/relsize/${TARGET} && sed 's/^/_/g' c_exported_list >> exported_list.txt + cd build/relsize/${TARGET} && echo '__ZNSt3__26chrono12system_clock9to_time_tERKNS0_10time_pointIS1_NS0_8durationIxNS_5ratioILx1ELx1000000EEEEEEE' >> exported_list.txt From 96d28f6d786bf1565c0add558c9b4faa0a23365b Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 13 May 2025 08:29:22 +0200 Subject: [PATCH 12/16] Fixup for getTempRet0: --- lib/CMakeLists.txt | 2 +- scripts/wasm_build_lib.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 16dced808..42bce6bd6 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -293,7 +293,7 @@ if(EMSCRIPTEN) -s MAXIMUM_MEMORY=4GB \ -s MODULARIZE=1 \ -s EXPORT_NAME='DuckDB' \ - -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\", \"createDyncallWrapper\"]' \ + -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\", \"createDyncallWrapper\", \"getTempRet0\", \"setTempRet0\"]' \ --js-library=${CMAKE_SOURCE_DIR}/js-stubs.js") endif() diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index 80a6eb5d4..a63e08931 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -92,6 +92,8 @@ fi js-beautify -v || npm install -g js-beautify js-beautify ${BUILD_DIR}/duckdb_wasm.js > ${BUILD_DIR}/beauty.js +sed -i '' 's/case \"__table_base\"/case \"getTempRet0\": return getTempRet0; case \"__table_base\"/g' ${BUILD_DIR}/beauty.js +cp ${BUILD_DIR}/beauty.js ${BUILD_DIR}/duckdb_wasm.js awk '{gsub(/get\(stubs, prop\) \{/,"get(stubs,prop) { if (prop.startsWith(\"invoke_\")) {return createDyncallWrapper(prop.substring(7));}"); print}' ${BUILD_DIR}/beauty.js > ${BUILD_DIR}/beauty2.js awk '!(/var .*wasmExports\[/ || /var [_a-z0-9A-Z]+ = Module\[\"[_a-z0-9A-Z]+\"\] = [0-9]+;/) || /var _duckdb_web/ || /var _main/ || /var _calloc/ || /var _malloc/ || /var _free/ || /var stack/ || /var ___dl_seterr/ || /var __em/ || /var _em/ || /var _pthread/' ${BUILD_DIR}/beauty2.js > ${BUILD_DIR}/duckdb_wasm.js From b4cf9f5fca4dd808db4cdd98fa52554524a0688a Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 13 May 2025 08:58:18 +0200 Subject: [PATCH 13/16] Fixup sed --- scripts/wasm_build_lib.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index a63e08931..f8b5494d6 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -92,7 +92,8 @@ fi js-beautify -v || npm install -g js-beautify js-beautify ${BUILD_DIR}/duckdb_wasm.js > ${BUILD_DIR}/beauty.js -sed -i '' 's/case \"__table_base\"/case \"getTempRet0\": return getTempRet0; case \"__table_base\"/g' ${BUILD_DIR}/beauty.js +sed 's/case \"__table_base\"/case \"getTempRet0\": return getTempRet0; case \"__table_base\"/g' ${BUILD_DIR}/beauty.js > ${BUILD_DIR}/beauty_sed.js +cp ${BUILD_DIR}/beauty_sed.js ${BUILD_DIR}/beauty.js cp ${BUILD_DIR}/beauty.js ${BUILD_DIR}/duckdb_wasm.js awk '{gsub(/get\(stubs, prop\) \{/,"get(stubs,prop) { if (prop.startsWith(\"invoke_\")) {return createDyncallWrapper(prop.substring(7));}"); print}' ${BUILD_DIR}/beauty.js > ${BUILD_DIR}/beauty2.js awk '!(/var .*wasmExports\[/ || /var [_a-z0-9A-Z]+ = Module\[\"[_a-z0-9A-Z]+\"\] = [0-9]+;/) || /var _duckdb_web/ || /var _main/ || /var _calloc/ || /var _malloc/ || /var _free/ || /var stack/ || /var ___dl_seterr/ || /var __em/ || /var _em/ || /var _pthread/' ${BUILD_DIR}/beauty2.js > ${BUILD_DIR}/duckdb_wasm.js From 51d37e80cf2b8542738f5d9b9382d1f63e025d08 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 13 May 2025 09:54:37 +0200 Subject: [PATCH 14/16] Move inner --- scripts/wasm_build_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index f8b5494d6..f697badf7 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -69,9 +69,9 @@ emmake make \ -j${CORES} \ duckdb_wasm +if [ "${USE_GENERATED_EXPORTED_LIST:-no}" == "yes" ]; then make TARGET=${FEATURES} update_exported_list -if [ "${USE_GENERATED_EXPORTED_LIST:-no}" == "yes" ]; then emcmake cmake \ -S${CPP_SOURCE_DIR} \ -B${BUILD_DIR} \ From 2412a6a74de7f6bcca624a1498beb6f295e43f58 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 13 May 2025 10:09:51 +0200 Subject: [PATCH 15/16] Remove unneeded wabt install --- .github/workflows/main.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 27a39ed60..040648b75 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -364,10 +364,6 @@ jobs: submodules: 'recursive' fetch-depth: 0 - - name: Install dependencies - run: | - sudo apt install wabt - - name: Prepare repository run: | make apply_patches @@ -411,10 +407,6 @@ jobs: submodules: 'recursive' fetch-depth: 0 - - name: Install dependencies - run: | - sudo apt install wabt - - name: Prepare repository run: | make apply_patches @@ -458,10 +450,6 @@ jobs: submodules: 'recursive' fetch-depth: 0 - - name: Install dependencies - run: | - sudo apt install wabt - - name: Prepare repository run: | make apply_patches From 350ea32da85a13cce177761f0cdca92f562937df Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 13 May 2025 10:33:54 +0200 Subject: [PATCH 16/16] Add base_exported_list also for no-extensions case --- lib/CMakeLists.txt | 2 +- lib/base_exported_list.txt | 47 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 lib/base_exported_list.txt diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 42bce6bd6..98c081337 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -158,7 +158,7 @@ if (DUCKDB_WASM_LOADABLE_EXTENSIONS) set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1") endif() else() - set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker'") + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker' -s EXPORTED_FUNCTIONS='@../../../lib/base_exported_list.txt'") endif() # --------------------------------------------------------------------------- diff --git a/lib/base_exported_list.txt b/lib/base_exported_list.txt new file mode 100644 index 000000000..fb149fef8 --- /dev/null +++ b/lib/base_exported_list.txt @@ -0,0 +1,47 @@ +_main +_malloc +_calloc +_free +_duckdb_web_clear_response +_duckdb_web_collect_file_stats +_duckdb_web_connect +_duckdb_web_copy_file_to_buffer +_duckdb_web_copy_file_to_path +_duckdb_web_disconnect +_duckdb_web_export_file_stats +_duckdb_web_fail_with +_duckdb_web_flush_file +_duckdb_web_flush_files +_duckdb_web_fs_drop_file +_duckdb_web_fs_drop_files +_duckdb_web_fs_get_file_info_by_id +_duckdb_web_fs_get_file_info_by_name +_duckdb_web_fs_glob_add_path +_duckdb_web_fs_glob_file_infos +_duckdb_web_fs_register_file_buffer +_duckdb_web_fs_register_file_url +_duckdb_web_get_feature_flags +_duckdb_web_get_global_file_info +_duckdb_web_get_tablenames +_duckdb_web_get_tablenames_buffer +_duckdb_web_get_version +_duckdb_web_insert_arrow_from_ipc_stream +_duckdb_web_insert_csv_from_path +_duckdb_web_insert_json_from_path +_duckdb_web_open +_duckdb_web_pending_query_cancel +_duckdb_web_pending_query_poll +_duckdb_web_pending_query_start +_duckdb_web_pending_query_start_buffer +_duckdb_web_prepared_close +_duckdb_web_prepared_create +_duckdb_web_prepared_create_buffer +_duckdb_web_prepared_run +_duckdb_web_prepared_send +_duckdb_web_query_fetch_results +_duckdb_web_query_run +_duckdb_web_query_run_buffer +_duckdb_web_reset +_duckdb_web_tokenize +_duckdb_web_tokenize_buffer +_duckdb_web_udf_scalar_create