diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c9d216edd..040648b75 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -494,6 +494,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches @@ -514,7 +518,7 @@ jobs: - name: Build Wasm module run: | - DUCKDB_PLATFORM="wasm_mvp" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize mvp + USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_mvp" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize mvp - name: Upload artifact uses: actions/upload-artifact@v4 @@ -537,6 +541,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches @@ -557,7 +565,7 @@ jobs: - name: Build Wasm module run: | - DUCKDB_PLATFORM="wasm_eh" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize eh + USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_eh" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize eh - name: Upload artifact uses: actions/upload-artifact@v4 @@ -580,6 +588,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches @@ -600,7 +612,7 @@ jobs: - name: Build Wasm module run: | - DUCKDB_PLATFORM="wasm_threads" DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize coi + USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_threads" DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize coi - name: Upload artifact uses: actions/upload-artifact@v4 diff --git a/Makefile b/Makefile index e4c96a669..ec31a55a2 100644 --- a/Makefile +++ b/Makefile @@ -357,7 +357,7 @@ app: wasm wasmpack shell docs js_tests_release yarn workspace @duckdb/duckdb-wasm-app build:release build_loadable: - DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET} + USE_GENERATED_EXPORTED_LIST=yes DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET} build_loadable_unsigned: build_loadable # need to propagate the unsigned flag @@ -447,3 +447,15 @@ submodules: build/bootstrap: submodules yarn_install mkdir -p build touch build/bootstrap + +update_exported_list: + cd build/relsize/${TARGET} && wasm2wat duckdb_wasm.wasm --enable-all -o duckdb-wasm.wat + cd build/relsize/${TARGET} && grep export duckdb-wasm.wat | cut -d \" -f2 | sed '$d' | grep -v "^orig" | grep -v "^dynCall_" > export_list.txt + ## filter list of c++ symbols + cd build/relsize/${TARGET} && cat export_list.txt | grep "^_" | grep -v "_Unwind_" | grep -v "__syscall_shutdown" | grep -v "0\\00\\0" | grep -v "^_ZZN5arrow" | grep -v "^_ZGVZN5arrow" | grep -v "^_ZN5arrow" | sort > cpp_list + cd build/relsize/${TARGET} && sed 's/^/_/g' cpp_list > exported_list.txt + ## filter list of c symbols + cd build/relsize/${TARGET} && cat export_list.txt | grep -v "^_" | grep -v "getTempRet" | grep -v "^sched_yield" | grep -v "emscripten_wget" | grep -v "0\\00\\0" | sort > c_exported_list + # prepend '_' + cd build/relsize/${TARGET} && sed 's/^/_/g' c_exported_list >> exported_list.txt + cd build/relsize/${TARGET} && echo '__ZNSt3__26chrono12system_clock9to_time_tERKNS0_10time_pointIS1_NS0_8durationIxNS_5ratioILx1ELx1000000EEEEEEE' >> exported_list.txt diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index f7df08312..98c081337 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -150,9 +150,15 @@ else() endif() if (DUCKDB_WASM_LOADABLE_EXTENSIONS) - set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1 -s FILESYSTEM=1 -s ENVIRONMENT='web,node,worker' -s ALLOW_TABLE_GROWTH -lembind") + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=1 -s ENVIRONMENT='web,node,worker' -s ALLOW_TABLE_GROWTH -lembind") + + if (USE_GENERATED_EXPORTED_LIST) + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=2 -s EXPORTED_FUNCTIONS='@exported_list.txt'") + else() + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1") + endif() else() - set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker'") + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker' -s EXPORTED_FUNCTIONS='@../../../lib/base_exported_list.txt'") endif() # --------------------------------------------------------------------------- @@ -287,56 +293,7 @@ if(EMSCRIPTEN) -s MAXIMUM_MEMORY=4GB \ -s MODULARIZE=1 \ -s EXPORT_NAME='DuckDB' \ - -s EXPORTED_FUNCTIONS='[ \ - _main, \ - _malloc, \ - _calloc, \ - _free, \ - _duckdb_web_clear_response, \ - _duckdb_web_collect_file_stats, \ - _duckdb_web_connect, \ - _duckdb_web_copy_file_to_buffer, \ - _duckdb_web_copy_file_to_path, \ - _duckdb_web_disconnect, \ - _duckdb_web_export_file_stats, \ - _duckdb_web_fail_with, \ - _duckdb_web_flush_file, \ - _duckdb_web_flush_files, \ - _duckdb_web_fs_drop_file, \ - _duckdb_web_fs_drop_files, \ - _duckdb_web_fs_get_file_info_by_id, \ - _duckdb_web_fs_get_file_info_by_name, \ - _duckdb_web_fs_glob_add_path, \ - _duckdb_web_fs_glob_file_infos, \ - _duckdb_web_fs_register_file_buffer, \ - _duckdb_web_fs_register_file_url, \ - _duckdb_web_get_feature_flags, \ - _duckdb_web_get_global_file_info, \ - _duckdb_web_get_tablenames, \ - _duckdb_web_get_tablenames_buffer, \ - _duckdb_web_get_version, \ - _duckdb_web_insert_arrow_from_ipc_stream, \ - _duckdb_web_insert_csv_from_path, \ - _duckdb_web_insert_json_from_path, \ - _duckdb_web_open, \ - _duckdb_web_pending_query_cancel, \ - _duckdb_web_pending_query_poll, \ - _duckdb_web_pending_query_start, \ - _duckdb_web_pending_query_start_buffer, \ - _duckdb_web_prepared_close, \ - _duckdb_web_prepared_create, \ - _duckdb_web_prepared_create_buffer, \ - _duckdb_web_prepared_run, \ - _duckdb_web_prepared_send, \ - _duckdb_web_query_fetch_results, \ - _duckdb_web_query_run, \ - _duckdb_web_query_run_buffer, \ - _duckdb_web_reset, \ - _duckdb_web_tokenize, \ - _duckdb_web_tokenize_buffer, \ - _duckdb_web_udf_scalar_create \ - ]' \ - -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\"]' \ + -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\", \"createDyncallWrapper\", \"getTempRet0\", \"setTempRet0\"]' \ --js-library=${CMAKE_SOURCE_DIR}/js-stubs.js") endif() diff --git a/lib/base_exported_list.txt b/lib/base_exported_list.txt new file mode 100644 index 000000000..fb149fef8 --- /dev/null +++ b/lib/base_exported_list.txt @@ -0,0 +1,47 @@ +_main +_malloc +_calloc +_free +_duckdb_web_clear_response +_duckdb_web_collect_file_stats +_duckdb_web_connect +_duckdb_web_copy_file_to_buffer +_duckdb_web_copy_file_to_path +_duckdb_web_disconnect +_duckdb_web_export_file_stats +_duckdb_web_fail_with +_duckdb_web_flush_file +_duckdb_web_flush_files +_duckdb_web_fs_drop_file +_duckdb_web_fs_drop_files +_duckdb_web_fs_get_file_info_by_id +_duckdb_web_fs_get_file_info_by_name +_duckdb_web_fs_glob_add_path +_duckdb_web_fs_glob_file_infos +_duckdb_web_fs_register_file_buffer +_duckdb_web_fs_register_file_url +_duckdb_web_get_feature_flags +_duckdb_web_get_global_file_info +_duckdb_web_get_tablenames +_duckdb_web_get_tablenames_buffer +_duckdb_web_get_version +_duckdb_web_insert_arrow_from_ipc_stream +_duckdb_web_insert_csv_from_path +_duckdb_web_insert_json_from_path +_duckdb_web_open +_duckdb_web_pending_query_cancel +_duckdb_web_pending_query_poll +_duckdb_web_pending_query_start +_duckdb_web_pending_query_start_buffer +_duckdb_web_prepared_close +_duckdb_web_prepared_create +_duckdb_web_prepared_create_buffer +_duckdb_web_prepared_run +_duckdb_web_prepared_send +_duckdb_web_query_fetch_results +_duckdb_web_query_run +_duckdb_web_query_run_buffer +_duckdb_web_reset +_duckdb_web_tokenize +_duckdb_web_tokenize_buffer +_duckdb_web_udf_scalar_create diff --git a/lib/src/utils/parking_lot.cc b/lib/src/utils/parking_lot.cc index 888272734..fd9162d3d 100644 --- a/lib/src/utils/parking_lot.cc +++ b/lib/src/utils/parking_lot.cc @@ -71,7 +71,7 @@ void ParkingLot::Park(const void* addr, std::function check, std::chrono bucket.cv.wait(lock); } } else { - auto stop = std::chrono::system_clock::now() + timeout; + auto stop = std::chrono::high_resolution_clock::now() + timeout; while (!check()) { if (bucket.cv.wait_until(lock, stop) == std::cv_status::timeout) break; } diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index cf2b268ad..f697badf7 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -69,8 +69,32 @@ emmake make \ -j${CORES} \ duckdb_wasm -npm install -g js-beautify +if [ "${USE_GENERATED_EXPORTED_LIST:-no}" == "yes" ]; then +make TARGET=${FEATURES} update_exported_list + +emcmake cmake \ + -S${CPP_SOURCE_DIR} \ + -B${BUILD_DIR} \ + -DDUCKDB_WASM_VERSION=${DUCKDB_WASM_VERSION_NAME} \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DDUCKDB_LOCATION=${DUCKDB_LOCATION} \ + -DWASM_LINK_FLAGS_EXT="${LINK_FLAGS}" \ + -DDUCKDB_EXTENSION_CONFIGS=extension_config_wasm.cmake \ + -DUSE_GENERATED_EXPORTED_LIST=1 \ + ${ADDITIONAL_FLAGS} + +emmake make \ + -C${BUILD_DIR} \ + -j${CORES} \ + duckdb_wasm +fi + +js-beautify -v || npm install -g js-beautify js-beautify ${BUILD_DIR}/duckdb_wasm.js > ${BUILD_DIR}/beauty.js +sed 's/case \"__table_base\"/case \"getTempRet0\": return getTempRet0; case \"__table_base\"/g' ${BUILD_DIR}/beauty.js > ${BUILD_DIR}/beauty_sed.js +cp ${BUILD_DIR}/beauty_sed.js ${BUILD_DIR}/beauty.js +cp ${BUILD_DIR}/beauty.js ${BUILD_DIR}/duckdb_wasm.js awk '{gsub(/get\(stubs, prop\) \{/,"get(stubs,prop) { if (prop.startsWith(\"invoke_\")) {return createDyncallWrapper(prop.substring(7));}"); print}' ${BUILD_DIR}/beauty.js > ${BUILD_DIR}/beauty2.js awk '!(/var .*wasmExports\[/ || /var [_a-z0-9A-Z]+ = Module\[\"[_a-z0-9A-Z]+\"\] = [0-9]+;/) || /var _duckdb_web/ || /var _main/ || /var _calloc/ || /var _malloc/ || /var _free/ || /var stack/ || /var ___dl_seterr/ || /var __em/ || /var _em/ || /var _pthread/' ${BUILD_DIR}/beauty2.js > ${BUILD_DIR}/duckdb_wasm.js