diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c9d216edd..040648b75 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -494,6 +494,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches @@ -514,7 +518,7 @@ jobs: - name: Build Wasm module run: | - DUCKDB_PLATFORM="wasm_mvp" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize mvp + USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_mvp" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize mvp - name: Upload artifact uses: actions/upload-artifact@v4 @@ -537,6 +541,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches @@ -557,7 +565,7 @@ jobs: - name: Build Wasm module run: | - DUCKDB_PLATFORM="wasm_eh" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize eh + USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_eh" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize eh - name: Upload artifact uses: actions/upload-artifact@v4 @@ -580,6 +588,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches @@ -600,7 +612,7 @@ jobs: - name: Build Wasm module run: | - DUCKDB_PLATFORM="wasm_threads" DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize coi + USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_threads" DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize coi - name: Upload artifact uses: actions/upload-artifact@v4 diff --git a/Makefile b/Makefile index e4c96a669..86f6216fb 100644 --- a/Makefile +++ b/Makefile @@ -357,7 +357,7 @@ app: wasm wasmpack shell docs js_tests_release yarn workspace @duckdb/duckdb-wasm-app build:release build_loadable: - DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET} + USE_GENERATED_EXPORTED_LIST=yes DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET} build_loadable_unsigned: build_loadable # need to propagate the unsigned flag @@ -436,6 +436,7 @@ build/docker_ci_image: patch_duckdb: (find patches/duckdb/* -type f -name '*.patch' -print0 | xargs -0 cat | patch -p1 --forward -d submodules/duckdb) || true (find patches/arrow/* -type f -name '*.patch' -print0 | xargs -0 cat | patch -p1 --forward -d submodules/arrow) || true + (find patches/rapidjson/* -type f -name '*.patch' -print0 | xargs -0 cat | patch -p1 --forward -d submodules/rapidjson) || true apply_patches: patch_duckdb @@ -447,3 +448,15 @@ submodules: build/bootstrap: submodules yarn_install mkdir -p build touch build/bootstrap + +update_exported_list: + cd build/relsize/${TARGET} && wasm2wat duckdb_wasm.wasm --enable-all -o duckdb-wasm.wat + cd build/relsize/${TARGET} && grep export duckdb-wasm.wat | cut -d \" -f2 | sed '$d' | grep -v "^orig" | grep -v "^dynCall_" > export_list.txt + ## filter list of c++ symbols + cd build/relsize/${TARGET} && cat export_list.txt | grep "^_" | grep -v "_Unwind_" | grep -v "__syscall_shutdown" | grep -v "0\\00\\0" | grep -v "^_ZZN5arrow" | grep -v "^_ZGVZN5arrow" | grep -v "^_ZN5arrow" | sort > cpp_list + cd build/relsize/${TARGET} && sed 's/^/_/g' cpp_list > exported_list.txt + ## filter list of c symbols + cd build/relsize/${TARGET} && cat export_list.txt | grep -v "^_" | grep -v "getTempRet" | grep -v "^sched_yield" | grep -v "emscripten_wget" | grep -v "0\\00\\0" | sort > c_exported_list + # prepend '_' + cd build/relsize/${TARGET} && sed 's/^/_/g' c_exported_list >> exported_list.txt + cd build/relsize/${TARGET} && echo '__ZNSt3__26chrono12system_clock9to_time_tERKNS0_10time_pointIS1_NS0_8durationIxNS_5ratioILx1ELx1000000EEEEEEE' >> exported_list.txt diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index f7df08312..98c081337 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -150,9 +150,15 @@ else() endif() if (DUCKDB_WASM_LOADABLE_EXTENSIONS) - set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1 -s FILESYSTEM=1 -s ENVIRONMENT='web,node,worker' -s ALLOW_TABLE_GROWTH -lembind") + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=1 -s ENVIRONMENT='web,node,worker' -s ALLOW_TABLE_GROWTH -lembind") + + if (USE_GENERATED_EXPORTED_LIST) + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=2 -s EXPORTED_FUNCTIONS='@exported_list.txt'") + else() + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1") + endif() else() - set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker'") + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker' -s EXPORTED_FUNCTIONS='@../../../lib/base_exported_list.txt'") endif() # --------------------------------------------------------------------------- @@ -287,56 +293,7 @@ if(EMSCRIPTEN) -s MAXIMUM_MEMORY=4GB \ -s MODULARIZE=1 \ -s EXPORT_NAME='DuckDB' \ - -s EXPORTED_FUNCTIONS='[ \ - _main, \ - _malloc, \ - _calloc, \ - _free, \ - _duckdb_web_clear_response, \ - _duckdb_web_collect_file_stats, \ - _duckdb_web_connect, \ - _duckdb_web_copy_file_to_buffer, \ - _duckdb_web_copy_file_to_path, \ - _duckdb_web_disconnect, \ - _duckdb_web_export_file_stats, \ - _duckdb_web_fail_with, \ - _duckdb_web_flush_file, \ - _duckdb_web_flush_files, \ - _duckdb_web_fs_drop_file, \ - _duckdb_web_fs_drop_files, \ - _duckdb_web_fs_get_file_info_by_id, \ - _duckdb_web_fs_get_file_info_by_name, \ - _duckdb_web_fs_glob_add_path, \ - _duckdb_web_fs_glob_file_infos, \ - _duckdb_web_fs_register_file_buffer, \ - _duckdb_web_fs_register_file_url, \ - _duckdb_web_get_feature_flags, \ - _duckdb_web_get_global_file_info, \ - _duckdb_web_get_tablenames, \ - _duckdb_web_get_tablenames_buffer, \ - _duckdb_web_get_version, \ - _duckdb_web_insert_arrow_from_ipc_stream, \ - _duckdb_web_insert_csv_from_path, \ - _duckdb_web_insert_json_from_path, \ - _duckdb_web_open, \ - _duckdb_web_pending_query_cancel, \ - _duckdb_web_pending_query_poll, \ - _duckdb_web_pending_query_start, \ - _duckdb_web_pending_query_start_buffer, \ - _duckdb_web_prepared_close, \ - _duckdb_web_prepared_create, \ - _duckdb_web_prepared_create_buffer, \ - _duckdb_web_prepared_run, \ - _duckdb_web_prepared_send, \ - _duckdb_web_query_fetch_results, \ - _duckdb_web_query_run, \ - _duckdb_web_query_run_buffer, \ - _duckdb_web_reset, \ - _duckdb_web_tokenize, \ - _duckdb_web_tokenize_buffer, \ - _duckdb_web_udf_scalar_create \ - ]' \ - -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\"]' \ + -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\", \"createDyncallWrapper\", \"getTempRet0\", \"setTempRet0\"]' \ --js-library=${CMAKE_SOURCE_DIR}/js-stubs.js") endif() diff --git a/lib/base_exported_list.txt b/lib/base_exported_list.txt new file mode 100644 index 000000000..fb149fef8 --- /dev/null +++ b/lib/base_exported_list.txt @@ -0,0 +1,47 @@ +_main +_malloc +_calloc +_free +_duckdb_web_clear_response +_duckdb_web_collect_file_stats +_duckdb_web_connect +_duckdb_web_copy_file_to_buffer +_duckdb_web_copy_file_to_path +_duckdb_web_disconnect +_duckdb_web_export_file_stats +_duckdb_web_fail_with +_duckdb_web_flush_file +_duckdb_web_flush_files +_duckdb_web_fs_drop_file +_duckdb_web_fs_drop_files +_duckdb_web_fs_get_file_info_by_id +_duckdb_web_fs_get_file_info_by_name +_duckdb_web_fs_glob_add_path +_duckdb_web_fs_glob_file_infos +_duckdb_web_fs_register_file_buffer +_duckdb_web_fs_register_file_url +_duckdb_web_get_feature_flags +_duckdb_web_get_global_file_info +_duckdb_web_get_tablenames +_duckdb_web_get_tablenames_buffer +_duckdb_web_get_version +_duckdb_web_insert_arrow_from_ipc_stream +_duckdb_web_insert_csv_from_path +_duckdb_web_insert_json_from_path +_duckdb_web_open +_duckdb_web_pending_query_cancel +_duckdb_web_pending_query_poll +_duckdb_web_pending_query_start +_duckdb_web_pending_query_start_buffer +_duckdb_web_prepared_close +_duckdb_web_prepared_create +_duckdb_web_prepared_create_buffer +_duckdb_web_prepared_run +_duckdb_web_prepared_send +_duckdb_web_query_fetch_results +_duckdb_web_query_run +_duckdb_web_query_run_buffer +_duckdb_web_reset +_duckdb_web_tokenize +_duckdb_web_tokenize_buffer +_duckdb_web_udf_scalar_create diff --git a/lib/include/duckdb/web/utils/wasm_response.h b/lib/include/duckdb/web/utils/wasm_response.h index 2cff60931..9edbca39e 100644 --- a/lib/include/duckdb/web/utils/wasm_response.h +++ b/lib/include/duckdb/web/utils/wasm_response.h @@ -8,6 +8,8 @@ namespace duckdb { namespace web { +struct DuckDBWasmResultsWrapper; + struct WASMResponse { /// The status code double statusCode = 1; @@ -35,6 +37,8 @@ class WASMResponseBuffer { /// Store the arrow status. /// Returns wheather the result was OK bool Store(WASMResponse& response, arrow::Status status); + /// Store a DuckDBWasmResultsWrapper + void Store(WASMResponse& response, DuckDBWasmResultsWrapper& value); /// Store a string void Store(WASMResponse& response, std::string value); /// Store a string view diff --git a/lib/include/duckdb/web/webdb.h b/lib/include/duckdb/web/webdb.h index f7d15841a..0cd847169 100644 --- a/lib/include/duckdb/web/webdb.h +++ b/lib/include/duckdb/web/webdb.h @@ -29,6 +29,22 @@ namespace web { struct BufferingArrowIPCStreamDecoder; +struct DuckDBWasmResultsWrapper { + // Additional ResponseStatuses to be >= 256, and mirrored to packages/duckdb-wasm/src/status.ts + // Missing mapping result in a throw, but they should eventually align (it's fine if typescript side only has a + // subset) + enum ResponseStatus : uint32_t { ARROW_BUFFER = 0, MAX_ARROW_ERROR = 255, DUCKDB_WASM_RETRY = 256 }; + DuckDBWasmResultsWrapper(arrow::Result> res, + ResponseStatus status = ResponseStatus::ARROW_BUFFER) + : arrow_buffer(res), status(status) {} + DuckDBWasmResultsWrapper(arrow::Status res, ResponseStatus status = ResponseStatus::ARROW_BUFFER) + : arrow_buffer(res), status(status) {} + DuckDBWasmResultsWrapper(ResponseStatus status = ResponseStatus::ARROW_BUFFER) + : DuckDBWasmResultsWrapper(nullptr, status) {} + arrow::Result> arrow_buffer; + ResponseStatus status; +}; + class WebDB { public: /// A connection @@ -93,7 +109,7 @@ class WebDB { /// Cancel a pending query bool CancelPendingQuery(); /// Fetch a data chunk from a pending query - arrow::Result> FetchQueryResults(); + DuckDBWasmResultsWrapper FetchQueryResults(); /// Get table names arrow::Result GetTableNames(std::string_view text); diff --git a/lib/src/utils/parking_lot.cc b/lib/src/utils/parking_lot.cc index 888272734..fd9162d3d 100644 --- a/lib/src/utils/parking_lot.cc +++ b/lib/src/utils/parking_lot.cc @@ -71,7 +71,7 @@ void ParkingLot::Park(const void* addr, std::function check, std::chrono bucket.cv.wait(lock); } } else { - auto stop = std::chrono::system_clock::now() + timeout; + auto stop = std::chrono::high_resolution_clock::now() + timeout; while (!check()) { if (bucket.cv.wait_until(lock, stop) == std::cv_status::timeout) break; } diff --git a/lib/src/utils/wasm_response.cc b/lib/src/utils/wasm_response.cc index d5645314e..61a062cea 100644 --- a/lib/src/utils/wasm_response.cc +++ b/lib/src/utils/wasm_response.cc @@ -3,6 +3,7 @@ #include #include "arrow/buffer.h" +#include "duckdb/web/webdb.h" namespace duckdb { namespace web { @@ -26,6 +27,15 @@ bool WASMResponseBuffer::Store(WASMResponse& response, arrow::Status status) { return true; } +void WASMResponseBuffer::Store(WASMResponse& response, DuckDBWasmResultsWrapper& value) { + if (value.status == DuckDBWasmResultsWrapper::ResponseStatus::ARROW_BUFFER) { + Store(response, std::move(value.arrow_buffer)); + } else { + Clear(); + response.statusCode = value.status; + } +} + void WASMResponseBuffer::Store(WASMResponse& response, std::string value) { result_str_ = std::move(value); response.statusCode = 0; diff --git a/lib/src/webdb.cc b/lib/src/webdb.cc index 90ee16133..56b9c717c 100644 --- a/lib/src/webdb.cc +++ b/lib/src/webdb.cc @@ -232,13 +232,53 @@ bool WebDB::Connection::CancelPendingQuery() { } } -arrow::Result> WebDB::Connection::FetchQueryResults() { +DuckDBWasmResultsWrapper WebDB::Connection::FetchQueryResults() { try { // Fetch data if a query is active duckdb::unique_ptr chunk; if (current_query_result_ == nullptr) { - return nullptr; + return DuckDBWasmResultsWrapper{nullptr}; + } + + if (current_query_result_->type == QueryResultType::STREAM_RESULT) { + auto& stream_result = current_query_result_->Cast(); + + auto before = std::chrono::steady_clock::now(); + uint64_t elapsed; + auto polling_interval = + webdb_.config_->query.query_polling_interval.value_or(DEFAULT_QUERY_POLLING_INTERVAL); + bool ready = false; + do { + switch (stream_result.ExecuteTask()) { + case StreamExecutionResult::EXECUTION_ERROR: + return arrow::Status{arrow::StatusCode::ExecutionError, + std::move(current_query_result_->GetError())}; + case StreamExecutionResult::EXECUTION_CANCELLED: + return arrow::Status{arrow::StatusCode::ExecutionError, + "The execution of the query was cancelled before it could finish, likely " + "caused by executing a different query"}; + case StreamExecutionResult::CHUNK_READY: + case StreamExecutionResult::EXECUTION_FINISHED: + ready = true; + break; + case StreamExecutionResult::BLOCKED: + stream_result.WaitForTask(); + return DuckDBWasmResultsWrapper::ResponseStatus::DUCKDB_WASM_RETRY; + case StreamExecutionResult::NO_TASKS_AVAILABLE: + return DuckDBWasmResultsWrapper::ResponseStatus::DUCKDB_WASM_RETRY; + case StreamExecutionResult::CHUNK_NOT_READY: + break; + } + + auto after = std::chrono::steady_clock::now(); + elapsed = std::chrono::duration_cast(after - before).count(); + } while (!ready && elapsed < polling_interval); + + if (!ready) { + return DuckDBWasmResultsWrapper::ResponseStatus::DUCKDB_WASM_RETRY; + } } + // Fetch next result chunk chunk = current_query_result_->Fetch(); if (current_query_result_->HasError()) { @@ -249,7 +289,7 @@ arrow::Result> WebDB::Connection::FetchQueryResul current_query_result_.reset(); current_schema_.reset(); current_schema_patched_.reset(); - return nullptr; + return DuckDBWasmResultsWrapper{nullptr}; } // Serialize the record batch diff --git a/lib/src/webdb_api.cc b/lib/src/webdb_api.cc index bad9c7f9e..bf3521333 100644 --- a/lib/src/webdb_api.cc +++ b/lib/src/webdb_api.cc @@ -241,7 +241,7 @@ bool duckdb_web_pending_query_cancel(ConnectionHdl connHdl, const char* script) void duckdb_web_query_fetch_results(WASMResponse* packed, ConnectionHdl connHdl) { auto c = reinterpret_cast(connHdl); auto r = c->FetchQueryResults(); - WASMResponseBuffer::Get().Store(*packed, std::move(r)); + WASMResponseBuffer::Get().Store(*packed, r); } /// Get table names void duckdb_web_get_tablenames(WASMResponse* packed, ConnectionHdl connHdl, const char* query) { diff --git a/packages/duckdb-wasm/karma/s3rver/s3rver.js b/packages/duckdb-wasm/karma/s3rver/s3rver.js index 72a8c51ec..4f31c8121 100644 --- a/packages/duckdb-wasm/karma/s3rver/s3rver.js +++ b/packages/duckdb-wasm/karma/s3rver/s3rver.js @@ -1,15 +1,16 @@ const S3rver = require('s3rver'); -const CORS_CONFIG = "\n" + - " \n" + - " *\n" + - " PUT\n" + - " GET\n" + - " HEAD\n" + - " *\n" + - " Content-Range\n" + - " \n" + - ""; +const CORS_CONFIG = + '\n' + + ' \n' + + ' *\n' + + ' PUT\n' + + ' GET\n' + + ' HEAD\n' + + ' *\n' + + ' Content-Range\n' + + ' \n' + + ''; var createS3rver = function (args, config, logger) { const log = logger.create('S3-test-server'); @@ -19,10 +20,10 @@ var createS3rver = function (args, config, logger) { address: 'localhost', silent: config.s3rver.silent, directory: './../../.tmp/s3rver', - configureBuckets: [{name: 'test-bucket', configs:[CORS_CONFIG]}] + configureBuckets: [{ name: 'test-bucket', configs: [CORS_CONFIG] }], }).run(); }; module.exports = { - 'framework:s3rver': ['factory', createS3rver] -}; \ No newline at end of file + 'framework:s3rver': ['factory', createS3rver], +}; diff --git a/packages/duckdb-wasm/package.json b/packages/duckdb-wasm/package.json index 7fba82009..9931a2401 100644 --- a/packages/duckdb-wasm/package.json +++ b/packages/duckdb-wasm/package.json @@ -62,6 +62,7 @@ "build:debug": "node bundle.mjs debug && tsc --emitDeclarationOnly", "build:release": "node bundle.mjs release && tsc --emitDeclarationOnly", "docs": "typedoc", + "format": "prettier --write \"**/*.+(js|ts)\"", "report": "node ./coverage.mjs", "test:node": "node --enable-source-maps ../../node_modules/jasmine/bin/jasmine ./dist/tests-node.cjs", "test:node:debug": "node --inspect-brk --enable-source-maps ../../node_modules/jasmine/bin/jasmine ./dist/tests-node.cjs", diff --git a/packages/duckdb-wasm/src/bindings/bindings_base.ts b/packages/duckdb-wasm/src/bindings/bindings_base.ts index f395bdb10..d73bfa2c2 100644 --- a/packages/duckdb-wasm/src/bindings/bindings_base.ts +++ b/packages/duckdb-wasm/src/bindings/bindings_base.ts @@ -4,7 +4,7 @@ import { Logger } from '../log'; import { InstantiationProgress } from './progress'; import { DuckDBBindings } from './bindings_interface'; import { DuckDBConnection } from './connection'; -import { StatusCode } from '../status'; +import { StatusCode, IsArrowBuffer, IsDuckDBWasmRetry } from '../status'; import { dropResponseBuffers, DuckDBRuntime, readString, callSRet, copyBuffer, DuckDBDataProtocol } from './runtime'; import { CSVInsertOptions, JSONInsertOptions, ArrowInsertOptions } from './insert_options'; import { ScriptTokens } from './tokens'; @@ -135,10 +135,15 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { /** Tokenize a script */ public tokenize(text: string): ScriptTokens { const BUF = TEXT_ENCODER.encode(text); - const bufferPtr = this.mod._malloc(BUF.length ); - const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length ); + const bufferPtr = this.mod._malloc(BUF.length); + const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length); bufferOfs.set(BUF); - const [s, d, n] = callSRet(this.mod, 'duckdb_web_tokenize_buffer', ['number', 'number'], [bufferPtr, BUF.length]); + const [s, d, n] = callSRet( + this.mod, + 'duckdb_web_tokenize_buffer', + ['number', 'number'], + [bufferPtr, BUF.length], + ); this.mod._free(bufferPtr); if (s !== StatusCode.SUCCESS) { throw new Error(readString(this.mod, d, n)); @@ -172,7 +177,12 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { const bufferPtr = this.mod._malloc(BUF.length); const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length); bufferOfs.set(BUF); - const [s, d, n] = callSRet(this.mod, 'duckdb_web_query_run_buffer', ['number', 'number', 'number'], [conn, bufferPtr, BUF.length]); + const [s, d, n] = callSRet( + this.mod, + 'duckdb_web_query_run_buffer', + ['number', 'number', 'number'], + [conn, bufferPtr, BUF.length], + ); this.mod._free(bufferPtr); if (s !== StatusCode.SUCCESS) { throw new Error(readString(this.mod, d, n)); @@ -189,10 +199,15 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { */ public startPendingQuery(conn: number, text: string, allowStreamResult: boolean = false): Uint8Array | null { const BUF = TEXT_ENCODER.encode(text); - const bufferPtr = this.mod._malloc(BUF.length ); - const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length ); + const bufferPtr = this.mod._malloc(BUF.length); + const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length); bufferOfs.set(BUF); - const [s, d, n] = callSRet(this.mod, 'duckdb_web_pending_query_start_buffer', ['number', 'number', 'number', 'boolean'], [conn, bufferPtr, BUF.length, allowStreamResult]); + const [s, d, n] = callSRet( + this.mod, + 'duckdb_web_pending_query_start_buffer', + ['number', 'number', 'number', 'boolean'], + [conn, bufferPtr, BUF.length, allowStreamResult], + ); this.mod._free(bufferPtr); if (s !== StatusCode.SUCCESS) { throw new Error(readString(this.mod, d, n)); @@ -222,11 +237,25 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { return this.mod.ccall('duckdb_web_pending_query_cancel', 'boolean', ['number'], [conn]); } /** Fetch query results */ - public fetchQueryResults(conn: number): Uint8Array { + public fetchQueryResults(conn: number): Uint8Array | null { const [s, d, n] = callSRet(this.mod, 'duckdb_web_query_fetch_results', ['number'], [conn]); + if (IsDuckDBWasmRetry(s)) { + dropResponseBuffers(this.mod); + return null; // Retry + } + + if (!IsArrowBuffer(s)) { + throw new Error( + 'Unexpected StatusCode from duckdb_web_query_fetch_results (' + + s + + ') and with self reported error as' + + readString(this.mod, d, n), + ); + } if (s !== StatusCode.SUCCESS) { throw new Error(readString(this.mod, d, n)); } + const res = copyBuffer(this.mod, d, n); dropResponseBuffers(this.mod); return res; @@ -237,7 +266,12 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { const bufferPtr = this.mod._malloc(BUF.length); const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length); bufferOfs.set(BUF); - const [s, d, n] = callSRet(this.mod, 'duckdb_web_get_tablenames_buffer', ['number', 'number', 'number'], [conn, bufferPtr, BUF.length]); + const [s, d, n] = callSRet( + this.mod, + 'duckdb_web_get_tablenames_buffer', + ['number', 'number', 'number'], + [conn, bufferPtr, BUF.length], + ); this.mod._free(bufferPtr); if (s !== StatusCode.SUCCESS) { throw new Error(readString(this.mod, d, n)); @@ -297,7 +331,12 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { const bufferPtr = this.mod._malloc(BUF.length); const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length); bufferOfs.set(BUF); - const [s, d, n] = callSRet(this.mod, 'duckdb_web_prepared_create_buffer', ['number', 'number', 'number'], [conn, bufferPtr, BUF.length]); + const [s, d, n] = callSRet( + this.mod, + 'duckdb_web_prepared_create_buffer', + ['number', 'number', 'number'], + [conn, bufferPtr, BUF.length], + ); this.mod._free(bufferPtr); if (s !== StatusCode.SUCCESS) { throw new Error(readString(this.mod, d, n)); @@ -504,28 +543,28 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { directIO: boolean, ): Promise { if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS) { - if( handle instanceof FileSystemSyncAccessHandle ){ + if (handle instanceof FileSystemSyncAccessHandle) { // already a handle is sync handle. - } else if( handle instanceof FileSystemFileHandle ){ + } else if (handle instanceof FileSystemFileHandle) { // handle is an async handle, should convert to sync handle const fileHandle: FileSystemFileHandle = handle as any; try { handle = (await fileHandle.createSyncAccessHandle()) as any; } catch (e: any) { - throw new Error( e.message + ":" + name ); + throw new Error(e.message + ':' + name); } - } else if( name != null ){ + } else if (name != null) { // should get sync handle from the file name. try { const opfsRoot = await navigator.storage.getDirectory(); const fileHandle = await opfsRoot.getFileHandle(name); handle = (await fileHandle.createSyncAccessHandle()) as any; } catch (e: any) { - throw new Error( e.message + ":" + name ); + throw new Error(e.message + ':' + name); } } } - return handle; + return handle; } /** Register a file object URL async */ public async registerFileHandleAsync( @@ -616,10 +655,10 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { } /** Enable tracking of file statistics */ public registerOPFSFileName(file: string): Promise { - if (file.startsWith("opfs://")) { - return this.prepareFileHandle(file, DuckDBDataProtocol.BROWSER_FSACCESS); - } else { - throw new Error("Not an OPFS file name: " + file); + if (file.startsWith('opfs://')) { + return this.prepareFileHandle(file, DuckDBDataProtocol.BROWSER_FSACCESS); + } else { + throw new Error('Not an OPFS file name: ' + file); } } public collectFileStatistics(file: string, enable: boolean): void { diff --git a/packages/duckdb-wasm/src/bindings/bindings_interface.ts b/packages/duckdb-wasm/src/bindings/bindings_interface.ts index 271a42ef9..2920972b2 100644 --- a/packages/duckdb-wasm/src/bindings/bindings_interface.ts +++ b/packages/duckdb-wasm/src/bindings/bindings_interface.ts @@ -19,7 +19,7 @@ export interface DuckDBBindings { startPendingQuery(conn: number, text: string, allowStreamResult: boolean): Uint8Array | null; pollPendingQuery(conn: number): Uint8Array | null; cancelPendingQuery(conn: number): boolean; - fetchQueryResults(conn: number): Uint8Array; + fetchQueryResults(conn: number): Uint8Array | null; getTableNames(conn: number, text: string): string[]; createPrepared(conn: number, text: string): number; diff --git a/packages/duckdb-wasm/src/bindings/connection.ts b/packages/duckdb-wasm/src/bindings/connection.ts index 70b5f018d..a80c12215 100644 --- a/packages/duckdb-wasm/src/bindings/connection.ts +++ b/packages/duckdb-wasm/src/bindings/connection.ts @@ -53,7 +53,6 @@ export class DuckDBConnection { //Otherwise, reject with the error reject(e); } - } }); } @@ -112,7 +111,11 @@ export class ResultStreamIterator implements Iterable { /** Reached end of stream? */ _depleted: boolean; - constructor(protected bindings: DuckDBBindings, protected conn: number, protected header: Uint8Array) { + constructor( + protected bindings: DuckDBBindings, + protected conn: number, + protected header: Uint8Array, + ) { this._first = true; this._depleted = false; } @@ -125,7 +128,10 @@ export class ResultStreamIterator implements Iterable { if (this._depleted) { return { done: true, value: null }; } - const bufferI8 = this.bindings.fetchQueryResults(this.conn); + let bufferI8 = null; + do { + bufferI8 = this.bindings.fetchQueryResults(this.conn); + } while (bufferI8 == null); this._depleted = bufferI8.length == 0; return { done: this._depleted, diff --git a/packages/duckdb-wasm/src/bindings/runtime.ts b/packages/duckdb-wasm/src/bindings/runtime.ts index f894dba46..86b27d992 100644 --- a/packages/duckdb-wasm/src/bindings/runtime.ts +++ b/packages/duckdb-wasm/src/bindings/runtime.ts @@ -140,7 +140,7 @@ export interface DuckDBRuntime { openFile(mod: DuckDBModule, fileId: number, flags: FileFlags): void; syncFile(mod: DuckDBModule, fileId: number): void; closeFile(mod: DuckDBModule, fileId: number): void; - dropFile(mod: DuckDBModule, fileNamePtr: number, fileNameLen:number): void; + dropFile(mod: DuckDBModule, fileNamePtr: number, fileNameLen: number): void; getLastFileModificationTime(mod: DuckDBModule, fileId: number): number; truncateFile(mod: DuckDBModule, fileId: number, newSize: number): void; readFile(mod: DuckDBModule, fileId: number, buffer: number, bytes: number, location: number): number; @@ -162,7 +162,7 @@ export interface DuckDBRuntime { prepareDBFileHandle?: (path: string, protocol: DuckDBDataProtocol) => Promise; // Internal API - experimental - progressUpdate(final: number, percentage: number, iteration:number): void; + progressUpdate(final: number, percentage: number, iteration: number): void; // Call a scalar UDF function callScalarUDF( @@ -184,7 +184,7 @@ export const DEFAULT_RUNTIME: DuckDBRuntime = { openFile: (_mod: DuckDBModule, _fileId: number, flags: FileFlags): void => {}, syncFile: (_mod: DuckDBModule, _fileId: number): void => {}, closeFile: (_mod: DuckDBModule, _fileId: number): void => {}, - dropFile: (_mod: DuckDBModule, _fileNamePtr: number, _fileNameLen:number): void => {}, + dropFile: (_mod: DuckDBModule, _fileNamePtr: number, _fileNameLen: number): void => {}, getLastFileModificationTime: (_mod: DuckDBModule, _fileId: number): number => { return 0; }, diff --git a/packages/duckdb-wasm/src/bindings/runtime_browser.ts b/packages/duckdb-wasm/src/bindings/runtime_browser.ts index b5069dac4..e9f88b2b4 100644 --- a/packages/duckdb-wasm/src/bindings/runtime_browser.ts +++ b/packages/duckdb-wasm/src/bindings/runtime_browser.ts @@ -1,8 +1,6 @@ -import {StatusCode} from '../status'; -import { - WorkerResponseType, -} from '../parallel/worker_request'; -import {addS3Headers, getHTTPUrl} from '../utils'; +import { StatusCode } from '../status'; +import { WorkerResponseType } from '../parallel/worker_request'; +import { addS3Headers, getHTTPUrl } from '../utils'; import { callSRet, @@ -22,7 +20,6 @@ import * as udf from './udf_runtime'; const OPFS_PREFIX_LEN = 'opfs://'.length; const PATH_SEP_REGEX = /\/|\\/; - export const BROWSER_RUNTIME: DuckDBRuntime & { _files: Map; _fileInfoCache: Map; @@ -100,7 +97,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { if (info == null) { return null; } - BROWSER_RUNTIME._globalFileInfo = { ...info, blob: null} as DuckDBGlobalFileInfo; + BROWSER_RUNTIME._globalFileInfo = { ...info, blob: null } as DuckDBGlobalFileInfo; return BROWSER_RUNTIME._globalFileInfo; } catch (e: any) { @@ -111,7 +108,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { async assignOPFSRoot(): Promise { if (!BROWSER_RUNTIME._opfsRoot) { BROWSER_RUNTIME._opfsRoot = await navigator.storage.getDirectory(); - } + } }, /** Prepare a file handle that could only be acquired aschronously */ async prepareFileHandles(filePaths: string[], protocol: DuckDBDataProtocol): Promise { @@ -157,7 +154,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { fromCached: false, }; } catch (e: any) { - throw new Error(e.message + ":" + name); + throw new Error(e.message + ':' + name); } }; const result: PreparedDBFileHandle[] = []; @@ -268,7 +265,6 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { mod.HEAPF64[(result >> 3) + 1] = 0; return result; } - } catch (e: any) { error = e; console.warn(`HEAD request with range header failed: ${e}`); @@ -318,13 +314,23 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { } } - if (xhr.status == 206 && contentLength2 !== null && +contentLength2 == 1 && presumedLength !== null) { + if ( + xhr.status == 206 && + contentLength2 !== null && + +contentLength2 == 1 && + presumedLength !== null + ) { const result = mod._malloc(2 * 8); mod.HEAPF64[(result >> 3) + 0] = +presumedLength; mod.HEAPF64[(result >> 3) + 1] = 0; return result; } - if (xhr.status == 200 && contentLength2 !== null && contentLength !== null && +contentLength2 == +contentLength) { + if ( + xhr.status == 200 && + contentLength2 !== null && + contentLength !== null && + +contentLength2 == +contentLength + ) { console.warn(`fall back to full HTTP read for: ${file.dataUrl}`); const data = mod._malloc(xhr.response.byteLength); const src = new Uint8Array(xhr.response, 0, xhr.response.byteLength); @@ -494,24 +500,24 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { closeFile: (mod: DuckDBModule, fileId: number) => { const file = BROWSER_RUNTIME.getFileInfo(mod, fileId); BROWSER_RUNTIME._fileInfoCache.delete(fileId); - try { - switch (file?.dataProtocol) { - case DuckDBDataProtocol.BUFFER: - case DuckDBDataProtocol.HTTP: - case DuckDBDataProtocol.S3: - break; - case DuckDBDataProtocol.NODE_FS: - case DuckDBDataProtocol.BROWSER_FILEREADER: - // XXX Remove from registry - return; - case DuckDBDataProtocol.BROWSER_FSACCESS: { - const handle: FileSystemSyncAccessHandle = BROWSER_RUNTIME._files?.get(file.fileName); - if (!handle) { - throw new Error(`No OPFS access handle registered with name: ${file.fileName}`); + try { + switch (file?.dataProtocol) { + case DuckDBDataProtocol.BUFFER: + case DuckDBDataProtocol.HTTP: + case DuckDBDataProtocol.S3: + break; + case DuckDBDataProtocol.NODE_FS: + case DuckDBDataProtocol.BROWSER_FILEREADER: + // XXX Remove from registry + return; + case DuckDBDataProtocol.BROWSER_FSACCESS: { + const handle: FileSystemSyncAccessHandle = BROWSER_RUNTIME._files?.get(file.fileName); + if (!handle) { + throw new Error(`No OPFS access handle registered with name: ${file.fileName}`); + } + return handle.flush(); } - return handle.flush(); } - } } catch (e: any) { console.log(e); failWith(mod, e.toString()); @@ -691,9 +697,13 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { return 0; }, progressUpdate: (done: number, percentage: number, repeat: number): void => { - if (postMessage) { - postMessage({requestId: 0, type: WorkerResponseType.PROGRESS_UPDATE, data: {status: done?"completed":"in-progress", percentage: percentage, repetitions: repeat}}); - } + if (postMessage) { + postMessage({ + requestId: 0, + type: WorkerResponseType.PROGRESS_UPDATE, + data: { status: done ? 'completed' : 'in-progress', percentage: percentage, repetitions: repeat }, + }); + } }, checkDirectory: (mod: DuckDBModule, pathPtr: number, pathLen: number) => { const path = readString(mod, pathPtr, pathLen); diff --git a/packages/duckdb-wasm/src/bindings/runtime_node.ts b/packages/duckdb-wasm/src/bindings/runtime_node.ts index 5037068a2..7f331d079 100644 --- a/packages/duckdb-wasm/src/bindings/runtime_node.ts +++ b/packages/duckdb-wasm/src/bindings/runtime_node.ts @@ -127,7 +127,7 @@ export const NODE_RUNTIME: DuckDBRuntime & { } return 0; }, - dropFile: (mod: DuckDBModule, _fileNamePtr: number, _fileNameLen:number) => {}, + dropFile: (mod: DuckDBModule, _fileNamePtr: number, _fileNameLen: number) => {}, truncateFile: (mod: DuckDBModule, fileId: number, newSize: number) => { try { const file = NODE_RUNTIME.resolveFileInfo(mod, fileId); diff --git a/packages/duckdb-wasm/src/log.ts b/packages/duckdb-wasm/src/log.ts index 4a4f836ca..c705e28b3 100644 --- a/packages/duckdb-wasm/src/log.ts +++ b/packages/duckdb-wasm/src/log.ts @@ -45,7 +45,7 @@ export type ProgressEntry = { readonly status: string; readonly percentage: string; readonly repetitions: string; -} +}; /** An execution progress handler */ export type ExecutionProgressHandler = (p: ProgressEntry) => void; diff --git a/packages/duckdb-wasm/src/parallel/async_bindings.ts b/packages/duckdb-wasm/src/parallel/async_bindings.ts index dc8a81e53..2e2b3dc72 100644 --- a/packages/duckdb-wasm/src/parallel/async_bindings.ts +++ b/packages/duckdb-wasm/src/parallel/async_bindings.ts @@ -129,7 +129,7 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { case WorkerResponseType.PROGRESS_UPDATE: { for (const p of this._onExecutionProgress) { p(response.data); - } + } return; } case WorkerResponseType.LOG: { @@ -442,8 +442,8 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { } /** Fetch query results */ - public async fetchQueryResults(conn: ConnectionID): Promise { - const task = new WorkerTask( + public async fetchQueryResults(conn: ConnectionID): Promise { + const task = new WorkerTask( WorkerRequestType.FETCH_QUERY_RESULTS, conn, ); @@ -524,7 +524,7 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { /** Register an empty file buffer. */ public async registerEmptyFileBuffer(name: string): Promise { -/* + /* const task = new WorkerTask( WorkerRequestType.REGISTER_FILE_BUFFER, [name, new Uint8Array()], diff --git a/packages/duckdb-wasm/src/parallel/async_bindings_interface.ts b/packages/duckdb-wasm/src/parallel/async_bindings_interface.ts index 97ba2b191..da1e7cbf6 100644 --- a/packages/duckdb-wasm/src/parallel/async_bindings_interface.ts +++ b/packages/duckdb-wasm/src/parallel/async_bindings_interface.ts @@ -22,7 +22,7 @@ export interface AsyncDuckDBBindings { startPendingQuery(conn: number, text: string, allowStreamResult: boolean): Promise; pollPendingQuery(conn: number): Promise; cancelPendingQuery(conn: number): Promise; - fetchQueryResults(conn: number): Promise; + fetchQueryResults(conn: number): Promise; createPrepared(conn: number, text: string): Promise; closePrepared(conn: number, statement: number): Promise; diff --git a/packages/duckdb-wasm/src/parallel/async_connection.ts b/packages/duckdb-wasm/src/parallel/async_connection.ts index 783a18bd5..89bec591f 100644 --- a/packages/duckdb-wasm/src/parallel/async_connection.ts +++ b/packages/duckdb-wasm/src/parallel/async_connection.ts @@ -42,8 +42,8 @@ export class AsyncDuckDBConnection { }); const buffer = await this._bindings.runQuery(this._conn, text); const reader = arrow.RecordBatchReader.from(buffer); - console.assert(reader.isSync(), "Reader is not sync"); - console.assert(reader.isFile(), "Reader is not file"); + console.assert(reader.isSync(), 'Reader is not sync'); + console.assert(reader.isFile(), 'Reader is not file'); return new arrow.Table(reader as arrow.RecordBatchFileReader); } @@ -115,7 +115,7 @@ export class AsyncResultStreamIterator implements AsyncIterable { /** Reached end of stream? */ protected _depleted: boolean; /** In-flight */ - protected _inFlight: Promise | null; + protected _inFlight: Promise | null; constructor( protected readonly db: AsyncDuckDB, @@ -135,17 +135,21 @@ export class AsyncResultStreamIterator implements AsyncIterable { if (this._depleted) { return { done: true, value: null }; } - let buffer: Uint8Array; + let buffer: Uint8Array | null = null; if (this._inFlight != null) { buffer = await this._inFlight; this._inFlight = null; - } else { + } + + while (buffer == null) { buffer = await this.db.fetchQueryResults(this.conn); } + this._depleted = buffer.length == 0; if (!this._depleted) { this._inFlight = this.db.fetchQueryResults(this.conn); } + return { done: this._depleted, value: buffer, diff --git a/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts b/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts index 3a5a8f295..b637b78b4 100644 --- a/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts +++ b/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts @@ -279,6 +279,7 @@ export abstract class AsyncDuckDBDispatcher implements Logger { } case WorkerRequestType.FETCH_QUERY_RESULTS: { const result = this._bindings.fetchQueryResults(request.data); + const transfer = result ? [result.buffer] : []; this.postMessage( { messageId: this._nextMessageId++, @@ -286,7 +287,7 @@ export abstract class AsyncDuckDBDispatcher implements Logger { type: WorkerResponseType.QUERY_RESULT_CHUNK, data: result, }, - [result.buffer], + transfer, ); break; } diff --git a/packages/duckdb-wasm/src/parallel/worker_request.ts b/packages/duckdb-wasm/src/parallel/worker_request.ts index 38502b7b6..071510d16 100644 --- a/packages/duckdb-wasm/src/parallel/worker_request.ts +++ b/packages/duckdb-wasm/src/parallel/worker_request.ts @@ -160,7 +160,7 @@ export type WorkerResponseVariant = | WorkerResponse | WorkerResponse | WorkerResponse - | WorkerResponse + | WorkerResponse | WorkerResponse | WorkerResponse | WorkerResponse @@ -180,7 +180,7 @@ export type WorkerTaskVariant = | WorkerTask | WorkerTask | WorkerTask - | WorkerTask + | WorkerTask | WorkerTask | WorkerTask | WorkerTask diff --git a/packages/duckdb-wasm/src/platform.ts b/packages/duckdb-wasm/src/platform.ts index 8f17c3ad6..8d175d69c 100644 --- a/packages/duckdb-wasm/src/platform.ts +++ b/packages/duckdb-wasm/src/platform.ts @@ -20,10 +20,10 @@ export const isFirefox = () => userAgent().includes('Firefox'); export const isSafari = () => /^((?!chrome|android).)*safari/i.test(userAgent()); /** Bundles have different characteristics: - * - MVP: minimum viable product (uses features from first stable version of WebAssembly standard) - * - EH: exception handling - * - COI: cross origin isolation - */ + * - MVP: minimum viable product (uses features from first stable version of WebAssembly standard) + * - EH: exception handling + * - COI: cross origin isolation + */ export interface DuckDBBundles { mvp: { mainModule: string; diff --git a/packages/duckdb-wasm/src/status.ts b/packages/duckdb-wasm/src/status.ts index 6390a5ec1..7b0557da1 100644 --- a/packages/duckdb-wasm/src/status.ts +++ b/packages/duckdb-wasm/src/status.ts @@ -1,3 +1,13 @@ export enum StatusCode { SUCCESS = 0, + MAX_ARROW_ERROR = 255, + DUCKDB_WASM_RETRY = 256, +} + +export function IsArrowBuffer(status: StatusCode): boolean { + return status <= StatusCode.MAX_ARROW_ERROR; +} + +export function IsDuckDBWasmRetry(status: StatusCode): boolean { + return status === StatusCode.DUCKDB_WASM_RETRY; } diff --git a/packages/duckdb-wasm/src/utils/s3_helper.ts b/packages/duckdb-wasm/src/utils/s3_helper.ts index aa0d908a2..72390a9e6 100644 --- a/packages/duckdb-wasm/src/utils/s3_helper.ts +++ b/packages/duckdb-wasm/src/utils/s3_helper.ts @@ -1,30 +1,30 @@ -import {S3Config} from "../bindings"; -import {sha256} from "js-sha256"; +import { S3Config } from '../bindings'; +import { sha256 } from 'js-sha256'; export interface S3Params { - url: string, - query: string, - host: string, - region: string, - service: string, - method: string, - accessKeyId: string, - secretAccessKey: string, - sessionToken: string, - dateNow: string, - datetimeNow: string + url: string; + query: string; + host: string; + region: string; + service: string; + method: string; + accessKeyId: string; + secretAccessKey: string; + sessionToken: string; + dateNow: string; + datetimeNow: string; } export interface S3PayloadParams { - contentHash: string | null, - contentType: string | null + contentHash: string | null; + contentType: string | null; } -const getHTTPHost = function (config : S3Config | undefined, url : string, bucket : string) : string { - if (config?.endpoint?.startsWith("http")) { +const getHTTPHost = function (config: S3Config | undefined, url: string, bucket: string): string { + if (config?.endpoint?.startsWith('http')) { // Endpoint is a full url, we append the bucket const httpHost = `${config?.endpoint}`; - const offset = httpHost.indexOf("://")+3; + const offset = httpHost.indexOf('://') + 3; return httpHost.substring(offset); } else if (config?.endpoint) { // Endpoint is not a full url and the https://{bucket}.{domain} format will be used @@ -33,53 +33,60 @@ const getHTTPHost = function (config : S3Config | undefined, url : string, bucke // Default aws s3 url return `${bucket}.s3.amazonaws.com`; } -} +}; -export function getS3Params (config : S3Config | undefined, url: string, method : string) : S3Params { +export function getS3Params(config: S3Config | undefined, url: string, method: string): S3Params { const parsedS3Url = parseS3Url(url); - // when using S3 path-style access, the signed URL should also include the bucket name, + // when using S3 path-style access, the signed URL should also include the bucket name, // as it is present in the HTTP URL path. // See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-bucket-intro.html#path-style-url-ex - let path = parsedS3Url.path; + let path = parsedS3Url.path; if (isPathStyleAccess(config)) { path = `/${parsedS3Url.bucket}${path}`; } return { url: path, - query: "", + query: '', host: getHTTPHost(config, url, parsedS3Url.bucket), - region: (config?.region) ?? "", - service: "s3", + region: config?.region ?? '', + service: 's3', method: method, - accessKeyId: (config?.accessKeyId) ?? "", - secretAccessKey: (config?.secretAccessKey) ?? "", - sessionToken: (config?.sessionToken) ?? "", - dateNow: new Date().toISOString().replace(/-/g,'').split('T')[0], - datetimeNow: new Date().toISOString().replace(/-/g,'').replace(/:/g,'').split('.')[0]+ 'Z', + accessKeyId: config?.accessKeyId ?? '', + secretAccessKey: config?.secretAccessKey ?? '', + sessionToken: config?.sessionToken ?? '', + dateNow: new Date().toISOString().replace(/-/g, '').split('T')[0], + datetimeNow: new Date().toISOString().replace(/-/g, '').replace(/:/g, '').split('.')[0] + 'Z', }; } -export function uriEncode(input : string, encode_slash = false) { +export function uriEncode(input: string, encode_slash = false) { // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html - const hexDigit = "0123456789ABCDEF"; - let result = ""; + const hexDigit = '0123456789ABCDEF'; + let result = ''; for (let i = 0; i < input.length; i++) { - const ch : string = input[i]; - - if ((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' || - ch == '-' || ch == '~' || ch == '.') { + const ch: string = input[i]; + + if ( + (ch >= 'A' && ch <= 'Z') || + (ch >= 'a' && ch <= 'z') || + (ch >= '0' && ch <= '9') || + ch == '_' || + ch == '-' || + ch == '~' || + ch == '.' + ) { result += ch; } else if (ch == '/') { if (encode_slash) { - result += "%2F"; + result += '%2F'; } else { result += ch; } } else { - result += "%"; + result += '%'; result += hexDigit[ch.charCodeAt(0) >> 4]; result += hexDigit[ch.charCodeAt(0) & 15]; } @@ -87,47 +94,57 @@ export function uriEncode(input : string, encode_slash = false) { return result; } -export function createS3Headers(params: S3Params, payloadParams : S3PayloadParams | null = null) : Map { +export function createS3Headers(params: S3Params, payloadParams: S3PayloadParams | null = null): Map { // this is the sha256 of the empty string, its useful since we have no payload for GET requests - const payloadHash = (payloadParams?.contentHash) ?? "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + const payloadHash = + payloadParams?.contentHash ?? 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; const res = new Map(); // res.set("host", params.host) - res.set("x-amz-date", params.datetimeNow); - res.set("x-amz-content-sha256", payloadHash); + res.set('x-amz-date', params.datetimeNow); + res.set('x-amz-content-sha256', payloadHash); if (params.sessionToken) { - res.set("x-amz-security-token", params.sessionToken); + res.set('x-amz-security-token', params.sessionToken); } // construct string to sign - let signedHeaders = ""; + let signedHeaders = ''; if (payloadParams?.contentType) { - signedHeaders += "content-type;"; + signedHeaders += 'content-type;'; } - signedHeaders += "host;x-amz-content-sha256;x-amz-date"; + signedHeaders += 'host;x-amz-content-sha256;x-amz-date'; if (params.sessionToken) { - signedHeaders += ";x-amz-security-token"; + signedHeaders += ';x-amz-security-token'; } - let canonicalRequest = params.method + "\n" + uriEncode(params.url) + "\n" + params.query; + let canonicalRequest = params.method + '\n' + uriEncode(params.url) + '\n' + params.query; if (payloadParams?.contentType) { - canonicalRequest += "\ncontent-type:" + payloadParams?.contentType; + canonicalRequest += '\ncontent-type:' + payloadParams?.contentType; } - canonicalRequest += "\nhost:" + params.host + - "\nx-amz-content-sha256:" + payloadHash + "\nx-amz-date:" + params.datetimeNow; + canonicalRequest += + '\nhost:' + params.host + '\nx-amz-content-sha256:' + payloadHash + '\nx-amz-date:' + params.datetimeNow; if (params.sessionToken && params.sessionToken.length > 0) { - canonicalRequest += "\nx-amz-security-token:" + params.sessionToken; + canonicalRequest += '\nx-amz-security-token:' + params.sessionToken; } - canonicalRequest += "\n\n" + signedHeaders + "\n" + payloadHash; + canonicalRequest += '\n\n' + signedHeaders + '\n' + payloadHash; const canonicalRequestHashStr = sha256(canonicalRequest); - const stringToSign = "AWS4-HMAC-SHA256\n" + params.datetimeNow + "\n" + params.dateNow + "/" + params.region + "/" + params.service + - "/aws4_request\n" + canonicalRequestHashStr; + const stringToSign = + 'AWS4-HMAC-SHA256\n' + + params.datetimeNow + + '\n' + + params.dateNow + + '/' + + params.region + + '/' + + params.service + + '/aws4_request\n' + + canonicalRequestHashStr; // ts-ignore's because library can accept array buffer as key, but TS arg is incorrect - const signKey = "AWS4" + params.secretAccessKey; + const signKey = 'AWS4' + params.secretAccessKey; const kDate = sha256.hmac.arrayBuffer(signKey, params.dateNow); // Note, js-sha256 has a bug in the TS interface that only supports strings as keys, while we need a bytearray @@ -137,31 +154,56 @@ export function createS3Headers(params: S3Params, payloadParams : S3PayloadParam const kRegion = sha256.hmac.arrayBuffer(kDate, params.region); // eslint-disable-next-line // @ts-ignore - const kService = sha256.hmac.arrayBuffer(kRegion, params.service,); + const kService = sha256.hmac.arrayBuffer(kRegion, params.service); // eslint-disable-next-line // @ts-ignore - const signingKey = sha256.hmac.arrayBuffer(kService, "aws4_request"); + const signingKey = sha256.hmac.arrayBuffer(kService, 'aws4_request'); // eslint-disable-next-line // @ts-ignore const signature = sha256.hmac(signingKey, stringToSign); - res.set("Authorization", "AWS4-HMAC-SHA256 Credential=" + params.accessKeyId + "/" + params.dateNow + "/" + params.region + "/" + - params.service + "/aws4_request, SignedHeaders=" + signedHeaders + - ", Signature=" + signature); + res.set( + 'Authorization', + 'AWS4-HMAC-SHA256 Credential=' + + params.accessKeyId + + '/' + + params.dateNow + + '/' + + params.region + + '/' + + params.service + + '/aws4_request, SignedHeaders=' + + signedHeaders + + ', Signature=' + + signature, + ); return res; } -const createS3HeadersFromS3Config = function (config : S3Config | undefined, url : string, method : string, contentType: string | null = null, payload : Uint8Array | null = null) : Map { +const createS3HeadersFromS3Config = function ( + config: S3Config | undefined, + url: string, + method: string, + contentType: string | null = null, + payload: Uint8Array | null = null, +): Map { const params = getS3Params(config, url, method); const payloadParams = { contentType: contentType, - contentHash: payload ? sha256.hex(payload!) : null + contentHash: payload ? sha256.hex(payload!) : null, } as S3PayloadParams; return createS3Headers(params, payloadParams); -} +}; -export function addS3Headers(xhr: XMLHttpRequest, config : S3Config | undefined, url : string, method: string, contentType: string | null = null, payload : Uint8Array | null = null) { +export function addS3Headers( + xhr: XMLHttpRequest, + config: S3Config | undefined, + url: string, + method: string, + contentType: string | null = null, + payload: Uint8Array | null = null, +) { if (config?.accessKeyId || config?.sessionToken) { const headers = createS3HeadersFromS3Config(config, url, method, contentType, payload); headers.forEach((value: string, header: string) => { @@ -169,14 +211,14 @@ export function addS3Headers(xhr: XMLHttpRequest, config : S3Config | undefined, }); if (contentType) { - xhr.setRequestHeader("content-type", contentType); + xhr.setRequestHeader('content-type', contentType); } } } -export function parseS3Url (url: string) : {bucket : string, path : string} { - if (url.indexOf("s3://") != 0) { - throw new Error("URL needs to start with s3://"); +export function parseS3Url(url: string): { bucket: string; path: string } { + if (url.indexOf('s3://') != 0) { + throw new Error('URL needs to start with s3://'); } const slashPos = url.indexOf('/', 5); @@ -186,28 +228,28 @@ export function parseS3Url (url: string) : {bucket : string, path : string} { const bucket = url.substring(5, slashPos); if (!bucket) { - throw new Error("URL needs to contain a bucket name"); + throw new Error('URL needs to contain a bucket name'); } const path = url.substring(slashPos); if (!path) { - throw new Error("URL needs to contain key"); + throw new Error('URL needs to contain key'); } - return {bucket: bucket, path: path} + return { bucket: bucket, path: path }; } -function isPathStyleAccess(config : S3Config | undefined) : boolean { - if (config?.endpoint?.startsWith("http")) { - return true +function isPathStyleAccess(config: S3Config | undefined): boolean { + if (config?.endpoint?.startsWith('http')) { + return true; } - return false + return false; } -export function getHTTPUrl(config : S3Config | undefined, url : string) : string { +export function getHTTPUrl(config: S3Config | undefined, url: string): string { const parsedUrl = parseS3Url(url); if (isPathStyleAccess(config)) { // Endpoint is a full url, we append the bucket return `${config?.endpoint}/${parsedUrl.bucket}` + parsedUrl.path; } return 'https://' + getHTTPHost(config, url, parsedUrl.bucket) + parsedUrl.path; -} \ No newline at end of file +} diff --git a/packages/duckdb-wasm/test/bindings.test.ts b/packages/duckdb-wasm/test/bindings.test.ts index e016ba13e..97a4447e0 100644 --- a/packages/duckdb-wasm/test/bindings.test.ts +++ b/packages/duckdb-wasm/test/bindings.test.ts @@ -45,12 +45,10 @@ export function testBindings(db: () => duckdb.DuckDBBindings, baseURL: string): it('Platform check', async () => { await db().reset(); conn = db().connect(); - const version = conn.query<{ name: arrow.Utf8 }>( - "PRAGMA platform;", - ); + const version = conn.query<{ name: arrow.Utf8 }>('PRAGMA platform;'); const rows = version.getChildAt(0)?.toArray(); expect(rows.length).toEqual(1); - expect(rows[0].toString().substr(0,5)).toEqual("wasm_"); + expect(rows[0].toString().substr(0, 5)).toEqual('wasm_'); await db().reset(); }); }); diff --git a/packages/duckdb-wasm/test/excel.test.ts b/packages/duckdb-wasm/test/excel.test.ts index 14e266f07..d293c0e05 100644 --- a/packages/duckdb-wasm/test/excel.test.ts +++ b/packages/duckdb-wasm/test/excel.test.ts @@ -14,15 +14,30 @@ export function testEXCEL(db: () => duckdb.DuckDBBindings): void { describe('EXCEL', () => { it('sample', async () => { - expect(conn.query("SELECT text(1234567.897, 'h:mm:ss.00')",).getChildAt(0)?.toArray()).toEqual(['21:31:40.80']); - expect(conn.query("SELECT text(1234567.897, 'm/d/yyyy h:mm AM/PM')",).getChildAt(0)?.toArray()).toEqual(['2/15/5280 9:31 PM']); - expect(conn.query("SELECT text(1234567.897, 'dddd, dd of MMMM of YYYY')",).getChildAt(0)?.toArray()).toEqual(['Thursday, 15 of February of 5280']); + expect(conn.query("SELECT text(1234567.897, 'h:mm:ss.00')").getChildAt(0)?.toArray()).toEqual([ + '21:31:40.80', + ]); + expect(conn.query("SELECT text(1234567.897, 'm/d/yyyy h:mm AM/PM')").getChildAt(0)?.toArray()).toEqual([ + '2/15/5280 9:31 PM', + ]); + expect(conn.query("SELECT text(1234567.897, 'dddd, dd of MMMM of YYYY')").getChildAt(0)?.toArray()).toEqual( + ['Thursday, 15 of February of 5280'], + ); - expect(conn.query("SELECT text(1234567.897, '# ??/??')",).getChildAt(0)?.toArray()).toEqual(['1234567 61/68']); + expect(conn.query("SELECT text(1234567.897, '# ??/??')").getChildAt(0)?.toArray()).toEqual([ + '1234567 61/68', + ]); - expect(conn.query("SELECT text(12345678912, '(###) ###-####')",).getChildAt(0)?.toArray()).toEqual(['(1234) 567-8912']); - expect(conn.query("SELECT text(1234567.897, '$#,##0')",).getChildAt(0)?.toArray()).toEqual(['$1,234,568']); - expect(conn.query("SELECT excel_text(123456789123, '[<=9999999]##-####;[>9999999](###) ###-####')",).getChildAt(0)?.toArray()).toEqual(['(12345) 678-9123']); + expect(conn.query("SELECT text(12345678912, '(###) ###-####')").getChildAt(0)?.toArray()).toEqual([ + '(1234) 567-8912', + ]); + expect(conn.query("SELECT text(1234567.897, '$#,##0')").getChildAt(0)?.toArray()).toEqual(['$1,234,568']); + expect( + conn + .query("SELECT excel_text(123456789123, '[<=9999999]##-####;[>9999999](###) ###-####')") + .getChildAt(0) + ?.toArray(), + ).toEqual(['(12345) 678-9123']); }); }); } diff --git a/packages/duckdb-wasm/test/httpfs_test.ts b/packages/duckdb-wasm/test/httpfs_test.ts index 6e2f719cd..9b9617ceb 100644 --- a/packages/duckdb-wasm/test/httpfs_test.ts +++ b/packages/duckdb-wasm/test/httpfs_test.ts @@ -2,7 +2,7 @@ import * as duckdb from '../src/'; import { getS3Params, S3Params, S3PayloadParams, createS3Headers, uriEncode, getHTTPUrl } from '../src/utils'; import { AsyncDuckDBConnection, DuckDBBindings, DuckDBBindingsBase, DuckDBModule } from '../src/'; import BROWSER_RUNTIME from '../src/bindings/runtime_browser'; -import {generateLongQueryString} from "./string_test_helper"; +import { generateLongQueryString } from './string_test_helper'; // S3 config for tests const BUCKET_NAME = 'test-bucket'; @@ -312,7 +312,7 @@ export function testHTTPFSAsync( `COPY (SELECT * FROM range(1000,1010) tbl(i)) TO 's3://${BUCKET_NAME}/test_written.csv' (FORMAT 'csv');`, ); const result = await conn!.query(`SELECT * FROM "s3://${BUCKET_NAME}/test_written.csv";`); - expect(Number((result.getChildAt(0)?.get(6)))).toEqual(Number(1006)); + expect(Number(result.getChildAt(0)?.get(6))).toEqual(Number(1006)); await expectAsync( conn!.query( `COPY (SELECT * FROM range(2000,2010) tbl(i)) TO 's3://${BUCKET_NAME}/test_written.csv' (FORMAT 'csv');`, @@ -330,7 +330,7 @@ export function testHTTPFSAsync( const result = await conn!.query( `SELECT * FROM "${S3_ENDPOINT}/${BUCKET_NAME}/correct_auth_test.parquet?${queryString}";`, ); - expect(Number((result.getChildAt(0)?.get(6)))).toEqual(Number(29120)); + expect(Number(result.getChildAt(0)?.get(6))).toEqual(Number(29120)); }); it('can read csv file from URL with long query string', async () => { @@ -343,7 +343,7 @@ export function testHTTPFSAsync( const result = await conn!.query( `SELECT * FROM "${S3_ENDPOINT}/${BUCKET_NAME}/correct_auth_test.csv?${queryString}";`, ); - expect(Number((result.getChildAt(0)?.get(6)))).toEqual(Number(29120)); + expect(Number(result.getChildAt(0)?.get(6))).toEqual(Number(29120)); }); }); } diff --git a/packages/duckdb-wasm/test/json.test.ts b/packages/duckdb-wasm/test/json.test.ts index 7e7cacfb1..46d115903 100644 --- a/packages/duckdb-wasm/test/json.test.ts +++ b/packages/duckdb-wasm/test/json.test.ts @@ -14,8 +14,8 @@ export function testJSON(db: () => duckdb.DuckDBBindings): void { describe('JSON', () => { it('sample', async () => { - expect(conn.query("select to_json({n: 42})",).getChildAt(0)?.toArray()).toEqual(['{"n":42}']); - expect(conn.query("select json_object('duck', 42)",).getChildAt(0)?.toArray()).toEqual(['{"duck":42}']); + expect(conn.query('select to_json({n: 42})').getChildAt(0)?.toArray()).toEqual(['{"n":42}']); + expect(conn.query("select json_object('duck', 42)").getChildAt(0)?.toArray()).toEqual(['{"duck":42}']); }); }); } diff --git a/packages/duckdb-wasm/test/long_queries.test.ts b/packages/duckdb-wasm/test/long_queries.test.ts index 5ff02daf6..87729d022 100644 --- a/packages/duckdb-wasm/test/long_queries.test.ts +++ b/packages/duckdb-wasm/test/long_queries.test.ts @@ -26,14 +26,13 @@ export function longQueries(db: () => duckdb.AsyncDuckDB): void { let str = `with big_expr as ( select `; let i = 1; - while (str.length < 1e6) { + while (str.length < 1e6) { str += ` ` + i + ` as col_` + i + `,`; i++; } - str += ` NULL as col_NULL) select 99;` + str += ` NULL as col_NULL) select 99;`; await conn.query(str); }); }); } - diff --git a/packages/duckdb-wasm/test/opfs.test.ts b/packages/duckdb-wasm/test/opfs.test.ts index eaf1a0fcc..f4dfb4e02 100644 --- a/packages/duckdb-wasm/test/opfs.test.ts +++ b/packages/duckdb-wasm/test/opfs.test.ts @@ -1,5 +1,5 @@ import * as duckdb from '../src/'; -import {LogLevel} from '../src/'; +import { LogLevel } from '../src/'; import * as arrow from 'apache-arrow'; export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): void { @@ -29,7 +29,7 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await db.instantiate(bundle().mainModule, bundle().pthreadWorker); await db.open({ path: 'opfs://test.db', - accessMode: duckdb.DuckDBAccessMode.READ_WRITE + accessMode: duckdb.DuckDBAccessMode.READ_WRITE, }); conn = await db.connect(); }); @@ -83,7 +83,7 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await db.instantiate(bundle().mainModule, bundle().pthreadWorker); await db.open({ path: 'opfs://test.db', - accessMode: duckdb.DuckDBAccessMode.READ_WRITE + accessMode: duckdb.DuckDBAccessMode.READ_WRITE, }); conn = await db.connect(); @@ -102,7 +102,7 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo res.arrayBuffer(), ); const opfsRoot = await navigator.storage.getDirectory(); - const fileHandle = await opfsRoot.getFileHandle('test.parquet', {create: true}); + const fileHandle = await opfsRoot.getFileHandle('test.parquet', { create: true }); const writable = await fileHandle.createWritable(); await writable.write(parquetBuffer); await writable.close(); @@ -126,8 +126,8 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo res.arrayBuffer(), ); const opfsRoot = await navigator.storage.getDirectory(); - const datadir = await opfsRoot.getDirectoryHandle("datadir", {create: true}); - const fileHandle = await datadir.getFileHandle('test.parquet', {create: true}); + const datadir = await opfsRoot.getDirectoryHandle('datadir', { create: true }); + const fileHandle = await datadir.getFileHandle('test.parquet', { create: true }); const writable = await fileHandle.createWritable(); await writable.write(parquetBuffer); await writable.close(); @@ -150,7 +150,7 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo res.arrayBuffer(), ); const opfsRoot = await navigator.storage.getDirectory(); - const fileHandle = await opfsRoot.getFileHandle('test.parquet', {create: true}); + const fileHandle = await opfsRoot.getFileHandle('test.parquet', { create: true }); const writable = await fileHandle.createWritable(); await writable.write(parquetBuffer); await writable.close(); @@ -192,12 +192,11 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo const table3 = await new arrow.Table<{ cnt: arrow.Int }>(batches3); expect(table3.getChildAt(0)?.get(0)).toBeGreaterThan(60_000); } - }); it('Drop File + Export as CSV to OPFS + Load CSV', async () => { const opfsRoot = await navigator.storage.getDirectory(); - const testHandle = await opfsRoot.getFileHandle('test.csv', {create: true}); + const testHandle = await opfsRoot.getFileHandle('test.csv', { create: true }); await db.registerFileHandle('test.csv', testHandle, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); await conn.send(`CREATE TABLE zzz AS SELECT * FROM "${baseDir}/tpch/0_01/parquet/lineitem.parquet"`); await conn.send(`COPY (SELECT * FROM zzz) TO 'test.csv'`); @@ -221,12 +220,11 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await db.dropFile('test.csv'); }); - it('Drop Files + Export as CSV to OPFS + Load CSV', async () => { const opfsRoot = await navigator.storage.getDirectory(); - const testHandle1 = await opfsRoot.getFileHandle('test1.csv', {create: true}); - const testHandle2 = await opfsRoot.getFileHandle('test2.csv', {create: true}); - const testHandle3 = await opfsRoot.getFileHandle('test3.csv', {create: true}); + const testHandle1 = await opfsRoot.getFileHandle('test1.csv', { create: true }); + const testHandle2 = await opfsRoot.getFileHandle('test2.csv', { create: true }); + const testHandle3 = await opfsRoot.getFileHandle('test3.csv', { create: true }); await db.registerFileHandle('test1.csv', testHandle1, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); await db.registerFileHandle('test2.csv', testHandle2, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); await db.registerFileHandle('test3.csv', testHandle3, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); @@ -280,28 +278,19 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo async function removeFiles() { const opfsRoot = await navigator.storage.getDirectory(); - await opfsRoot.removeEntry('test.db').catch(() => { - }); - await opfsRoot.removeEntry('test.db.wal').catch(() => { - }); - await opfsRoot.removeEntry('test.csv').catch(() => { - }); - await opfsRoot.removeEntry('test1.csv').catch(() => { - }); - await opfsRoot.removeEntry('test2.csv').catch(() => { - }); - await opfsRoot.removeEntry('test3.csv').catch(() => { - }); - await opfsRoot.removeEntry('test.parquet').catch(() => { - }); + await opfsRoot.removeEntry('test.db').catch(() => {}); + await opfsRoot.removeEntry('test.db.wal').catch(() => {}); + await opfsRoot.removeEntry('test.csv').catch(() => {}); + await opfsRoot.removeEntry('test1.csv').catch(() => {}); + await opfsRoot.removeEntry('test2.csv').catch(() => {}); + await opfsRoot.removeEntry('test3.csv').catch(() => {}); + await opfsRoot.removeEntry('test.parquet').catch(() => {}); try { const datadir = await opfsRoot.getDirectoryHandle('datadir'); - datadir.removeEntry('test.parquet').catch(() => { - }); + datadir.removeEntry('test.parquet').catch(() => {}); } catch (e) { // } - await opfsRoot.removeEntry('datadir').catch(() => { - }); + await opfsRoot.removeEntry('datadir').catch(() => {}); } } diff --git a/packages/duckdb-wasm/test/regression/github_1833.test.ts b/packages/duckdb-wasm/test/regression/github_1833.test.ts index 4e597ed2c..ca89e0497 100644 --- a/packages/duckdb-wasm/test/regression/github_1833.test.ts +++ b/packages/duckdb-wasm/test/regression/github_1833.test.ts @@ -14,17 +14,17 @@ export function test1833(db: () => duckdb.AsyncDuckDB): void { }); describe('GitHub issues', () => { it('1833', async () => { - await conn.query(` + await conn.query(` CREATE TABLE "Test" (value VARCHAR) `); - const stmt = await conn.prepare(` + const stmt = await conn.prepare(` INSERT INTO "Test" (value) VALUES (?) `); - await stmt.query('🦆🦆🦆🦆🦆'); - await stmt.query('goo␀se'); - await stmt.query('goo\u0000se'); - const result = await conn.query(` + await stmt.query('🦆🦆🦆🦆🦆'); + await stmt.query('goo␀se'); + await stmt.query('goo\u0000se'); + const result = await conn.query(` SELECT * FROM "Test" `); expect(result.schema.fields.length).toBe(1); diff --git a/packages/duckdb-wasm/test/string_test_helper.ts b/packages/duckdb-wasm/test/string_test_helper.ts index 96e83b823..96078f0d4 100644 --- a/packages/duckdb-wasm/test/string_test_helper.ts +++ b/packages/duckdb-wasm/test/string_test_helper.ts @@ -5,16 +5,18 @@ export function generateLongQueryString(): string { const eee = repeatCharacter('E', 256); const ggg = repeatCharacter('G', 128); - return `test=inline` + + return ( + `test=inline` + `&Test-Security-Token=${aaa}` + `&Test-Algorithm=${ccc}` + `&Test-Date=${ddd}` + `&Test-SignedHeaders=host` + `&Test-Expires=43200` + `&Test-Credential=${eee}` + - `&Test-Signature=${ggg}`; + `&Test-Signature=${ggg}` + ); } export function repeatCharacter(char: string, length: number): string { return char.repeat(length); -} \ No newline at end of file +} diff --git a/patches/duckdb/virtualized_file_system.patch b/patches/duckdb/virtualized_file_system.patch deleted file mode 100644 index 028fc3709..000000000 --- a/patches/duckdb/virtualized_file_system.patch +++ /dev/null @@ -1,26 +0,0 @@ -diff --git a/src/common/virtual_file_system.cpp b/src/common/virtual_file_system.cpp -index 74892a4e05..c3d1c4f333 100644 ---- a/src/common/virtual_file_system.cpp -+++ b/src/common/virtual_file_system.cpp -@@ -5,7 +5,7 @@ - - namespace duckdb { - --VirtualFileSystem::VirtualFileSystem() : default_fs(FileSystem::CreateLocal()) { -+VirtualFileSystem::VirtualFileSystem(unique_ptr inner) : default_fs(std::move(inner)) { - VirtualFileSystem::RegisterSubSystem(FileCompressionType::GZIP, make_uniq()); - } - -diff --git a/src/include/duckdb/common/virtual_file_system.hpp b/src/include/duckdb/common/virtual_file_system.hpp -index 110ad04877..30a7eb29d3 100644 ---- a/src/include/duckdb/common/virtual_file_system.hpp -+++ b/src/include/duckdb/common/virtual_file_system.hpp -@@ -17,7 +17,7 @@ namespace duckdb { - // bunch of wrappers to allow registering protocol handlers - class VirtualFileSystem : public FileSystem { - public: -- VirtualFileSystem(); -+ VirtualFileSystem(unique_ptr inner_file_system = nullptr); - - unique_ptr OpenFile(const string &path, FileOpenFlags flags, - optional_ptr opener = nullptr) override; diff --git a/patches/rapidjson/cmake_minimum_required.patch b/patches/rapidjson/cmake_minimum_required.patch new file mode 100644 index 000000000..2b436915d --- /dev/null +++ b/patches/rapidjson/cmake_minimum_required.patch @@ -0,0 +1,10 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index dd1f173..2351bad 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -1,4 +1,4 @@ +-CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12) ++CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12...3.29) + if(POLICY CMP0025) + # detect Apple's Clang + cmake_policy(SET CMP0025 NEW) diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index cf2b268ad..f697badf7 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -69,8 +69,32 @@ emmake make \ -j${CORES} \ duckdb_wasm -npm install -g js-beautify +if [ "${USE_GENERATED_EXPORTED_LIST:-no}" == "yes" ]; then +make TARGET=${FEATURES} update_exported_list + +emcmake cmake \ + -S${CPP_SOURCE_DIR} \ + -B${BUILD_DIR} \ + -DDUCKDB_WASM_VERSION=${DUCKDB_WASM_VERSION_NAME} \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DDUCKDB_LOCATION=${DUCKDB_LOCATION} \ + -DWASM_LINK_FLAGS_EXT="${LINK_FLAGS}" \ + -DDUCKDB_EXTENSION_CONFIGS=extension_config_wasm.cmake \ + -DUSE_GENERATED_EXPORTED_LIST=1 \ + ${ADDITIONAL_FLAGS} + +emmake make \ + -C${BUILD_DIR} \ + -j${CORES} \ + duckdb_wasm +fi + +js-beautify -v || npm install -g js-beautify js-beautify ${BUILD_DIR}/duckdb_wasm.js > ${BUILD_DIR}/beauty.js +sed 's/case \"__table_base\"/case \"getTempRet0\": return getTempRet0; case \"__table_base\"/g' ${BUILD_DIR}/beauty.js > ${BUILD_DIR}/beauty_sed.js +cp ${BUILD_DIR}/beauty_sed.js ${BUILD_DIR}/beauty.js +cp ${BUILD_DIR}/beauty.js ${BUILD_DIR}/duckdb_wasm.js awk '{gsub(/get\(stubs, prop\) \{/,"get(stubs,prop) { if (prop.startsWith(\"invoke_\")) {return createDyncallWrapper(prop.substring(7));}"); print}' ${BUILD_DIR}/beauty.js > ${BUILD_DIR}/beauty2.js awk '!(/var .*wasmExports\[/ || /var [_a-z0-9A-Z]+ = Module\[\"[_a-z0-9A-Z]+\"\] = [0-9]+;/) || /var _duckdb_web/ || /var _main/ || /var _calloc/ || /var _malloc/ || /var _free/ || /var stack/ || /var ___dl_seterr/ || /var __em/ || /var _em/ || /var _pthread/' ${BUILD_DIR}/beauty2.js > ${BUILD_DIR}/duckdb_wasm.js diff --git a/submodules/duckdb b/submodules/duckdb index 57cbd519a..93fda3591 160000 --- a/submodules/duckdb +++ b/submodules/duckdb @@ -1 +1 @@ -Subproject commit 57cbd519ad94a5c5b11bde40a86bde8e074c3c91 +Subproject commit 93fda3591f4298414fa362c59219c09e03f718ab