From 3ef9993f238642be57cda769cc5fa84415c13b0c Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 21 Jan 2025 16:46:46 +0100 Subject: [PATCH 1/5] Add GetProgressBar to Connections --- lib/include/duckdb/web/webdb.h | 2 ++ lib/src/webdb.cc | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/lib/include/duckdb/web/webdb.h b/lib/include/duckdb/web/webdb.h index 18c6cbe74..f7d15841a 100644 --- a/lib/include/duckdb/web/webdb.h +++ b/lib/include/duckdb/web/webdb.h @@ -33,6 +33,8 @@ class WebDB { public: /// A connection class Connection { + friend WebDB; + protected: /// The webdb WebDB& webdb_; diff --git a/lib/src/webdb.cc b/lib/src/webdb.cc index d940344ca..f1dbfa604 100644 --- a/lib/src/webdb.cc +++ b/lib/src/webdb.cc @@ -783,11 +783,29 @@ std::string WebDB::Tokenize(std::string_view text) { /// Get the version std::string_view WebDB::GetVersion() { return database_->LibraryVersion(); } +class ProgressBarCustom: public ProgressBarDisplay { +public: + ProgressBarCustom() { + } + ~ProgressBarCustom() {} +public: + void Update(double percentage) { + std::cout << "ProgressBar::Update() called with " << percentage << "\n"; + } + void Finish() { + std::cout << "Finish() called\n"; + } + static unique_ptr GetProgressBar() { + return make_uniq(); + } +}; + /// Create a session WebDB::Connection* WebDB::Connect() { auto conn = duckdb::make_uniq(*this); auto conn_ptr = conn.get(); connections_.insert({conn_ptr, std::move(conn)}); + ClientConfig::GetConfig(*conn_ptr->connection_.context).display_create_func = ProgressBarCustom::GetProgressBar; return conn_ptr; } From 473cdf9067478678c07582340843bba1038a3f51 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 21 Jan 2025 18:14:08 +0100 Subject: [PATCH 2/5] Pass down from C++ to JS layer --- lib/src/webdb.cc | 54 ++++++++++++++----- packages/duckdb-wasm/src/bindings/runtime.ts | 4 ++ .../src/bindings/runtime_browser.ts | 4 ++ .../duckdb-wasm/src/bindings/runtime_node.ts | 3 ++ 4 files changed, 51 insertions(+), 14 deletions(-) diff --git a/lib/src/webdb.cc b/lib/src/webdb.cc index f1dbfa604..2d2914dec 100644 --- a/lib/src/webdb.cc +++ b/lib/src/webdb.cc @@ -2,6 +2,8 @@ #include "duckdb/web/webdb.h" +#include + #include #include #include @@ -783,21 +785,44 @@ std::string WebDB::Tokenize(std::string_view text) { /// Get the version std::string_view WebDB::GetVersion() { return database_->LibraryVersion(); } -class ProgressBarCustom: public ProgressBarDisplay { -public: - ProgressBarCustom() { +class ProgressBarCustom : public ProgressBarDisplay { + double value{0.0}; + double times{0.0}; + double to_send{1.0}; + + public: + ProgressBarCustom() { + value = 0.0; + times = 0.0; + to_send = 1.0; + } + ~ProgressBarCustom() {} + static void SendMessage(bool end, double percentage, double times) { + emscripten::val::global("DUCKDB_RUNTIME").call("progressUpdate", end, percentage, times); + } + + public: + void Update(double percentage) { + if (percentage >= value + 1.0) { + value = percentage; + times = 1.0; + SendMessage(false, percentage, times); + to_send = 10.0; + } else { + times += 1.0; + if (times >= to_send) { + SendMessage(false, percentage, times); + to_send *= 10.0; + } } - ~ProgressBarCustom() {} -public: - void Update(double percentage) { - std::cout << "ProgressBar::Update() called with " << percentage << "\n"; - } - void Finish() { - std::cout << "Finish() called\n"; - } - static unique_ptr GetProgressBar() { - return make_uniq(); - } + } + void Finish() { + SendMessage(true, value, times); + value = 0.0; + times = 0.0; + to_send = 1.0; + } + static unique_ptr GetProgressBar() { return make_uniq(); } }; /// Create a session @@ -805,6 +830,7 @@ WebDB::Connection* WebDB::Connect() { auto conn = duckdb::make_uniq(*this); auto conn_ptr = conn.get(); connections_.insert({conn_ptr, std::move(conn)}); + ClientConfig::GetConfig(*conn_ptr->connection_.context).wait_time = 1; ClientConfig::GetConfig(*conn_ptr->connection_.context).display_create_func = ProgressBarCustom::GetProgressBar; return conn_ptr; } diff --git a/packages/duckdb-wasm/src/bindings/runtime.ts b/packages/duckdb-wasm/src/bindings/runtime.ts index b720e6877..5f88f5594 100644 --- a/packages/duckdb-wasm/src/bindings/runtime.ts +++ b/packages/duckdb-wasm/src/bindings/runtime.ts @@ -142,6 +142,7 @@ export interface DuckDBRuntime { closeFile(mod: DuckDBModule, fileId: number): void; dropFile(mod: DuckDBModule, fileNamePtr: number, fileNameLen:number): void; getLastFileModificationTime(mod: DuckDBModule, fileId: number): number; + progressUpdate(mod: DuckDBModule, final: number, a: number, b:number): void; truncateFile(mod: DuckDBModule, fileId: number, newSize: number): void; readFile(mod: DuckDBModule, fileId: number, buffer: number, bytes: number, location: number): number; writeFile(mod: DuckDBModule, fileId: number, buffer: number, bytes: number, location: number): number; @@ -185,6 +186,9 @@ export const DEFAULT_RUNTIME: DuckDBRuntime = { getLastFileModificationTime: (_mod: DuckDBModule, _fileId: number): number => { return 0; }, + progressUpdate: (_mod: DuckDBModule, _fileId: number, a: number, b: number): void => { + return; + }, truncateFile: (_mod: DuckDBModule, _fileId: number, _newSize: number): void => {}, readFile: (_mod: DuckDBModule, _fileId: number, _buffer: number, _bytes: number, _location: number): number => { return 0; diff --git a/packages/duckdb-wasm/src/bindings/runtime_browser.ts b/packages/duckdb-wasm/src/bindings/runtime_browser.ts index 0b4ebedca..7a3a26057 100644 --- a/packages/duckdb-wasm/src/bindings/runtime_browser.ts +++ b/packages/duckdb-wasm/src/bindings/runtime_browser.ts @@ -687,6 +687,10 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { } return 0; }, + progressUpdate: (_mod: DuckDBModule, done: number, a: number, b: number): void => { + //postMessage(""); + console.log("Update progress: ", done, a, b); + }, checkDirectory: (mod: DuckDBModule, pathPtr: number, pathLen: number) => { const path = readString(mod, pathPtr, pathLen); console.log(`checkDirectory: ${path}`); diff --git a/packages/duckdb-wasm/src/bindings/runtime_node.ts b/packages/duckdb-wasm/src/bindings/runtime_node.ts index 2f92368bc..db07e6645 100644 --- a/packages/duckdb-wasm/src/bindings/runtime_node.ts +++ b/packages/duckdb-wasm/src/bindings/runtime_node.ts @@ -197,6 +197,9 @@ export const NODE_RUNTIME: DuckDBRuntime & { } return 0; }, + progressUpdate: (_mod: DuckDBModule, _fileId: number, a: number, b: number): void => { + return; + }, getLastFileModificationTime: (mod: DuckDBModule, fileId: number) => { try { const file = NODE_RUNTIME.resolveFileInfo(mod, fileId); From 13274dfdd17a15b8728e3017c1d08a88802b37b0 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 17 Mar 2025 13:55:12 +0100 Subject: [PATCH 3/5] ProgressUpdate --- lib/src/webdb.cc | 4 ++-- packages/duckdb-wasm/src/bindings/runtime.ts | 6 ++++-- packages/duckdb-wasm/src/bindings/runtime_browser.ts | 10 +++++++--- packages/duckdb-wasm/src/bindings/runtime_node.ts | 2 +- packages/duckdb-wasm/src/log.ts | 6 ++++++ packages/duckdb-wasm/src/parallel/async_bindings.ts | 7 +++++++ packages/duckdb-wasm/src/parallel/worker_request.ts | 4 +++- 7 files changed, 30 insertions(+), 9 deletions(-) diff --git a/lib/src/webdb.cc b/lib/src/webdb.cc index 2d2914dec..345c9ac67 100644 --- a/lib/src/webdb.cc +++ b/lib/src/webdb.cc @@ -797,8 +797,8 @@ class ProgressBarCustom : public ProgressBarDisplay { to_send = 1.0; } ~ProgressBarCustom() {} - static void SendMessage(bool end, double percentage, double times) { - emscripten::val::global("DUCKDB_RUNTIME").call("progressUpdate", end, percentage, times); + static void SendMessage(double end, double percentage, double times) { + emscripten::val::global("DUCKDB_RUNTIME").call("progressUpdate", end ? 1.0 : 0.0, percentage, times); } public: diff --git a/packages/duckdb-wasm/src/bindings/runtime.ts b/packages/duckdb-wasm/src/bindings/runtime.ts index 5f88f5594..f894dba46 100644 --- a/packages/duckdb-wasm/src/bindings/runtime.ts +++ b/packages/duckdb-wasm/src/bindings/runtime.ts @@ -142,7 +142,6 @@ export interface DuckDBRuntime { closeFile(mod: DuckDBModule, fileId: number): void; dropFile(mod: DuckDBModule, fileNamePtr: number, fileNameLen:number): void; getLastFileModificationTime(mod: DuckDBModule, fileId: number): number; - progressUpdate(mod: DuckDBModule, final: number, a: number, b:number): void; truncateFile(mod: DuckDBModule, fileId: number, newSize: number): void; readFile(mod: DuckDBModule, fileId: number, buffer: number, bytes: number, location: number): number; writeFile(mod: DuckDBModule, fileId: number, buffer: number, bytes: number, location: number): number; @@ -162,6 +161,9 @@ export interface DuckDBRuntime { prepareFileHandles?: (path: string[], protocol: DuckDBDataProtocol) => Promise; prepareDBFileHandle?: (path: string, protocol: DuckDBDataProtocol) => Promise; + // Internal API - experimental + progressUpdate(final: number, percentage: number, iteration:number): void; + // Call a scalar UDF function callScalarUDF( mod: DuckDBModule, @@ -186,7 +188,7 @@ export const DEFAULT_RUNTIME: DuckDBRuntime = { getLastFileModificationTime: (_mod: DuckDBModule, _fileId: number): number => { return 0; }, - progressUpdate: (_mod: DuckDBModule, _fileId: number, a: number, b: number): void => { + progressUpdate: (_final: number, _percentage: number, _iteration: number): void => { return; }, truncateFile: (_mod: DuckDBModule, _fileId: number, _newSize: number): void => {}, diff --git a/packages/duckdb-wasm/src/bindings/runtime_browser.ts b/packages/duckdb-wasm/src/bindings/runtime_browser.ts index 7a3a26057..dc8f7c3aa 100644 --- a/packages/duckdb-wasm/src/bindings/runtime_browser.ts +++ b/packages/duckdb-wasm/src/bindings/runtime_browser.ts @@ -1,4 +1,7 @@ import {StatusCode} from '../status'; +import { + WorkerResponseType, +} from '../parallel/worker_request'; import {addS3Headers, getHTTPUrl} from '../utils'; import { @@ -687,9 +690,10 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { } return 0; }, - progressUpdate: (_mod: DuckDBModule, done: number, a: number, b: number): void => { - //postMessage(""); - console.log("Update progress: ", done, a, b); + progressUpdate: (done: number, percentage: number, repeat: number): void => { + if (postMessage) { + postMessage({requestId: 0, type: WorkerResponseType.PROGRESS_UPDATE, data: {status: done?"completed":"in-progress", percentage: percentage, repetitions: repeat}}); + } }, checkDirectory: (mod: DuckDBModule, pathPtr: number, pathLen: number) => { const path = readString(mod, pathPtr, pathLen); diff --git a/packages/duckdb-wasm/src/bindings/runtime_node.ts b/packages/duckdb-wasm/src/bindings/runtime_node.ts index db07e6645..5037068a2 100644 --- a/packages/duckdb-wasm/src/bindings/runtime_node.ts +++ b/packages/duckdb-wasm/src/bindings/runtime_node.ts @@ -197,7 +197,7 @@ export const NODE_RUNTIME: DuckDBRuntime & { } return 0; }, - progressUpdate: (_mod: DuckDBModule, _fileId: number, a: number, b: number): void => { + progressUpdate: (_final: number, _percentage: number, _iteration: number): void => { return; }, getLastFileModificationTime: (mod: DuckDBModule, fileId: number) => { diff --git a/packages/duckdb-wasm/src/log.ts b/packages/duckdb-wasm/src/log.ts index e6ff577ca..2fad1cac6 100644 --- a/packages/duckdb-wasm/src/log.ts +++ b/packages/duckdb-wasm/src/log.ts @@ -41,6 +41,12 @@ export type LogEntry = { readonly value: V; }; +export type ProgressEntry = { + readonly status: string; + readonly percentage: string; + readonly repetitions: string; +} + export type LogEntryVariant = | LogEntry | LogEntry diff --git a/packages/duckdb-wasm/src/parallel/async_bindings.ts b/packages/duckdb-wasm/src/parallel/async_bindings.ts index 4aaf3a6fa..c6fa54c12 100644 --- a/packages/duckdb-wasm/src/parallel/async_bindings.ts +++ b/packages/duckdb-wasm/src/parallel/async_bindings.ts @@ -32,6 +32,9 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { /** Instantiate the module */ protected _onInstantiationProgress: ((p: InstantiationProgress) => void)[] = []; + /** Progress callbacks */ + //protected _onProgressCallback: ((p: InstantiationProgress) => void)[] = []; + /** The logger */ protected readonly _logger: Logger; /** The worker */ @@ -122,6 +125,10 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { const response = event.data as WorkerResponseVariant; switch (response.type) { // Request failed? + case WorkerResponseType.PROGRESS_UPDATE: { + console.log(response.data); + return; + } case WorkerResponseType.LOG: { this._logger.log(response.data); return; diff --git a/packages/duckdb-wasm/src/parallel/worker_request.ts b/packages/duckdb-wasm/src/parallel/worker_request.ts index 9b9df0634..38502b7b6 100644 --- a/packages/duckdb-wasm/src/parallel/worker_request.ts +++ b/packages/duckdb-wasm/src/parallel/worker_request.ts @@ -1,5 +1,5 @@ import { CSVInsertOptions, JSONInsertOptions, ArrowInsertOptions } from '../bindings/insert_options'; -import { LogEntryVariant } from '../log'; +import { LogEntryVariant, ProgressEntry } from '../log'; import { ScriptTokens } from '../bindings/tokens'; import { FileStatistics } from '../bindings/file_stats'; import { DuckDBConfig } from '../bindings/config'; @@ -57,6 +57,7 @@ export enum WorkerResponseType { FILE_STATISTICS = 'FILE_STATISTICS', INSTANTIATE_PROGRESS = 'INSTANTIATE_PROGRESS', LOG = 'LOG', + PROGRESS_UPDATE = 'PROGRESS_UPDATE', OK = 'OK', PREPARED_STATEMENT_ID = 'PREPARED_STATEMENT_ID', QUERY_PLAN = 'QUERY_PLAN', @@ -154,6 +155,7 @@ export type WorkerResponseVariant = | WorkerResponse | WorkerResponse | WorkerResponse + | WorkerResponse | WorkerResponse | WorkerResponse | WorkerResponse From 76cd5bcfea6db294b3df748486c58b0726d69c42 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 22 Jan 2025 16:10:50 +0100 Subject: [PATCH 4/5] Expose as _onExecutionProgress --- packages/duckdb-wasm/src/log.ts | 3 +++ packages/duckdb-wasm/src/parallel/async_bindings.ts | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/duckdb-wasm/src/log.ts b/packages/duckdb-wasm/src/log.ts index 2fad1cac6..4a4f836ca 100644 --- a/packages/duckdb-wasm/src/log.ts +++ b/packages/duckdb-wasm/src/log.ts @@ -47,6 +47,9 @@ export type ProgressEntry = { readonly repetitions: string; } +/** An execution progress handler */ +export type ExecutionProgressHandler = (p: ProgressEntry) => void; + export type LogEntryVariant = | LogEntry | LogEntry diff --git a/packages/duckdb-wasm/src/parallel/async_bindings.ts b/packages/duckdb-wasm/src/parallel/async_bindings.ts index c6fa54c12..299b421a7 100644 --- a/packages/duckdb-wasm/src/parallel/async_bindings.ts +++ b/packages/duckdb-wasm/src/parallel/async_bindings.ts @@ -18,6 +18,7 @@ import { InstantiationProgress } from '../bindings/progress'; import { arrowToSQLField } from '../json_typedef'; import { WebFile } from '../bindings/web_file'; import { DuckDBDataProtocol } from '../bindings'; +import { ProgressEntry } from '../log'; const TEXT_ENCODER = new TextEncoder(); @@ -33,7 +34,7 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { protected _onInstantiationProgress: ((p: InstantiationProgress) => void)[] = []; /** Progress callbacks */ - //protected _onProgressCallback: ((p: InstantiationProgress) => void)[] = []; + protected _onExecutionProgress: ((p: ProgressEntry) => void)[] = []; /** The logger */ protected readonly _logger: Logger; @@ -126,7 +127,9 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { switch (response.type) { // Request failed? case WorkerResponseType.PROGRESS_UPDATE: { - console.log(response.data); + for (const p of this._onExecutionProgress) { + p(response.data); + } return; } case WorkerResponseType.LOG: { From 3178a1133dc7d22e0eb606ed254e8e6f09c8b07e Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 22 Jan 2025 16:18:30 +0100 Subject: [PATCH 5/5] Format fix --- packages/duckdb-wasm/src/bindings/runtime_browser.ts | 2 +- packages/duckdb-wasm/src/parallel/async_bindings.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/duckdb-wasm/src/bindings/runtime_browser.ts b/packages/duckdb-wasm/src/bindings/runtime_browser.ts index dc8f7c3aa..b5069dac4 100644 --- a/packages/duckdb-wasm/src/bindings/runtime_browser.ts +++ b/packages/duckdb-wasm/src/bindings/runtime_browser.ts @@ -692,7 +692,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { }, progressUpdate: (done: number, percentage: number, repeat: number): void => { if (postMessage) { - postMessage({requestId: 0, type: WorkerResponseType.PROGRESS_UPDATE, data: {status: done?"completed":"in-progress", percentage: percentage, repetitions: repeat}}); + postMessage({requestId: 0, type: WorkerResponseType.PROGRESS_UPDATE, data: {status: done?"completed":"in-progress", percentage: percentage, repetitions: repeat}}); } }, checkDirectory: (mod: DuckDBModule, pathPtr: number, pathLen: number) => { diff --git a/packages/duckdb-wasm/src/parallel/async_bindings.ts b/packages/duckdb-wasm/src/parallel/async_bindings.ts index 299b421a7..dc8a81e53 100644 --- a/packages/duckdb-wasm/src/parallel/async_bindings.ts +++ b/packages/duckdb-wasm/src/parallel/async_bindings.ts @@ -128,10 +128,10 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { // Request failed? case WorkerResponseType.PROGRESS_UPDATE: { for (const p of this._onExecutionProgress) { - p(response.data); + p(response.data); } return; - } + } case WorkerResponseType.LOG: { this._logger.log(response.data); return;