Skip to content

Add forceFullHTTPReads filesystem config flag. #2060

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 4, 2025
Merged

Add forceFullHTTPReads filesystem config flag. #2060

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion lib/include/duckdb/web/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ struct DuckDBConfigOptions {
struct FileSystemConfig {
/// Allow falling back to full HTTP reads if the server does not support range requests
std::optional<bool> allow_full_http_reads = std::nullopt;
/// Force full HTTP reads, suppressing use of range requests
std::optional<bool> force_full_http_reads = std::nullopt;
std::optional<bool> reliable_head_requests = std::nullopt;
};

Expand All @@ -91,7 +93,11 @@ struct WebDBConfig {
.cast_decimal_to_double = std::nullopt,
};
/// The filesystem
FileSystemConfig filesystem = {.allow_full_http_reads = std::nullopt, .reliable_head_requests = std::nullopt};
FileSystemConfig filesystem = {
.allow_full_http_reads = std::nullopt,
.force_full_http_reads = std::nullopt,
.reliable_head_requests = std::nullopt,
};

/// These options are fetched from DuckDB
DuckDBConfigOptions duckdb_config_options = {
Expand Down
4 changes: 4 additions & 0 deletions lib/src/config.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ WebDBConfig WebDBConfig::ReadFrom(std::string_view args_json) {
.filesystem =
FileSystemConfig{
.allow_full_http_reads = std::nullopt,
.force_full_http_reads = std::nullopt,
.reliable_head_requests = std::nullopt,
},
.duckdb_config_options =
Expand Down Expand Up @@ -106,6 +107,9 @@ WebDBConfig WebDBConfig::ReadFrom(std::string_view args_json) {
if (fs.HasMember("allowFullHTTPReads") && fs["allowFullHTTPReads"].IsBool()) {
config.filesystem.allow_full_http_reads = fs["allowFullHTTPReads"].GetBool();
}
if (fs.HasMember("forceFullHTTPReads") && fs["forceFullHTTPReads"].IsBool()) {
config.filesystem.force_full_http_reads = fs["forceFullHTTPReads"].GetBool();
}
if (fs.HasMember("reliableHeadRequests") && fs["reliableHeadRequests"].IsBool()) {
config.filesystem.reliable_head_requests = fs["reliableHeadRequests"].GetBool();
}
Expand Down
8 changes: 7 additions & 1 deletion lib/src/io/web_filesystem.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,10 @@ rapidjson::Value WebFileSystem::WebFile::WriteInfo(rapidjson::Document &doc) con
filesystem_.config_->filesystem.allow_full_http_reads.value_or(true)) {
value.AddMember("allowFullHttpReads", true, allocator);
}

if ((data_protocol_ == DataProtocol::HTTP || data_protocol_ == DataProtocol::S3) &&
filesystem_.config_->filesystem.force_full_http_reads.value_or(true)) {
value.AddMember("forceFullHttpReads", true, allocator);
}
if ((data_protocol_ == DataProtocol::HTTP || data_protocol_ == DataProtocol::S3)) {
if (filesystem_.config_->duckdb_config_options.reliable_head_requests)
value.AddMember("reliableHeadRequests", true, allocator);
Expand Down Expand Up @@ -518,6 +521,9 @@ rapidjson::Value WebFileSystem::WriteGlobalFileInfo(rapidjson::Document &doc, ui
if (config_->filesystem.allow_full_http_reads.value_or(true)) {
value.AddMember("allowFullHttpReads", true, allocator);
}
if (config_->filesystem.force_full_http_reads.value_or(true)) {
value.AddMember("forceFullHttpReads", true, allocator);
}
if (config_->filesystem.reliable_head_requests.value_or(true)) {
value.AddMember("reliableHeadRequests", true, allocator);
} else {
Expand Down
6 changes: 5 additions & 1 deletion packages/duckdb-wasm/src/bindings/config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,15 @@ export interface DuckDBQueryConfig {
}

export interface DuckDBFilesystemConfig {
reliableHeadRequests?: boolean;
/**
* Allow falling back to full HTTP reads if the server does not support range requests.
*/
reliableHeadRequests?: boolean;
allowFullHTTPReads?: boolean;
/**
* Force use of full HTTP reads, suppressing range requests.
*/
forceFullHTTPReads?: boolean;
}

export enum DuckDBAccessMode {
Expand Down
2 changes: 2 additions & 0 deletions packages/duckdb-wasm/src/bindings/runtime.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ export interface DuckDBFileInfo {
dataUrl: string | null;
reliableHeadRequests?: boolean;
allowFullHttpReads?: boolean;
forceFullHttpReads?: boolean;
s3Config?: S3Config;
}

Expand All @@ -91,6 +92,7 @@ export interface DuckDBGlobalFileInfo {
cacheEpoch: number;
reliableHeadRequests?: boolean;
allowFullHttpReads?: boolean;
forceFullHttpReads?: boolean;
s3Config?: S3Config;
}

Expand Down
6 changes: 3 additions & 3 deletions packages/duckdb-wasm/src/bindings/runtime_browser.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
// Supports ranges?
let contentLength = null;
let error: any | null = null;
if (file.reliableHeadRequests || !file.allowFullHttpReads) {
if (!file.forceFullHttpReads && (file.reliableHeadRequests || !file.allowFullHttpReads)) {
try {
// Send a dummy HEAD request with range protocol
// -> good IFF status is 206 and contentLenght is present
Expand Down Expand Up @@ -278,7 +278,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {

// Try to fallback to full read?
if (file.allowFullHttpReads) {
{
if (!file.forceFullHttpReads) {
// 2. Send a dummy GET range request querying the first byte of the file
// -> good IFF status is 206 and contentLenght2 is 1
// -> otherwise, iff 200 and contentLenght2 == contentLenght
Expand Down Expand Up @@ -354,8 +354,8 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
mod.HEAPF64[(result >> 3) + 2] = +modification_time;
return result;
}
console.warn(`falling back to full HTTP read for: ${file.dataUrl}`);
}
console.warn(`falling back to full HTTP read for: ${file.dataUrl}`);
// 3. Send non-range request
const xhr = new XMLHttpRequest();
if (file.dataProtocol == DuckDBDataProtocol.S3) {
Expand Down