diff --git a/lib/src/webdb.cc b/lib/src/webdb.cc index 88aaa1bb8..921427110 100644 --- a/lib/src/webdb.cc +++ b/lib/src/webdb.cc @@ -32,6 +32,7 @@ #include "duckdb/common/arrow/arrow.hpp" #include "duckdb/common/arrow/arrow_converter.hpp" #include "duckdb/common/file_system.hpp" +#include "duckdb/common/http_util.hpp" #include "duckdb/common/types.hpp" #include "duckdb/common/types/data_chunk.hpp" #include "duckdb/common/types/vector.hpp" @@ -70,6 +71,9 @@ #include "rapidjson/writer.h" namespace duckdb { + +bool preloaded_httpfs{true}; + namespace web { static constexpr int64_t DEFAULT_QUERY_POLLING_INTERVAL = 100; @@ -753,6 +757,9 @@ void WebDB::RegisterCustomExtensionOptions(shared_ptr database) // Register S3 Config parameters if (webfs) { + auto callback_builtin_httpfs = [](ClientContext& context, SetScope scope, Value& parameter) { + preloaded_httpfs = BooleanValue::Get(parameter); + }; auto callback_s3_region = [](ClientContext& context, SetScope scope, Value& parameter) { auto webfs = io::WebFileSystem::Get(); webfs->Config()->duckdb_config_options.s3_region = StringValue::Get(parameter); @@ -784,6 +791,8 @@ void WebDB::RegisterCustomExtensionOptions(shared_ptr database) webfs->IncrementCacheEpoch(); }; + config.AddExtensionOption("builtin_httpfs", "Use built-in HTTPS support", LogicalType::BOOLEAN, true, + callback_builtin_httpfs); config.AddExtensionOption("s3_region", "S3 Region", LogicalType::VARCHAR, Value(), callback_s3_region); config.AddExtensionOption("s3_access_key_id", "S3 Access Key ID", LogicalType::VARCHAR, Value(), callback_s3_access_key_id); diff --git a/packages/duckdb-wasm/src/bindings/runtime.ts b/packages/duckdb-wasm/src/bindings/runtime.ts index 86b27d992..d8e3f1357 100644 --- a/packages/duckdb-wasm/src/bindings/runtime.ts +++ b/packages/duckdb-wasm/src/bindings/runtime.ts @@ -58,6 +58,11 @@ export enum FileFlags { FILE_FLAGS_FILE_CREATE_NEW = 1 << 4, //! Open file in append mode FILE_FLAGS_APPEND = 1 << 5, + FILE_FLAGS_PRIVATE = 1 << 6, + FILE_FLAGS_NULL_IF_NOT_EXISTS = 1 << 7, + FILE_FLAGS_PARALLEL_ACCESS = 1 << 8, + FILE_FLAGS_EXCLUSIVE_CREATE = 1 << 9, + FILE_FLAGS_NULL_IF_EXISTS = 1 << 10 } /** Configuration for the AWS S3 Filesystem */ diff --git a/packages/duckdb-wasm/src/bindings/runtime_browser.ts b/packages/duckdb-wasm/src/bindings/runtime_browser.ts index 5aebb03ef..b549716b6 100644 --- a/packages/duckdb-wasm/src/bindings/runtime_browser.ts +++ b/packages/duckdb-wasm/src/bindings/runtime_browser.ts @@ -381,6 +381,11 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { return result; } + // Depending on file flags, return nullptr + if (flags & FileFlags.FILE_FLAGS_NULL_IF_NOT_EXISTS) { + return 0; + } + // Fall back to empty buffered file in the browser console.warn(`Buffering missing file: ${file.fileName}`); const result = mod._malloc(2 * 8); diff --git a/packages/duckdb-wasm/src/bindings/runtime_node.ts b/packages/duckdb-wasm/src/bindings/runtime_node.ts index 7f331d079..193d84af0 100644 --- a/packages/duckdb-wasm/src/bindings/runtime_node.ts +++ b/packages/duckdb-wasm/src/bindings/runtime_node.ts @@ -76,6 +76,11 @@ export const NODE_RUNTIME: DuckDBRuntime & { case DuckDBDataProtocol.NODE_FS: { let fd = NODE_RUNTIME._files?.get(file.dataUrl!); if (fd === null || fd === undefined) { + // Depending on file flags, return nullptr + if (flags & FileFlags.FILE_FLAGS_NULL_IF_NOT_EXISTS) { + return 0; + } + fd = fs.openSync( file.dataUrl!, fs.constants.O_CREAT | fs.constants.O_RDWR, diff --git a/patches/duckdb/extension_install_rework.patch b/patches/duckdb/extension_install_rework.patch index b1b49f367..ec00f7809 100644 --- a/patches/duckdb/extension_install_rework.patch +++ b/patches/duckdb/extension_install_rework.patch @@ -1,15 +1,16 @@ diff --git a/src/include/duckdb/main/database.hpp b/src/include/duckdb/main/database.hpp -index d3c5fb9bd5..b3d0aaa09e 100644 +index d3c5fb9bd5..eace7b4af1 100644 --- a/src/include/duckdb/main/database.hpp +++ b/src/include/duckdb/main/database.hpp -@@ -100,6 +100,10 @@ private: +@@ -100,6 +100,11 @@ private: unique_ptr external_file_cache; duckdb_ext_api_v1 (*create_api_v1)(); ++ +public: -+ static void SetPreferredRepository(const string& extension, const string &repository); -+ static string GetPreferredRepository(const string& extension); -+ static unordered_map extensionsRepos; ++ static void SetPreferredRepository(const string& extension, const string &repository); ++ static string GetPreferredRepository(const string& extension); ++ static unordered_map extensionsRepos; }; //! The database object. This object holds the catalog and all the @@ -31,10 +32,24 @@ index 6ccd1a1156..8040f537b6 100644 //! Debugging repositories (target local, relative paths that are produced by DuckDB's build system) static constexpr const char *BUILD_DEBUG_REPOSITORY_PATH = "./build/debug/repository"; diff --git a/src/main/database.cpp b/src/main/database.cpp -index db6e1ed445..d495aab058 100644 +index 773f8d967a..fa089a47ee 100644 --- a/src/main/database.cpp +++ b/src/main/database.cpp -@@ -356,6 +356,28 @@ DuckDB::DuckDB(DatabaseInstance &instance_p) : instance(instance_p.shared_from_t +@@ -1,5 +1,4 @@ + #include "duckdb/main/database.hpp" +- + #include "duckdb/catalog/catalog.hpp" + #include "duckdb/common/virtual_file_system.hpp" + #include "duckdb/execution/index/index_type_set.hpp" +@@ -35,6 +34,7 @@ + #include "duckdb/common/thread.hpp" + #endif + ++ + namespace duckdb { + + DBConfig::DBConfig() { +@@ -357,6 +357,28 @@ DuckDB::DuckDB(DatabaseInstance &instance_p) : instance(instance_p.shared_from_t DuckDB::~DuckDB() { } @@ -63,11 +78,11 @@ index db6e1ed445..d495aab058 100644 SecretManager &DatabaseInstance::GetSecretManager() { return *config.secret_manager; } -@@ -507,6 +529,10 @@ idx_t DuckDB::NumberOfThreads() { +@@ -508,6 +530,10 @@ idx_t DuckDB::NumberOfThreads() { bool DatabaseInstance::ExtensionIsLoaded(const std::string &name) { auto extension_name = ExtensionHelper::GetExtensionName(name); -+ if (extension_name == "httpfs") { ++ if (extension_name == "httpfs" && preloaded_httpfs) { + ExtensionInstallInfo info; + SetExtensionLoaded(extension_name, info); + } diff --git a/patches/duckdb/fix_load_database.patch b/patches/duckdb/fix_load_database.patch index 5399bfc4d..1790e5027 100644 --- a/patches/duckdb/fix_load_database.patch +++ b/patches/duckdb/fix_load_database.patch @@ -1,20 +1,23 @@ diff --git a/src/storage/storage_manager.cpp b/src/storage/storage_manager.cpp -index cb6c654e5f..a6b2af3b85 100644 +index 45d42174f1..10b3ce47b1 100644 --- a/src/storage/storage_manager.cpp +++ b/src/storage/storage_manager.cpp -@@ -160,9 +160,12 @@ void SingleFileStorageManager::LoadDatabase(StorageOptions storage_options) { +@@ -162,9 +162,15 @@ void SingleFileStorageManager::LoadDatabase(StorageOptions storage_options) { row_group_size, STANDARD_VECTOR_SIZE); } } - // Check if the database file already exists. - // Note: a file can also exist if there was a ROLLBACK on a previous transaction creating that file. - if (!read_only && !fs.FileExists(path)) { -+ auto db_file_handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS); -+ bool is_empty_file = db_file_handle->GetFileSize() == 0; -+ db_file_handle.reset(); ++ bool is_empty_file = true; ++ auto db_file_handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS); ++ if (db_file_handle && db_file_handle->GetFileSize() != 0) { ++ is_empty_file = false; ++ db_file_handle.reset(); ++ } + -+ // first check if the database exists -+ if (!read_only && ( !fs.FileExists(path) || ( options.use_direct_io && is_empty_file )) ) { ++ // first check if the database exists ++ if (!read_only && (!fs.FileExists(path) || (options.use_direct_io && is_empty_file))) { // file does not exist and we are in read-write mode // create a new file diff --git a/patches/duckdb/preloaded_httpfs.patch b/patches/duckdb/preloaded_httpfs.patch new file mode 100644 index 000000000..94f4e0e8d --- /dev/null +++ b/patches/duckdb/preloaded_httpfs.patch @@ -0,0 +1,14 @@ +diff --git a/src/include/duckdb/main/database.hpp b/src/include/duckdb/main/database.hpp +index d3c5fb9bd5..adc2ac42e6 100644 +--- a/src/include/duckdb/main/database.hpp ++++ b/src/include/duckdb/main/database.hpp +@@ -17,6 +17,9 @@ + #include "duckdb/main/valid_checker.hpp" + + namespace duckdb { ++ ++extern bool preloaded_httpfs; ++ + class BufferManager; + class DatabaseManager; + class StorageManager;