diff --git a/lib/src/webdb.cc b/lib/src/webdb.cc index 345c9ac67..f1cdd8cde 100644 --- a/lib/src/webdb.cc +++ b/lib/src/webdb.cc @@ -36,6 +36,7 @@ #include "duckdb/common/types/data_chunk.hpp" #include "duckdb/common/types/vector.hpp" #include "duckdb/common/types/vector_buffer.hpp" +#include "duckdb/common/virtual_file_system.hpp" #include "duckdb/main/query_result.hpp" #include "duckdb/parser/expression/constant_expression.hpp" #include "duckdb/parser/parser.hpp" @@ -866,7 +867,7 @@ arrow::Status WebDB::Open(std::string_view args_json) { auto buffered_fs_ptr = buffered_fs.get(); duckdb::DBConfig db_config; - db_config.file_system = std::move(buffered_fs); + db_config.file_system = std::move(make_uniq(std::move(buffered_fs))); db_config.options.allow_unsigned_extensions = config_->allow_unsigned_extensions; db_config.options.maximum_threads = config_->maximum_threads; db_config.options.use_temporary_directory = false; diff --git a/packages/duckdb-wasm/test/httpfs_test.ts b/packages/duckdb-wasm/test/httpfs_test.ts index 4ffe2ac0b..6e2f719cd 100644 --- a/packages/duckdb-wasm/test/httpfs_test.ts +++ b/packages/duckdb-wasm/test/httpfs_test.ts @@ -257,6 +257,12 @@ export function testHTTPFSAsync( expect(BigInt(results.getChildAt(2)?.get(2))).toEqual(BigInt(9n)); }); + it('can fetch over https csv.gz', async () => { + await conn!.query( + `select * from "https://raw.githubusercontent.com/duckdb/duckdb/v1.2.2/data/csv/test_apple_financial.csv.gz";`, + ); + }); + it('can read and write csv file from S3 with correct auth credentials', async () => { let data = await resolveData('/uni/studenten.parquet'); await setAwsConfig(conn!); diff --git a/patches/duckdb/virtualized_file_system.patch b/patches/duckdb/virtualized_file_system.patch new file mode 100644 index 000000000..028fc3709 --- /dev/null +++ b/patches/duckdb/virtualized_file_system.patch @@ -0,0 +1,26 @@ +diff --git a/src/common/virtual_file_system.cpp b/src/common/virtual_file_system.cpp +index 74892a4e05..c3d1c4f333 100644 +--- a/src/common/virtual_file_system.cpp ++++ b/src/common/virtual_file_system.cpp +@@ -5,7 +5,7 @@ + + namespace duckdb { + +-VirtualFileSystem::VirtualFileSystem() : default_fs(FileSystem::CreateLocal()) { ++VirtualFileSystem::VirtualFileSystem(unique_ptr inner) : default_fs(std::move(inner)) { + VirtualFileSystem::RegisterSubSystem(FileCompressionType::GZIP, make_uniq()); + } + +diff --git a/src/include/duckdb/common/virtual_file_system.hpp b/src/include/duckdb/common/virtual_file_system.hpp +index 110ad04877..30a7eb29d3 100644 +--- a/src/include/duckdb/common/virtual_file_system.hpp ++++ b/src/include/duckdb/common/virtual_file_system.hpp +@@ -17,7 +17,7 @@ namespace duckdb { + // bunch of wrappers to allow registering protocol handlers + class VirtualFileSystem : public FileSystem { + public: +- VirtualFileSystem(); ++ VirtualFileSystem(unique_ptr inner_file_system = nullptr); + + unique_ptr OpenFile(const string &path, FileOpenFlags flags, + optional_ptr opener = nullptr) override;