diff --git a/src/duckdb/src/common/file_system.cpp b/src/duckdb/src/common/file_system.cpp index f00bbadc0..27160adc3 100644 --- a/src/duckdb/src/common/file_system.cpp +++ b/src/duckdb/src/common/file_system.cpp @@ -623,7 +623,7 @@ FileType FileHandle::GetType() { } bool FileSystem::IsRemoteFile(const string &path) { - const string prefixes[] = {"http://", "https://", "s3://", "s3a://", "s3n://", "gcs://", "gs://", "r2://"}; + const string prefixes[] = {"http://", "https://", "s3://", "s3a://", "s3n://", "gcs://", "gs://", "r2://", "hf://"}; for (auto &prefix : prefixes) { if (StringUtil::StartsWith(path, prefix)) { return true; diff --git a/src/duckdb/src/function/table/version/pragma_version.cpp b/src/duckdb/src/function/table/version/pragma_version.cpp index a590dfbd9..f1b2c357b 100644 --- a/src/duckdb/src/function/table/version/pragma_version.cpp +++ b/src/duckdb/src/function/table/version/pragma_version.cpp @@ -1,5 +1,5 @@ #ifndef DUCKDB_PATCH_VERSION -#define DUCKDB_PATCH_VERSION "3-dev876" +#define DUCKDB_PATCH_VERSION "3-dev897" #endif #ifndef DUCKDB_MINOR_VERSION #define DUCKDB_MINOR_VERSION 10 @@ -8,10 +8,10 @@ #define DUCKDB_MAJOR_VERSION 0 #endif #ifndef DUCKDB_VERSION -#define DUCKDB_VERSION "v0.10.3-dev876" +#define DUCKDB_VERSION "v0.10.3-dev897" #endif #ifndef DUCKDB_SOURCE_ID -#define DUCKDB_SOURCE_ID "ffb3f9789f" +#define DUCKDB_SOURCE_ID "b9ecd8da3a" #endif #include "duckdb/function/table/system_functions.hpp" #include "duckdb/main/database.hpp" diff --git a/src/duckdb/src/include/duckdb/common/file_opener.hpp b/src/duckdb/src/include/duckdb/common/file_opener.hpp index 2c4dbc396..6f3e5a8e4 100644 --- a/src/duckdb/src/include/duckdb/common/file_opener.hpp +++ b/src/duckdb/src/include/duckdb/common/file_opener.hpp @@ -14,6 +14,8 @@ namespace duckdb { +struct CatalogTransaction; +class SecretManager; class ClientContext; class Value; @@ -33,8 +35,10 @@ class FileOpener { virtual optional_ptr TryGetClientContext() = 0; virtual optional_ptr TryGetDatabase() = 0; + DUCKDB_API static unique_ptr TryGetCatalogTransaction(optional_ptr opener); DUCKDB_API static optional_ptr TryGetClientContext(optional_ptr opener); DUCKDB_API static optional_ptr TryGetDatabase(optional_ptr opener); + DUCKDB_API static optional_ptr TryGetSecretManager(optional_ptr opener); DUCKDB_API static SettingLookupResult TryGetCurrentSetting(optional_ptr opener, const string &key, Value &result); DUCKDB_API static SettingLookupResult TryGetCurrentSetting(optional_ptr opener, const string &key, diff --git a/src/duckdb/src/include/duckdb/main/extension_entries.hpp b/src/duckdb/src/include/duckdb/main/extension_entries.hpp index ead02c699..51c467a6d 100644 --- a/src/duckdb/src/include/duckdb/main/extension_entries.hpp +++ b/src/duckdb/src/include/duckdb/main/extension_entries.hpp @@ -255,6 +255,7 @@ static constexpr ExtensionEntry EXTENSION_SETTINGS[] = { {"calendar", "icu"}, {"enable_server_cert_verification", "httpfs"}, {"force_download", "httpfs"}, + {"hf_max_per_page", "httpfs"}, {"http_keep_alive", "httpfs"}, {"http_retries", "httpfs"}, {"http_retry_backoff", "httpfs"}, @@ -326,9 +327,9 @@ static constexpr ExtensionEntry EXTENSION_COLLATIONS[] = { // Note: these are currently hardcoded in scripts/generate_extensions_function.py // TODO: automate by passing though to script via duckdb static constexpr ExtensionEntry EXTENSION_FILE_PREFIXES[] = { - {"http://", "httpfs"}, {"https://", "httpfs"}, {"s3://", "httpfs"}, {"s3a://", "httpfs"}, - {"s3n://", "httpfs"}, {"gcs://", "httpfs"}, {"gs://", "httpfs"}, {"r2://", "httpfs"}, - {"azure://", "azure"}, {"az://", "azure"}, {"abfss://", "azure"}}; // END_OF_EXTENSION_FILE_PREFIXES + {"http://", "httpfs"}, {"https://", "httpfs"}, {"s3://", "httpfs"}, {"s3a://", "httpfs"}, {"s3n://", "httpfs"}, + {"gcs://", "httpfs"}, {"gs://", "httpfs"}, {"r2://", "httpfs"}, {"azure://", "azure"}, {"az://", "azure"}, + {"abfss://", "azure"}, {"hf://", "httpfs"}}; // END_OF_EXTENSION_FILE_PREFIXES // Note: these are currently hardcoded in scripts/generate_extensions_function.py // TODO: automate by passing though to script via duckdb @@ -346,14 +347,18 @@ static constexpr ExtensionEntry EXTENSION_FILE_CONTAINS[] = {{".parquet?", "parq // Note: these are currently hardcoded in scripts/generate_extensions_function.py // TODO: automate by passing though to script via duckdb static constexpr ExtensionEntry EXTENSION_SECRET_TYPES[] = { - {"s3", "httpfs"}, {"r2", "httpfs"}, {"gcs", "httpfs"}, {"azure", "azure"}}; // EXTENSION_SECRET_TYPES + {"s3", "httpfs"}, {"r2", "httpfs"}, {"gcs", "httpfs"}, + {"azure", "azure"}, {"huggingface", "httpfs"}, {"bearer", "httpfs"}}; // EXTENSION_SECRET_TYPES // Note: these are currently hardcoded in scripts/generate_extensions_function.py // TODO: automate by passing though to script via duckdb static constexpr ExtensionEntry EXTENSION_SECRET_PROVIDERS[] = { - {"s3/config", "httpfs"}, {"gcs/config", "httpfs"}, {"r2/config", "httpfs"}, - {"s3/credential_chain", "aws"}, {"gcs/credential_chain", "aws"}, {"r2/credential_chain", "aws"}, - {"azure/config", "azure"}, {"azure/credential_chain", "azure"}}; // EXTENSION_SECRET_PROVIDERS + {"s3/config", "httpfs"}, {"gcs/config", "httpfs"}, + {"r2/config", "httpfs"}, {"s3/credential_chain", "aws"}, + {"gcs/credential_chain", "aws"}, {"r2/credential_chain", "aws"}, + {"azure/config", "azure"}, {"azure/credential_chain", "azure"}, + {"huggingface/config", "httfps"}, {"huggingface/cache", "httpfs"}, + {"bearer/config", "httpfs"}}; // EXTENSION_SECRET_PROVIDERS static constexpr const char *AUTOLOADABLE_EXTENSIONS[] = { "aws", "azure", "autocomplete", "excel", "fts", "httpfs", "inet", diff --git a/src/duckdb/src/main/client_context_file_opener.cpp b/src/duckdb/src/main/client_context_file_opener.cpp index 17ddef4f0..5c7387b86 100644 --- a/src/duckdb/src/main/client_context_file_opener.cpp +++ b/src/duckdb/src/main/client_context_file_opener.cpp @@ -1,5 +1,7 @@ #include "duckdb/main/client_context_file_opener.hpp" +#include "duckdb/catalog/catalog_transaction.hpp" +#include "duckdb/main/database.hpp" #include "duckdb/common/file_opener.hpp" #include "duckdb/main/client_context.hpp" @@ -18,6 +20,22 @@ optional_ptr ClientContextFileOpener::TryGetDatabase() { return context.db.get(); } +unique_ptr FileOpener::TryGetCatalogTransaction(optional_ptr opener) { + if (!opener) { + return nullptr; + } + auto context = opener->TryGetClientContext(); + if (context) { + return make_uniq(CatalogTransaction::GetSystemCatalogTransaction(*context)); + } + + auto db = opener->TryGetDatabase(); + if (db) { + return make_uniq(CatalogTransaction::GetSystemTransaction(*db)); + } + return nullptr; +} + optional_ptr FileOpener::TryGetClientContext(optional_ptr opener) { if (!opener) { return nullptr; @@ -32,6 +50,19 @@ optional_ptr FileOpener::TryGetDatabase(optional_ptrTryGetDatabase(); } +optional_ptr FileOpener::TryGetSecretManager(optional_ptr opener) { + if (!opener) { + return nullptr; + } + + auto db = opener->TryGetDatabase(); + if (!db) { + return nullptr; + } + + return &db->GetSecretManager(); +} + SettingLookupResult FileOpener::TryGetCurrentSetting(optional_ptr opener, const string &key, Value &result) { if (!opener) {