Merge pull request #9817 from samansmink/add-s3a-s3n

Add s3a s3n protocols for httpfs
duckdb · Nov 28, 2023 · e2e2897 · e2e2897
2 parents 2d710de + ee1d3cf
commit e2e2897
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 10 deletions.
diff --git a/extension/httpfs/s3fs.cpp b/extension/httpfs/s3fs.cpp
@@ -569,7 +569,7 @@ void S3FileSystem::ReadQueryParams(const string &url_query_param, S3AuthParams &
 }
 
 static string GetPrefix(string url) {
-	const string prefixes[] = {"s3://", "gcs://", "r2://"};
+	const string prefixes[] = {"s3://", "s3a://", "s3n://", "gcs://", "r2://"};
 	for (auto &prefix : prefixes) {
 		if (StringUtil::StartsWith(url, prefix)) {
 			return prefix;
@@ -834,7 +834,9 @@ void S3FileHandle::Initialize(FileOpener *opener) {
 }
 
 bool S3FileSystem::CanHandleFile(const string &fpath) {
-	return fpath.rfind("s3://", 0) * fpath.rfind("gcs://", 0) * fpath.rfind("r2://", 0) == 0;
+	return fpath.rfind("s3://", 0) * fpath.rfind("s3a://", 0) * fpath.rfind("s3n://", 0) * fpath.rfind("gcs://", 0) *
+	           fpath.rfind("r2://", 0) ==
+	       0;
 }
 
 void S3FileSystem::FileSync(FileHandle &handle) {

diff --git a/scripts/generate_extensions_function.py b/scripts/generate_extensions_function.py
@@ -271,6 +271,8 @@ def print_map_diff(d1, d2):
     {"http://", "httpfs"},
     {"https://", "httpfs"},
     {"s3://", "httpfs"},
+    {"s3a://", "httpfs"},
+    {"s3n://", "httpfs"},
     {"gcs://", "httpfs"},
     {"r2://", "httpfs"}
 //    {"azure://", "azure"}

diff --git a/src/common/file_system.cpp b/src/common/file_system.cpp
@@ -544,7 +544,7 @@ FileType FileHandle::GetType() {
 }
 
 bool FileSystem::IsRemoteFile(const string &path) {
-	const string prefixes[] = {"http://", "https://", "s3://", "gcs://", "r2://"};
+	const string prefixes[] = {"http://", "https://", "s3://", "s3a://", "s3n://", "gcs://", "r2://"};
 	for (auto &prefix : prefixes) {
 		if (StringUtil::StartsWith(path, prefix)) {
 			return true;

diff --git a/src/include/duckdb/main/extension_entries.hpp b/src/include/duckdb/main/extension_entries.hpp
@@ -275,12 +275,9 @@ static constexpr ExtensionEntry EXTENSION_COLLATIONS[] = {
 // Note: these are currently hardcoded in scripts/generate_extensions_function.py
 // TODO: automate by passing though to script via duckdb
 static constexpr ExtensionEntry EXTENSION_FILE_PREFIXES[] = {
-    {"http://", "httpfs"},
-    {"https://", "httpfs"},
-    {"s3://", "httpfs"},
-    {"gcs://", "httpfs"},
-    {"r2://", "httpfs"} // , {"azure://", "azure"}
-};                      // END_OF_EXTENSION_FILE_PREFIXES
+    {"http://", "httpfs"}, {"https://", "httpfs"}, {"s3://", "httpfs"}, {"s3a://", "httpfs"},
+    {"s3n://", "httpfs"},  {"gcs://", "httpfs"},   {"r2://", "httpfs"} // , {"azure://", "azure"}
+};                                                                     // END_OF_EXTENSION_FILE_PREFIXES
 
 // Note: these are currently hardcoded in scripts/generate_extensions_function.py
 // TODO: automate by passing though to script via duckdb

diff --git a/test/sql/copy/s3/url_encode.test b/test/sql/copy/s3/url_encode.test
@@ -29,7 +29,7 @@ CREATE TABLE test_2 as (SELECT 2 FROM range(0,5));
 CREATE TABLE test_3 as (SELECT 3 FROM range(0,5));
 CREATE TABLE test_4 as (SELECT 4 FROM range(0,5));
 
-foreach prefix s3:// r2:// gcs://
+foreach prefix s3:// r2:// gcs:// s3a:// s3n://
 
 statement ok
 COPY test_1 TO '${prefix}test-bucket-public/url_encode/just because you can doesnt mean you should.parquet' (FORMAT 'parquet');