Force download doesn't require to do a head request

duckdb · Nov 25, 2024 · 112a5aa · 112a5aa
1 parent 1d4c3d1
commit 112a5aa
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 1 deletion.
diff --git a/extension/httpfs/httpfs.cpp b/extension/httpfs/httpfs.cpp
@@ -700,7 +700,7 @@ void HTTPFileHandle::Initialize(optional_ptr<FileOpener> opener) {
 			}
 			length = 0;
 			return;
-		} else {
+		} else if (!http_params.force_download){
 			// HEAD request fail, use Range request for another try (read only one byte)
 			if (flags.OpenForReading() && res->code != 404) {
 				auto range_res = hfs.GetRangeRequest(*this, path, {}, 0, nullptr, 2);

diff --git a/test/sql/copy/test_remote_head_forbidden.test b/test/sql/copy/test_remote_head_forbidden.test
@@ -0,0 +1,15 @@
+# name: test/sql/copy/test_remote_head_forbidden.test
+# description: Test Force download with server that doesn't want to give us the head
+# group: [parquet]
+
+require httpfs
+
+require json
+
+statement ok
+SET force_download=true;
+
+query II
+FROM read_json('https://api.spring.io/projects/spring-boot/generations')
+----
+{'generations': [{'name': 1.5.x, 'initialReleaseDate': 2017-01-30, 'ossSupportEndDate': 2019-08-06, 'commercialSupportEndDate': 2020-11-06, '_links': {'self': {'href': https://api.spring.io/projects/spring-boot/generations/1.5.x}, 'project': {'href': https://api.spring.io/projects/spring-boot}}}, {'name': 2.0.x, 'initialReleaseDate': 2018-03-01, 'ossSupportEndDate': 2019-03-01, 'commercialSupportEndDate': 2020-06-01, '_links': {'self': {'href': https://api.spring.io/projects/spring-boot/generations/2.0.x}, 'project': {'href': https://api.spring.io/projects/spring-boot}}}, {'name': 2.1.x, 'initialReleaseDate': 2018-10-30, 'ossSupportEndDate': 2019-10-30, 'commercialSupportEndDate': 2021-01-30, '_links': {'self': {'href': https://api.spring.io/projects/spring-boot/generations/2.1.x}, 'project': {'href': https://api.spring.io/projects/spring-boot}}}, {'name': 2.2.x, 'initialReleaseDate': 2019-10-16, 'ossSupportEndDate': 2020-10-16, 'commercialSupportEndDate': 2022-01-16, '_links': {'self': {'href': https://api.spring.io/projects/spring-boot/generations/2.2.x}, 'project': {'href': https://api.spring.io/projects/spring-boot}}}, {'name': 2.3.x, 'initialReleaseDate': 2020-05-15, 'ossSupportEndDate': 2021-05-20, 'commercialSupportEndDate': 2022-08-20, '_links': {'self': {'href': https://api.spring.io/projects/spring-boot/generations/2.3.x}, 'project': {'href': https://api.spring.io/projects/spring-boot}}}, {'name': 2.4.x, 'initialReleaseDate': 2020-11-12, 'ossSupportEndDate': 2021-11-18, 'commercialSupportEndDate': 2023-02-23, '_links': {'self': {'href': https://api.spring.io/projects/spring-boot/generations/2.4.x}, 'project': {'href': https://api.spring.io/projects/spring-boot}}}, {'name': 2.5.x, 'initialReleaseDate': 2021-05-20, 'ossSupportEndDate': 2022-05-19, 'commercialSupportEndDate': 2023-08-24, '_links': {'self': {'href': https://api.spring.io/projects/spring-boot/generations/2.5.x}, 'project': {'href': https://api.spring.io/projects/spring-boot}}}, {'name': 2.6.x, 'initialReleaseDate': 2021-11-17, 'ossSupportEndDate': 2022-11-24, 'commercialSupportEndDate': 2024-02-24, '_links': {'self': {'href': https://api.spring.io/projects/spring-boot/generations/2.6.x}, 'project': {'href': https://api.spring.io/projects/spring-boot}}}, {'name': 2.7.x, 'initialReleaseDate': 2022-05-19, 'ossSupportEndDate': 2023-11-24, 'commercialSupportEndDate': 2026-12-31, '_links': {'self': {'href': https://api.spring.io/projects/spring-boot/generations/2.7.x}, 'project': {'href': https://api.spring.io/projects/spring-boot}}}, {'name': 3.0.x, 'initialReleaseDate': 2022-11-24, 'ossSupportEndDate': 2023-11-24, 'commercialSupportEndDate': 2025-02-24, '_links': {'self': {'href': https://api.spring.io/projects/spring-boot/generations/3.0.x}, 'project': {'href': https://api.spring.io/projects/spring-boot}}}, {'name': 3.1.x, 'initialReleaseDate': 2023-05-18, 'ossSupportEndDate': 2024-05-18, 'commercialSupportEndDate': 2025-08-18, '_links': {'self': {'href': https://api.spring.io/projects/spring-boot/generations/3.1.x}, 'project': {'href': https://api.spring.io/projects/spring-boot}}}, {'name': 3.2.x, 'initialReleaseDate': 2023-11-23, 'ossSupportEndDate': 2024-11-23, 'commercialSupportEndDate': 2025-02-23, '_links': {'self': {'href': https://api.spring.io/projects/spring-boot/generations/3.2.x}, 'project': {'href': https://api.spring.io/projects/spring-boot}}}, {'name': 3.3.x, 'initialReleaseDate': 2024-05-23, 'ossSupportEndDate': 2025-05-23, 'commercialSupportEndDate': 2025-08-23, '_links': {'self': {'href': https://api.spring.io/projects/spring-boot/generations/3.3.x}, 'project': {'href': https://api.spring.io/projects/spring-boot}}}, {'name': 3.4.x, 'initialReleaseDate': 2024-11-21, 'ossSupportEndDate': 2025-11-21, 'commercialSupportEndDate': 2026-02-21, '_links': {'self': {'href': https://api.spring.io/projects/spring-boot/generations/3.4.x}, 'project': {'href': https://api.spring.io/projects/spring-boot}}}]}	{'project': {'href': https://api.spring.io/projects/spring-boot}}