add option for keep_alive setting

extension/httpfs/httpfs.cpp

@@ -34,6 +34,8 @@ HTTPParams HTTPParams::ReadFrom(FileOpener *opener) {
 	uint64_t retry_wait_ms = DEFAULT_RETRY_WAIT_MS;
 	float retry_backoff = DEFAULT_RETRY_BACKOFF;
 	bool force_download = DEFAULT_FORCE_DOWNLOAD;
+	bool keep_alive = DEFAULT_KEEP_ALIVE;
+
 	Value value;
 	if (FileOpener::TryGetCurrentSetting(opener, "http_timeout", value)) {
 		timeout = value.GetValue<uint64_t>();
@@ -50,8 +52,11 @@ HTTPParams HTTPParams::ReadFrom(FileOpener *opener) {
 	if (FileOpener::TryGetCurrentSetting(opener, "http_retry_backoff", value)) {
 		retry_backoff = value.GetValue<float>();
 	}
+	if (FileOpener::TryGetCurrentSetting(opener, "http_keep_alive", value)) {
+		keep_alive = value.GetValue<bool>();
+	}
 
-	return {timeout, retries, retry_wait_ms, retry_backoff, force_download};
+	return {timeout, retries, retry_wait_ms, retry_backoff, force_download, keep_alive};
 }
 
 void HTTPFileSystem::ParseUrl(string &url, string &path_out, string &proto_host_port_out) {
@@ -181,7 +186,7 @@ unique_ptr<duckdb_httplib_openssl::Client> HTTPFileSystem::GetClient(const HTTPP
                                                                      const char *proto_host_port) {
 	auto client = make_uniq<duckdb_httplib_openssl::Client>(proto_host_port);
 	client->set_follow_location(true);
-	client->set_keep_alive(true);
+	client->set_keep_alive(http_params.keep_alive);
 	client->enable_server_certificate_verification(false);
 	client->set_write_timeout(http_params.timeout);
 	client->set_read_timeout(http_params.timeout);

extension/httpfs/httpfs_extension.cpp

@@ -28,6 +28,10 @@ static void LoadInternal(DatabaseInstance &instance) {
 	config.AddExtensionOption("http_retry_backoff",
 	                          "Backoff factor for exponentially increasing retry wait time (default 4)",
 	                          LogicalType::FLOAT, Value(4));
+	config.AddExtensionOption(
+	    "http_keep_alive",
+	    "Keep alive connections. Setting this to false can help when running into connection failures",
+	    LogicalType::BOOLEAN, Value(true));
 	// Global S3 config
 	config.AddExtensionOption("s3_region", "S3 Region", LogicalType::VARCHAR);
 	config.AddExtensionOption("s3_access_key_id", "S3 Access Key ID", LogicalType::VARCHAR);

extension/httpfs/include/httpfs.hpp

@@ -35,12 +35,14 @@ struct HTTPParams {
 	static constexpr uint64_t DEFAULT_RETRY_WAIT_MS = 100;
 	static constexpr float DEFAULT_RETRY_BACKOFF = 4;
 	static constexpr bool DEFAULT_FORCE_DOWNLOAD = false;
+	static constexpr bool DEFAULT_KEEP_ALIVE = true;
 
 	uint64_t timeout;
 	uint64_t retries;
 	uint64_t retry_wait_ms;
 	float retry_backoff;
 	bool force_download;
+	bool keep_alive;
 
 	static HTTPParams ReadFrom(FileOpener *opener);
 };

src/include/duckdb/main/extension_entries.hpp

@@ -210,6 +210,7 @@ static constexpr ExtensionEntry EXTENSION_SETTINGS[] = {
     {"http_retry_backoff", "httpfs"},
     {"http_retry_wait_ms", "httpfs"},
     {"http_timeout", "httpfs"},
+    {"http_keep_alive", "httpfs"},
     {"pg_debug_show_queries", "postgres_scanner"},
     {"pg_use_binary_copy", "postgres_scanner"},
     {"pg_experimental_filter_pushdown", "postgres_scanner"},

test/sql/copy/s3/download_config.test

@@ -48,6 +48,9 @@ SET http_retry_backoff=1;
 statement ok
 SET http_timeout=50000;
 
+statement ok
+SET http_keep_alive=false;
+
 # Test the vhost style urls (this is the default)
 statement ok
 SET s3_url_style='${url_style}';