Exposes additional configuration options for S3Client (#204)

include/aws/s3/private/s3_client_impl.h

@@ -15,7 +15,9 @@
 #include <aws/common/mutex.h>
 #include <aws/common/ref_count.h>
 #include <aws/common/task_scheduler.h>
+#include <aws/http/connection.h>
 #include <aws/http/connection_manager.h>
+#include <aws/http/proxy.h>
 
 /* TODO automate this value in the future to prevent it from becoming out-of-sync. */
 #define AWS_S3_CLIENT_VERSION "0.1.x"
@@ -58,6 +60,40 @@ struct aws_s3_endpoint_options {
 
     /* HTTP port override. If zero, determine port based on TLS context */
     uint16_t port;
+
+    /**
+     * Optional.
+     * Proxy configuration for http connection.
+     */
+    struct aws_http_proxy_config *proxy_config;
+
+    /**
+     * Optional.
+     * Configuration for fetching proxy configuration from environment.
+     * By Default proxy_ev_settings.aws_http_proxy_env_var_type is set to AWS_HPEV_ENABLE which means read proxy
+     * configuration from environment.
+     * Only works when proxy_config is not set. If both are set, configuration from proxy_config is used.
+     */
+    struct proxy_env_var_settings *proxy_ev_settings;
+
+    /**
+     * Optional.
+     * If set to 0, default value is used.
+     */
+    uint32_t connect_timeout_ms;
+
+    /**
+     * Optional.
+     * Set keepalive to periodically transmit messages for detecting a disconnected peer.
+     */
+    struct aws_s3_tcp_keep_alive_options *tcp_keep_alive_options;
+
+    /**
+     * Optional.
+     * Configuration options for connection monitoring.
+     * If the transfer speed falls below the specified minimum_throughput_bytes_per_second, the operation is aborted.
+     */
+    struct aws_http_connection_monitoring_options *monitoring_options;
 };
 
 struct aws_s3_endpoint {
@@ -179,6 +215,43 @@ struct aws_s3_client {
     /* Retry strategy used for scheduling request retries. */
     struct aws_retry_strategy *retry_strategy;
 
+    /**
+     * Optional.
+     * Proxy configuration for http connection.
+     */
+    struct aws_http_proxy_config *proxy_config;
+
+    /**
+     * Optional.
+     * Configuration for fetching proxy configuration from environment.
+     * By Default proxy_ev_settings.aws_http_proxy_env_var_type is set to AWS_HPEV_ENABLE which means read proxy
+     * configuration from environment.
+     * Only works when proxy_config is not set. If both are set, configuration from proxy_config is used.
+     */
+    struct proxy_env_var_settings *proxy_ev_settings;
+
+    /**
+     * Optional.
+     * If set to 0, default value is used.
+     */
+    uint32_t connect_timeout_ms;
+
+    /**
+     * Optional.
+     * Set keepalive to periodically transmit messages for detecting a disconnected peer.
+     */
+    struct aws_s3_tcp_keep_alive_options *tcp_keep_alive_options;
+
+    /**
+     * Optional.
+     * Configuration options for connection monitoring.
+     * If the transfer speed falls below the specified minimum_throughput_bytes_per_second, the operation is aborted.
+     */
+    struct aws_http_connection_monitoring_options *monitoring_options;
+
+    /* tls options from proxy environment settings. */
+    struct aws_tls_connection_options *proxy_ev_tls_options;
+
     /* Shutdown callbacks to notify when the client is completely cleaned up. */
     aws_s3_client_shutdown_complete_callback_fn *shutdown_callback;
     void *shutdown_callback_user_data;

include/aws/s3/private/s3_meta_request_impl.h

@@ -51,7 +51,7 @@ struct aws_s3_prepare_request_payload {
 };
 
 struct aws_s3_meta_request_vtable {
-    /* Update the meta request.  out_request is required to be non-null. Returns true if there is there is any work in
+    /* Update the meta request.  out_request is required to be non-null. Returns true if there is any work in
      * progress, false if there is not. */
     bool (*update)(struct aws_s3_meta_request *meta_request, uint32_t flags, struct aws_s3_request **out_request);
 

include/aws/s3/s3_client.h

@@ -6,11 +6,10 @@
  * SPDX-License-Identifier: Apache-2.0.
  */
 
+#include <aws/auth/signing_config.h>
 #include <aws/io/retry_strategy.h>
 #include <aws/s3/s3.h>
 
-#include <aws/auth/signing_config.h>
-
 struct aws_allocator;
 
 struct aws_http_stream;
@@ -150,6 +149,19 @@ enum aws_s3_checksum_algorithm {
     AWS_SCA_COUNT,
 };
 
+/* Keepalive properties are TCP only.
+ * If interval or timeout are zero, then default values are used.
+ */
+struct aws_s3_tcp_keep_alive_options {
+
+    uint16_t keep_alive_interval_sec;
+    uint16_t keep_alive_timeout_sec;
+
+    /* If set, sets the number of keep alive probes allowed to fail before the connection is considered
+     * lost. If zero OS defaults are used. On Windows, this option is meaningless until Windows 10 1703.*/
+    uint16_t keep_alive_max_failed_probes;
+};
+
 /* Options for a new client. */
 struct aws_s3_client_config {
 
@@ -200,6 +212,40 @@ struct aws_s3_client_config {
     /* Callback and associated user data for when the client has completed its shutdown process. */
     aws_s3_client_shutdown_complete_callback_fn *shutdown_callback;
     void *shutdown_callback_user_data;
+
+    /**
+     * Optional.
+     * Proxy configuration for http connection.
+     */
+    struct aws_http_proxy_options *proxy_options;
+
+    /**
+     * Optional.
+     * Configuration for fetching proxy configuration from environment.
+     * By Default proxy_ev_settings.aws_http_proxy_env_var_type is set to AWS_HPEV_ENABLE which means read proxy
+     * configuration from environment.
+     * Only works when proxy_options is not set. If both are set, configuration from proxy_options is used.
+     */
+    struct proxy_env_var_settings *proxy_ev_settings;
+
+    /**
+     * Optional.
+     * If set to 0, default value is used.
+     */
+    uint32_t connect_timeout_ms;
+
+    /**
+     * Optional.
+     * Set keepalive to periodically transmit messages for detecting a disconnected peer.
+     */
+    struct aws_s3_tcp_keep_alive_options *tcp_keep_alive_options;
+
+    /**
+     * Optional.
+     * Configuration options for connection monitoring.
+     * If the transfer speed falls below the specified minimum_throughput_bytes_per_second, the operation is aborted.
+     */
+    struct aws_http_connection_monitoring_options *monitoring_options;
 };
 
 /* Options for a new meta request, ie, file transfer that will be handled by the high performance client. */

source/s3_auto_ranged_get.c

@@ -135,7 +135,7 @@ static bool s_s3_auto_ranged_get_update(
     {
         aws_s3_meta_request_lock_synced_data(meta_request);
 
-        /* If nothing has set the the "finish result" then this meta request is still in progress and we can potentially
+        /* If nothing has set the "finish result" then this meta request is still in progress, and we can potentially
          * send additional requests. */
         if (!aws_s3_meta_request_has_finish_result_synced(meta_request)) {
 

source/s3_client.c

@@ -304,6 +304,37 @@ struct aws_s3_client *aws_s3_client_new(
         *((size_t *)&client_config->max_part_size) = client_config->part_size;
     }
 
+    if (client_config->proxy_options) {
+        client->proxy_config = aws_http_proxy_config_new_from_proxy_options(allocator, client_config->proxy_options);
+        if (client->proxy_config == NULL) {
+            goto on_error;
+        }
+    }
+    client->connect_timeout_ms = client_config->connect_timeout_ms;
+    if (client_config->proxy_ev_settings) {
+        client->proxy_ev_settings = aws_mem_calloc(allocator, 1, sizeof(struct proxy_env_var_settings));
+        *client->proxy_ev_settings = *client_config->proxy_ev_settings;
+
+        if (client_config->proxy_ev_settings->tls_options) {
+            client->proxy_ev_tls_options = aws_mem_calloc(allocator, 1, sizeof(struct aws_tls_connection_options));
+            if (aws_tls_connection_options_copy(client->proxy_ev_tls_options, client->proxy_ev_settings->tls_options)) {
+                goto on_error;
+            }
+            client->proxy_ev_settings->tls_options = client->proxy_ev_tls_options;
+        }
+    }
+
+    if (client_config->tcp_keep_alive_options) {
+        client->tcp_keep_alive_options = aws_mem_calloc(allocator, 1, sizeof(struct aws_s3_tcp_keep_alive_options));
+        *client->tcp_keep_alive_options = *client_config->tcp_keep_alive_options;
+    }
+
+    if (client_config->monitoring_options) {
+        client->monitoring_options =
+            aws_mem_calloc(allocator, 1, sizeof(struct aws_http_connection_monitoring_options));
+        *client->monitoring_options = *client_config->monitoring_options;
+    }
+
     if (client_config->tls_mode == AWS_MR_TLS_ENABLED) {
         client->tls_connection_options =
             aws_mem_calloc(client->allocator, 1, sizeof(struct aws_tls_connection_options));
@@ -421,6 +452,18 @@ struct aws_s3_client *aws_s3_client_new(
         aws_mem_release(client->allocator, client->tls_connection_options);
         client->tls_connection_options = NULL;
     }
+    if (client->proxy_config) {
+        aws_http_proxy_config_destroy(client->proxy_config);
+    }
+    if (client->proxy_ev_tls_options) {
+        aws_tls_connection_options_clean_up(client->proxy_ev_tls_options);
+        aws_mem_release(client->allocator, client->proxy_ev_tls_options);
+        client->proxy_ev_settings->tls_options = NULL;
+    }
+    aws_mem_release(client->allocator, client->proxy_ev_settings);
+    aws_mem_release(client->allocator, client->monitoring_options);
+    aws_mem_release(client->allocator, client->tcp_keep_alive_options);
+
     aws_event_loop_group_release(client->client_bootstrap->event_loop_group);
     aws_client_bootstrap_release(client->client_bootstrap);
     aws_mutex_clean_up(&client->synced_data.lock);
@@ -495,6 +538,19 @@ static void s_s3_client_finish_destroy_default(struct aws_s3_client *client) {
         client->tls_connection_options = NULL;
     }
 
+    if (client->proxy_config) {
+        aws_http_proxy_config_destroy(client->proxy_config);
+    }
+
+    if (client->proxy_ev_tls_options) {
+        aws_tls_connection_options_clean_up(client->proxy_ev_tls_options);
+        aws_mem_release(client->allocator, client->proxy_ev_tls_options);
+        client->proxy_ev_settings->tls_options = NULL;
+    }
+    aws_mem_release(client->allocator, client->proxy_ev_settings);
+    aws_mem_release(client->allocator, client->monitoring_options);
+    aws_mem_release(client->allocator, client->tcp_keep_alive_options);
+
     aws_mutex_clean_up(&client->synced_data.lock);
 
     AWS_ASSERT(aws_linked_list_empty(&client->synced_data.pending_meta_request_work));
@@ -667,7 +723,6 @@ struct aws_s3_meta_request *aws_s3_client_make_meta_request(
         struct aws_hash_element *endpoint_hash_element = NULL;
 
         int was_created = 0;
-
         if (aws_hash_table_create(
                 &client->synced_data.endpoints, endpoint_host_name, &endpoint_hash_element, &was_created)) {
             error_occurred = true;
@@ -684,7 +739,11 @@ struct aws_s3_meta_request *aws_s3_client_make_meta_request(
                 .user_data = client,
                 .max_connections = aws_s3_client_get_max_active_connections(client, NULL),
                 .port = port,
-            };
+                .proxy_config = client->proxy_config,
+                .proxy_ev_settings = client->proxy_ev_settings,
+                .connect_timeout_ms = client->connect_timeout_ms,
+                .tcp_keep_alive_options = client->tcp_keep_alive_options,
+                .monitoring_options = client->monitoring_options};
 
             endpoint = aws_s3_endpoint_new(client->allocator, &endpoint_options);
 

source/s3_endpoint.c

@@ -20,7 +20,6 @@
 #include <aws/common/system_info.h>
 #include <aws/http/connection.h>
 #include <aws/http/connection_manager.h>
-#include <aws/http/proxy.h>
 #include <aws/http/request_response.h>
 #include <aws/io/channel_bootstrap.h>
 #include <aws/io/event_loop.h>
@@ -51,7 +50,12 @@ static struct aws_http_connection_manager *s_s3_endpoint_create_http_connection_
     struct aws_client_bootstrap *client_bootstrap,
     const struct aws_tls_connection_options *tls_connection_options,
     uint32_t max_connections,
-    uint16_t port);
+    uint16_t port,
+    const struct aws_http_proxy_config *proxy_config,
+    const struct proxy_env_var_settings *proxy_ev_settings,
+    uint32_t connect_timeout_ms,
+    const struct aws_s3_tcp_keep_alive_options *tcp_keep_alive_options,
+    const struct aws_http_connection_monitoring_options *monitoring_options);
 
 static void s_s3_endpoint_http_connection_manager_shutdown_callback(void *user_data);
 
@@ -98,7 +102,12 @@ struct aws_s3_endpoint *aws_s3_endpoint_new(
         options->client_bootstrap,
         options->tls_connection_options,
         options->max_connections,
-        options->port);
+        options->port,
+        options->proxy_config,
+        options->proxy_ev_settings,
+        options->connect_timeout_ms,
+        options->tcp_keep_alive_options,
+        options->monitoring_options);
 
     if (endpoint->http_connection_manager == NULL) {
         goto error_cleanup;
@@ -124,7 +133,13 @@ static struct aws_http_connection_manager *s_s3_endpoint_create_http_connection_
     struct aws_client_bootstrap *client_bootstrap,
     const struct aws_tls_connection_options *tls_connection_options,
     uint32_t max_connections,
-    uint16_t port) {
+    uint16_t port,
+    const struct aws_http_proxy_config *proxy_config,
+    const struct proxy_env_var_settings *proxy_ev_settings,
+    uint32_t connect_timeout_ms,
+    const struct aws_s3_tcp_keep_alive_options *tcp_keep_alive_options,
+    const struct aws_http_connection_monitoring_options *monitoring_options) {
+
     AWS_PRECONDITION(endpoint);
     AWS_PRECONDITION(client_bootstrap);
     AWS_PRECONDITION(host_name);
@@ -136,11 +151,20 @@ static struct aws_http_connection_manager *s_s3_endpoint_create_http_connection_
     AWS_ZERO_STRUCT(socket_options);
     socket_options.type = AWS_SOCKET_STREAM;
     socket_options.domain = AWS_SOCKET_IPV4;
-    socket_options.connect_timeout_ms = s_connection_timeout_ms;
-    struct proxy_env_var_settings proxy_ev_settings;
-    AWS_ZERO_STRUCT(proxy_ev_settings);
+    socket_options.connect_timeout_ms = connect_timeout_ms == 0 ? s_connection_timeout_ms : connect_timeout_ms;
+    if (tcp_keep_alive_options != NULL) {
+        socket_options.keepalive = true;
+        socket_options.keep_alive_interval_sec = tcp_keep_alive_options->keep_alive_interval_sec;
+        socket_options.keep_alive_timeout_sec = tcp_keep_alive_options->keep_alive_timeout_sec;
+        socket_options.keep_alive_max_failed_probes = tcp_keep_alive_options->keep_alive_max_failed_probes;
+    }
+    struct proxy_env_var_settings proxy_ev_settings_default;
     /* Turn on envrionment variable for proxy by default */
-    proxy_ev_settings.env_var_type = AWS_HPEV_ENABLE;
+    if (proxy_ev_settings == NULL) {
+        AWS_ZERO_STRUCT(proxy_ev_settings_default);
+        proxy_ev_settings_default.env_var_type = AWS_HPEV_ENABLE;
+        proxy_ev_settings = &proxy_ev_settings_default;
+    }
 
     struct aws_http_connection_manager_options manager_options;
     AWS_ZERO_STRUCT(manager_options);
@@ -151,7 +175,16 @@ static struct aws_http_connection_manager *s_s3_endpoint_create_http_connection_
     manager_options.max_connections = max_connections;
     manager_options.shutdown_complete_callback = s_s3_endpoint_http_connection_manager_shutdown_callback;
     manager_options.shutdown_complete_user_data = endpoint;
-    manager_options.proxy_ev_settings = &proxy_ev_settings;
+    manager_options.proxy_ev_settings = proxy_ev_settings;
+    if (monitoring_options != NULL) {
+        manager_options.monitoring_options = monitoring_options;
+    }
+
+    struct aws_http_proxy_options proxy_options;
+    if (proxy_config != NULL) {
+        aws_http_proxy_options_init_from_config(&proxy_options, proxy_config);
+        manager_options.proxy_options = &proxy_options;
+    }
 
     struct aws_tls_connection_options *manager_tls_options = NULL;