From c21122e16e8d8fbfe0b24641f5cc62ede1ef91a7 Mon Sep 17 00:00:00 2001 From: Mukund Thakur Date: Tue, 19 Dec 2023 14:17:41 -0600 Subject: [PATCH 1/3] HADOOP-19015. Increase fs.s3a.connection.maximum to 500 to minimize risk of Timeout waiting for connection from pool --- .../hadoop-common/src/main/resources/core-default.xml | 3 ++- .../src/main/java/org/apache/hadoop/fs/s3a/Constants.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 5a5171056d048..00d700392797f 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1530,7 +1530,7 @@ fs.s3a.connection.maximum - 200 + 500 Controls the maximum number of simultaneous connections to S3. This must be bigger than the value of fs.s3a.threads.max so as to stop threads being blocked waiting for new HTTPS connections. @@ -1538,6 +1538,7 @@ + fs.s3a.connection.ssl.enabled true diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index fb4f22cedb9ba..05e6a43a05597 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -173,7 +173,7 @@ private Constants() { * Future releases are likely to increase this value. * Keep in sync with the value in {@code core-default.xml} */ - public static final int DEFAULT_MAXIMUM_CONNECTIONS = 200; + public static final int DEFAULT_MAXIMUM_CONNECTIONS = 500; /** * Configuration option to configure expiration time of From 6c3585459d3ab93a4410bd5e58e52cc6930f5421 Mon Sep 17 00:00:00 2001 From: Mukund Thakur Date: Thu, 21 Dec 2023 10:50:22 -0500 Subject: [PATCH 2/3] add extra line for yetus to rerun --- .../hadoop-common/src/main/resources/core-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 00d700392797f..29ec06db65989 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1538,7 +1538,7 @@ - + fs.s3a.connection.ssl.enabled true From a9703efedd6e66b2b433cee8e7c013f7a3a0e7a6 Mon Sep 17 00:00:00 2001 From: Mukund Thakur Date: Tue, 2 Jan 2024 17:45:45 -0600 Subject: [PATCH 3/3] updating performance.md doc --- .../src/site/markdown/tools/hadoop-aws/performance.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md index 37cf472277d27..9b36bb79d9ee2 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md @@ -225,7 +225,7 @@ The choice is yours. Generally recovery is better, but sometimes fail-fast is mo | Property | Default | V2 | Meaning | |-----------------------------------------|---------|:----|-------------------------------------------------------| -| `fs.s3a.connection.maximum` | `200` | | Connection pool size | +| `fs.s3a.connection.maximum` | `500` | | Connection pool size | | `fs.s3a.connection.keepalive` | `false` | `*` | Use TCP keepalive on open channels | | `fs.s3a.connection.acquisition.timeout` | `60s` | `*` | Timeout for waiting for a connection from the pool. | | `fs.s3a.connection.establish.timeout` | `30s` | | Time to establish the TCP/TLS connection |