From f6ead98655623f4c52967f31fda6c5e639f597be Mon Sep 17 00:00:00 2001 From: Sneha Vijayarajan Date: Mon, 15 Nov 2021 09:52:42 -0800 Subject: [PATCH 1/4] Configurable HTTP connection and read timeouts --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 16 ++++++++ .../azurebfs/constants/ConfigurationKeys.java | 2 + .../constants/FileSystemConfigurations.java | 2 + .../fs/azurebfs/services/AbfsClient.java | 8 ++++ .../azurebfs/services/AbfsHttpOperation.java | 19 +++++---- .../azurebfs/services/AbfsRestOperation.java | 4 +- .../azurebfs/ITestAzureBlobFileSystemE2E.java | 39 +++++++++++++++++++ ...TestAbfsConfigurationFieldsValidation.java | 4 ++ .../MockDelegationSASTokenProvider.java | 6 ++- .../services/TestAbfsPerfTracker.java | 39 ++++++++++++++----- 10 files changed, 121 insertions(+), 18 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index fafc30372b4a5..ef3a57e3f0abb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -146,6 +146,14 @@ public class AbfsConfiguration{ DefaultValue = DEFAULT_MAX_RETRY_ATTEMPTS) private int maxIoRetries; + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_HTTP_CONNECTION_TIMEOUT, + DefaultValue = DEFAULT_HTTP_CONNECTION_TIMEOUT) + private int httpConnectionTimeout; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_HTTP_READ_TIMEOUT, + DefaultValue = DEFAULT_HTTP_READ_TIMEOUT) + private int httpReadTimeout; + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, MinValue = 0, DefaultValue = DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT) @@ -641,6 +649,14 @@ public int getCustomTokenFetchRetryCount() { return this.customTokenFetchRetryCount; } + public int getHttpConnectionTimeout() { + return this.httpConnectionTimeout; + } + + public int getHttpReadTimeout() { + return this.httpReadTimeout; + } + public long getAzureBlockSize() { return this.azureBlockSize; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 9d3b2d5e82c6e..9f1ff30d7bd09 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -45,6 +45,8 @@ public final class ConfigurationKeys { public static final String AZURE_BACKOFF_INTERVAL = "fs.azure.io.retry.backoff.interval"; public static final String AZURE_MAX_IO_RETRIES = "fs.azure.io.retry.max.retries"; public static final String AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT = "fs.azure.custom.token.fetch.retry.count"; + public static final String AZURE_HTTP_CONNECTION_TIMEOUT = "fs.azure.http.connection.timeout"; + public static final String AZURE_HTTP_READ_TIMEOUT = "fs.azure.http.read.timeout"; // Retry strategy for getToken calls public static final String AZURE_OAUTH_TOKEN_FETCH_RETRY_COUNT = "fs.azure.oauth.token.fetch.retry.max.retries"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 63d62a33b1819..c9ed524b8c7ba 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -43,6 +43,8 @@ public final class FileSystemConfigurations { public static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30; public static final int DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT = 3; + public static final int DEFAULT_HTTP_CONNECTION_TIMEOUT = 30 * 1000; // 30 secs + public static final int DEFAULT_HTTP_READ_TIMEOUT = 30 * 1000; // 30 secs // Retry parameter defaults. public static final int DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS = 5; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index b701037d0fc41..a58c787df0c8d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1243,4 +1243,12 @@ public ListenableFuture submit(Runnable runnable) { public void addCallback(ListenableFuture future, FutureCallback callback) { Futures.addCallback(future, callback, executorService); } + + /** + * Gets the current active configuration instance + * @return + */ + public AbfsConfiguration getAbfsConfiguration() { + return abfsConfiguration; + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index 413bf3686898b..867775a7192c8 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -49,9 +49,6 @@ public class AbfsHttpOperation implements AbfsPerfLoggable { private static final Logger LOG = LoggerFactory.getLogger(AbfsHttpOperation.class); - private static final int CONNECT_TIMEOUT = 30 * 1000; - private static final int READ_TIMEOUT = 30 * 1000; - private static final int CLEAN_UP_BUFFER_SIZE = 64 * 1024; private static final int ONE_THOUSAND = 1000; @@ -257,11 +254,17 @@ public String getMaskedEncodedUrl() { * * @param url The full URL including query string parameters. * @param method The HTTP method (PUT, PATCH, POST, GET, HEAD, or DELETE). - * @param requestHeaders The HTTP request headers.READ_TIMEOUT + * @param requestHeaders The HTTP request headers. + * @param connectionTimeout timeout for connection establishment. + * @param readTimeout timeout for read streaming from server. * * @throws IOException if an error occurs. */ - public AbfsHttpOperation(final URL url, final String method, final List requestHeaders) + public AbfsHttpOperation(final URL url, + final String method, + final List requestHeaders, + final int connectionTimeout, + final int readTimeout) throws IOException { this.isTraceEnabled = LOG.isTraceEnabled(); this.url = url; @@ -276,8 +279,10 @@ public AbfsHttpOperation(final URL url, final String method, final List"); - AbfsHttpOperation op = new AbfsHttpOperation(url, method, requestHeaders); + AbfsHttpOperation op = new AbfsHttpOperation(url, method, requestHeaders, + DEFAULT_HTTP_CONNECTION_TIMEOUT, DEFAULT_HTTP_READ_TIMEOUT); byte[] requestBuffer = requestBody.toString().getBytes(StandardCharsets.UTF_8.toString()); op.sendRequest(requestBuffer, 0, requestBuffer.length); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsPerfTracker.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsPerfTracker.java index 191d6e77ae09b..4e2c6c1dc6ae9 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsPerfTracker.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsPerfTracker.java @@ -38,6 +38,9 @@ import static org.assertj.core.api.Assertions.assertThat; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_HTTP_CONNECTION_TIMEOUT; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_HTTP_READ_TIMEOUT; + /** * Test the latency tracker for ABFS. * @@ -74,7 +77,9 @@ public void verifyDisablingOfTracker() throws Exception { try (AbfsPerfInfo tracker = new AbfsPerfInfo(abfsPerfTracker, "disablingCaller", "disablingCallee")) { - AbfsHttpOperation op = new AbfsHttpOperation(url, "GET", new ArrayList<>()); + AbfsHttpOperation op = new AbfsHttpOperation(url, "GET", + new ArrayList<>(), DEFAULT_HTTP_CONNECTION_TIMEOUT, + DEFAULT_HTTP_READ_TIMEOUT); tracker.registerResult(op).registerSuccess(true); } @@ -92,7 +97,9 @@ public void verifyTrackingForSingletonLatencyRecords() throws Exception { assertThat(latencyDetails).describedAs("AbfsPerfTracker should be empty").isNull(); List> tasks = new ArrayList<>(); - AbfsHttpOperation httpOperation = new AbfsHttpOperation(url, "GET", new ArrayList<>()); + AbfsHttpOperation httpOperation = new AbfsHttpOperation(url, "GET", + new ArrayList<>(), DEFAULT_HTTP_CONNECTION_TIMEOUT, + DEFAULT_HTTP_READ_TIMEOUT); for (int i = 0; i < numTasks; i++) { tasks.add(() -> { @@ -131,7 +138,9 @@ public void verifyTrackingForAggregateLatencyRecords() throws Exception { assertThat(latencyDetails).describedAs("AbfsPerfTracker should be empty").isNull(); List> tasks = new ArrayList<>(); - AbfsHttpOperation httpOperation = new AbfsHttpOperation(url, "GET", new ArrayList<>()); + AbfsHttpOperation httpOperation = new AbfsHttpOperation(url, "GET", + new ArrayList<>(), DEFAULT_HTTP_CONNECTION_TIMEOUT, + DEFAULT_HTTP_READ_TIMEOUT); for (int i = 0; i < numTasks; i++) { tasks.add(() -> { @@ -170,7 +179,9 @@ public void verifyRecordingSingletonLatencyIsCheapWhenDisabled() throws Exceptio long aggregateLatency = 0; AbfsPerfTracker abfsPerfTracker = new AbfsPerfTracker(accountName, filesystemName, false); List> tasks = new ArrayList<>(); - final AbfsHttpOperation httpOperation = new AbfsHttpOperation(url, "GET", new ArrayList<>()); + final AbfsHttpOperation httpOperation = new AbfsHttpOperation(url, "GET", + new ArrayList<>(), DEFAULT_HTTP_CONNECTION_TIMEOUT, + DEFAULT_HTTP_READ_TIMEOUT); for (int i = 0; i < numTasks; i++) { tasks.add(() -> { @@ -205,7 +216,9 @@ public void verifyRecordingAggregateLatencyIsCheapWhenDisabled() throws Exceptio long aggregateLatency = 0; AbfsPerfTracker abfsPerfTracker = new AbfsPerfTracker(accountName, filesystemName, false); List> tasks = new ArrayList<>(); - final AbfsHttpOperation httpOperation = new AbfsHttpOperation(url, "GET", new ArrayList<>()); + final AbfsHttpOperation httpOperation = new AbfsHttpOperation(url, "GET", + new ArrayList<>(), DEFAULT_HTTP_CONNECTION_TIMEOUT, + DEFAULT_HTTP_READ_TIMEOUT); for (int i = 0; i < numTasks; i++) { tasks.add(() -> { @@ -269,7 +282,9 @@ public void verifyRecordingSingletonLatencyIsCheapWhenEnabled() throws Exception long aggregateLatency = 0; AbfsPerfTracker abfsPerfTracker = new AbfsPerfTracker(accountName, filesystemName, true); List> tasks = new ArrayList<>(); - final AbfsHttpOperation httpOperation = new AbfsHttpOperation(url, "GET", new ArrayList<>()); + final AbfsHttpOperation httpOperation = new AbfsHttpOperation(url, "GET", + new ArrayList<>(), DEFAULT_HTTP_CONNECTION_TIMEOUT, + DEFAULT_HTTP_READ_TIMEOUT); for (int i = 0; i < numTasks; i++) { tasks.add(() -> { @@ -303,7 +318,9 @@ public void verifyRecordingAggregateLatencyIsCheapWhenEnabled() throws Exception long aggregateLatency = 0; AbfsPerfTracker abfsPerfTracker = new AbfsPerfTracker(accountName, filesystemName, true); List> tasks = new ArrayList<>(); - final AbfsHttpOperation httpOperation = new AbfsHttpOperation(url, "GET", new ArrayList<>()); + final AbfsHttpOperation httpOperation = new AbfsHttpOperation(url, "GET", + new ArrayList<>(), DEFAULT_HTTP_CONNECTION_TIMEOUT, + DEFAULT_HTTP_READ_TIMEOUT); for (int i = 0; i < numTasks; i++) { tasks.add(() -> { @@ -363,7 +380,9 @@ public void verifyNoExceptionOnInvalidInput() throws Exception { Instant testInstant = Instant.now(); AbfsPerfTracker abfsPerfTrackerDisabled = new AbfsPerfTracker(accountName, filesystemName, false); AbfsPerfTracker abfsPerfTrackerEnabled = new AbfsPerfTracker(accountName, filesystemName, true); - final AbfsHttpOperation httpOperation = new AbfsHttpOperation(url, "GET", new ArrayList()); + final AbfsHttpOperation httpOperation = new AbfsHttpOperation(url, "GET", + new ArrayList(), DEFAULT_HTTP_CONNECTION_TIMEOUT, + DEFAULT_HTTP_READ_TIMEOUT); verifyNoException(abfsPerfTrackerDisabled); verifyNoException(abfsPerfTrackerEnabled); @@ -371,7 +390,9 @@ public void verifyNoExceptionOnInvalidInput() throws Exception { private void verifyNoException(AbfsPerfTracker abfsPerfTracker) throws Exception { Instant testInstant = Instant.now(); - final AbfsHttpOperation httpOperation = new AbfsHttpOperation(url, "GET", new ArrayList()); + final AbfsHttpOperation httpOperation = new AbfsHttpOperation(url, "GET", + new ArrayList(), DEFAULT_HTTP_CONNECTION_TIMEOUT, + DEFAULT_HTTP_READ_TIMEOUT); try ( AbfsPerfInfo tracker01 = new AbfsPerfInfo(abfsPerfTracker, null, null); From 57c5fbbd2b7802dcb42ed66417073a457635002c Mon Sep 17 00:00:00 2001 From: Sneha Vijayarajan Date: Wed, 11 May 2022 13:19:20 -0700 Subject: [PATCH 2/4] Updating default connection timeout from TPCDS run observations. + Yetus fixes --- .../constants/FileSystemConfigurations.java | 2 +- .../fs/azurebfs/services/AbfsClient.java | 4 ++-- .../azurebfs/ITestAzureBlobFileSystemE2E.java | 23 +++++++++++++------ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index c9ed524b8c7ba..7fc43d0d1b53f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -43,7 +43,7 @@ public final class FileSystemConfigurations { public static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30; public static final int DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT = 3; - public static final int DEFAULT_HTTP_CONNECTION_TIMEOUT = 30 * 1000; // 30 secs + public static final int DEFAULT_HTTP_CONNECTION_TIMEOUT = 500; // 500ms public static final int DEFAULT_HTTP_READ_TIMEOUT = 30 * 1000; // 30 secs // Retry parameter defaults. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index a58c787df0c8d..05ed3f5316518 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1245,8 +1245,8 @@ public void addCallback(ListenableFuture future, FutureCallback callba } /** - * Gets the current active configuration instance - * @return + * Gets the current active configuration instance. + * @return active AbfsConfiguration instance. */ public AbfsConfiguration getAbfsConfiguration() { return abfsConfiguration; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java index 1d36a64a76182..1b22480082c8d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java @@ -51,6 +51,9 @@ public class ITestAzureBlobFileSystemE2E extends AbstractAbfsIntegrationTest { private static final int TEST_OFFSET = 100; private static final int TEST_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024; private static final int TEST_DEFAULT_READ_BUFFER_SIZE = 1023900; + private static final int TEST_STABLE_DEFAULT_CONNECTION_TIMEOUT_MS = 500; + private static final int TEST_STABLE_DEFAULT_READ_TIMEOUT_MS = 30000; + private static final int TEST_UNSTABLE_READ_TIMEOUT_MS = 1; public ITestAzureBlobFileSystemE2E() throws Exception { super(); @@ -238,15 +241,19 @@ private void testWriteOneByteToFile(Path testFilePath) throws Exception { @Test public void testHttpConnectionTimeout() throws Exception { - // Not seeing connection failures while testing with - // 1 ms connection timeout - testHttpTimeouts(1, 30000); + // Not seeing connection failures while testing with 1 ms connection + // timeout itself and on repeated TPCDS runs when cluster + // and account are in same region, 10 ms is seen stable. + // 500 ms is seen stable for cross region. + testHttpTimeouts(TEST_STABLE_DEFAULT_CONNECTION_TIMEOUT_MS, + TEST_STABLE_DEFAULT_READ_TIMEOUT_MS); } @Test(expected = InvalidAbfsRestOperationException.class) public void testHttpReadTimeout() throws Exception { - // Seeing consistent connection failures with small read timeout - testHttpTimeouts(30000, 1); + // Small read timeout is bound to make the request fail. + testHttpTimeouts(TEST_STABLE_DEFAULT_CONNECTION_TIMEOUT_MS, + TEST_UNSTABLE_READ_TIMEOUT_MS); } public void testHttpTimeouts(int connectionTimeoutMs, int readTimeoutMs) @@ -258,10 +265,12 @@ public void testHttpTimeouts(int connectionTimeoutMs, int readTimeoutMs) // Reduce retry count to reduce test run time conf.setInt(AZURE_MAX_IO_RETRIES, 1); final AzureBlobFileSystem fs = getFileSystem(conf); - Assertions.assertThat(fs.getAbfsStore().getAbfsConfiguration().getHttpConnectionTimeout()) + Assertions.assertThat( + fs.getAbfsStore().getAbfsConfiguration().getHttpConnectionTimeout()) .describedAs("HTTP connection time should be picked from config") .isEqualTo(connectionTimeoutMs); - Assertions.assertThat(fs.getAbfsStore().getAbfsConfiguration().getHttpReadTimeout()) + Assertions.assertThat( + fs.getAbfsStore().getAbfsConfiguration().getHttpReadTimeout()) .describedAs("HTTP Read time should be picked from config") .isEqualTo(readTimeoutMs); Path testPath = path(methodName.getMethodName()); From 44553599f173d6ca61f96f0feaeb23886c05e5d8 Mon Sep 17 00:00:00 2001 From: Sneha Vijayarajan Date: Mon, 15 Nov 2021 09:52:42 -0800 Subject: [PATCH 3/4] Configurable HTTP connection and read timeouts --- .../constants/FileSystemConfigurations.java | 4 ++++ .../fs/azurebfs/services/AbfsClient.java | 5 +++++ .../azurebfs/ITestAzureBlobFileSystemE2E.java | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 7fc43d0d1b53f..ef37f104e5e5f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -43,7 +43,11 @@ public final class FileSystemConfigurations { public static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30; public static final int DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT = 3; +<<<<<<< HEAD public static final int DEFAULT_HTTP_CONNECTION_TIMEOUT = 500; // 500ms +======= + public static final int DEFAULT_HTTP_CONNECTION_TIMEOUT = 30 * 1000; // 30 secs +>>>>>>> Configurable HTTP connection and read timeouts public static final int DEFAULT_HTTP_READ_TIMEOUT = 30 * 1000; // 30 secs // Retry parameter defaults. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 05ed3f5316518..1cc47b3589521 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1245,8 +1245,13 @@ public void addCallback(ListenableFuture future, FutureCallback callba } /** +<<<<<<< HEAD * Gets the current active configuration instance. * @return active AbfsConfiguration instance. +======= + * Gets the current active configuration instance + * @return +>>>>>>> Configurable HTTP connection and read timeouts */ public AbfsConfiguration getAbfsConfiguration() { return abfsConfiguration; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java index 1b22480082c8d..83258934bd2bd 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java @@ -241,19 +241,30 @@ private void testWriteOneByteToFile(Path testFilePath) throws Exception { @Test public void testHttpConnectionTimeout() throws Exception { +<<<<<<< HEAD // Not seeing connection failures while testing with 1 ms connection // timeout itself and on repeated TPCDS runs when cluster // and account are in same region, 10 ms is seen stable. // 500 ms is seen stable for cross region. testHttpTimeouts(TEST_STABLE_DEFAULT_CONNECTION_TIMEOUT_MS, TEST_STABLE_DEFAULT_READ_TIMEOUT_MS); +======= + // Not seeing connection failures while testing with + // 1 ms connection timeout + testHttpTimeouts(1, 30000); +>>>>>>> Configurable HTTP connection and read timeouts } @Test(expected = InvalidAbfsRestOperationException.class) public void testHttpReadTimeout() throws Exception { +<<<<<<< HEAD // Small read timeout is bound to make the request fail. testHttpTimeouts(TEST_STABLE_DEFAULT_CONNECTION_TIMEOUT_MS, TEST_UNSTABLE_READ_TIMEOUT_MS); +======= + // Seeing consistent connection failures with small read timeout + testHttpTimeouts(30000, 1); +>>>>>>> Configurable HTTP connection and read timeouts } public void testHttpTimeouts(int connectionTimeoutMs, int readTimeoutMs) @@ -265,12 +276,19 @@ public void testHttpTimeouts(int connectionTimeoutMs, int readTimeoutMs) // Reduce retry count to reduce test run time conf.setInt(AZURE_MAX_IO_RETRIES, 1); final AzureBlobFileSystem fs = getFileSystem(conf); +<<<<<<< HEAD Assertions.assertThat( fs.getAbfsStore().getAbfsConfiguration().getHttpConnectionTimeout()) .describedAs("HTTP connection time should be picked from config") .isEqualTo(connectionTimeoutMs); Assertions.assertThat( fs.getAbfsStore().getAbfsConfiguration().getHttpReadTimeout()) +======= + Assertions.assertThat(fs.getAbfsStore().getAbfsConfiguration().getHttpConnectionTimeout()) + .describedAs("HTTP connection time should be picked from config") + .isEqualTo(connectionTimeoutMs); + Assertions.assertThat(fs.getAbfsStore().getAbfsConfiguration().getHttpReadTimeout()) +>>>>>>> Configurable HTTP connection and read timeouts .describedAs("HTTP Read time should be picked from config") .isEqualTo(readTimeoutMs); Path testPath = path(methodName.getMethodName()); From e8d5895704589547fa4127b2d31cfea3e5dafa6c Mon Sep 17 00:00:00 2001 From: Sneha Vijayarajan Date: Wed, 11 May 2022 16:22:27 -0700 Subject: [PATCH 4/4] Conflict fixes --- .../constants/FileSystemConfigurations.java | 4 ---- .../fs/azurebfs/services/AbfsClient.java | 5 ----- .../azurebfs/ITestAzureBlobFileSystemE2E.java | 18 ------------------ 3 files changed, 27 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index ef37f104e5e5f..7fc43d0d1b53f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -43,11 +43,7 @@ public final class FileSystemConfigurations { public static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30; public static final int DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT = 3; -<<<<<<< HEAD public static final int DEFAULT_HTTP_CONNECTION_TIMEOUT = 500; // 500ms -======= - public static final int DEFAULT_HTTP_CONNECTION_TIMEOUT = 30 * 1000; // 30 secs ->>>>>>> Configurable HTTP connection and read timeouts public static final int DEFAULT_HTTP_READ_TIMEOUT = 30 * 1000; // 30 secs // Retry parameter defaults. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 1cc47b3589521..05ed3f5316518 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1245,13 +1245,8 @@ public void addCallback(ListenableFuture future, FutureCallback callba } /** -<<<<<<< HEAD * Gets the current active configuration instance. * @return active AbfsConfiguration instance. -======= - * Gets the current active configuration instance - * @return ->>>>>>> Configurable HTTP connection and read timeouts */ public AbfsConfiguration getAbfsConfiguration() { return abfsConfiguration; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java index 83258934bd2bd..1b22480082c8d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java @@ -241,30 +241,19 @@ private void testWriteOneByteToFile(Path testFilePath) throws Exception { @Test public void testHttpConnectionTimeout() throws Exception { -<<<<<<< HEAD // Not seeing connection failures while testing with 1 ms connection // timeout itself and on repeated TPCDS runs when cluster // and account are in same region, 10 ms is seen stable. // 500 ms is seen stable for cross region. testHttpTimeouts(TEST_STABLE_DEFAULT_CONNECTION_TIMEOUT_MS, TEST_STABLE_DEFAULT_READ_TIMEOUT_MS); -======= - // Not seeing connection failures while testing with - // 1 ms connection timeout - testHttpTimeouts(1, 30000); ->>>>>>> Configurable HTTP connection and read timeouts } @Test(expected = InvalidAbfsRestOperationException.class) public void testHttpReadTimeout() throws Exception { -<<<<<<< HEAD // Small read timeout is bound to make the request fail. testHttpTimeouts(TEST_STABLE_DEFAULT_CONNECTION_TIMEOUT_MS, TEST_UNSTABLE_READ_TIMEOUT_MS); -======= - // Seeing consistent connection failures with small read timeout - testHttpTimeouts(30000, 1); ->>>>>>> Configurable HTTP connection and read timeouts } public void testHttpTimeouts(int connectionTimeoutMs, int readTimeoutMs) @@ -276,19 +265,12 @@ public void testHttpTimeouts(int connectionTimeoutMs, int readTimeoutMs) // Reduce retry count to reduce test run time conf.setInt(AZURE_MAX_IO_RETRIES, 1); final AzureBlobFileSystem fs = getFileSystem(conf); -<<<<<<< HEAD Assertions.assertThat( fs.getAbfsStore().getAbfsConfiguration().getHttpConnectionTimeout()) .describedAs("HTTP connection time should be picked from config") .isEqualTo(connectionTimeoutMs); Assertions.assertThat( fs.getAbfsStore().getAbfsConfiguration().getHttpReadTimeout()) -======= - Assertions.assertThat(fs.getAbfsStore().getAbfsConfiguration().getHttpConnectionTimeout()) - .describedAs("HTTP connection time should be picked from config") - .isEqualTo(connectionTimeoutMs); - Assertions.assertThat(fs.getAbfsStore().getAbfsConfiguration().getHttpReadTimeout()) ->>>>>>> Configurable HTTP connection and read timeouts .describedAs("HTTP Read time should be picked from config") .isEqualTo(readTimeoutMs); Path testPath = path(methodName.getMethodName());