From 1f76e74796deb30fe334c1f2ef320d8fbee266b0 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Mon, 1 Dec 2025 11:20:30 +0000 Subject: [PATCH 1/4] fix failing tests --- .../hadoop/fs/s3a/impl/streams/TestStreamFactories.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/streams/TestStreamFactories.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/streams/TestStreamFactories.java index 76b18c04339c3..4ff00e0ae4436 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/streams/TestStreamFactories.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/streams/TestStreamFactories.java @@ -54,14 +54,14 @@ public class TestStreamFactories extends AbstractHadoopTestBase { /** - * The empty string and "default" both map to the classic stream. + * The empty string and "default" both map to the analytics stream. */ @Test public void testDefaultFactoryCreation() throws Throwable { load("", DEFAULT_STREAM_TYPE, - ClassicObjectInputStreamFactory.class); + AnalyticsStreamFactory.class); load(INPUT_STREAM_TYPE_DEFAULT, DEFAULT_STREAM_TYPE, - ClassicObjectInputStreamFactory.class); + AnalyticsStreamFactory.class); } /** @@ -70,7 +70,7 @@ public void testDefaultFactoryCreation() throws Throwable { @Test public void testClassicFactoryCreation() throws Throwable { final ClassicObjectInputStreamFactory f = - load(INPUT_STREAM_TYPE_CLASSIC, DEFAULT_STREAM_TYPE, + load(INPUT_STREAM_TYPE_CLASSIC, InputStreamType.Classic, ClassicObjectInputStreamFactory.class); final StreamFactoryRequirements requirements = f.factoryRequirements(); assertThat(requirements.requiresFuturePool()) From 6da5cd1e9e33cdab5c8806274b25141fa6de34c5 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Mon, 1 Dec 2025 12:45:54 +0000 Subject: [PATCH 2/4] adds leak detection stat --- .../apache/hadoop/fs/s3a/impl/streams/AnalyticsStream.java | 1 + .../org/apache/hadoop/fs/s3a/ITestS3AInputStreamLeakage.java | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/AnalyticsStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/AnalyticsStream.java index 954ee3a0e48ae..48c2a920dd8d2 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/AnalyticsStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/AnalyticsStream.java @@ -227,6 +227,7 @@ protected boolean isClosed() { @Override protected void abortInFinalizer() { + getS3AStreamStatistics().streamLeaked(); try { close(); } catch (IOException ignored) { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AInputStreamLeakage.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AInputStreamLeakage.java index 31fe270a8b9e3..cf3bc3c4aaeff 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AInputStreamLeakage.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AInputStreamLeakage.java @@ -90,10 +90,6 @@ public void setup() throws Exception { @Test public void testFinalizer() throws Throwable { Path path = methodPath(); - // Analytics accelerator currently does not support stream leak detection. This work is tracked - // in https://issues.apache.org/jira/browse/HADOOP-19451 - skipIfAnalyticsAcceleratorEnabled(getConfiguration(), - "Analytics Accelerator currently does not support leak detection"); final S3AFileSystem fs = getFileSystem(); From 16907a7ff297681304052e7f97ef2143e3a086f2 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Mon, 1 Dec 2025 12:50:01 +0000 Subject: [PATCH 3/4] skip if unbuffer --- .../src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 1eb302a0ca137..e5205d648545e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -624,7 +624,7 @@ public static void skipIfAnalyticsAcceleratorEnabled( public static boolean isAnalyticsAcceleratorEnabled(final Configuration conf) { return conf.get(INPUT_STREAM_TYPE, - INPUT_STREAM_TYPE_CLASSIC).equals(INPUT_STREAM_TYPE_ANALYTICS); + INPUT_STREAM_TYPE_ANALYTICS).equals(INPUT_STREAM_TYPE_ANALYTICS); } /** From 80d2c89c46a67496ab0a391d24dc14d204457b04 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Wed, 3 Dec 2025 12:08:26 +0000 Subject: [PATCH 4/4] removes unused import --- .../org/apache/hadoop/fs/s3a/ITestS3AInputStreamLeakage.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AInputStreamLeakage.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AInputStreamLeakage.java index cf3bc3c4aaeff..0aca12ee337e3 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AInputStreamLeakage.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AInputStreamLeakage.java @@ -37,7 +37,6 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfAnalyticsAcceleratorEnabled; import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticCounter; import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_LEAKS; import static org.apache.hadoop.test.GenericTestUtils.LogCapturer.captureLogs;