From 7854d647b6d58db2acd9e78c55f8b9df3a54092e Mon Sep 17 00:00:00 2001 From: Chris Bevard Date: Mon, 17 Apr 2023 10:40:19 -0400 Subject: [PATCH 01/12] downgrade to flush() instead of no-op. --- .../org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java | 3 ++- .../org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java index 43a2b7e0dbd5b..e79d215b62a2a 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java @@ -729,7 +729,7 @@ public void hsync() throws IOException { /** * Shared processing of Syncable operation reporting/downgrade. */ - private void handleSyncableInvocation() { + private void handleSyncableInvocation() throws IOException { final UnsupportedOperationException ex = new UnsupportedOperationException(E_NOT_SYNCABLE); if (!downgradeSyncableExceptions) { @@ -741,6 +741,7 @@ private void handleSyncableInvocation() { key); // and log at debug LOG.debug("Downgrading Syncable call", ex); + flush(); } @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java index ffa2c81e58adf..27f49f2c658ef 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java @@ -38,7 +38,10 @@ import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** @@ -156,6 +159,7 @@ public void testSyncableUnsupported() throws Exception { stream = spy(new S3ABlockOutputStream(builder)); intercept(UnsupportedOperationException.class, () -> stream.hflush()); intercept(UnsupportedOperationException.class, () -> stream.hsync()); + verify(stream, never()).flush(); } /** @@ -169,8 +173,11 @@ public void testSyncableDowngrade() throws Exception { builder.withDowngradeSyncableExceptions(true); stream = spy(new S3ABlockOutputStream(builder)); + verify(stream, never()).flush(); stream.hflush(); + verify(stream, times(1)).flush(); stream.hsync(); + verify(stream, times(2)).flush(); } } From 85e580e82c1143915398fc6d0c974b69827cca34 Mon Sep 17 00:00:00 2001 From: Chris Bevard Date: Mon, 17 Apr 2023 12:31:02 -0400 Subject: [PATCH 02/12] include key in s3a disk buffer file name --- .../hadoop/fs/s3a/S3ABlockOutputStream.java | 2 +- .../apache/hadoop/fs/s3a/S3ADataBlocks.java | 26 +++++++++++++------ .../fs/s3a/ITestS3ABlockOutputArray.java | 2 +- .../apache/hadoop/fs/s3a/TestDataBlocks.java | 2 +- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java index e79d215b62a2a..ce7b1256290bc 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java @@ -232,7 +232,7 @@ private synchronized S3ADataBlocks.DataBlock createBlockIfNeeded() LOG.error("Number of partitions in stream exceeds limit for S3: " + Constants.MAX_MULTIPART_COUNT + " write may fail."); } - activeBlock = blockFactory.create(blockCount, this.blockSize, statistics); + activeBlock = blockFactory.create(key, blockCount, this.blockSize, statistics); } return activeBlock; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java index b20d8e859aa88..eaa14f5fd72ce 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java @@ -175,12 +175,13 @@ protected BlockFactory(S3AFileSystem owner) { /** * Create a block. * + * @param key of s3 object being written to * @param index index of block * @param limit limit of the block. * @param statistics stats to work with * @return a new block. */ - abstract DataBlock create(long index, long limit, + abstract DataBlock create(String key, long index, long limit, BlockOutputStreamStatistics statistics) throws IOException; @@ -391,11 +392,11 @@ static class ArrayBlockFactory extends BlockFactory { } @Override - DataBlock create(long index, long limit, + DataBlock create(String key, long index, long limit, BlockOutputStreamStatistics statistics) throws IOException { Preconditions.checkArgument(limit > 0, - "Invalid block size: %d", limit); + "Invalid block size: %d [%s]", limit, key); return new ByteArrayBlock(0, limit, statistics); } @@ -516,11 +517,11 @@ static class ByteBufferBlockFactory extends BlockFactory { } @Override - ByteBufferBlock create(long index, long limit, + ByteBufferBlock create(String key, long index, long limit, BlockOutputStreamStatistics statistics) throws IOException { Preconditions.checkArgument(limit > 0, - "Invalid block size: %d", limit); + "Invalid block size: %d [%s]", limit, key); return new ByteBufferBlock(index, limit, statistics); } @@ -798,6 +799,8 @@ public String toString() { * Buffer blocks to disk. */ static class DiskBlockFactory extends BlockFactory { + private static final String ESCAPED_FORWARD_SLASH = "EFS"; + private static final String ESCAPED_BACKSLASH = "EBS"; DiskBlockFactory(S3AFileSystem owner) { super(owner); @@ -806,6 +809,7 @@ static class DiskBlockFactory extends BlockFactory { /** * Create a temp file and a {@link DiskBlock} instance to manage it. * + * @param key of the s3 object being written * @param index block index * @param limit limit of the block. -1 means "no limit" * @param statistics statistics to update @@ -813,17 +817,23 @@ static class DiskBlockFactory extends BlockFactory { * @throws IOException IO problems */ @Override - DataBlock create(long index, + DataBlock create(String key, long index, long limit, BlockOutputStreamStatistics statistics) throws IOException { Preconditions.checkArgument(limit != 0, - "Invalid block size: %d", limit); + "Invalid block size: %d [%s]", limit, key); File destFile = getOwner() - .createTmpFileForWrite(String.format("s3ablock-%04d-", index), + .createTmpFileForWrite(String.format("s3ablock-%04d-%s-", index, escapeS3Key(key)), limit, getOwner().getConf()); return new DiskBlock(destFile, limit, index, statistics); } + + protected static String escapeS3Key(String key) { + return key + .replace("\\", ESCAPED_BACKSLASH) + .replace("/", ESCAPED_FORWARD_SLASH); + } } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java index 53fa0d83b55a7..aeb1da248ea7c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java @@ -136,7 +136,7 @@ private void markAndResetDatablock(S3ADataBlocks.BlockFactory factory) new S3AInstrumentation(new URI("s3a://example")); BlockOutputStreamStatistics outstats = instrumentation.newOutputStreamStatistics(null); - S3ADataBlocks.DataBlock block = factory.create(1, BLOCK_SIZE, outstats); + S3ADataBlocks.DataBlock block = factory.create("object/key", 1, BLOCK_SIZE, outstats); block.write(dataset, 0, dataset.length); S3ADataBlocks.BlockUploadData uploadData = block.startUpload(); InputStream stream = uploadData.getUploadStream(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java index 700ef5ced3d8a..97cfff7ff24c7 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java @@ -51,7 +51,7 @@ public void testByteBufferIO() throws Throwable { new S3ADataBlocks.ByteBufferBlockFactory(null)) { int limit = 128; S3ADataBlocks.ByteBufferBlockFactory.ByteBufferBlock block - = factory.create(1, limit, null); + = factory.create("object/key", 1, limit, null); assertOutstandingBuffers(factory, 1); byte[] buffer = ContractTestUtils.toAsciiByteArray("test data"); From d1b69947c23420595368ac1bd84d7252a758d3f6 Mon Sep 17 00:00:00 2001 From: Chris Bevard Date: Tue, 18 Apr 2023 12:16:40 -0400 Subject: [PATCH 03/12] included the audit span ID in the s3ablock file name --- .../apache/hadoop/fs/s3a/S3ABlockOutputStream.java | 2 +- .../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 14 ++++++++------ .../apache/hadoop/fs/s3a/WriteOperationHelper.java | 1 + .../org/apache/hadoop/fs/s3a/WriteOperations.java | 7 +++++++ .../hadoop/fs/s3a/ITestS3ABlockOutputArray.java | 2 +- .../org/apache/hadoop/fs/s3a/TestDataBlocks.java | 2 +- 6 files changed, 19 insertions(+), 9 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java index ce7b1256290bc..1977452ef1355 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java @@ -232,7 +232,7 @@ private synchronized S3ADataBlocks.DataBlock createBlockIfNeeded() LOG.error("Number of partitions in stream exceeds limit for S3: " + Constants.MAX_MULTIPART_COUNT + " write may fail."); } - activeBlock = blockFactory.create(key, blockCount, this.blockSize, statistics); + activeBlock = blockFactory.create(writeOperationHelper.getAuditSpan().getSpanId(), key, blockCount, this.blockSize, statistics); } return activeBlock; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java index eaa14f5fd72ce..a898f9987a8e3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java @@ -175,13 +175,14 @@ protected BlockFactory(S3AFileSystem owner) { /** * Create a block. * - * @param key of s3 object being written to + * @param spanId id of the audit span + * @param key key of s3 object being written to * @param index index of block * @param limit limit of the block. * @param statistics stats to work with * @return a new block. */ - abstract DataBlock create(String key, long index, long limit, + abstract DataBlock create(String spanId, String key, long index, long limit, BlockOutputStreamStatistics statistics) throws IOException; @@ -392,7 +393,7 @@ static class ArrayBlockFactory extends BlockFactory { } @Override - DataBlock create(String key, long index, long limit, + DataBlock create(String spanId, String key, long index, long limit, BlockOutputStreamStatistics statistics) throws IOException { Preconditions.checkArgument(limit > 0, @@ -517,7 +518,7 @@ static class ByteBufferBlockFactory extends BlockFactory { } @Override - ByteBufferBlock create(String key, long index, long limit, + ByteBufferBlock create(String spanId, String key, long index, long limit, BlockOutputStreamStatistics statistics) throws IOException { Preconditions.checkArgument(limit > 0, @@ -809,6 +810,7 @@ static class DiskBlockFactory extends BlockFactory { /** * Create a temp file and a {@link DiskBlock} instance to manage it. * + * @param spanId id of the audit span * @param key of the s3 object being written * @param index block index * @param limit limit of the block. -1 means "no limit" @@ -817,14 +819,14 @@ static class DiskBlockFactory extends BlockFactory { * @throws IOException IO problems */ @Override - DataBlock create(String key, long index, + DataBlock create(String spanId, String key, long index, long limit, BlockOutputStreamStatistics statistics) throws IOException { Preconditions.checkArgument(limit != 0, "Invalid block size: %d [%s]", limit, key); File destFile = getOwner() - .createTmpFileForWrite(String.format("s3ablock-%04d-%s-", index, escapeS3Key(key)), + .createTmpFileForWrite(String.format("s3ablock-%04d-%s-%s-", index, spanId, escapeS3Key(key)), limit, getOwner().getConf()); return new DiskBlock(destFile, limit, index, statistics); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java index 8e15a10944c38..3f42d2caf4a0b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java @@ -217,6 +217,7 @@ public T retry(String action, * Get the audit span this object was created with. * @return the audit span */ + @Override public AuditSpan getAuditSpan() { return auditSpan; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java index bd6e5ac14731f..0c34371983595 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java @@ -43,6 +43,7 @@ import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.statistics.DurationTrackerFactory; +import org.apache.hadoop.fs.store.audit.AuditSpan; import org.apache.hadoop.fs.store.audit.AuditSpanSource; import org.apache.hadoop.util.functional.CallableRaisingIOE; @@ -305,6 +306,12 @@ UploadPartResult uploadPart(UploadPartRequest request, */ Configuration getConf(); + /** + * Get the audit span this object was created with. + * @return the audit span + */ + AuditSpan getAuditSpan(); + /** * Create a S3 Select request for the destination path. * This does not build the query. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java index aeb1da248ea7c..e21e755bbe547 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java @@ -136,7 +136,7 @@ private void markAndResetDatablock(S3ADataBlocks.BlockFactory factory) new S3AInstrumentation(new URI("s3a://example")); BlockOutputStreamStatistics outstats = instrumentation.newOutputStreamStatistics(null); - S3ADataBlocks.DataBlock block = factory.create("object/key", 1, BLOCK_SIZE, outstats); + S3ADataBlocks.DataBlock block = factory.create("spanId", "object/key", 1, BLOCK_SIZE, outstats); block.write(dataset, 0, dataset.length); S3ADataBlocks.BlockUploadData uploadData = block.startUpload(); InputStream stream = uploadData.getUploadStream(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java index 97cfff7ff24c7..d2ea0218acd9f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java @@ -51,7 +51,7 @@ public void testByteBufferIO() throws Throwable { new S3ADataBlocks.ByteBufferBlockFactory(null)) { int limit = 128; S3ADataBlocks.ByteBufferBlockFactory.ByteBufferBlock block - = factory.create("object/key", 1, limit, null); + = factory.create("spanId", "s3\\object/key", 1, limit, null); assertOutstandingBuffers(factory, 1); byte[] buffer = ContractTestUtils.toAsciiByteArray("test data"); From 0f2ff5b220562990bd735d125b5224e110b7ecc7 Mon Sep 17 00:00:00 2001 From: Chris Bevard Date: Tue, 18 Apr 2023 14:29:17 -0400 Subject: [PATCH 04/12] updated javadocs and test mock --- .../java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java | 5 +++++ .../org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java index 1977452ef1355..1ef4fc29dd50f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java @@ -728,6 +728,11 @@ public void hsync() throws IOException { /** * Shared processing of Syncable operation reporting/downgrade. + * + * Syncable API is not supported, so calls to hsync/hflush will throw an UnsupportedOperationException unless the + * stream was constructed with {@link #downgradeSyncableExceptions} set to true, in which case the stream is flushed. + * @throws IOException IO Problem + * @throws UnsupportedOperationException if downgrade syncable exceptions is set to false */ private void handleSyncableInvocation() throws IOException { final UnsupportedOperationException ex diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java index 27f49f2c658ef..beeeb39fc4852 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; import org.apache.hadoop.fs.s3a.test.MinimalWriteOperationHelperCallbacks; import org.apache.hadoop.fs.statistics.IOStatisticsContext; +import org.apache.hadoop.fs.store.audit.AuditSpan; import org.apache.hadoop.util.Progressable; import org.junit.Before; import org.junit.Test; @@ -62,6 +63,9 @@ private S3ABlockOutputStream.BlockOutputStreamBuilder mockS3ABuilder() { mock(S3ADataBlocks.BlockFactory.class); long blockSize = Constants.DEFAULT_MULTIPART_SIZE; WriteOperationHelper oHelper = mock(WriteOperationHelper.class); + AuditSpan auditSpan = mock(AuditSpan.class); + when(auditSpan.getSpanId()).thenReturn("spanId"); + when(oHelper.getAuditSpan()).thenReturn(auditSpan); PutTracker putTracker = mock(PutTracker.class); final S3ABlockOutputStream.BlockOutputStreamBuilder builder = S3ABlockOutputStream.builder() From 2a8458e4226315dcf90b375616f908b75d072162 Mon Sep 17 00:00:00 2001 From: Chris Bevard Date: Fri, 21 Apr 2023 12:30:49 -0400 Subject: [PATCH 05/12] added a test for datablock tmp file name length --- .../hadoop/fs/s3a/S3ABlockOutputStream.java | 9 +++-- .../apache/hadoop/fs/s3a/S3ADataBlocks.java | 5 +-- .../fs/s3a/ITestS3ABlockOutputArray.java | 37 +++++++++++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java index 1ef4fc29dd50f..2febc87aec36d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java @@ -232,7 +232,9 @@ private synchronized S3ADataBlocks.DataBlock createBlockIfNeeded() LOG.error("Number of partitions in stream exceeds limit for S3: " + Constants.MAX_MULTIPART_COUNT + " write may fail."); } - activeBlock = blockFactory.create(writeOperationHelper.getAuditSpan().getSpanId(), key, blockCount, this.blockSize, statistics); + activeBlock = blockFactory.create( + writeOperationHelper.getAuditSpan().getSpanId(), + key, blockCount, this.blockSize, statistics); } return activeBlock; } @@ -729,8 +731,9 @@ public void hsync() throws IOException { /** * Shared processing of Syncable operation reporting/downgrade. * - * Syncable API is not supported, so calls to hsync/hflush will throw an UnsupportedOperationException unless the - * stream was constructed with {@link #downgradeSyncableExceptions} set to true, in which case the stream is flushed. + * Syncable API is not supported, so calls to hsync/hflush will throw an + * UnsupportedOperationException unless the stream was constructed with + * {@link #downgradeSyncableExceptions} set to true, in which case the stream is flushed. * @throws IOException IO Problem * @throws UnsupportedOperationException if downgrade syncable exceptions is set to false */ diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java index a898f9987a8e3..2299892b35b94 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java @@ -825,9 +825,8 @@ DataBlock create(String spanId, String key, long index, throws IOException { Preconditions.checkArgument(limit != 0, "Invalid block size: %d [%s]", limit, key); - File destFile = getOwner() - .createTmpFileForWrite(String.format("s3ablock-%04d-%s-%s-", index, spanId, escapeS3Key(key)), - limit, getOwner().getConf()); + String prefix = String.format("s3ablock-%04d-%s-%s-", index, spanId, escapeS3Key(key)); + File destFile = getOwner().createTmpFileForWrite(prefix, limit, getOwner().getConf()); return new DiskBlock(destFile, limit, index, statistics); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java index e21e755bbe547..4a28fa2d321aa 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java @@ -29,6 +29,7 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -79,6 +80,42 @@ public void testRegularUpload() throws IOException { verifyUpload("regular", 1024); } + /** + * Test that the DiskBlock's local file doesn't result in error when the S3 key exceeds the max + * char limit of the local file system. Currently + * {@link java.io.File#createTempFile(String, String, File)} is being relied on to handle the + * truncation. + * @throws IOException + */ + @Test + public void testDiskBlockCreate() throws IOException { + S3ADataBlocks.BlockFactory diskBlockFactory = + new S3ADataBlocks.DiskBlockFactory(getFileSystem()); + String s3Key = // 1024 char + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key"; + S3ADataBlocks.DataBlock dataBlock = diskBlockFactory.create("spanId", s3Key, 1, + getFileSystem().getDefaultBlockSize(), null); + LOG.info(dataBlock.toString()); // block file name and location can be viewed in failsafe-report + + // delete the block file + dataBlock.innerClose(); + diskBlockFactory.close(); + } + @Test(expected = IOException.class) public void testWriteAfterStreamClose() throws Throwable { Path dest = path("testWriteAfterStreamClose"); From e481be6071b96e1a8a90df66498cee96efc28ccf Mon Sep 17 00:00:00 2001 From: Chris Bevard Date: Wed, 26 Apr 2023 14:01:03 -0400 Subject: [PATCH 06/12] added check to make sure the tmp file was created --- .../fs/s3a/ITestS3ABlockOutputArray.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java index 4a28fa2d321aa..eb32f117a68ed 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java @@ -33,6 +33,9 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Objects; import static org.apache.hadoop.fs.StreamCapabilities.ABORTABLE_STREAM; import static org.apache.hadoop.fs.s3a.Constants.*; @@ -89,8 +92,6 @@ public void testRegularUpload() throws IOException { */ @Test public void testDiskBlockCreate() throws IOException { - S3ADataBlocks.BlockFactory diskBlockFactory = - new S3ADataBlocks.DiskBlockFactory(getFileSystem()); String s3Key = // 1024 char "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + @@ -107,13 +108,18 @@ public void testDiskBlockCreate() throws IOException { "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + "very_long_s3_key"; - S3ADataBlocks.DataBlock dataBlock = diskBlockFactory.create("spanId", s3Key, 1, - getFileSystem().getDefaultBlockSize(), null); - LOG.info(dataBlock.toString()); // block file name and location can be viewed in failsafe-report - - // delete the block file - dataBlock.innerClose(); - diskBlockFactory.close(); + long blockSize = getFileSystem().getDefaultBlockSize(); + try (S3ADataBlocks.BlockFactory diskBlockFactory = + new S3ADataBlocks.DiskBlockFactory(getFileSystem()); + S3ADataBlocks.DataBlock dataBlock = + diskBlockFactory.create("spanId", s3Key, 1, blockSize, null); + ) { + boolean created = Arrays.stream( + Objects.requireNonNull(new File(getConfiguration().get("hadoop.tmp.dir")).listFiles())) + .anyMatch(f -> f.getName().contains("very_long_s3_key")); + assertTrue(created); + LOG.info(dataBlock.toString()); // block file name/location can be viewed in failsafe-report + } } @Test(expected = IOException.class) From fd736d01b03bad2889ee11da094efd789d9f7198 Mon Sep 17 00:00:00 2001 From: Chris Bevard Date: Thu, 27 Apr 2023 10:01:50 -0400 Subject: [PATCH 07/12] added message to assertion for easier debugging --- .../org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java index eb32f117a68ed..cea432337a523 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java @@ -114,10 +114,11 @@ public void testDiskBlockCreate() throws IOException { S3ADataBlocks.DataBlock dataBlock = diskBlockFactory.create("spanId", s3Key, 1, blockSize, null); ) { + String tmpDir = getConfiguration().get("hadoop.tmp.dir"); boolean created = Arrays.stream( - Objects.requireNonNull(new File(getConfiguration().get("hadoop.tmp.dir")).listFiles())) + Objects.requireNonNull(new File(tmpDir).listFiles())) .anyMatch(f -> f.getName().contains("very_long_s3_key")); - assertTrue(created); + assertTrue(String.format("tmp file should have been created locally in %s", tmpDir), created); LOG.info(dataBlock.toString()); // block file name/location can be viewed in failsafe-report } } From 4f41cf1d0d6427d59e42b58a94b333b326938b83 Mon Sep 17 00:00:00 2001 From: Chris Bevard Date: Thu, 4 May 2023 09:06:06 -0400 Subject: [PATCH 08/12] checkstyle corrections --- .../org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java index cea432337a523..82ec36c1aaf4c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java @@ -33,7 +33,6 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; -import java.nio.file.Files; import java.util.Arrays; import java.util.Objects; @@ -93,7 +92,7 @@ public void testRegularUpload() throws IOException { @Test public void testDiskBlockCreate() throws IOException { String s3Key = // 1024 char - "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + @@ -112,11 +111,11 @@ public void testDiskBlockCreate() throws IOException { try (S3ADataBlocks.BlockFactory diskBlockFactory = new S3ADataBlocks.DiskBlockFactory(getFileSystem()); S3ADataBlocks.DataBlock dataBlock = - diskBlockFactory.create("spanId", s3Key, 1, blockSize, null); + diskBlockFactory.create("spanId", s3Key, 1, blockSize, null); ) { String tmpDir = getConfiguration().get("hadoop.tmp.dir"); boolean created = Arrays.stream( - Objects.requireNonNull(new File(tmpDir).listFiles())) + Objects.requireNonNull(new File(tmpDir).listFiles())) .anyMatch(f -> f.getName().contains("very_long_s3_key")); assertTrue(String.format("tmp file should have been created locally in %s", tmpDir), created); LOG.info(dataBlock.toString()); // block file name/location can be viewed in failsafe-report From 7b5f48a240105c6f8e203780d00e31852a11c92b Mon Sep 17 00:00:00 2001 From: Chris Bevard Date: Thu, 4 May 2023 15:19:46 -0400 Subject: [PATCH 09/12] checkstyle again --- .../java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java index 82ec36c1aaf4c..bfeaa25f16ca5 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java @@ -111,7 +111,7 @@ public void testDiskBlockCreate() throws IOException { try (S3ADataBlocks.BlockFactory diskBlockFactory = new S3ADataBlocks.DiskBlockFactory(getFileSystem()); S3ADataBlocks.DataBlock dataBlock = - diskBlockFactory.create("spanId", s3Key, 1, blockSize, null); + diskBlockFactory.create("spanId", s3Key, 1, blockSize, null); ) { String tmpDir = getConfiguration().get("hadoop.tmp.dir"); boolean created = Arrays.stream( From 3efa795b3d35da1b0de39f835787195331e6ad03 Mon Sep 17 00:00:00 2001 From: Chris Bevard Date: Fri, 16 Jun 2023 10:06:03 -0400 Subject: [PATCH 10/12] add createTmpFile validation to fix 'name too long' bug introduced by the s3a recovery changes --- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 999186f8cd5ae..6feeee4d1cdb0 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -1372,11 +1372,55 @@ File createTmpFileForWrite(String pathStr, long size, Path path = directoryAllocator.getLocalPathForWrite(pathStr, size, conf); File dir = new File(path.getParent().toUri().getPath()); - String prefix = path.getName(); +// String prefix = path.getName(); + String prefix = validateTmpFilePrefix(pathStr.toString(), null); // create a temp file on this directory return File.createTempFile(prefix, null, dir); } + /** + * Ensure that the temp file prefix doesn't exceed the maximum number of characters allowed by + * the underlying file system. This validation isn't required in Java 9+ since + * {@link java.io.File#createTempFile(String, String, File)} automatically truncates file names. + * @param prefix + * @param suffix + * @return validated prefix + * @throws IOException + */ + static String validateTmpFilePrefix(String prefix, String suffix) throws IOException + { + if(suffix == null) { + suffix = ".tmp"; + } + // Use only the file name from the supplied prefix + prefix = (new File(prefix)).getName(); + + int prefixLength = prefix.length(); + int maxRandomSuffixLen = 19; // Long.toUnsignedString(Long.MAX_VALUE).length() + int suffixLength = suffix.length();; + + String name; + int nameMax = 255; // unable to access the underlying FS directly, so assume 255 + int excess = prefixLength + maxRandomSuffixLen + suffixLength - nameMax; + if (excess > 0) { + // Name exceeds the maximum path component length: shorten it + + // Attempt to shorten the prefix length to no less than 3 + prefixLength = shortenSubName(prefixLength, excess, 3); + prefix = prefix.substring(0, prefixLength); + } + return prefix; + } + + private static int shortenSubName(int subNameLength, int excess, + int nameMin) { + int newLength = Math.max(nameMin, subNameLength - excess); + if (newLength < subNameLength) { + return newLength; + } + return subNameLength; + } + /** * Initialize dir allocator if not already initialized. * From d02215e22ee903df0ea2ec0f7c29f79c59361d3d Mon Sep 17 00:00:00 2001 From: Chris Bevard Date: Thu, 22 Jun 2023 09:14:50 -0400 Subject: [PATCH 11/12] skip validation when running in JVM versions 9 or higher --- .../main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 6feeee4d1cdb0..174c9ce327b51 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -1389,6 +1389,12 @@ File createTmpFileForWrite(String pathStr, long size, */ static String validateTmpFilePrefix(String prefix, String suffix) throws IOException { + // avoid validating multiple times. + // if the jvm running is version 9+ then defer to java.io.File validation implementation + if(Float.valueOf(System.getProperty("java.class.version")) > 52) { + return prefix; + } + if(suffix == null) { suffix = ".tmp"; } From 95138e0dedfb8934695a766e5b72360a90b44517 Mon Sep 17 00:00:00 2001 From: Chris Bevard Date: Thu, 22 Jun 2023 10:47:13 -0400 Subject: [PATCH 12/12] unit tests unit tests --- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 47 ++++---- .../hadoop/fs/s3a/TestS3AFileSystem.java | 110 ++++++++++++++++++ 2 files changed, 137 insertions(+), 20 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileSystem.java diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 174c9ce327b51..f985261f64c6e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -1369,32 +1369,32 @@ public S3AEncryptionMethods getS3EncryptionAlgorithm() { File createTmpFileForWrite(String pathStr, long size, Configuration conf) throws IOException { initLocalDirAllocatorIfNotInitialized(conf); - Path path = directoryAllocator.getLocalPathForWrite(pathStr, - size, conf); + Path path = directoryAllocator.getLocalPathForWrite(pathStr, size, conf); File dir = new File(path.getParent().toUri().getPath()); -// String prefix = path.getName(); - String prefix = validateTmpFilePrefix(pathStr.toString(), null); - // create a temp file on this directory - return File.createTempFile(prefix, null, dir); + return safeCreateTempFile(pathStr, null, dir); } + // TODO remove this method when hadoop upgrades to a newer version of java than 1.8 /** - * Ensure that the temp file prefix doesn't exceed the maximum number of characters allowed by - * the underlying file system. This validation isn't required in Java 9+ since + * Ensure that the temp file prefix and suffix don't exceed the maximum number of characters + * allowed by the underlying file system. This validation isn't required in Java 9+ since * {@link java.io.File#createTempFile(String, String, File)} automatically truncates file names. - * @param prefix - * @param suffix - * @return validated prefix + * + * @param prefix prefix for the temporary file + * @param suffix suffix for the temporary file + * @param dir directory to create the temporary file in + * @return a unique temporary file * @throws IOException */ - static String validateTmpFilePrefix(String prefix, String suffix) throws IOException + static File safeCreateTempFile(String prefix, String suffix, File dir) throws IOException { // avoid validating multiple times. // if the jvm running is version 9+ then defer to java.io.File validation implementation - if(Float.valueOf(System.getProperty("java.class.version")) > 52) { - return prefix; + if(Float.parseFloat(System.getProperty("java.class.version")) >= 53) { + return File.createTempFile(prefix, null, dir); } + // if no suffix was defined assume the default if(suffix == null) { suffix = ".tmp"; } @@ -1402,24 +1402,31 @@ static String validateTmpFilePrefix(String prefix, String suffix) throws IOExcep prefix = (new File(prefix)).getName(); int prefixLength = prefix.length(); + int suffixLength = suffix.length(); int maxRandomSuffixLen = 19; // Long.toUnsignedString(Long.MAX_VALUE).length() - int suffixLength = suffix.length();; String name; int nameMax = 255; // unable to access the underlying FS directly, so assume 255 int excess = prefixLength + maxRandomSuffixLen + suffixLength - nameMax; - if (excess > 0) { - // Name exceeds the maximum path component length: shorten it + // shorten the prefix length if the file name exceeds 255 chars + if (excess > 0) { // Attempt to shorten the prefix length to no less than 3 prefixLength = shortenSubName(prefixLength, excess, 3); prefix = prefix.substring(0, prefixLength); } - return prefix; + // shorten the suffix if the file name still exceeds 255 chars + excess = prefixLength + maxRandomSuffixLen + suffixLength - nameMax; + if (excess > 0) { + // Attempt to shorten the suffix length to no less than 3 + suffixLength = shortenSubName(suffixLength, excess, 3); + suffix = suffix.substring(0, suffixLength); + } + + return File.createTempFile(prefix, suffix, dir); } - private static int shortenSubName(int subNameLength, int excess, - int nameMin) { + private static int shortenSubName(int subNameLength, int excess, int nameMin) { int newLength = Math.max(nameMin, subNameLength - excess); if (newLength < subNameLength) { return newLength; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileSystem.java new file mode 100644 index 0000000000000..5d3331be88b67 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileSystem.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.regex.Pattern; + +/** + * Unit tests for {@link S3AFileSystem}. + */ +public class TestS3AFileSystem extends Assert { + final File TEMP_DIR = new File("target/build/test/TestS3AFileSystem"); + final String longStr = // 1024 char + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key__very_long_s3_key__very_long_s3_key__very_long_s3_key__" + + "very_long_s3_key"; + final String longStrTruncated = "very_long_s3_key__very_long_s3_key__"; + + @Rule + public Timeout testTimeout = new Timeout(30 * 1000); + + @Before + public void init() throws IOException { + Files.createDirectories(TEMP_DIR.toPath()); + } + + @After + public void teardown() throws IOException { + File[] testOutputFiles = TEMP_DIR.listFiles(); + for(File file: testOutputFiles) { + Files.delete(file.toPath()); + } + Files.deleteIfExists(TEMP_DIR.toPath()); + } + + @Before + public void nameThread() { + Thread.currentThread().setName("JUnit"); + } + + /** + * Test the {@link S3AFileSystem#safeCreateTempFile(String, String, File)}. + * The code verifies that the input prefix and suffix don't exceed the file system's max name + * length and cause an exception. + * + * This test verifies the basic contract of the process. + */ + @Test + public void testSafeCreateTempFile() throws Throwable { + // fitting name isn't changed + File noChangesRequired = S3AFileSystem.safeCreateTempFile("noChangesRequired", ".tmp", TEMP_DIR); + assertTrue(noChangesRequired.exists()); + String noChangesRequiredName = noChangesRequired.getName(); + assertTrue(noChangesRequiredName.startsWith("noChangesRequired")); + assertTrue(noChangesRequiredName.endsWith(".tmp")); + + // a long prefix should be truncated + File excessivelyLongPrefix = S3AFileSystem.safeCreateTempFile(longStr, ".tmp", TEMP_DIR); + assertTrue(excessivelyLongPrefix.exists()); + String excessivelyLongPrefixName = excessivelyLongPrefix.getName(); + assertTrue(excessivelyLongPrefixName.startsWith(longStrTruncated)); + assertTrue(excessivelyLongPrefixName.endsWith(".tmp")); + + // a long suffix should be truncated + File excessivelyLongSuffix = S3AFileSystem.safeCreateTempFile("excessivelyLongSuffix", "." + longStr, TEMP_DIR); + assertTrue(excessivelyLongSuffix.exists()); + String excessivelyLongSuffixName = excessivelyLongSuffix.getName(); + // the prefix should have been truncated first + assertTrue(excessivelyLongSuffixName.startsWith("exc")); + Pattern p = Pattern.compile("^exc\\d{1,19}\\.very_long_s3_key__very_long_s3_key__"); + assertTrue(p.matcher(excessivelyLongSuffixName).find()); + } +}