From 5967565190331b361a9c0f025b20f82f4e2014ce Mon Sep 17 00:00:00 2001 From: Abhishek Modi Date: Sun, 23 Oct 2016 20:02:28 +0530 Subject: [PATCH 1/4] HADOOP-13680. fs.s3a.readahead.range to use getLongBytes --- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 32 ++++++++++++++++++- .../hadoop/fs/s3a/ITestS3AConfiguration.java | 11 +++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 9908ba7426d34..acd03913b87af 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -182,7 +182,7 @@ public void initialize(URI name, Configuration conf) throws IOException { longOption(conf, FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE, 1); enableMultiObjectsDelete = conf.getBoolean(ENABLE_MULTI_DELETE, true); - readAhead = longOption(conf, READAHEAD_RANGE, DEFAULT_READAHEAD_RANGE, 0); + readAhead = longBytesOption(conf, READAHEAD_RANGE, DEFAULT_READAHEAD_RANGE, 0); storageStatistics = (S3AStorageStatistics) GlobalStorageStatistics.INSTANCE .put(S3AStorageStatistics.NAME, @@ -353,6 +353,16 @@ AmazonS3 getAmazonS3Client() { return s3; } + /** + * Returns the read ahead range value used by this filesystem + * @return + */ + + @VisibleForTesting + long getReadAheadRange() { + return readAhead; + } + /** * Get the input policy for this FS instance. * @return the input policy @@ -2264,4 +2274,24 @@ public String toString() { } } + /** + * Get a longBytes option >= the minimum allowed value. + * @param conf configuration + * @param key key to look up + * @param defVal default value + * @param min minimum value + * @return the value + * @throws IllegalArgumentException if the value is below the minimum + */ + static long longBytesOption(Configuration conf, + String key, + long defVal, + long min) { + long v = conf.getLongBytes(key, defVal); + Preconditions.checkArgument(v >= min, + String.format("Value of %s: %d is below the minimum value %d", + key, v, min)); + return v; + } + } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 30d4bf66baf53..6a89e776e019e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -446,6 +446,17 @@ public void testDirectoryAllocatorRR() throws Throwable { tmp1.getParent(), tmp2.getParent()); } + @Test + public void testReadAheadRange() throws Exception { + conf = new Configuration(); + conf.set(Constants.READAHEAD_RANGE, "300k"); + fs = S3ATestUtils.createTestFileSystem(conf); + assertNotNull(fs); + long readAheadRange = fs.getReadAheadRange(); + assertNotNull(readAheadRange); + assertEquals("Read Ahead Range Incorrect.", readAheadRange, 300000); + } + /** * Reads and returns a field from an object using reflection. If the field * cannot be found, is null, or is not the expected type, then this method From 872bf8896bad1a5625b0f7f796d9861a5bda5851 Mon Sep 17 00:00:00 2001 From: Abhishek Modi Date: Sun, 23 Oct 2016 20:25:33 +0530 Subject: [PATCH 2/4] Fixing the test case --- .../java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 6a89e776e019e..b10a032bc9f04 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -454,7 +454,7 @@ public void testReadAheadRange() throws Exception { assertNotNull(fs); long readAheadRange = fs.getReadAheadRange(); assertNotNull(readAheadRange); - assertEquals("Read Ahead Range Incorrect.", readAheadRange, 300000); + assertEquals("Read Ahead Range Incorrect.", readAheadRange, 300*1024); } /** From f32742ac11d40a12290cc906d04fe089e555bbab Mon Sep 17 00:00:00 2001 From: Abhishek Modi Date: Mon, 24 Oct 2016 22:04:53 +0530 Subject: [PATCH 3/4] Updating documentation. Incorporating fixes done in 681e412c7d59e5bc83702b2dc3715da8054b02f4 in this PR --- .../src/main/resources/core-default.xml | 4 ++-- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 22 +----------------- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 23 ++++++++++++++++++- .../site/markdown/tools/hadoop-aws/index.md | 4 ++-- .../hadoop/fs/s3a/ITestS3AConfiguration.java | 2 +- 5 files changed, 28 insertions(+), 27 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 59d939bd83756..1423809a9a840 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1098,7 +1098,7 @@ fs.s3a.block.size - 33554432 + 32M Block size to use when reading files using s3a: file system. @@ -1161,7 +1161,7 @@ fs.s3a.readahead.range - 65536 + 64K Bytes to read ahead during a seek() before closing and re-opening the S3 HTTP connection. This option will be overridden if any call to setReadahead() is made to an open stream. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index acd03913b87af..91ed7d5a8af7f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -1880,7 +1880,7 @@ private ObjectMetadata cloneObjectMetadata(ObjectMetadata source) { */ @Deprecated public long getDefaultBlockSize() { - return getConf().getLong(FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE); + return getConf().getLongBytes(FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE); } @Override @@ -2274,24 +2274,4 @@ public String toString() { } } - /** - * Get a longBytes option >= the minimum allowed value. - * @param conf configuration - * @param key key to look up - * @param defVal default value - * @param min minimum value - * @return the value - * @throws IllegalArgumentException if the value is below the minimum - */ - static long longBytesOption(Configuration conf, - String key, - long defVal, - long min) { - long v = conf.getLongBytes(key, defVal); - Preconditions.checkArgument(v >= min, - String.format("Value of %s: %d is below the minimum value %d", - key, v, min)); - return v; - } - } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index c89f6904cdaa6..b96201dc3c4cc 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -462,6 +462,27 @@ static long longOption(Configuration conf, return v; } + /** + * Get a long option >= the minimum allowed value, supporting memory + * prefixes K,M,G,T,P. + * @param conf configuration + * @param key key to look up + * @param defVal default value + * @param min minimum value + * @return the value + * @throws IllegalArgumentException if the value is below the minimum + */ + static long longBytesOption(Configuration conf, + String key, + long defVal, + long min) { + long v = conf.getLongBytes(key, defVal); + Preconditions.checkArgument(v >= min, + String.format("Value of %s: %d is below the minimum value %d", + key, v, min)); + return v; + } + /** * Get a size property from the configuration: this property must * be at least equal to {@link Constants#MULTIPART_MIN_SIZE}. @@ -474,7 +495,7 @@ static long longOption(Configuration conf, */ public static long getMultipartSizeProperty(Configuration conf, String property, long defVal) { - long partSize = conf.getLong(property, defVal); + long partSize = conf.getLongBytes(property, defVal); if (partSize < MULTIPART_MIN_SIZE) { LOG.warn("{} must be at least 5 MB; configured value is {}", property, partSize); diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index c23e782d38074..54cb2899ad740 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -759,7 +759,7 @@ from placing its declaration on the command line. fs.s3a.block.size - 33554432 + 32M Block size to use when reading files using s3a: file system. @@ -793,7 +793,7 @@ from placing its declaration on the command line. fs.s3a.readahead.range - 65536 + 64K Bytes to read ahead during a seek() before closing and re-opening the S3 HTTP connection. This option will be overridden if any call to setReadahead() is made to an open stream. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index b10a032bc9f04..79bbb5ecf5993 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -449,7 +449,7 @@ public void testDirectoryAllocatorRR() throws Throwable { @Test public void testReadAheadRange() throws Exception { conf = new Configuration(); - conf.set(Constants.READAHEAD_RANGE, "300k"); + conf.set(Constants.READAHEAD_RANGE, "300K"); fs = S3ATestUtils.createTestFileSystem(conf); assertNotNull(fs); long readAheadRange = fs.getReadAheadRange(); From 40383e157fbd11cd392e5692618c102bfea0acba Mon Sep 17 00:00:00 2001 From: Abhishek Modi Date: Mon, 24 Oct 2016 22:53:35 +0530 Subject: [PATCH 4/4] Fixing test cases broken due to change in default block size for S3A --- .../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 +- .../java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 91ed7d5a8af7f..d6cf29164c983 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -179,7 +179,7 @@ public void initialize(URI name, Configuration conf) throws IOException { MIN_MULTIPART_THRESHOLD, DEFAULT_MIN_MULTIPART_THRESHOLD); //check but do not store the block size - longOption(conf, FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE, 1); + longBytesOption(conf, FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE, 1); enableMultiObjectsDelete = conf.getBoolean(ENABLE_MULTI_DELETE, true); readAhead = longBytesOption(conf, READAHEAD_RANGE, DEFAULT_READAHEAD_RANGE, 0); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 79bbb5ecf5993..4cf076ca7e623 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -375,7 +375,7 @@ public void shouldBeAbleToSwitchOnS3PathStyleAccessViaConfigProperty() byte[] file = ContractTestUtils.toAsciiByteArray("test file"); ContractTestUtils.writeAndRead(fs, new Path("/path/style/access/testFile"), file, file.length, - conf.getInt(Constants.FS_S3A_BLOCK_SIZE, file.length), false, true); + (int) conf.getLongBytes(Constants.FS_S3A_BLOCK_SIZE, file.length), false, true); } catch (final AWSS3IOException e) { LOG.error("Caught exception: ", e); // Catch/pass standard path style access behaviour when live bucket