From 9a50dfedd920a44efce367a9a619b668e7d12bc3 Mon Sep 17 00:00:00 2001 From: wenxin he Date: Thu, 8 Jun 2017 20:03:41 +0800 Subject: [PATCH 1/2] HADOOP-14508. TestDFSIO throws NPE when set -sequential argument. --- .../java/org/apache/hadoop/fs/TestDFSIO.java | 72 +++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java index 34eac83ae512d..79bf63506e63f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java @@ -108,7 +108,8 @@ public class TestDFSIO implements Tool { " [-size Size[B|KB|MB|GB|TB]]" + " [-resFile resultFileName] [-bufferSize Bytes]" + " [-storagePolicy storagePolicyName]" + - " [-erasureCodePolicy erasureCodePolicyName]"; + " [-erasureCodePolicy erasureCodePolicyName]" + + " [-sequential]"; private Configuration config; private static final String STORAGE_POLICY_NAME_KEY = @@ -288,6 +289,61 @@ public void testTruncate() throws Exception { bench.analyzeResult(fs, TestType.TEST_TYPE_TRUNCATE, execTime); } + @Test (timeout = 10000) + public void testSequentialWrite() throws Exception { + FileSystem fs = cluster.getFileSystem(); + bench.sequentialTest(fs, TestType.TEST_TYPE_WRITE, + DEFAULT_NR_BYTES, DEFAULT_NR_FILES); + } + + @Test (timeout = 10000) + public void testSequentialRead() throws Exception { + FileSystem fs = cluster.getFileSystem(); + bench.sequentialTest(fs, TestType.TEST_TYPE_READ, + DEFAULT_NR_BYTES, DEFAULT_NR_FILES); + } + + @Test (timeout = 10000) + public void testSequentialReadRandom() throws Exception { + FileSystem fs = cluster.getFileSystem(); + bench.getConf().setLong("test.io.skip.size", 0); + bench.sequentialTest(fs, TestType.TEST_TYPE_READ_RANDOM, + DEFAULT_NR_BYTES, DEFAULT_NR_FILES); + } + + @Test (timeout = 10000) + public void testSequentialAppend() throws Exception { + FileSystem fs = cluster.getFileSystem(); + bench.getConf().setLong("test.io.skip.size", 0); + bench.sequentialTest(fs, TestType.TEST_TYPE_READ_RANDOM, + DEFAULT_NR_BYTES, DEFAULT_NR_FILES); + } + + @Test (timeout = 10000) + public void testSequentialReadBackward() throws Exception { + FileSystem fs = cluster.getFileSystem(); + bench.getConf().setLong("test.io.skip.size", -DEFAULT_BUFFER_SIZE); + bench.sequentialTest(fs, TestType.TEST_TYPE_READ_RANDOM, + DEFAULT_NR_BYTES, DEFAULT_NR_FILES); + } + + @Test (timeout = 10000) + public void testSequentialReadSkip() throws Exception { + FileSystem fs = cluster.getFileSystem(); + bench.sequentialTest(fs, TestType.TEST_TYPE_APPEND, + DEFAULT_NR_BYTES, DEFAULT_NR_FILES); + } + + @Test(timeout = 60000) + public void testSequentialTruncate() throws Exception { + FileSystem fs = cluster.getFileSystem(); + bench.sequentialTest(fs, TestType.TEST_TYPE_WRITE, + DEFAULT_NR_BYTES, + DEFAULT_NR_FILES); // make sure the length of files to be truncated is DEFAULT_NR_BYTES + bench.sequentialTest(fs, TestType.TEST_TYPE_TRUNCATE, + DEFAULT_NR_BYTES, DEFAULT_NR_FILES); + } + @SuppressWarnings("deprecation") private void createControlFile(FileSystem fs, long nrBytes, // in bytes @@ -718,10 +774,16 @@ private void sequentialTest(FileSystem fs, default: return; } - for(int i=0; i < nrFiles; i++) - ioer.doIO(Reporter.NULL, - BASE_FILE_NAME+Integer.toString(i), - fileSize); + for(int i=0; i < nrFiles; i++) { + ioer.configure(new JobConf(config, TestDFSIO.class)); + String fileName = getFileName(i); + ioer.stream = ioer.getIOStream(fileName); + try { + ioer.doIO(Reporter.NULL, fileName, fileSize); + } finally { + if(ioer.stream != null) ioer.stream.close(); + } + } } public static void main(String[] args) { From 2039eb7e86d2c1a1c957a4aeeff5e2d047359648 Mon Sep 17 00:00:00 2001 From: wenxin he Date: Sat, 17 Jun 2017 11:30:18 +0800 Subject: [PATCH 2/2] fix checkstyle --- .../src/test/java/org/apache/hadoop/fs/TestDFSIO.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java index 79bf63506e63f..b5e6998569008 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java @@ -337,9 +337,10 @@ public void testSequentialReadSkip() throws Exception { @Test(timeout = 60000) public void testSequentialTruncate() throws Exception { FileSystem fs = cluster.getFileSystem(); + // given DEFAULT_NR_BYTES files to be truncated bench.sequentialTest(fs, TestType.TEST_TYPE_WRITE, - DEFAULT_NR_BYTES, - DEFAULT_NR_FILES); // make sure the length of files to be truncated is DEFAULT_NR_BYTES + DEFAULT_NR_BYTES, DEFAULT_NR_FILES); + // test truncate bench.sequentialTest(fs, TestType.TEST_TYPE_TRUNCATE, DEFAULT_NR_BYTES, DEFAULT_NR_FILES); } @@ -774,14 +775,16 @@ private void sequentialTest(FileSystem fs, default: return; } - for(int i=0; i < nrFiles; i++) { + for (int i = 0; i < nrFiles; i++) { ioer.configure(new JobConf(config, TestDFSIO.class)); String fileName = getFileName(i); ioer.stream = ioer.getIOStream(fileName); try { ioer.doIO(Reporter.NULL, fileName, fileSize); } finally { - if(ioer.stream != null) ioer.stream.close(); + if (ioer.stream != null) { + ioer.stream.close(); + } } } }