diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index 6801ff48e6..c519a53ea1 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -254,15 +254,16 @@ public MiniDFSCluster startMiniDFSCluster(int servers, final String hosts[]) throws IOException { - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dir.toString()); + conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dir.getAbsolutePath()); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, false); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_DU_RESERVED_KEY, 0); + MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(new HdfsConfiguration(conf)); builder.hosts(hosts); builder.numDataNodes(servers); builder.format(true); - builder.manageNameDfsDirs(true); - builder.manageDataDfsDirs(true); + builder.storagesPerDatanode(1); builder.waitSafeMode(true); this.dfsCluster = builder.build(); diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml index ed006a7191..bea3486615 100644 --- a/tajo-core/pom.xml +++ b/tajo-core/pom.xml @@ -280,7 +280,7 @@ org.mortbay.jetty - jsp-2.1 + jsp-2.1-jetty io.dropwizard.metrics diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index f8a82d056f..d32f991e32 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -40,7 +40,7 @@ 1.1.0 4.0.33.Final 2.6 - 6.1.14 + 6.1.26 ${project.parent.relativePath}/.. src/main/hadoop-${hadoop.version} @@ -485,6 +485,9 @@ org.apache.maven.plugins maven-surefire-plugin 2.19 + + false + org.codehaus.mojo @@ -1190,7 +1193,7 @@ org.mortbay.jetty - jsp-2.1 + jsp-2.1-jetty ${jetty.version} @@ -1199,6 +1202,17 @@ + + org.mortbay.jetty + jsp-2.1-glassfish + 2.1.v20091210 + + + org.eclipse.jdt.core.compiler + ecj + + + diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/HttpFileServerHandler.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/HttpFileServerHandler.java index 78902f33b1..a58cb83649 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/HttpFileServerHandler.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/HttpFileServerHandler.java @@ -25,6 +25,8 @@ import io.netty.handler.ssl.SslHandler; import io.netty.handler.stream.ChunkedFile; import io.netty.util.CharsetUtil; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import java.io.File; import java.io.FileNotFoundException; @@ -32,9 +34,6 @@ import java.io.UnsupportedEncodingException; import java.net.URLDecoder; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - public class HttpFileServerHandler extends SimpleChannelInboundHandler { private final Log LOG = LogFactory.getLog(HttpFileServerHandler.class); @@ -84,11 +83,12 @@ public void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) thr ChannelFuture lastContentFuture; if (ctx.pipeline().get(SslHandler.class) != null) { // Cannot use zero-copy with HTTPS. - lastContentFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedFile(raf, 0, fileLength, 8192))); + lastContentFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedFile(raf, 0, fileLength, 8192)), + ctx.newProgressivePromise()); } else { // No encryption - use zero-copy. final FileRegion region = new DefaultFileRegion(raf.getChannel(), 0, fileLength); - writeFuture = ctx.write(region); + writeFuture = ctx.write(region, ctx.newProgressivePromise()); lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT); writeFuture.addListener(new ChannelProgressiveFutureListener() { @Override @@ -99,7 +99,7 @@ public void operationProgressed(ChannelProgressiveFuture future, long progress, @Override public void operationComplete(ChannelProgressiveFuture future) throws Exception { - region.release(); + LOG.trace(future.channel() + " Transfer complete."); } }); } diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java index 8472095698..2cee3bb292 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java @@ -21,9 +21,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.text.ByteBufLineReader; @@ -35,7 +33,6 @@ import java.io.File; import java.io.FileInputStream; import java.nio.charset.Charset; -import java.util.UUID; import static org.junit.Assert.*; @@ -85,16 +82,12 @@ public void testReaderWithLocalFS() throws Exception { fs.close(); } - @Test(timeout = 60000) + @Test(timeout = 120000) public void testReaderWithDFS() throws Exception { - final Configuration conf = new HdfsConfiguration(); - String testDataPath = TEST_PATH + "/" + UUID.randomUUID().toString(); - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, testDataPath); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0); - conf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, true); + final Configuration conf = TestFileTablespace.getTestHdfsConfiguration(); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(2).waitSafeMode(true).build(); + .numDataNodes(2).storagesPerDatanode(1).format(true).build(); TajoConf tajoConf = new TajoConf(conf); tajoConf.setVar(TajoConf.ConfVars.ROOT_DIR, cluster.getFileSystem().getUri() + "/tajo"); diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java index c99e0dc589..037030266a 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java @@ -39,6 +39,7 @@ import org.junit.Before; import org.junit.Test; +import java.io.File; import java.io.IOException; import java.net.URI; import java.util.List; @@ -48,7 +49,7 @@ public class TestFileTablespace { private TajoConf conf; - private static String TEST_PATH = "target/test-data/TestFileTablespace"; + private static String TEST_PATH = "target/test-data/hdfs"; private Path testDir; private FileSystem localFs; @@ -63,6 +64,21 @@ public void setUp() throws Exception { public void tearDown() throws Exception { } + public static HdfsConfiguration getTestHdfsConfiguration() { + HdfsConfiguration conf = new HdfsConfiguration(); + String testDataPath = new File(TEST_PATH + "/" + UUID.randomUUID().toString()).getAbsolutePath(); + + String namenodeDir = new File(testDataPath, "name").getAbsolutePath(); + conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, testDataPath); + conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, namenodeDir); + conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, namenodeDir); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_DU_RESERVED_KEY, 0); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0); + conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); + + return conf; + } + @Test public final void testGetScannerAndAppender() throws IOException { Schema schema = new Schema(); @@ -102,19 +118,11 @@ public final void testGetScannerAndAppender() throws IOException { localFs.delete(path, true); } - @Test(timeout = 60000) + @Test(timeout = 120000) public void testGetSplit() throws Exception { - final Configuration conf = new HdfsConfiguration(); - String testDataPath = TEST_PATH + "/" + UUID.randomUUID().toString(); - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, testDataPath); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0); - conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); - conf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, false); - - final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(1).build(); - TajoConf tajoConf = new TajoConf(conf); - tajoConf.setVar(TajoConf.ConfVars.ROOT_DIR, cluster.getFileSystem().getUri() + "/tajo"); + final Configuration hdfsConf = getTestHdfsConfiguration(); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(hdfsConf) + .numDataNodes(1).storagesPerDatanode(1).format(true).build(); int testCount = 10; Path tablePath = new Path("/testGetSplit"); @@ -131,7 +139,9 @@ public void testGetSplit() throws Exception { assertTrue(fs.exists(tablePath)); FileTablespace space = new FileTablespace("testGetSplit", fs.getUri(), null); - space.init(new TajoConf(conf)); + space.init(conf); + + TablespaceManager.addTableSpaceForTest(space); assertEquals(fs.getUri(), space.getUri()); Schema schema = new Schema(); @@ -153,24 +163,18 @@ public void testGetSplit() throws Exception { assertEquals(testCount / 2, splits.size()); assertEquals(1, splits.get(0).getHosts().length); assertEquals(-1, ((FileFragment)splits.get(0)).getDiskIds()[0]); + fs.close(); } finally { cluster.shutdown(); } } - @Test(timeout = 60000) + @Test(timeout = 120000) public void testZeroLengthSplit() throws Exception { - final Configuration conf = new HdfsConfiguration(); - String testDataPath = TEST_PATH + "/" + UUID.randomUUID().toString(); - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, testDataPath); - conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0); - - final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(1).build(); - TajoConf tajoConf = new TajoConf(conf); - tajoConf.setVar(TajoConf.ConfVars.ROOT_DIR, cluster.getFileSystem().getUri() + "/tajo"); + final Configuration hdfsConf = getTestHdfsConfiguration(); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(hdfsConf) + .numDataNodes(1).storagesPerDatanode(1).format(true).build(); int testCount = 10; Path tablePath = new Path("/testZeroLengthSplit"); @@ -188,40 +192,37 @@ public void testZeroLengthSplit() throws Exception { } assertTrue(fs.exists(tablePath)); + FileTablespace space = new FileTablespace("testZeroLengthSplit", fs.getUri(), new JSONObject()); - space.init(new TajoConf(conf)); + space.init(conf); + TablespaceManager.addTableSpaceForTest(space); assertEquals(fs.getUri(), space.getUri()); Schema schema = new Schema(); schema.addColumn("id", Type.INT4); schema.addColumn("age",Type.INT4); schema.addColumn("name",Type.TEXT); - TableMeta meta = CatalogUtil.newTableMeta("TEXT"); + TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT); List splits = Lists.newArrayList(); // Get FileFragments in partition batch splits.addAll(space.getSplits("data", meta, schema, partitions.toArray(new Path[partitions.size()]))); assertEquals(0, splits.size()); + fs.close(); } finally { cluster.shutdown(); } } - @Test(timeout = 60000) + @Test(timeout = 120000) public void testGetSplitWithBlockStorageLocationsBatching() throws Exception { - final Configuration conf = new HdfsConfiguration(); - String testDataPath = TEST_PATH + "/" + UUID.randomUUID().toString(); - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, testDataPath); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0); - conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2); - conf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, true); - - final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(2).build(); + final Configuration hdfsConf = getTestHdfsConfiguration(); - TajoConf tajoConf = new TajoConf(conf); - tajoConf.setVar(TajoConf.ConfVars.ROOT_DIR, cluster.getFileSystem().getUri() + "/tajo"); + hdfsConf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2); + hdfsConf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, true); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(hdfsConf) + .numDataNodes(2).storagesPerDatanode(1).format(true).build(); int testCount = 10; Path tablePath = new Path("/testGetSplitWithBlockStorageLocationsBatching"); @@ -236,15 +237,15 @@ public void testGetSplitWithBlockStorageLocationsBatching() throws Exception { assertTrue(fs.exists(tablePath)); FileTablespace sm = new FileTablespace("testGetSplitWithBlockStorageLocationsBatching", fs.getUri(), null); - sm.init(new TajoConf(conf)); - + sm.init(new TajoConf(hdfsConf)); + TablespaceManager.addTableSpaceForTest(sm); assertEquals(fs.getUri(), sm.getUri()); Schema schema = new Schema(); schema.addColumn("id", Type.INT4); schema.addColumn("age", Type.INT4); schema.addColumn("name", Type.TEXT); - TableMeta meta = CatalogUtil.newTableMeta("TEXT"); + TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT); List splits = Lists.newArrayList(); splits.addAll(sm.getSplits("data", meta, schema, tablePath)); @@ -253,23 +254,19 @@ public void testGetSplitWithBlockStorageLocationsBatching() throws Exception { assertEquals(2, splits.get(0).getHosts().length); assertEquals(2, ((FileFragment)splits.get(0)).getDiskIds().length); assertNotEquals(-1, ((FileFragment)splits.get(0)).getDiskIds()[0]); + fs.close(); } finally { cluster.shutdown(); } } - @Test(timeout = 60000) + @Test(timeout = 120000) public void testGetFileTablespace() throws Exception { - final Configuration hdfsConf = new HdfsConfiguration(); - String testDataPath = TEST_PATH + "/" + UUID.randomUUID().toString(); - hdfsConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, testDataPath); - hdfsConf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0); - hdfsConf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); - hdfsConf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, true); + final Configuration hdfsConf = getTestHdfsConfiguration(); - final MiniDFSCluster cluster = - new MiniDFSCluster.Builder(hdfsConf).numDataNodes(1).build(); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(hdfsConf) + .numDataNodes(1).storagesPerDatanode(1).format(true).build(); URI uri = URI.create(cluster.getFileSystem().getUri() + "/tajo"); try { diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java index 78e3390e87..d027fb8a1e 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java @@ -24,8 +24,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.tajo.BuiltinStorages; import org.apache.tajo.catalog.CatalogUtil; @@ -37,6 +35,7 @@ import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.ProtobufDatum; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; +import org.apache.tajo.storage.TestFileTablespace; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.rawfile.DirectRawFileScanner; @@ -46,9 +45,11 @@ import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.ProtoUtil; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.Timeout; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -56,6 +57,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.UUID; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.*; @@ -66,13 +68,16 @@ public class TestDirectRawFile { public static Schema schema; private static String TEST_PATH = "target/test-data/TestDirectRawFile"; - private static MiniDFSCluster cluster; - private static FileSystem dfs; - private static FileSystem localFs; + private MiniDFSCluster cluster; + private FileSystem fs; + private boolean isLocal; private TajoConf tajoConf; private Path testDir; + @Rule + public Timeout timeout = new Timeout(120, TimeUnit.SECONDS); + @Parameterized.Parameters public static Collection generateParameters() throws IOException { return Arrays.asList(new Object[][]{ @@ -83,41 +88,35 @@ public static Collection generateParameters() throws IOException { public TestDirectRawFile(boolean isLocal) throws IOException { - FileSystem fs; + this.isLocal = isLocal; + } + + @Before + public void setup() throws IOException { if (isLocal) { - fs = localFs; + fs = FileSystem.getLocal(new TajoConf()); } else { - fs = dfs; + final Configuration conf = TestFileTablespace.getTestHdfsConfiguration(); + + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(1) + .format(true) + .storagesPerDatanode(1).build(); + + fs = cluster.getFileSystem(); } this.tajoConf = new TajoConf(fs.getConf()); this.testDir = getTestDir(fs, TEST_PATH); } - @BeforeClass - public static void setUpClass() throws IOException, InterruptedException { - final Configuration conf = new HdfsConfiguration(); - String testDataPath = TEST_PATH + "/" + UUID.randomUUID().toString(); - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, testDataPath); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0); - conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); - conf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, false); - - MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(new HdfsConfiguration(conf)); - builder.numDataNodes(1); - builder.format(true); - builder.manageNameDfsDirs(true); - builder.manageDataDfsDirs(true); - builder.waitSafeMode(true); - cluster = builder.build(); - - dfs = cluster.getFileSystem(); - localFs = FileSystem.getLocal(new TajoConf()); - } - - @AfterClass - public static void tearDownClass() throws InterruptedException { - cluster.shutdown(); + @After + public void tearDown() throws IOException { + if (isLocal) { + fs.delete(testDir, true); + } else { + cluster.shutdown(); + } } public Path getTestDir(FileSystem fs, String dir) throws IOException { @@ -168,7 +167,7 @@ public FileStatus writeRowBlock(TajoConf conf, TableMeta meta, MemoryRowBlock ro return writeRowBlock(conf, meta, rowBlock, outputFile); } - @Test(timeout = 60000) + @Test public void testRWForAllTypesWithNextTuple() throws IOException { int rowNum = 10000; @@ -198,7 +197,7 @@ public void testRWForAllTypesWithNextTuple() throws IOException { assertEquals(rowNum, j); } - @Test(timeout = 60000) + @Test public void testRepeatedScan() throws IOException { int rowNum = 2; @@ -226,7 +225,7 @@ public void testRepeatedScan() throws IOException { reader.close(); } - @Test(timeout = 60000) + @Test public void testReset() throws IOException { int rowNum = 2;