From af62c81faf66966305596fb6effb90c7e34f6094 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 17 Sep 2015 12:56:12 -0700 Subject: [PATCH 01/10] TAJO-1868: Allow TablespaceManager::get to return unregistered tablespace. --- .../tajo/engine/query/TestHBaseTable.java | 341 ++++++++++-------- .../tajo/storage/TablespaceManager.java | 22 ++ .../pgsql/TestPgSQLJdbcTableSpace.java | 14 +- .../storage/pgsql/TestPgSQLQueryTests.java | 2 + 4 files changed, 228 insertions(+), 151 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index 760fa62ef0..ceae92bb3e 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.query; +import com.google.common.base.Optional; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileStatus; @@ -151,15 +152,22 @@ public void testCreateHBaseTable() throws Exception { @Test public void testCreateNotExistsExternalHBaseTable() throws Exception { - String sql = String.format( - "CREATE EXTERNAL TABLE external_hbase_mapped_table1 (col1 text, col2 text, col3 text, col4 text) " + - "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col2:a,col3:,col2:b') " + - "LOCATION '%s/external_hbase_table'", tableSpaceUri); + Optional existing = TablespaceManager.removeTablespaceForTest("cluster1"); + assertTrue(existing.isPresent()); + try { - executeString(sql).close(); - fail("External table should be a existed table."); - } catch (Throwable e) { - assertTrue(e.getMessage().indexOf("External table should be a existed table.") >= 0); + String sql = String.format( + "CREATE EXTERNAL TABLE external_hbase_mapped_table1 (col1 text, col2 text, col3 text, col4 text) " + + "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col2:a,col3:,col2:b') " + + "LOCATION '%s/external_hbase_table'", tableSpaceUri); + try { + executeString(sql).close(); + fail("External table should be a existed table."); + } catch (Throwable e) { + assertTrue(e.getMessage().indexOf("External table should be a existed table.") >= 0); + } + } finally { + TablespaceManager.addTableSpaceForTest(existing.get()); } } @@ -177,195 +185,230 @@ public void testCreateRowFieldWithNonText() throws Exception { @Test public void testCreateExternalHBaseTable() throws Exception { - HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table_not_purge")); - hTableDesc.addFamily(new HColumnDescriptor("col1")); - hTableDesc.addFamily(new HColumnDescriptor("col2")); - hTableDesc.addFamily(new HColumnDescriptor("col3")); - testingCluster.getHBaseUtil().createTable(hTableDesc); - - String sql = String.format( - "CREATE EXTERNAL TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " + - "USING hbase WITH ('table'='external_hbase_table_not_purge', 'columns'=':key,col1:a,col2:,col3:b') " + - "LOCATION '%s/external_hbase_table'", tableSpaceUri); - executeString(sql).close(); + Optional existing = TablespaceManager.removeTablespaceForTest("cluster1"); + assertTrue(existing.isPresent()); - assertTableExists("external_hbase_mapped_table"); + try { + HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table_not_purge")); + hTableDesc.addFamily(new HColumnDescriptor("col1")); + hTableDesc.addFamily(new HColumnDescriptor("col2")); + hTableDesc.addFamily(new HColumnDescriptor("col3")); + testingCluster.getHBaseUtil().createTable(hTableDesc); + + String sql = String.format( + "CREATE EXTERNAL TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " + + "USING hbase WITH ('table'='external_hbase_table_not_purge', 'columns'=':key,col1:a,col2:,col3:b') " + + "LOCATION '%s/external_hbase_table'", tableSpaceUri); + executeString(sql).close(); - executeString("DROP TABLE external_hbase_mapped_table").close(); + assertTableExists("external_hbase_mapped_table"); + + executeString("DROP TABLE external_hbase_mapped_table").close(); + + HBaseAdmin hAdmin = new HBaseAdmin(testingCluster.getHBaseUtil().getConf()); + try { + assertTrue(hAdmin.tableExists("external_hbase_table_not_purge")); + hAdmin.disableTable("external_hbase_table_not_purge"); + hAdmin.deleteTable("external_hbase_table_not_purge"); + } finally { + hAdmin.close(); + } - HBaseAdmin hAdmin = new HBaseAdmin(testingCluster.getHBaseUtil().getConf()); - try { - assertTrue(hAdmin.tableExists("external_hbase_table_not_purge")); - hAdmin.disableTable("external_hbase_table_not_purge"); - hAdmin.deleteTable("external_hbase_table_not_purge"); } finally { - hAdmin.close(); + TablespaceManager.addTableSpaceForTest(existing.get()); } + } @Test public void testSimpleSelectQuery() throws Exception { - HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); - hTableDesc.addFamily(new HColumnDescriptor("col1")); - hTableDesc.addFamily(new HColumnDescriptor("col2")); - hTableDesc.addFamily(new HColumnDescriptor("col3")); - testingCluster.getHBaseUtil().createTable(hTableDesc); - - String sql = String.format( - "CREATE EXTERNAL TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " + - "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col1:a,col2:,col3:b') " + - "LOCATION '%s/external_hbase_table'", tableSpaceUri); - executeString(sql).close(); + Optional existing = TablespaceManager.removeTablespaceForTest("cluster1"); + assertTrue(existing.isPresent()); - assertTableExists("external_hbase_mapped_table"); + try { + HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); + hTableDesc.addFamily(new HColumnDescriptor("col1")); + hTableDesc.addFamily(new HColumnDescriptor("col2")); + hTableDesc.addFamily(new HColumnDescriptor("col3")); + testingCluster.getHBaseUtil().createTable(hTableDesc); + + String sql = String.format( + "CREATE EXTERNAL TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " + + "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col1:a,col2:,col3:b') " + + "LOCATION '%s/external_hbase_table'", tableSpaceUri); + executeString(sql).close(); - HBaseTablespace space = (HBaseTablespace) TablespaceManager.getByName("cluster1").get(); - HConnection hconn = space.getConnection(); - HTableInterface htable = hconn.getTable("external_hbase_table"); + assertTableExists("external_hbase_mapped_table"); - try { - for (int i = 0; i < 100; i++) { - Put put = new Put(String.valueOf(i).getBytes()); - put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); - put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); - put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); - put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); - put.add("col3".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); - htable.put(put); - } + HConnection hconn = ((HBaseTablespace)existing.get()).getConnection(); + HTableInterface htable = hconn.getTable("external_hbase_table"); - ResultSet res = executeString("select * from external_hbase_mapped_table where rk > '20'"); - assertResultSet(res); - cleanupQuery(res); + try { + for (int i = 0; i < 100; i++) { + Put put = new Put(String.valueOf(i).getBytes()); + put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); + put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); + put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); + put.add("col3".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + htable.put(put); + } + + ResultSet res = executeString("select * from external_hbase_mapped_table where rk > '20'"); + assertResultSet(res); + cleanupQuery(res); + } finally { + executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); + htable.close(); + } } finally { - executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); - htable.close(); + TablespaceManager.addTableSpaceForTest(existing.get()); } } @Test public void testBinaryMappedQuery() throws Exception { - HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); - hTableDesc.addFamily(new HColumnDescriptor("col1")); - hTableDesc.addFamily(new HColumnDescriptor("col2")); - hTableDesc.addFamily(new HColumnDescriptor("col3")); - testingCluster.getHBaseUtil().createTable(hTableDesc); - - String sql = String.format( - "CREATE EXTERNAL TABLE external_hbase_mapped_table (rk int8, col1 text, col2 text, col3 int4)\n " + - "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key#b,col1:a,col2:,col3:b#b') " + - "LOCATION '%s/external_hbase_table'", tableSpaceUri); - executeString(sql).close(); - - assertTableExists("external_hbase_mapped_table"); - - HBaseTablespace space = (HBaseTablespace) TablespaceManager.getByName("cluster1").get(); - HConnection hconn = space.getConnection(); - HTableInterface htable = hconn.getTable("external_hbase_table"); + //Optional existing = TablespaceManager.removeTablespaceForTest("cluster1"); + //assertTrue(existing.isPresent()); try { - for (int i = 0; i < 100; i++) { - Put put = new Put(Bytes.toBytes((long) i)); - put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); - put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); - put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); - put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); - put.add("col3".getBytes(), "b".getBytes(), Bytes.toBytes(i)); - htable.put(put); - } + HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); + hTableDesc.addFamily(new HColumnDescriptor("col1")); + hTableDesc.addFamily(new HColumnDescriptor("col2")); + hTableDesc.addFamily(new HColumnDescriptor("col3")); + testingCluster.getHBaseUtil().createTable(hTableDesc); + + String sql = String.format( + "CREATE EXTERNAL TABLE external_hbase_mapped_table (rk int8, col1 text, col2 text, col3 int4)\n " + + "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key#b,col1:a,col2:,col3:b#b') " + + "LOCATION '%s/external_hbase_table'", tableSpaceUri); + executeString(sql).close(); - ResultSet res = executeString("select * from external_hbase_mapped_table where rk > 20"); - assertResultSet(res); - res.close(); + assertTableExists("external_hbase_mapped_table"); + + HBaseTablespace space = (HBaseTablespace) TablespaceManager.getByName("cluster1").get(); + HConnection hconn = space.getConnection(); + HTableInterface htable = hconn.getTable("external_hbase_table"); + + try { + for (int i = 0; i < 100; i++) { + Put put = new Put(Bytes.toBytes((long) i)); + put.add("col1".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); + put.add("col1".getBytes(), "b".getBytes(), ("b-" + i).getBytes()); + put.add("col2".getBytes(), "k1".getBytes(), ("k1-" + i).getBytes()); + put.add("col2".getBytes(), "k2".getBytes(), ("k2-" + i).getBytes()); + put.add("col3".getBytes(), "b".getBytes(), Bytes.toBytes(i)); + htable.put(put); + } - //Projection - res = executeString("select col3, col2, rk from external_hbase_mapped_table where rk > 95"); + ResultSet res = executeString("select * from external_hbase_mapped_table where rk > 20"); + assertResultSet(res); + res.close(); - String expected = "col3,col2,rk\n" + - "-------------------------------\n" + - "96,{\"k1\":\"k1-96\", \"k2\":\"k2-96\"},96\n" + - "97,{\"k1\":\"k1-97\", \"k2\":\"k2-97\"},97\n" + - "98,{\"k1\":\"k1-98\", \"k2\":\"k2-98\"},98\n" + - "99,{\"k1\":\"k1-99\", \"k2\":\"k2-99\"},99\n"; + //Projection + res = executeString("select col3, col2, rk from external_hbase_mapped_table where rk > 95"); - assertEquals(expected, resultSetToString(res)); - res.close(); + String expected = "col3,col2,rk\n" + + "-------------------------------\n" + + "96,{\"k1\":\"k1-96\", \"k2\":\"k2-96\"},96\n" + + "97,{\"k1\":\"k1-97\", \"k2\":\"k2-97\"},97\n" + + "98,{\"k1\":\"k1-98\", \"k2\":\"k2-98\"},98\n" + + "99,{\"k1\":\"k1-99\", \"k2\":\"k2-99\"},99\n"; + assertEquals(expected, resultSetToString(res)); + res.close(); + + } finally { + executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); + htable.close(); + } } finally { - executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); - htable.close(); + //TablespaceManager.addTableSpaceForTest(existing.get()); } } @Test public void testColumnKeyValueSelectQuery() throws Exception { - HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); - hTableDesc.addFamily(new HColumnDescriptor("col2")); - hTableDesc.addFamily(new HColumnDescriptor("col3")); - testingCluster.getHBaseUtil().createTable(hTableDesc); - - String sql = String.format( - "CREATE EXTERNAL TABLE external_hbase_mapped_table (rk1 text, col2_key text, col2_value text, col3 text) " + - "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col2:key:,col2:value:,col3:', " + - "'hbase.rowkey.delimiter'='_') LOCATION '%s/external_hbase_table'", tableSpaceUri); - executeString(sql).close(); + Optional existing = TablespaceManager.removeTablespaceForTest("cluster1"); + assertTrue(existing.isPresent()); - assertTableExists("external_hbase_mapped_table"); + try { + HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); + hTableDesc.addFamily(new HColumnDescriptor("col2")); + hTableDesc.addFamily(new HColumnDescriptor("col3")); + testingCluster.getHBaseUtil().createTable(hTableDesc); + + String sql = String.format( + "CREATE EXTERNAL TABLE external_hbase_mapped_table (rk1 text, col2_key text, col2_value text, col3 text) " + + "USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col2:key:,col2:value:,col3:', " + + "'hbase.rowkey.delimiter'='_') LOCATION '%s/external_hbase_table'", tableSpaceUri); + executeString(sql).close(); - HBaseTablespace space = (HBaseTablespace) TablespaceManager.getByName("cluster1").get(); - HConnection hconn = space.getConnection(); - HTableInterface htable = hconn.getTable("external_hbase_table"); + assertTableExists("external_hbase_mapped_table"); - try { - for (int i = 0; i < 10; i++) { - Put put = new Put(Bytes.toBytes("rk-" + i)); - for (int j = 0; j < 5; j++) { - put.add("col2".getBytes(), ("key-" + j).getBytes(), Bytes.toBytes("value-" + j)); + HConnection hconn = ((HBaseTablespace)existing.get()).getConnection(); + HTableInterface htable = hconn.getTable("external_hbase_table"); + + try { + for (int i = 0; i < 10; i++) { + Put put = new Put(Bytes.toBytes("rk-" + i)); + for (int j = 0; j < 5; j++) { + put.add("col2".getBytes(), ("key-" + j).getBytes(), Bytes.toBytes("value-" + j)); + } + put.add("col3".getBytes(), "".getBytes(), ("col3-value-" + i).getBytes()); + htable.put(put); } - put.add("col3".getBytes(), "".getBytes(), ("col3-value-" + i).getBytes()); - htable.put(put); - } - ResultSet res = executeString("select * from external_hbase_mapped_table where rk1 >= 'rk-0'"); - assertResultSet(res); - cleanupQuery(res); + ResultSet res = executeString("select * from external_hbase_mapped_table where rk1 >= 'rk-0'"); + assertResultSet(res); + cleanupQuery(res); + } finally { + executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); + htable.close(); + } } finally { - executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); - htable.close(); + TablespaceManager.addTableSpaceForTest(existing.get()); } } @Test public void testRowFieldSelectQuery() throws Exception { - HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); - hTableDesc.addFamily(new HColumnDescriptor("col3")); - testingCluster.getHBaseUtil().createTable(hTableDesc); - - String sql = String.format( - "CREATE EXTERNAL TABLE external_hbase_mapped_table (rk1 text, rk2 text, col3 text) " + - "USING hbase WITH ('table'='external_hbase_table', 'columns'='0:key,1:key,col3:a', " + - "'hbase.rowkey.delimiter'='_') LOCATION '%s/external_hbase_table'", tableSpaceUri); - executeString(sql).close(); + Optional existing = TablespaceManager.removeTablespaceForTest("cluster1"); + assertTrue(existing.isPresent()); - assertTableExists("external_hbase_mapped_table"); + try { + HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); + hTableDesc.addFamily(new HColumnDescriptor("col3")); + testingCluster.getHBaseUtil().createTable(hTableDesc); + + String sql = String.format( + "CREATE EXTERNAL TABLE external_hbase_mapped_table (rk1 text, rk2 text, col3 text) " + + "USING hbase WITH ('table'='external_hbase_table', 'columns'='0:key,1:key,col3:a', " + + "'hbase.rowkey.delimiter'='_') LOCATION '%s/external_hbase_table'", tableSpaceUri); + executeString(sql).close(); - HBaseTablespace space = (HBaseTablespace) TablespaceManager.getByName("cluster1").get(); - HConnection hconn = space.getConnection(); - HTableInterface htable = hconn.getTable("external_hbase_table"); + assertTableExists("external_hbase_mapped_table"); - try { - for (int i = 0; i < 100; i++) { - Put put = new Put(("field1-" + i + "_field2-" + i).getBytes()); - put.add("col3".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); - htable.put(put); - } - ResultSet res = executeString("select * from external_hbase_mapped_table where rk1 > 'field1-20'"); - assertResultSet(res); - cleanupQuery(res); + HConnection hconn = ((HBaseTablespace)existing.get()).getConnection(); + HTableInterface htable = hconn.getTable("external_hbase_table"); + + try { + for (int i = 0; i < 100; i++) { + Put put = new Put(("field1-" + i + "_field2-" + i).getBytes()); + put.add("col3".getBytes(), "a".getBytes(), ("a-" + i).getBytes()); + htable.put(put); + } + + ResultSet res = executeString("select * from external_hbase_mapped_table where rk1 > 'field1-20'"); + assertResultSet(res); + cleanupQuery(res); + } finally { + executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); + htable.close(); + } } finally { - executeString("DROP TABLE external_hbase_mapped_table PURGE").close(); - htable.close(); + TablespaceManager.addTableSpaceForTest(existing.get()); } } diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java index a1fa8570e3..c86537ad9b 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java @@ -319,6 +319,18 @@ public static Optional addTableSpaceForTest(Tablespace space) { return Optional.fromNullable(existing); } + @VisibleForTesting + public static Optional removeTablespaceForTest(String name) { + Tablespace existing = null; + synchronized (SPACES_URIS_MAP) { + URI uri = SPACES_URIS_MAP.remove(name); + if (uri != null) { + existing = TABLE_SPACES.remove(uri); + } + } + return Optional.fromNullable(existing); + } + public Iterable getSupportSchemes() { return TABLE_SPACE_HANDLERS.keySet(); } @@ -345,6 +357,16 @@ public static Optional get(@Nullable String uri) { lastOne = entry.getValue(); } } + + if (lastOne == null) { + lastOne = initializeTableSpace(UUID.randomUUID().toString(), URI.create(uri), new JSONObject()); + try { + lastOne.init(systemConf); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return (Optional) Optional.fromNullable(lastOne); } diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java index cf4fe123e5..574697db03 100644 --- a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java +++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java @@ -18,6 +18,7 @@ package org.apache.tajo.storage.pgsql; +import com.google.common.base.Optional; import net.minidev.json.JSONObject; import org.apache.tajo.catalog.MetadataProvider; import org.apache.tajo.catalog.TableDesc; @@ -111,8 +112,6 @@ public void testConnPropertiesNegative() throws Exception { public static JSONObject getJsonTablespace(Map connProperties) throws IOException { - String uri = PgSQLTestServer.getInstance().getJdbcUrl().split("\\?")[0]; - JSONObject configElements = new JSONObject(); configElements.put(JdbcTablespace.CONFIG_KEY_MAPPED_DATABASE, PgSQLTestServer.DATABASE_NAME); @@ -124,4 +123,15 @@ public static JSONObject getJsonTablespace(Map connProperties) return configElements; } + + @Test + public void testTemporaryTablespace() { + Optional ts = TablespaceManager.removeTablespaceForTest("pgsql_cluster"); + assertTrue(ts.isPresent()); + + Optional tempTs = TablespaceManager.get(jdbcUrl); + assertTrue(tempTs.isPresent()); + + TablespaceManager.addTableSpaceForTest(ts.get()); + } } diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java index 9655968874..ddae4b5a87 100644 --- a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java +++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java @@ -30,6 +30,8 @@ import java.util.HashMap; import java.util.Map; +import static org.junit.Assert.assertTrue; + public class TestPgSQLQueryTests extends QueryTestCaseBase { @SuppressWarnings("unused") // This should be invoked for initializing PgSQLTestServer From 3b9a1d1f4e37f153f72421119d381c1e66d954f6 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 17 Sep 2015 22:31:35 -0700 Subject: [PATCH 02/10] Restored some codes and removed unused import. --- .../java/org/apache/tajo/engine/query/TestHBaseTable.java | 6 +++--- .../org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java | 3 --- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index ceae92bb3e..ca06b5c794 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -268,8 +268,8 @@ public void testSimpleSelectQuery() throws Exception { @Test public void testBinaryMappedQuery() throws Exception { - //Optional existing = TablespaceManager.removeTablespaceForTest("cluster1"); - //assertTrue(existing.isPresent()); + Optional existing = TablespaceManager.removeTablespaceForTest("cluster1"); + assertTrue(existing.isPresent()); try { HTableDescriptor hTableDesc = new HTableDescriptor(TableName.valueOf("external_hbase_table")); @@ -323,7 +323,7 @@ public void testBinaryMappedQuery() throws Exception { htable.close(); } } finally { - //TablespaceManager.addTableSpaceForTest(existing.get()); + TablespaceManager.addTableSpaceForTest(existing.get()); } } diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java index ddae4b5a87..5b52105cd7 100644 --- a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java +++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java @@ -26,12 +26,9 @@ import org.junit.BeforeClass; import org.junit.Test; -import java.io.IOException; import java.util.HashMap; import java.util.Map; -import static org.junit.Assert.assertTrue; - public class TestPgSQLQueryTests extends QueryTestCaseBase { @SuppressWarnings("unused") // This should be invoked for initializing PgSQLTestServer From a44022dbc071f19624328796fa136a4aa2bd3aae Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 17 Sep 2015 22:36:51 -0700 Subject: [PATCH 03/10] fixed unit test failure. --- .../test/java/org/apache/tajo/engine/query/TestHBaseTable.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index ca06b5c794..16ddb769b9 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -286,8 +286,7 @@ public void testBinaryMappedQuery() throws Exception { assertTableExists("external_hbase_mapped_table"); - HBaseTablespace space = (HBaseTablespace) TablespaceManager.getByName("cluster1").get(); - HConnection hconn = space.getConnection(); + HConnection hconn = ((HBaseTablespace)existing.get()).getConnection(); HTableInterface htable = hconn.getTable("external_hbase_table"); try { From f19c0901de2de094ff1152c3f6927f3087919038 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 18 Sep 2015 01:12:46 -0700 Subject: [PATCH 04/10] Add s3a and s3n. --- .../src/main/resources/storage-default.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json index 3ede2d4eb5..4cef1a5657 100644 --- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json +++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json @@ -12,6 +12,14 @@ "handler": "org.apache.tajo.storage.FileTablespace", "default-format": "text" }, + "s3a": { + "handler": "org.apache.tajo.storage.FileTablespace", + "default-format": "text" + }, + "s3n": { + "handler": "org.apache.tajo.storage.FileTablespace", + "default-format": "text" + }, "hbase:zk": { "handler": "org.apache.tajo.storage.hbase.HBaseTablespace", "default-format": "hbase" From 62d954de4a92aca5a813dd89e101a846036d28f5 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 18 Sep 2015 01:17:14 -0700 Subject: [PATCH 05/10] Add switft scheme to storage-default.json. --- .../src/main/resources/storage-default.json | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json index 4cef1a5657..a24f301332 100644 --- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json +++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json @@ -1,13 +1,21 @@ { "storages": { - "hdfs": { + "file": { "handler": "org.apache.tajo.storage.FileTablespace", "default-format": "text" }, - "file": { + "hbase:zk": { + "handler": "org.apache.tajo.storage.hbase.HBaseTablespace", + "default-format": "hbase" + }, + "hdfs": { "handler": "org.apache.tajo.storage.FileTablespace", "default-format": "text" }, + "jdbc:postgresql": { + "handler": "org.apache.tajo.storage.pgsql.PgSQLTablespace", + "default-format": "rowstore" + }, "s3": { "handler": "org.apache.tajo.storage.FileTablespace", "default-format": "text" @@ -20,13 +28,9 @@ "handler": "org.apache.tajo.storage.FileTablespace", "default-format": "text" }, - "hbase:zk": { - "handler": "org.apache.tajo.storage.hbase.HBaseTablespace", - "default-format": "hbase" - }, - "jdbc:postgresql": { - "handler": "org.apache.tajo.storage.pgsql.PgSQLTablespace", - "default-format": "rowstore" + "swift": { + "handler": "org.apache.tajo.storage.FileTablespace", + "default-format": "text" } } } \ No newline at end of file From 8cb3bc004ecc8bb6859fe117ec7667a9c283f659 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 21 Sep 2015 23:18:37 -0700 Subject: [PATCH 06/10] Removed optional and changed them to throw exceptions. --- .../apache/tajo/exception/ErrorMessages.java | 1 + .../apache/tajo/exception/ExceptionUtil.java | 1 + .../tajo/exception/SQLExceptionUtil.java | 1 + .../UndefinedTablespaceHandlerException.java | 34 +++++++++++++++++++ tajo-common/src/main/proto/errors.proto | 1 + .../org/apache/tajo/storage/TestRowFile.java | 2 +- .../engine/planner/PhysicalPlannerImpl.java | 2 +- .../physical/ColPartitionStoreExec.java | 2 +- .../planner/physical/InsertRowsExec.java | 2 +- .../engine/planner/physical/SeqScanExec.java | 2 +- .../planner/physical/StoreTableExec.java | 9 ++--- .../org/apache/tajo/master/GlobalEngine.java | 2 +- .../tajo/master/exec/CreateTableExecutor.java | 7 +--- .../apache/tajo/master/exec/DDLExecutor.java | 4 +-- .../NonForwardQueryResultFileScanner.java | 2 +- .../tajo/master/exec/QueryExecutor.java | 6 ++-- .../org/apache/tajo/querymaster/Query.java | 4 +-- .../tajo/querymaster/QueryMasterTask.java | 4 +-- .../tajo/querymaster/Repartitioner.java | 6 ++-- .../org/apache/tajo/querymaster/Stage.java | 2 +- .../java/org/apache/tajo/worker/TaskImpl.java | 2 +- .../tajo/storage/TablespaceManager.java | 23 +++++++------ .../tajo/storage/hbase/HBasePutAppender.java | 2 +- .../tajo/storage/hbase/HBaseScanner.java | 4 +-- .../storage/hbase/TestHBaseTableSpace.java | 2 +- .../org/apache/tajo/storage/FileAppender.java | 8 +---- .../apache/tajo/storage/FileTablespace.java | 7 +--- .../storage/HashShuffleAppenderManager.java | 2 +- .../tajo/storage/TestFileTablespace.java | 2 +- .../pgsql/TestPgSQLJdbcTableSpace.java | 12 +++---- .../pgsql/TestPgSQLMetadataProvider.java | 10 +++--- 31 files changed, 94 insertions(+), 74 deletions(-) create mode 100644 tajo-common/src/main/java/org/apache/tajo/exception/UndefinedTablespaceHandlerException.java diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java index 96496447e1..b7fe0b87b3 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java @@ -72,6 +72,7 @@ public class ErrorMessages { ADD_MESSAGE(UNDEFINED_INDEX_FOR_TABLE, "index ''%s' does not exist", 1); ADD_MESSAGE(UNDEFINED_INDEX_FOR_COLUMNS, "index does not exist for '%s' columns of '%s' table", 2); ADD_MESSAGE(UNDEFINED_INDEX_NAME, "index name '%s' does not exist", 1); + ADD_MESSAGE(UNDEFINED_TABLESPACE_HANDLER, "No tablespace for URI scheme '%s'", 1); ADD_MESSAGE(DUPLICATE_TABLESPACE, "tablespace '%s' already exists", 1); ADD_MESSAGE(DUPLICATE_DATABASE, "database '%s' already exists", 1); diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java index eea85fb70a..59c57765fa 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java @@ -59,6 +59,7 @@ public class ExceptionUtil { ADD_EXCEPTION(UNDEFINED_PARTITION_KEY, UndefinedPartitionKeyException.class); ADD_EXCEPTION(UNDEFINED_OPERATOR, UndefinedOperatorException.class); ADD_EXCEPTION(UNDEFINED_INDEX_NAME, UndefinedIndexException.class); + ADD_EXCEPTION(UNDEFINED_TABLESPACE_HANDLER, UndefinedTablespaceHandlerException.class); ADD_EXCEPTION(DUPLICATE_TABLESPACE, DuplicateTablespaceException.class); ADD_EXCEPTION(DUPLICATE_DATABASE, DuplicateDatabaseException.class); diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/SQLExceptionUtil.java b/tajo-common/src/main/java/org/apache/tajo/exception/SQLExceptionUtil.java index b5708e13b4..4b4ec97af4 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/SQLExceptionUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/SQLExceptionUtil.java @@ -72,6 +72,7 @@ public class SQLExceptionUtil { SQLSTATES.put(ResultCode.UNDEFINED_PARTITION_METHOD, "42T06"); SQLSTATES.put(ResultCode.UNDEFINED_OPERATOR, "42883"); // == UNDEFINED_FUNCTION SQLSTATES.put(ResultCode.UNDEFINED_PARTITION_KEY, "42T07"); + SQLSTATES.put(ResultCode.UNDEFINED_TABLESPACE_HANDLER, "42T11"); SQLSTATES.put(ResultCode.DUPLICATE_TABLESPACE, "42T08"); SQLSTATES.put(ResultCode.DUPLICATE_DATABASE, "42P04"); diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedTablespaceHandlerException.java b/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedTablespaceHandlerException.java new file mode 100644 index 0000000000..f1090d366a --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/exception/UndefinedTablespaceHandlerException.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.exception; + +import org.apache.tajo.error.Errors; +import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState; + +public class UndefinedTablespaceHandlerException extends TajoException { + private static final long serialVersionUID = 277182608283894937L; + + public UndefinedTablespaceHandlerException(ReturnState state) { + super(state); + } + + public UndefinedTablespaceHandlerException(String spaceName) { + super(Errors.ResultCode.UNDEFINED_TABLESPACE_HANDLER, spaceName); + } +} diff --git a/tajo-common/src/main/proto/errors.proto b/tajo-common/src/main/proto/errors.proto index 6a1780be17..bb973f2afb 100644 --- a/tajo-common/src/main/proto/errors.proto +++ b/tajo-common/src/main/proto/errors.proto @@ -113,6 +113,7 @@ enum ResultCode { UNDEFINED_PARTITION_METHOD = 521; // ? UNDEFINED_OPERATOR = 522; // SQLState: 42883 (=UNDEFINED_FUNCTION) UNDEFINED_PARTITION_KEY = 523; // ? + UNDEFINED_TABLESPACE_HANDLER = 524; // SQLState: 42T11 - No Tablespace Handler for the URI scheme DUPLICATE_TABLESPACE = 531; DUPLICATE_DATABASE = 532; // SQLState: 42P04 diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java b/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java index c1d5a97b34..011803d4c0 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java @@ -71,7 +71,7 @@ public void test() throws IOException { TableMeta meta = CatalogUtil.newTableMeta("ROWFILE"); - FileTablespace sm = (FileTablespace) TablespaceManager.get(cluster.getDefaultFileSystem().getUri()).get(); + FileTablespace sm = (FileTablespace) TablespaceManager.get(cluster.getDefaultFileSystem().getUri()); Path tablePath = new Path("/test"); Path dataPath = new Path(tablePath, "test.tbl"); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 36d80da8e1..461e4055bb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -932,7 +932,7 @@ public PhysicalExec createScanPlan(TaskAttemptContext ctx, ScanNode scanNode, St PartitionedTableScanNode partitionedTableScanNode = (PartitionedTableScanNode) scanNode; List fileFragments = TUtil.newList(); - FileTablespace space = (FileTablespace) TablespaceManager.get(scanNode.getTableDesc().getUri()).get(); + FileTablespace space = (FileTablespace) TablespaceManager.get(scanNode.getTableDesc().getUri()); for (Path path : partitionedTableScanNode.getInputPaths()) { fileFragments.addAll(TUtil.newList(space.split(scanNode.getCanonicalName(), path))); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java index a8a1c787ac..01d3c0fd9d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java @@ -197,7 +197,7 @@ public void openAppender(int suffixId) throws IOException { actualFilePath = new Path(lastFileName + "_" + suffixId); } - appender = ((FileTablespace) TablespaceManager.get(lastFileName.toUri()).get()) + appender = ((FileTablespace) TablespaceManager.get(lastFileName.toUri())) .getAppender(meta, outSchema, actualFilePath); appender.enableStats(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/InsertRowsExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/InsertRowsExec.java index f3a24a70e6..6c2f7a5e98 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/InsertRowsExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/InsertRowsExec.java @@ -65,7 +65,7 @@ public void init() throws IOException { sumStats = new TableStats(); StoreTableNode storeTableNode = (StoreTableNode) plan; - appender = TablespaceManager.get(storeTableNode.getUri()).get().getAppenderForInsertRow( + appender = TablespaceManager.get(storeTableNode.getUri()).getAppenderForInsertRow( context.getQueryContext(), context.getTaskId(), meta, storeTableNode.getTableSchema(), context.getOutputPath()); appender.enableStats(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java index c2a6e996a4..a41d20acfc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java @@ -268,7 +268,7 @@ private void initScanner(Schema projected) throws IOException { } else { - Tablespace tablespace = TablespaceManager.get(table.getUri()).get(); + Tablespace tablespace = TablespaceManager.get(table.getUri()); this.scanner = tablespace.getScanner( meta, plan.getPhysicalSchema(), diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index c317f7fda5..8db1f5d0b7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -93,12 +93,7 @@ public void openNewFile(int suffixId) throws IOException { lastFileName = new Path(lastFileName + "_" + suffixId); } - Optional spaceRes = TablespaceManager.get(lastFileName.toUri()); - if (!spaceRes.isPresent()) { - throw new IllegalStateException("No Tablespace for " + lastFileName.toUri()); - } - - FileTablespace space = spaceRes.get(); + FileTablespace space = TablespaceManager.get(lastFileName.toUri()); appender = space.getAppender(meta, appenderSchema, lastFileName); if (suffixId > 0) { @@ -107,7 +102,7 @@ public void openNewFile(int suffixId) throws IOException { } } else { Path stagingDir = context.getQueryContext().getStagingDir(); - appender = TablespaceManager.get(stagingDir.toUri()).get().getAppender( + appender = TablespaceManager.get(stagingDir.toUri()).getAppender( context.getQueryContext(), context.getTaskId(), meta, diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 0d14d9a0a7..d5e5dd5fef 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -312,7 +312,7 @@ private void verifyInsertTableSchema(VerificationState state, LogicalPlan plan) InsertNode iNode = rootNode.getChild(); Schema outSchema = iNode.getChild().getOutSchema(); - TablespaceManager.get(tableDesc.getUri()).get().verifySchemaToWrite(tableDesc, outSchema); + TablespaceManager.get(tableDesc.getUri()).verifySchemaToWrite(tableDesc, outSchema); } catch (TajoException t) { state.addVerification(t); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java index 24f420cc3f..f907677e3e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java @@ -143,12 +143,7 @@ private Tablespace getTablespaceHandler(@Nullable String tableSpaceName, @Nullab throw new UndefinedTablespaceException(tableSpaceName); } } else if (tableUri != null) { - Optional ts = TablespaceManager.get(tableUri); - if (ts.isPresent()) { - return ts.get(); - } else { - throw new UndefinedTablespaceException(tableUri.toString()); - } + return TablespaceManager.get(tableUri); } else { return TablespaceManager.getDefault(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index c5d583878c..15abf9e120 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -311,7 +311,7 @@ public void dropTable(QueryContext queryContext, String tableName, boolean ifExi if (purge) { try { - TablespaceManager.get(tableDesc.getUri()).get().purgeTable(tableDesc); + TablespaceManager.get(tableDesc.getUri()).purgeTable(tableDesc); } catch (IOException e) { throw new InternalError(e.getMessage()); } @@ -355,7 +355,7 @@ public void truncateTable(final QueryContext queryContext, final TruncateTableNo new UnsupportedException("table truncation on an external table '" + eachTableName + "'")); } - Tablespace space = TablespaceManager.get(tableDesc.getUri()).get(); + Tablespace space = TablespaceManager.get(tableDesc.getUri()); if (space instanceof FileTablespace) { tableDescList.add(tableDesc); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java index ac40c4fc56..e520cdbc56 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java @@ -102,7 +102,7 @@ public void init() throws IOException, TajoException { } private void initSeqScanExec() throws IOException, TajoException { - Tablespace tablespace = TablespaceManager.get(tableDesc.getUri()).get(); + Tablespace tablespace = TablespaceManager.get(tableDesc.getUri()); List fragments = Lists.newArrayList(); if (tableDesc.hasPartition()) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index ffdb3f01a6..d5341d80db 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -458,7 +458,7 @@ private void insertRowValues(QueryContext queryContext, String queryId = nodeUniqName + "_" + System.currentTimeMillis(); URI finalOutputUri = insertNode.getUri(); - Tablespace space = TablespaceManager.get(finalOutputUri).get(); + Tablespace space = TablespaceManager.get(finalOutputUri); TableMeta tableMeta = new TableMeta(insertNode.getStorageType(), insertNode.getOptions()); tableMeta.putOption(StorageConstants.INSERT_DIRECTLY, Boolean.TRUE.toString()); @@ -564,7 +564,7 @@ private void prepareForCreateTableOrInsert(CatalogService catalog, LogicalPlan p TableDesc tableDesc = PlannerUtil.getTableDesc(catalog, plan.getRootBlock().getRoot()); if (tableDesc != null) { - Tablespace space = TablespaceManager.get(tableDesc.getUri()).get(); + Tablespace space = TablespaceManager.get(tableDesc.getUri()); FormatProperty formatProperty = space.getFormatProperty(tableDesc.getMeta()); if (!formatProperty.isInsertable()) { @@ -603,7 +603,7 @@ public MasterPlan compileMasterPlan(LogicalPlan plan, QueryContext context, Glob TableDesc tableDesc = PlannerUtil.getTableDesc(planner.getCatalog(), rootNode.getChild()); if (tableDesc != null) { - Tablespace space = TablespaceManager.get(tableDesc.getUri()).get(); + Tablespace space = TablespaceManager.get(tableDesc.getUri()); space.rewritePlan(context, plan); } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index 1ec45d183d..f06d28c20c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -470,7 +470,7 @@ private void handleQueryFailure(Query query, Stage lastStage) { QueryContext context = query.context.getQueryContext(); if (lastStage != null && context.hasOutputTableUri()) { - Tablespace space = TablespaceManager.get(context.getOutputTableUri()).get(); + Tablespace space = TablespaceManager.get(context.getOutputTableUri()); try { LogicalRootNode rootNode = lastStage.getMasterPlan().getLogicalPlan().getRootBlock().getRoot(); space.rollbackTable(rootNode.getChild()); @@ -493,7 +493,7 @@ private QueryState finalizeQuery(Query query, QueryCompletedEvent event) { // If there is not tabledesc, it is a select query without insert or ctas. // In this case, we should use default tablespace. - Tablespace space = TablespaceManager.get(queryContext.get(QueryVars.OUTPUT_TABLE_URI, "")).get(); + Tablespace space = TablespaceManager.get(queryContext.get(QueryVars.OUTPUT_TABLE_URI, "")); Path finalOutputDir = space.commitTable( query.context.getQueryContext(), diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java index 1313dade48..46e48e6876 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java @@ -322,7 +322,7 @@ public synchronized void startQuery() { optimizer.optimize(queryContext, plan); // when a given uri is null, TablespaceManager.get will return the default tablespace. - space = TablespaceManager.get(queryContext.get(QueryVars.OUTPUT_TABLE_URI, "")).get(); + space = TablespaceManager.get(queryContext.get(QueryVars.OUTPUT_TABLE_URI, "")); space.rewritePlan(queryContext, plan); initStagingDir(); @@ -379,7 +379,7 @@ private void initStagingDir() throws IOException { URI stagingDir; try { - Tablespace tablespace = TablespaceManager.get(queryContext.get(QueryVars.OUTPUT_TABLE_URI, "")).get(); + Tablespace tablespace = TablespaceManager.get(queryContext.get(QueryVars.OUTPUT_TABLE_URI, "")); TableDesc desc = PlannerUtil.getOutputTableDesc(plan); FormatProperty formatProperty = tablespace.getFormatProperty(desc.getMeta()); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java index 3ce86b155c..bddf198a84 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java @@ -113,7 +113,7 @@ public static void scheduleFragmentsForJoinQuery(TaskSchedulerContext schedulerC // if table has no data, tablespace will return empty FileFragment. // So, we need to handle FileFragment by its size. // If we don't check its size, it can cause IndexOutOfBoundsException. - Tablespace space = TablespaceManager.get(tableDesc.getUri()).get(); + Tablespace space = TablespaceManager.get(tableDesc.getUri()); List fileFragments = space.getSplits(scans[i].getCanonicalName(), tableDesc, null); if (fileFragments.size() > 0) { fragments[i] = fileFragments.get(0); @@ -378,7 +378,7 @@ private static void scheduleSymmetricRepartitionJoin(QueryMasterTask.QueryMaster Path[] partitionScanPaths = null; TableDesc tableDesc = masterContext.getTableDesc(eachScan); - Tablespace space = TablespaceManager.get(tableDesc.getUri()).get(); + Tablespace space = TablespaceManager.get(tableDesc.getUri()); if (eachScan.getType() == NodeType.PARTITIONS_SCAN) { @@ -507,7 +507,7 @@ private static void scheduleLeafTasksWithBroadcastTable(TaskSchedulerContext sch Path[] partitionScanPaths = null; - Tablespace space = TablespaceManager.get(desc.getUri()).get(); + Tablespace space = TablespaceManager.get(desc.getUri()); if (scan.getType() == NodeType.PARTITIONS_SCAN) { PartitionedTableScanNode partitionScan = (PartitionedTableScanNode) scan; diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index 125c83cca9..282edcc81c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -1132,7 +1132,7 @@ private static void scheduleFragmentsForLeafQuery(Stage stage) throws IOExceptio TableDesc table = stage.context.getTableDesc(scan); Collection fragments; - Tablespace tablespace = TablespaceManager.get(scan.getTableDesc().getUri()).get(); + Tablespace tablespace = TablespaceManager.get(scan.getTableDesc().getUri()); // Depending on scanner node's type, it creates fragments. If scan is for // a partitioned table, It will creates lots fragments for all partitions. diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java index 4e3a8bfc63..a8e874c029 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java @@ -142,7 +142,7 @@ public void initPlan() throws IOException { this.sortComp = new BaseTupleComparator(finalSchema, sortNode.getSortKeys()); } } else { - Path outFilePath = ((FileTablespace) TablespaceManager.get(queryContext.getStagingDir().toUri()).get()) + Path outFilePath = ((FileTablespace) TablespaceManager.get(queryContext.getStagingDir().toUri())) .getAppenderFilePath(getId(), queryContext.getStagingDir()); LOG.info("Output File Path: " + outFilePath); context.setOutputPath(outFilePath); diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java index c86537ad9b..f881b2862c 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java @@ -33,6 +33,8 @@ import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.MetadataProvider; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UndefinedTablespaceHandlerException; import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.JavaResourceUtil; @@ -276,10 +278,11 @@ public static long guessFragmentVolume(TajoConf conf, Fragment fragment) { public static final String KEY_SPACES = "spaces"; private static Tablespace initializeTableSpace(String spaceName, URI uri, JSONObject spaceDesc) { - Class clazz = TABLE_SPACE_HANDLERS.get(UriUtil.getScheme(uri)); + final String scheme = UriUtil.getScheme(uri); + Class clazz = TABLE_SPACE_HANDLERS.get(scheme); if (clazz == null) { - throw new RuntimeException("Not found Tablespace handler for " + uri.toString()); + throw new TajoRuntimeException(new UndefinedTablespaceHandlerException(scheme)); } try { @@ -342,10 +345,10 @@ public Iterable getSupportSchemes() { * @param Tablespace class type * @return Tablespace. If uri is null, the default tablespace will be returned. */ - public static Optional get(@Nullable String uri) { + public static T get(@Nullable String uri) { if (uri == null || uri.isEmpty()) { - return (Optional) Optional.of(getDefault()); + return getDefault(); } Tablespace lastOne = null; @@ -367,7 +370,7 @@ public static Optional get(@Nullable String uri) { } } - return (Optional) Optional.fromNullable(lastOne); + return (T) lastOne; } /** @@ -377,11 +380,11 @@ public static Optional get(@Nullable String uri) { * @param Tablespace class type * @return Tablespace. If uri is null, the default tablespace will be returned. */ - public static Optional get(@Nullable URI uri) { + public static T get(@Nullable URI uri) { if (uri == null) { - return (Optional) Optional.of(getDefault()); + return getDefault(); } else { - return (Optional) get(uri.toString()); + return (T) get(uri.toString()); } } @@ -395,7 +398,7 @@ public static T getDefault() { } public static T getLocalFs() { - return (T) get(LOCAL_FS_URI).get(); + return (T) get(LOCAL_FS_URI); } public static Optional getByName(String name) { @@ -426,7 +429,7 @@ public URI getTableURI(@Nullable String spaceName, String databaseName, String t @Override public long getTableVolumn(URI tableUri) throws UnsupportedException { - return get(tableUri).get().getTableVolume(tableUri); + return get(tableUri).getTableVolume(tableUri); } public static Iterable getAllTablespaces() { diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java index 7943134beb..20b1a088ab 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java @@ -47,7 +47,7 @@ public HBasePutAppender(Configuration conf, URI uri, TaskAttemptId taskAttemptId public void init() throws IOException { super.init(); - HBaseTablespace space = (HBaseTablespace) TablespaceManager.get(uri).get(); + HBaseTablespace space = (HBaseTablespace) TablespaceManager.get(uri); HConnection hconn = space.getConnection(); htable = hconn.getTable(columnMapping.getHbaseTableName()); htable.setAutoFlushTo(false); diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java index 90f7aa0105..55d27a1b6b 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java @@ -144,7 +144,7 @@ public void init() throws IOException { rowKeyDelimiter = columnMapping.getRowKeyDelimiter(); rowKeyFieldIndexes = columnMapping.getRowKeyFieldIndexes(); - HBaseTablespace space = (HBaseTablespace) TablespaceManager.get(fragment.getUri()).get(); + HBaseTablespace space = (HBaseTablespace) TablespaceManager.get(fragment.getUri()); hbaseConf = space.getHbaseConf(); initScanner(); } @@ -192,7 +192,7 @@ private void initScanner() throws IOException { } if (htable == null) { - HConnection hconn = ((HBaseTablespace) TablespaceManager.get(fragment.getUri()).get()).getConnection(); + HConnection hconn = ((HBaseTablespace) TablespaceManager.get(fragment.getUri())).getConnection(); htable = hconn.getTable(fragment.getHbaseTableName()); } scanner = htable.getScanner(scan); diff --git a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java index 56ca9be884..fb72e57b77 100644 --- a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java +++ b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java @@ -59,7 +59,7 @@ public void testExtractQuorum() { @Test public void testTablespaceHandler() throws Exception { assertTrue((TablespaceManager.getByName("cluster1").get()) instanceof HBaseTablespace); - assertTrue((TablespaceManager.get(URI.create("hbase:zk://host1:2171")).get()) + assertTrue((TablespaceManager.get(URI.create("hbase:zk://host1:2171"))) instanceof HBaseTablespace); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileAppender.java index efe2bfdc6d..eeeadcba7d 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileAppender.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileAppender.java @@ -58,13 +58,7 @@ public FileAppender(Configuration conf, TaskAttemptId taskAttemptId, Schema sche throw new IllegalArgumentException("Configuration must be an instance of TajoConf"); } - Optional spaceResult = TablespaceManager.get(workDir.toUri()); - - if (!spaceResult.isPresent()) { - throw new IllegalStateException("No TableSpace for " + workDir.toUri()); - } - - FileTablespace space = spaceResult.get(); + FileTablespace space = TablespaceManager.get(workDir.toUri()); this.path = space.getAppenderFilePath(taskAttemptId, workDir); } else { diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java index af981bbf83..1b0952ea8e 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java @@ -720,12 +720,7 @@ public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta // for temporarily written in the storage directory stagingDir = fs.makeQualified(new Path(stagingRootPath, queryId)); } else { - Optional spaceResult = TablespaceManager.get(outputPath); - if (!spaceResult.isPresent()) { - throw new IOException("No registered Tablespace for " + outputPath); - } - - Tablespace space = spaceResult.get(); + Tablespace space = TablespaceManager.get(outputPath); if (space.getProperty().isMovable()) { // checking if this tablespace allows MOVE operation // If this space allows move operation, the staging directory will be underneath the final output table uri. stagingDir = fs.makeQualified(StorageUtil.concatPath(outputPath, TMP_STAGING_DIR_PREFIX, queryId)); diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java index 1d32291535..b118044623 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java @@ -85,7 +85,7 @@ public HashShuffleAppender getAppender(TajoConf tajoConf, ExecutionBlockId ebId, fs.mkdirs(dataFile.getParent()); } - FileTablespace space = (FileTablespace) TablespaceManager.get(dataFile.toUri()).get(); + FileTablespace space = (FileTablespace) TablespaceManager.get(dataFile.toUri()); FileAppender appender = (FileAppender) space.getAppender(meta, outSchema, dataFile); appender.enableStats(); appender.init(); diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java index f536514203..7ff535cd22 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java @@ -282,7 +282,7 @@ public void testGetFileTablespace() throws Exception { existingTs = TablespaceManager.addTableSpaceForTest(distTablespace); /* Distributed FileSystem */ - space = (FileTablespace) TablespaceManager.get(uri).get(); + space = (FileTablespace) TablespaceManager.get(uri); assertEquals(cluster.getFileSystem().getUri(), space.getFileSystem().getUri()); space = (FileTablespace) TablespaceManager.getByName("testGetFileTablespace").get(); diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java index 574697db03..090780d8d3 100644 --- a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java +++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java @@ -48,11 +48,11 @@ public void testTablespaceHandler() throws Exception { assertTrue((TablespaceManager.getByName("pgsql_cluster").get()) instanceof PgSQLTablespace); assertEquals("pgsql_cluster", (TablespaceManager.getByName("pgsql_cluster").get().getName())); - assertTrue((TablespaceManager.get(jdbcUrl).get()) instanceof PgSQLTablespace); - assertTrue((TablespaceManager.get(jdbcUrl + "&table=tb1").get()) instanceof PgSQLTablespace); + assertTrue((TablespaceManager.get(jdbcUrl)) instanceof PgSQLTablespace); + assertTrue((TablespaceManager.get(jdbcUrl + "&table=tb1")) instanceof PgSQLTablespace); - assertEquals(jdbcUrl, TablespaceManager.get(jdbcUrl).get().getUri().toASCIIString()); - assertTrue(TablespaceManager.get(jdbcUrl).get().getMetadataProvider() instanceof PgSQLMetadataProvider); + assertEquals(jdbcUrl, TablespaceManager.get(jdbcUrl).getUri().toASCIIString()); + assertTrue(TablespaceManager.get(jdbcUrl).getMetadataProvider() instanceof PgSQLMetadataProvider); } @Test(timeout = 1000, expected = TajoRuntimeException.class) @@ -129,8 +129,8 @@ public void testTemporaryTablespace() { Optional ts = TablespaceManager.removeTablespaceForTest("pgsql_cluster"); assertTrue(ts.isPresent()); - Optional tempTs = TablespaceManager.get(jdbcUrl); - assertTrue(tempTs.isPresent()); + Tablespace tempTs = TablespaceManager.get(jdbcUrl); + assertNotNull(tempTs); TablespaceManager.addTableSpaceForTest(ts.get()); } diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLMetadataProvider.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLMetadataProvider.java index 2221f94e33..2af41792a8 100644 --- a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLMetadataProvider.java +++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLMetadataProvider.java @@ -39,28 +39,28 @@ public static void setUp() throws Exception { @Test public void testGetTablespaceName() throws Exception { - Tablespace tablespace = TablespaceManager.get(jdbcUrl).get(); + Tablespace tablespace = TablespaceManager.get(jdbcUrl); MetadataProvider provider = tablespace.getMetadataProvider(); assertEquals("pgsql_cluster", provider.getTablespaceName()); } @Test public void testGetDatabaseName() throws Exception { - Tablespace tablespace = TablespaceManager.get(jdbcUrl).get(); + Tablespace tablespace = TablespaceManager.get(jdbcUrl); MetadataProvider provider = tablespace.getMetadataProvider(); assertEquals("tpch", provider.getDatabaseName()); } @Test public void testGetSchemas() throws Exception { - Tablespace tablespace = TablespaceManager.get(jdbcUrl).get(); + Tablespace tablespace = TablespaceManager.get(jdbcUrl); MetadataProvider provider = tablespace.getMetadataProvider(); assertTrue(provider.getSchemas().isEmpty()); } @Test public void testGetTables() throws Exception { - Tablespace tablespace = TablespaceManager.get(jdbcUrl).get(); + Tablespace tablespace = TablespaceManager.get(jdbcUrl); MetadataProvider provider = tablespace.getMetadataProvider(); final Set expected = Sets.newHashSet(PgSQLTestServer.TPCH_TABLES); @@ -72,7 +72,7 @@ public void testGetTables() throws Exception { @Test public void testGetTableDescriptor() throws Exception { - Tablespace tablespace = TablespaceManager.get(jdbcUrl).get(); + Tablespace tablespace = TablespaceManager.get(jdbcUrl); MetadataProvider provider = tablespace.getMetadataProvider(); for (String tableName : PgSQLTestServer.TPCH_TABLES) { From 77cf14e25b614439b81434bdc6e19f16911c8219 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 22 Sep 2015 00:05:41 -0700 Subject: [PATCH 07/10] Removed Optional from TablespaceManager::getByName(). --- .../java/org/apache/tajo/client/TestTajoClient.java | 6 ++++++ .../org/apache/tajo/engine/query/TestCreateTable.java | 2 +- .../org/apache/tajo/engine/query/TestHBaseTable.java | 2 +- .../negative/create_table_on_absent_tablespace.sql | 1 + .../apache/tajo/master/exec/CreateTableExecutor.java | 7 +------ .../org/apache/tajo/storage/TablespaceManager.java | 11 ++++++----- .../tajo/storage/hbase/TestHBaseTableSpace.java | 4 ++-- .../org/apache/tajo/storage/TestFileTablespace.java | 2 +- .../tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java | 10 +++++----- 9 files changed, 24 insertions(+), 21 deletions(-) create mode 100644 tajo-core-tests/src/test/resources/queries/TestCreateTable/negative/create_table_on_absent_tablespace.sql diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java index 38819f1580..5616d539b9 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -35,6 +35,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.TajoException; +import org.apache.tajo.exception.UndefinedTablespaceException; import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.ipc.ClientProtos; import org.apache.tajo.ipc.ClientProtos.QueryHistoryProto; @@ -797,4 +798,9 @@ public void testClientRPCInterference() throws Exception { rpcClient2.getChannel().eventLoop().terminationFuture().sync(); assertTrue(rpcClient2.getChannel().eventLoop().isTerminated()); } + + @Test(expected = UndefinedTablespaceException.class) + public void testCreateTableOnAbsentTablespace() throws TajoException { + client.updateQuery("CREATE TABLE testCreateTableOnAbsentTablespace (AGE INT) TABLESPACE unknown123"); + } } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java index ed1026eb36..e33f1735a8 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java @@ -415,7 +415,7 @@ private boolean isClonedTable(String orignalTable, String newTable) throws Excep } if(isClonedSchema(origPartMethod.getExpressionSchema(), newPartMethod.getExpressionSchema()) == false) { - fail("Partition columns of input tables do not match"); + fail("Partition columns of input tables do not match"); return false; } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index 16ddb769b9..59cb3ec57a 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -525,7 +525,7 @@ private void assertIndexPredication(boolean isCompositeRowKey) throws Exception EvalNode evalNodeEq = new BinaryEval(EvalType.EQUAL, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), new ConstEval(new TextDatum("021"))); scanNode.setQual(evalNodeEq); - Tablespace tablespace = TablespaceManager.getByName("cluster1").get(); + Tablespace tablespace = TablespaceManager.getByName("cluster1"); List fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode.getQual()); assertEquals(1, fragments.size()); assertEquals("021", new String(((HBaseFragment)fragments.get(0)).getStartRow())); diff --git a/tajo-core-tests/src/test/resources/queries/TestCreateTable/negative/create_table_on_absent_tablespace.sql b/tajo-core-tests/src/test/resources/queries/TestCreateTable/negative/create_table_on_absent_tablespace.sql new file mode 100644 index 0000000000..8db9868c94 --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestCreateTable/negative/create_table_on_absent_tablespace.sql @@ -0,0 +1 @@ +CREATE TABLE default.LINEITEM (XXX TEXT) TABLESPACE unknown; \ No newline at end of file diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java index f907677e3e..b599d36b52 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java @@ -136,12 +136,7 @@ private Tablespace getTablespaceHandler(@Nullable String tableSpaceName, @Nullab throws UndefinedTablespaceException { if (tableSpaceName != null) { - Optional ts = (Optional) TablespaceManager.getByName(tableSpaceName); - if (ts.isPresent()) { - return ts.get(); - } else { - throw new UndefinedTablespaceException(tableSpaceName); - } + return TablespaceManager.getByName(tableSpaceName); } else if (tableUri != null) { return TablespaceManager.get(tableUri); } else { diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java index f881b2862c..85d5d519b4 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java @@ -34,6 +34,7 @@ import org.apache.tajo.catalog.MetadataProvider; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UndefinedTablespaceException; import org.apache.tajo.exception.UndefinedTablespaceHandlerException; import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.storage.fragment.Fragment; @@ -394,19 +395,19 @@ public static T get(@Nullable URI uri) { * @return */ public static T getDefault() { - return (T) getByName(DEFAULT_TABLESPACE_NAME).get(); + return (T) getByName(DEFAULT_TABLESPACE_NAME); } public static T getLocalFs() { return (T) get(LOCAL_FS_URI); } - public static Optional getByName(String name) { + public static T getByName(String name) { URI uri = SPACES_URIS_MAP.get(name); if (uri != null) { - return Optional.of(TABLE_SPACES.get(uri)); + return (T) TABLE_SPACES.get(uri); } else { - return Optional.absent(); + throw new TajoRuntimeException(new UndefinedTablespaceException(name)); } } @@ -423,7 +424,7 @@ public static Optional getAnyByScheme(String scheme) { @Override public URI getTableURI(@Nullable String spaceName, String databaseName, String tableName) { - Tablespace space = spaceName == null ? getDefault() : getByName(spaceName).get(); + Tablespace space = spaceName == null ? getDefault() : getByName(spaceName); return space.getTableUri(databaseName, tableName); } diff --git a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java index fb72e57b77..913c586408 100644 --- a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java +++ b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java @@ -58,7 +58,7 @@ public void testExtractQuorum() { @Test public void testTablespaceHandler() throws Exception { - assertTrue((TablespaceManager.getByName("cluster1").get()) instanceof HBaseTablespace); + assertTrue((TablespaceManager.getByName("cluster1")) instanceof HBaseTablespace); assertTrue((TablespaceManager.get(URI.create("hbase:zk://host1:2171"))) instanceof HBaseTablespace); } @@ -73,7 +73,7 @@ public void testGetIndexPredications() throws Exception { EvalNode evalNodeA = new BinaryEval(EvalType.AND, evalNode1, evalNode2); scanNode.setQual(evalNodeA); - HBaseTablespace storageManager = (HBaseTablespace) TablespaceManager.getByName("cluster1").get(); + HBaseTablespace storageManager = (HBaseTablespace) TablespaceManager.getByName("cluster1"); List> indexEvals = storageManager.findIndexablePredicateSet(scanNode.getQual(), new Column[]{rowkeyColumn}); assertNotNull(indexEvals); diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java index 7ff535cd22..627e4ed564 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java @@ -285,7 +285,7 @@ public void testGetFileTablespace() throws Exception { space = (FileTablespace) TablespaceManager.get(uri); assertEquals(cluster.getFileSystem().getUri(), space.getFileSystem().getUri()); - space = (FileTablespace) TablespaceManager.getByName("testGetFileTablespace").get(); + space = (FileTablespace) TablespaceManager.getByName("testGetFileTablespace"); assertEquals(cluster.getFileSystem().getUri(), space.getFileSystem().getUri()); } finally { diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java index 090780d8d3..05b26e8f08 100644 --- a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java +++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java @@ -45,8 +45,8 @@ public class TestPgSQLJdbcTableSpace { @Test(timeout = 1000) public void testTablespaceHandler() throws Exception { - assertTrue((TablespaceManager.getByName("pgsql_cluster").get()) instanceof PgSQLTablespace); - assertEquals("pgsql_cluster", (TablespaceManager.getByName("pgsql_cluster").get().getName())); + assertTrue((TablespaceManager.getByName("pgsql_cluster")) instanceof PgSQLTablespace); + assertEquals("pgsql_cluster", (TablespaceManager.getByName("pgsql_cluster").getName())); assertTrue((TablespaceManager.get(jdbcUrl)) instanceof PgSQLTablespace); assertTrue((TablespaceManager.get(jdbcUrl + "&table=tb1")) instanceof PgSQLTablespace); @@ -57,19 +57,19 @@ public void testTablespaceHandler() throws Exception { @Test(timeout = 1000, expected = TajoRuntimeException.class) public void testCreateTable() throws IOException, TajoException { - Tablespace space = TablespaceManager.getByName("pgsql_cluster").get(); + Tablespace space = TablespaceManager.getByName("pgsql_cluster"); space.createTable(null, false); } @Test(timeout = 1000, expected = TajoRuntimeException.class) public void testDropTable() throws IOException, TajoException { - Tablespace space = TablespaceManager.getByName("pgsql_cluster").get(); + Tablespace space = TablespaceManager.getByName("pgsql_cluster"); space.purgeTable(null); } @Test(timeout = 1000) public void testGetSplits() throws IOException, TajoException { - Tablespace space = TablespaceManager.getByName("pgsql_cluster").get(); + Tablespace space = TablespaceManager.getByName("pgsql_cluster"); MetadataProvider provider = space.getMetadataProvider(); TableDesc table = provider.getTableDesc(null, "lineitem"); List fragments = space.getSplits("lineitem", table, null); From 00498c581ee8c542f7e3ce26a0be58820d27f0c9 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 22 Sep 2015 02:48:31 -0700 Subject: [PATCH 08/10] Improved message for UNDEFINED_TABLESPACE_HANDLER. --- .../src/main/java/org/apache/tajo/exception/ErrorMessages.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java index b7fe0b87b3..ed84aa7f43 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java @@ -72,7 +72,7 @@ public class ErrorMessages { ADD_MESSAGE(UNDEFINED_INDEX_FOR_TABLE, "index ''%s' does not exist", 1); ADD_MESSAGE(UNDEFINED_INDEX_FOR_COLUMNS, "index does not exist for '%s' columns of '%s' table", 2); ADD_MESSAGE(UNDEFINED_INDEX_NAME, "index name '%s' does not exist", 1); - ADD_MESSAGE(UNDEFINED_TABLESPACE_HANDLER, "No tablespace for URI scheme '%s'", 1); + ADD_MESSAGE(UNDEFINED_TABLESPACE_HANDLER, "No tablespace handler for URI scheme '%s'", 1); ADD_MESSAGE(DUPLICATE_TABLESPACE, "tablespace '%s' already exists", 1); ADD_MESSAGE(DUPLICATE_DATABASE, "database '%s' already exists", 1); From 5b6155a6c290e0c2363e099cbde1cd980f27ab05 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 22 Sep 2015 03:14:55 -0700 Subject: [PATCH 09/10] Fixed exception and add more unit tests. --- .../test/java/org/apache/tajo/client/TestTajoClient.java | 7 +++++++ .../org/apache/tajo/master/exec/CreateTableExecutor.java | 4 +--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java index 5616d539b9..60f04db84f 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -36,6 +36,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.TajoException; import org.apache.tajo.exception.UndefinedTablespaceException; +import org.apache.tajo.exception.UndefinedTablespaceHandlerException; import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.ipc.ClientProtos; import org.apache.tajo.ipc.ClientProtos.QueryHistoryProto; @@ -803,4 +804,10 @@ public void testClientRPCInterference() throws Exception { public void testCreateTableOnAbsentTablespace() throws TajoException { client.updateQuery("CREATE TABLE testCreateTableOnAbsentTablespace (AGE INT) TABLESPACE unknown123"); } + + @Test(expected = UndefinedTablespaceHandlerException.class) + public void testCreateTableWithAbsentTablespaceHandler() throws TajoException { + client.updateQuery( + "CREATE EXTERNAL TABLE testCreateTableWithAbsentTablespaceHandler (AGE INT) USING TEXT LOCATION 'hdfx://tajo'"); + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java index b599d36b52..f7b7bf27c4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java @@ -132,9 +132,7 @@ private Pair getQualifiedName(String currentDatabase, String tab } } - private Tablespace getTablespaceHandler(@Nullable String tableSpaceName, @Nullable URI tableUri) - throws UndefinedTablespaceException { - + private Tablespace getTablespaceHandler(@Nullable String tableSpaceName, @Nullable URI tableUri) { if (tableSpaceName != null) { return TablespaceManager.getByName(tableSpaceName); } else if (tableUri != null) { From 0b65b8e2a9dd7cc8d8be1cf71e1b5c8e76971534 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 22 Sep 2015 04:10:51 -0700 Subject: [PATCH 10/10] Add change log. --- CHANGES | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES b/CHANGES index 057104fce7..fb0c8a6dbd 100644 --- a/CHANGES +++ b/CHANGES @@ -38,6 +38,9 @@ Release 0.11.0 - unreleased IMPROVEMENT + TAJO-1868: Allow TablespaceManager::get to return a unregistered + tablespace. (hyunsik) + TAJO-1867: TajoMaster should handle the change of ${tajo.root}. (hyunsik) TAJO-1134: start-tajo.sh should display WEB UI URL and TajoMaster RPC