From 73075899a03ad91f21d19ca6a3ee328201f50274 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 17 Sep 2015 11:40:31 -0700 Subject: [PATCH] TAJO-1867: TajoMaster should handle the change of ${tajo.root}. --- .../src/main/proto/CatalogProtos.proto | 6 +-- .../apache/tajo/catalog/CatalogServer.java | 4 +- .../tajo/catalog/store/AbstractDBStore.java | 4 +- .../org/apache/tajo/catalog/TestCatalog.java | 5 +-- .../TestCatalogAgainstCaseSensitivity.java | 7 +--- .../tajo/catalog/TestCatalogExceptions.java | 8 +--- .../org/apache/tajo/master/TajoMaster.java | 41 +++++++++++++++++-- .../apache/tajo/master/exec/DDLExecutor.java | 2 +- 8 files changed, 50 insertions(+), 27 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index cfac82f06d..cec528a203 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -292,13 +292,9 @@ message AlterTablespaceProto { LOCATION = 0; } - message SetLocation { - required string uri = 1; - } - message AlterTablespaceCommand { required AlterTablespaceType type = 1; - optional SetLocation location = 2; + optional string location = 2; } } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 8f4e0e88f5..a60dddab34 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -374,11 +374,11 @@ public ReturnState alterTablespace(RpcController controller, AlterTablespaceProt for (AlterTablespaceCommand command : request.getCommandList()) { if (command.getType() == AlterTablespaceProto.AlterTablespaceType.LOCATION) { try { - URI uri = URI.create(command.getLocation().getUri()); + URI uri = URI.create(command.getLocation()); Preconditions.checkArgument(uri.getScheme() != null); } catch (Exception e) { throw new ServiceException("ALTER TABLESPACE's LOCATION must be a URI form (scheme:///.../), but " - + command.getLocation().getUri()); + + command.getLocation()); } } } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 0f07149e3c..c4d1828d48 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -544,12 +544,12 @@ public void alterTablespace(AlterTablespaceProto alterProto) throws UndefinedTab if (alterProto.getCommandList().size() == 1) { AlterTablespaceCommand command = alterProto.getCommand(0); if (command.getType() == AlterTablespaceProto.AlterTablespaceType.LOCATION) { - AlterTablespaceProto.SetLocation setLocation = command.getLocation(); + final String uri = command.getLocation(); try { String sql = "UPDATE " + TB_SPACES + " SET SPACE_URI=? WHERE SPACE_NAME=?"; pstmt = conn.prepareStatement(sql); - pstmt.setString(1, setLocation.getUri()); + pstmt.setString(1, uri); pstmt.setString(2, alterProto.getSpaceName()); pstmt.executeUpdate(); } catch (SQLException se) { diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 8720105f24..a5e4861dee 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -47,7 +47,6 @@ import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceType; -import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.SetLocation; import static org.junit.Assert.*; public class TestCatalog { @@ -99,7 +98,7 @@ public void testGetTablespace() throws Exception { AlterTablespaceProto.AlterTablespaceCommand.Builder commandBuilder = AlterTablespaceProto.AlterTablespaceCommand.newBuilder(); commandBuilder.setType(AlterTablespaceType.LOCATION); - commandBuilder.setLocation(SetLocation.newBuilder().setUri("hdfs://zzz.com/warehouse")); + commandBuilder.setLocation("hdfs://zzz.com/warehouse"); AlterTablespaceProto.Builder alter = AlterTablespaceProto.newBuilder(); alter.setSpaceName("space1"); alter.addCommand(commandBuilder.build()); @@ -122,7 +121,7 @@ public void testGetTablespace() throws Exception { // ALTER TABLESPACE space1 LOCATION 'hdfs://zzz.com/warehouse'; commandBuilder = AlterTablespaceProto.AlterTablespaceCommand.newBuilder(); commandBuilder.setType(AlterTablespaceType.LOCATION); - commandBuilder.setLocation(SetLocation.newBuilder().setUri("hdfs://www.com/warehouse")); + commandBuilder.setLocation("hdfs://www.com/warehouse"); alter = AlterTablespaceProto.newBuilder(); alter.setSpaceName("space2"); alter.addCommand(commandBuilder.build()); diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java index bfff6b4850..594f5ce3e8 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java @@ -22,7 +22,6 @@ import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand; import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceType; -import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.SetLocation; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.exception.UndefinedPartitionException; @@ -68,16 +67,14 @@ public void testTablespace() throws Exception { addCommand( AlterTablespaceCommand.newBuilder(). setType(AlterTablespaceType.LOCATION). - setLocation(SetLocation.newBuilder() - .setUri("hdfs://zzz.com/warehouse"))).build()); + setLocation("hdfs://zzz.com/warehouse")).build()); catalog.alterTablespace(AlterTablespaceProto.newBuilder(). setSpaceName("SpAcE1"). addCommand( AlterTablespaceCommand.newBuilder(). setType(AlterTablespaceType.LOCATION). - setLocation(SetLocation.newBuilder() - .setUri("hdfs://zzz.com/warehouse"))).build()); + setLocation("hdfs://zzz.com/warehouse")).build()); Set tablespaceProtos = new HashSet<>(); for (String tablespaceName : catalog.getAllTablespaceNames()) { diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java index 7be34d87e2..6586cec808 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java @@ -24,7 +24,6 @@ import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto; import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand; import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceType; -import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.SetLocation; import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto; import org.apache.tajo.catalog.proto.CatalogProtos.UpdateTableStatsProto; @@ -37,7 +36,6 @@ import org.junit.Test; import java.net.URI; -import java.util.UUID; public class TestCatalogExceptions { @@ -85,8 +83,7 @@ public void testAlterTablespaceWithWrongUri() throws Exception { addCommand( AlterTablespaceCommand.newBuilder(). setType(AlterTablespaceType.LOCATION). - setLocation(SetLocation.newBuilder() - .setUri("hdfs:"))).build()); + setLocation("hdfs:")).build()); } @Test(expected = UndefinedTablespaceException.class) @@ -96,8 +93,7 @@ public void testAlterUndefinedTablespace() throws Exception { addCommand( AlterTablespaceCommand.newBuilder(). setType(AlterTablespaceType.LOCATION). - setLocation(SetLocation.newBuilder() - .setUri("hdfs://zzz.com/warehouse"))).build()); + setLocation("hdfs://zzz.com/warehouse")).build()); } @Test(expected = DuplicateDatabaseException.class) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index 326bc86d2d..2f37ceec0f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -34,17 +34,20 @@ import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.RackResolver; import org.apache.hadoop.yarn.util.SystemClock; +import org.apache.tajo.algebra.AlterTablespace; import org.apache.tajo.catalog.CatalogServer; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.LocalCatalogWrapper; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto; +import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand; import org.apache.tajo.catalog.store.AbstractDBStore; import org.apache.tajo.catalog.store.DerbyStore; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.engine.function.FunctionLoader; -import org.apache.tajo.exception.DuplicateDatabaseException; -import org.apache.tajo.exception.DuplicateTablespaceException; +import org.apache.tajo.exception.*; import org.apache.tajo.function.FunctionSignature; import org.apache.tajo.master.rm.TajoResourceManager; import org.apache.tajo.metrics.ClusterResourceMetricSet; @@ -382,7 +385,39 @@ private void writeSystemConf() throws IOException { private void checkBaseTBSpaceAndDatabase() throws IOException, DuplicateDatabaseException, DuplicateTablespaceException { - if (!catalog.existTablespace(DEFAULT_TABLESPACE_NAME)) { + if (catalog.existTablespace(DEFAULT_TABLESPACE_NAME)) { // if default tablespace already exists + + CatalogProtos.TablespaceProto tablespace = null; + try { + tablespace = catalog.getTablespace(DEFAULT_TABLESPACE_NAME); + } catch (UndefinedTablespaceException e) { + throw new TajoInternalError(e); + } + + // if warehouse directory and the location of default tablespace are different from each other + if (!tablespace.getUri().equals(context.getConf().getVar(ConfVars.WAREHOUSE_DIR))) { + AlterTablespaceCommand.Builder alterCommand = + AlterTablespaceCommand.newBuilder() + .setType(AlterTablespaceProto.AlterTablespaceType.LOCATION) + .setLocation(context.getConf().getVar(ConfVars.WAREHOUSE_DIR)); + + AlterTablespaceProto alterTablespace = AlterTablespaceProto.newBuilder() + .setSpaceName(DEFAULT_TABLESPACE_NAME) + .addCommand(alterCommand).build(); + + // update the location of default tablespace + try { + catalog.alterTablespace(alterTablespace); + } catch (TajoException e) { + throw new TajoInternalError(e); + } + + LOG.warn( + "The location of default tablespace has been changed. " + + "You may not accept existing managed tables stored in the previous default tablespace"); + } + + } else if (!catalog.existTablespace(DEFAULT_TABLESPACE_NAME)) { // if the default tablespace does not exists catalog.createTablespace(DEFAULT_TABLESPACE_NAME, context.getConf().getVar(ConfVars.WAREHOUSE_DIR)); } else { LOG.info(String.format("Default tablespace (%s) is already prepared.", DEFAULT_TABLESPACE_NAME)); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 9f80802ed6..c5d583878c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -206,7 +206,7 @@ public static void alterTablespace(final TajoMaster.MasterContext context, final AlterTablespaceProto.AlterTablespaceCommand.Builder commandBuilder = AlterTablespaceProto.AlterTablespaceCommand.newBuilder(); commandBuilder.setType(AlterTablespaceProto.AlterTablespaceType.LOCATION); - commandBuilder.setLocation(AlterTablespaceProto.SetLocation.newBuilder().setUri(alterTablespace.getLocation())); + commandBuilder.setLocation(alterTablespace.getLocation()); commandBuilder.build(); builder.addCommand(commandBuilder); } else {