From 6ff19b8cf8df6f037c5d9cce398a55c1740f96d7 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 30 Sep 2015 15:31:55 +0900 Subject: [PATCH 1/4] TAJO-1901: Repair partition throws ArrayIndexOutOfBoundsException occasionally. --- .../tajo/engine/query/TestAlterTable.java | 36 ++++++++++++++++++- .../apache/tajo/master/exec/DDLExecutor.java | 14 ++++---- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 9a30012a2a..5a8b7bc5a9 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -35,6 +35,7 @@ import java.sql.ResultSet; import java.util.List; +import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.junit.Assert.*; @Category(IntegrationTest.class) @@ -58,7 +59,7 @@ public final void testAlterTableColumnName() throws Exception { public final void testAlterTableAddNewColumn() throws Exception { List createdNames = executeDDL("table1_ddl.sql", "table1.tbl", "EFG"); executeDDL("alter_table_add_new_column_ddl.sql", null); - assertColumnExists(createdNames.get(0),"cool"); + assertColumnExists(createdNames.get(0), "cool"); } @Test @@ -199,6 +200,39 @@ public final void testAlterTableRepairPartition() throws Exception { catalog.dropTable(tableName); } + @Test + public final void testDatabaseNameIncludeTableNameWithRepairPartition() throws Exception { + String databaseName = "test_repair_partition"; + String tableName = "part"; + String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); + + executeString("create database " + databaseName).close(); + executeString("create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(key float8) " + + " as select l_orderkey, l_partkey, l_quantity from default.lineitem"); + + TableDesc tableDesc = catalog.getTableDesc(databaseName, tableName); + assertNotNull(tableDesc); + + verifyPartitionCount(databaseName, tableName, 5); + + // Remove all partitions + executeString("ALTER TABLE " + canonicalTableName + " DROP PARTITION (key = 17.0)").close(); + executeString("ALTER TABLE " + canonicalTableName + " DROP PARTITION (key = 36.0)").close(); + executeString("ALTER TABLE " + canonicalTableName + " DROP PARTITION (key = 38.0)").close(); + executeString("ALTER TABLE " + canonicalTableName + " DROP PARTITION (key = 45.0)").close(); + executeString("ALTER TABLE " + canonicalTableName + " DROP PARTITION (key = 49.0)").close(); + + verifyPartitionCount(databaseName, tableName, 0); + + executeString("ALTER TABLE " + canonicalTableName + " REPAIR PARTITION").close(); + + verifyPartitionCount(databaseName, tableName, 5); + + executeString("DROP TABLE " + canonicalTableName + " PURGE").close(); + executeString("DROP database " + databaseName).close(); + + } + private void verifyPartitionCount(String databaseName, String tableName, int expectedCount) throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException, UndefinedPartitionException { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 8d9facbd8c..f626506309 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -50,6 +50,7 @@ import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; +import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -625,7 +626,7 @@ public void repairPartition(TajoMaster.MasterContext context, final QueryContext // Find missing partitions from CatalogStore List targetPartitions = TUtil.newList(); for(Path filteredPath : filteredPaths) { - PartitionDescProto targetPartition = getPartitionDesc(simpleTableName, filteredPath); + PartitionDescProto targetPartition = getPartitionDesc(tablePath, filteredPath); if (!existingPartitionNames.contains(targetPartition.getPartitionName())) { if (LOG.isDebugEnabled()) { LOG.debug("Partitions not in CatalogStore:" + targetPartition.getPartitionName()); @@ -645,12 +646,11 @@ public void repairPartition(TajoMaster.MasterContext context, final QueryContext LOG.info("Total added partitions to CatalogStore: " + targetPartitions.size()); } - private PartitionDescProto getPartitionDesc(String tableName, Path path) throws IOException { - String partitionPath = path.toString(); + private PartitionDescProto getPartitionDesc(Path tablePath, Path partitionPath) throws IOException { + String partitionName = StringUtils.unescapePathName(partitionPath.toString()); - String partitionName = StringUtils.unescapePathName(partitionPath); - int startIndex = partitionPath.indexOf(tableName); - partitionName = partitionName.substring(startIndex + tableName.length() + 1, partitionPath.length()); + int startIndex = partitionName.indexOf(tablePath.toString()) + tablePath.toString().length(); + partitionName = partitionName.substring(startIndex + File.separator.length()); CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder(); builder.setPartitionName(partitionName); @@ -668,7 +668,7 @@ private PartitionDescProto getPartitionDesc(String tableName, Path path) throws builder.addPartitionKeys(keyBuilder.build()); } - builder.setPath(partitionPath); + builder.setPath(partitionPath.toString()); return builder.build(); } From ff971343790f6c0796505497a6ef2b054a4314cf Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 30 Sep 2015 15:50:34 +0900 Subject: [PATCH 2/4] Remove unnecessary updates --- .../test/java/org/apache/tajo/engine/query/TestAlterTable.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 5a8b7bc5a9..d5371c916a 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -35,7 +35,6 @@ import java.sql.ResultSet; import java.util.List; -import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.junit.Assert.*; @Category(IntegrationTest.class) @@ -59,7 +58,7 @@ public final void testAlterTableColumnName() throws Exception { public final void testAlterTableAddNewColumn() throws Exception { List createdNames = executeDDL("table1_ddl.sql", "table1.tbl", "EFG"); executeDDL("alter_table_add_new_column_ddl.sql", null); - assertColumnExists(createdNames.get(0), "cool"); + assertColumnExists(createdNames.get(0),"cool"); } @Test From d1b5fd9cc8b75abc57ee8bc30a035f4877cd0e4e Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 1 Oct 2015 18:53:31 +0900 Subject: [PATCH 3/4] Add more unit test cases --- .../tajo/engine/query/TestAlterTable.java | 324 +++++++++++++++++- 1 file changed, 312 insertions(+), 12 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index d5371c916a..58ceb74371 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -23,22 +23,35 @@ import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.exception.UndefinedDatabaseException; -import org.apache.tajo.exception.UndefinedPartitionException; -import org.apache.tajo.exception.UndefinedPartitionMethodException; -import org.apache.tajo.exception.UndefinedTableException; +import org.apache.tajo.exception.*; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; +import java.io.File; import java.sql.ResultSet; +import java.sql.SQLException; import java.util.List; import static org.junit.Assert.*; @Category(IntegrationTest.class) public class TestAlterTable extends QueryTestCaseBase { + + @Before + public void setUp() throws Exception { + executeString("create database " + getCurrentDatabase()).close(); + } + + @After + public void tearDown() throws Exception { + executeString("drop database " + getCurrentDatabase()).close(); + } + @Test public final void testAlterTableName() throws Exception { List createdNames = executeDDL("table1_ddl.sql", "table1.tbl", "ABC"); @@ -58,7 +71,7 @@ public final void testAlterTableColumnName() throws Exception { public final void testAlterTableAddNewColumn() throws Exception { List createdNames = executeDDL("table1_ddl.sql", "table1.tbl", "EFG"); executeDDL("alter_table_add_new_column_ddl.sql", null); - assertColumnExists(createdNames.get(0),"cool"); + assertColumnExists(createdNames.get(0), "cool"); } @Test @@ -200,26 +213,34 @@ public final void testAlterTableRepairPartition() throws Exception { } @Test - public final void testDatabaseNameIncludeTableNameWithRepairPartition() throws Exception { + public final void testRepairPartitionWithDatabaseNameIncludeTableName() throws Exception { String databaseName = "test_repair_partition"; String tableName = "part"; String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); executeString("create database " + databaseName).close(); executeString("create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(key float8) " - + " as select l_orderkey, l_partkey, l_quantity from default.lineitem"); + + " as select l_orderkey, l_partkey, l_quantity from default.lineitem").close(); TableDesc tableDesc = catalog.getTableDesc(databaseName, tableName); assertNotNull(tableDesc); verifyPartitionCount(databaseName, tableName, 5); + ResultSet res = executeString("SELECT * FROM " + canonicalTableName + " ORDER BY col1, col2 desc, key desc;"); + String result = resultSetToString(res); + String expectedResult = "col1,col2,key\n" + + "-------------------------------\n" + + "1,1,36.0\n" + + "1,1,17.0\n" + + "2,2,38.0\n" + + "3,3,49.0\n" + + "3,2,45.0\n"; + res.close(); + assertEquals(expectedResult, result); + // Remove all partitions - executeString("ALTER TABLE " + canonicalTableName + " DROP PARTITION (key = 17.0)").close(); - executeString("ALTER TABLE " + canonicalTableName + " DROP PARTITION (key = 36.0)").close(); - executeString("ALTER TABLE " + canonicalTableName + " DROP PARTITION (key = 38.0)").close(); - executeString("ALTER TABLE " + canonicalTableName + " DROP PARTITION (key = 45.0)").close(); - executeString("ALTER TABLE " + canonicalTableName + " DROP PARTITION (key = 49.0)").close(); + dropPartitions(databaseName, tableName, tableDesc.getPartitionMethod().getExpressionSchema().getAllColumns()); verifyPartitionCount(databaseName, tableName, 0); @@ -227,11 +248,254 @@ public final void testDatabaseNameIncludeTableNameWithRepairPartition() throws E verifyPartitionCount(databaseName, tableName, 5); + res = executeString("SELECT * FROM " + canonicalTableName + " ORDER BY col1, col2 desc, key desc;"); + result = resultSetToString(res); + res.close(); + assertEquals(expectedResult, result); + executeString("DROP TABLE " + canonicalTableName + " PURGE").close(); executeString("DROP database " + databaseName).close(); + } + + @Test + public void testRepairPartitionWithAbnormalDirectories() throws Exception { + String databaseName = getCurrentDatabase().toLowerCase(); + String tableName = "testRepairPartitionWithAbnormalDirectories".toLowerCase(); + String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); + + executeString("create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(key float8) " + + " as select l_orderkey, l_partkey, l_quantity from default.lineitem").close(); + + TableDesc tableDesc = catalog.getTableDesc(databaseName, tableName); + assertNotNull(tableDesc); + + verifyPartitionCount(databaseName, tableName, 5); + + ResultSet res = executeString("SELECT * FROM " + canonicalTableName + " ORDER BY col1, col2 desc, key desc;"); + String result = resultSetToString(res); + String expectedResult = "col1,col2,key\n" + + "-------------------------------\n" + + "1,1,36.0\n" + + "1,1,17.0\n" + + "2,2,38.0\n" + + "3,3,49.0\n" + + "3,2,45.0\n"; + res.close(); + assertEquals(expectedResult, result); + + // Remove all partitions + dropPartitions(databaseName, tableName, tableDesc.getPartitionMethod().getExpressionSchema().getAllColumns()); + + verifyPartitionCount(databaseName, tableName, 0); + + // Make abnormal directories + FileSystem fs = FileSystem.get(conf); + Path path = new Path(tableDesc.getUri().getPath(), "key=100.0"); + fs.mkdirs(path); + path = new Path(tableDesc.getUri().getPath(), "key=110.0"); + fs.mkdirs(path); + path = new Path(tableDesc.getUri().getPath(), "key="); + fs.mkdirs(path); + path = new Path(tableDesc.getUri().getPath(), "col1=a"); + fs.mkdirs(path); + assertEquals(9, fs.listStatus(path.getParent()).length); + + executeString("ALTER TABLE " + canonicalTableName + " REPAIR PARTITION").close(); + + verifyPartitionCount(databaseName, tableName, 7); + + res = executeString("SELECT * FROM " + canonicalTableName + " ORDER BY col1, col2 desc, key desc;"); + result = resultSetToString(res); + res.close(); + assertEquals(expectedResult, result); + executeString("DROP TABLE " + canonicalTableName + " PURGE").close(); + } + + @Test + public void testRepairPartitionWithDatePartitionColumn() throws Exception { + String databaseName = getCurrentDatabase().toLowerCase(); + String tableName = "testRepairPartitionWithDatePartitionColumn".toLowerCase(); + String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); + + executeString( + "create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(key date) " + + " as select l_orderkey, l_partkey, l_shipdate::date from default.lineitem").close(); + + TableDesc tableDesc = catalog.getTableDesc(databaseName, tableName); + assertNotNull(tableDesc); + + verifyPartitionCount(databaseName, tableName, 5); + + ResultSet res = executeString("SELECT * FROM " + canonicalTableName + " ORDER BY col1, col2 desc, key desc;"); + String result = resultSetToString(res); + String expectedResult = "col1,col2,key\n" + + "-------------------------------\n" + + "1,1,1996-04-12\n" + + "1,1,1996-03-13\n" + + "2,2,1997-01-28\n" + + "3,3,1993-11-09\n" + + "3,2,1994-02-02\n"; + res.close(); + assertEquals(expectedResult, result); + + // Remove all partitions + dropPartitions(databaseName, tableName, tableDesc.getPartitionMethod().getExpressionSchema().getAllColumns()); + + verifyPartitionCount(databaseName, tableName, 0); + + executeString("ALTER TABLE " + canonicalTableName + " REPAIR PARTITION").close(); + + verifyPartitionCount(databaseName, tableName, 5); + + res = executeString("SELECT * FROM " + canonicalTableName + " ORDER BY col1, col2 desc, key desc;"); + result = resultSetToString(res); + res.close(); + assertEquals(expectedResult, result); + + executeString("DROP TABLE " + canonicalTableName + " PURGE").close(); } + @Test + public void testRepairPartitionWithTimestampPartitionColumn() throws Exception { + String databaseName = getCurrentDatabase().toLowerCase(); + String tableName = "testRepairPartitionWithTimestampPartitionColumn".toLowerCase(); + String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); + + executeString( + "create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(key timestamp) " + + " as select l_orderkey, l_partkey, to_timestamp(l_shipdate, 'YYYY-MM-DD') from default.lineitem"); + + TableDesc tableDesc = catalog.getTableDesc(databaseName, tableName); + assertNotNull(tableDesc); + + verifyPartitionCount(databaseName, tableName, 5); + + ResultSet res = executeString("SELECT * FROM " + canonicalTableName + " ORDER BY col1, col2 desc, key desc;"); + String result = resultSetToString(res); + String expectedResult = "col1,col2,key\n" + + "-------------------------------\n" + + "1,1,1996-04-12 00:00:00\n" + + "1,1,1996-03-13 00:00:00\n" + + "2,2,1997-01-28 00:00:00\n" + + "3,3,1993-11-09 00:00:00\n" + + "3,2,1994-02-02 00:00:00\n"; + res.close(); + assertEquals(expectedResult, result); + + // Remove all partitions + dropPartitions(databaseName, tableName, tableDesc.getPartitionMethod().getExpressionSchema().getAllColumns()); + + verifyPartitionCount(databaseName, tableName, 0); + + executeString("ALTER TABLE " + canonicalTableName + " REPAIR PARTITION").close(); + + verifyPartitionCount(databaseName, tableName, 5); + + res = executeString("SELECT * FROM " + canonicalTableName + " ORDER BY col1, col2 desc, key desc;"); + result = resultSetToString(res); + res.close(); + assertEquals(expectedResult, result); + + executeString("DROP TABLE " + canonicalTableName + " PURGE").close(); + } + + @Test + public void testRepairPartitionWithTimesPartitionColumn() throws Exception { + String databaseName = getCurrentDatabase().toLowerCase(); + String tableName = "testRepairPartitionWithTimesPartitionColumn".toLowerCase(); + String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); + + executeString( + "create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(key time) " + + " as select l_orderkey, l_partkey " + + " , CASE l_shipdate WHEN '1996-03-13' THEN cast ('11:20:40' as time) " + + " WHEN '1997-01-28' THEN cast ('12:10:20' as time) " + + " WHEN '1994-02-02' THEN cast ('12:10:30' as time) " + + " ELSE cast ('00:00:00' as time) END " + + " from default.lineitem"); + + TableDesc tableDesc = catalog.getTableDesc(databaseName, tableName); + assertNotNull(tableDesc); + + ResultSet res = executeString("SELECT * FROM " + canonicalTableName + " ORDER BY col1, col2 desc, key desc;"); + String result = resultSetToString(res); + String expectedResult = "col1,col2,key\n" + + "-------------------------------\n" + + "1,1,11:20:40\n" + + "1,1,00:00:00\n" + + "2,2,12:10:20\n" + + "3,3,00:00:00\n" + + "3,2,12:10:30\n"; + res.close(); + assertEquals(expectedResult, result); + + verifyPartitionCount(databaseName, tableName, 4); + + // Remove all partitions + dropPartitions(databaseName, tableName, tableDesc.getPartitionMethod().getExpressionSchema().getAllColumns()); + + verifyPartitionCount(databaseName, tableName, 0); + + executeString("ALTER TABLE " + canonicalTableName + " REPAIR PARTITION").close(); + + verifyPartitionCount(databaseName, tableName, 4); + + res = executeString("SELECT * FROM " + canonicalTableName + " ORDER BY col1, col2 desc, key desc;"); + result = resultSetToString(res); + res.close(); + assertEquals(expectedResult, result); + + executeString("DROP TABLE " + canonicalTableName + " PURGE").close(); + } + + + @Test + public void testRepairPartitionWithMutiplePartitionColumn() throws Exception { + String databaseName = getCurrentDatabase().toLowerCase(); + String tableName = "testRepairPartitionWithMutiplePartitionColumn".toLowerCase(); + String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); + + executeString("create table " + canonicalTableName + " (col4 text) " + + " partition by column(col1 int4, col2 int4, col3 float8) as select l_returnflag, l_orderkey, l_partkey, " + + "l_quantity from default.lineitem"); + + TableDesc tableDesc = catalog.getTableDesc(databaseName, tableName); + assertNotNull(tableDesc); + + ResultSet res = executeString("SELECT * FROM " + canonicalTableName + + " ORDER BY col1, col2 desc, col3 desc, col4;"); + String result = resultSetToString(res); + String expectedResult = "col4,col1,col2,col3\n" + + "-------------------------------\n" + + "N,1,1,36.0\n" + + "N,1,1,17.0\n" + + "N,2,2,38.0\n" + + "R,3,3,49.0\n" + + "R,3,2,45.0\n"; + res.close(); + assertEquals(expectedResult, result); + + verifyPartitionCount(databaseName, tableName, 5); + + // Remove all partitions + dropPartitions(databaseName, tableName, tableDesc.getPartitionMethod().getExpressionSchema().getAllColumns()); + + verifyPartitionCount(databaseName, tableName, 0); + + executeString("ALTER TABLE " + canonicalTableName + " REPAIR PARTITION").close(); + + verifyPartitionCount(databaseName, tableName, 5); + + res = executeString("SELECT * FROM " + canonicalTableName + + " ORDER BY col1, col2 desc, col3 desc, col4;"); result = resultSetToString(res); + res.close(); + assertEquals(expectedResult, result); + + executeString("DROP TABLE " + canonicalTableName + " PURGE").close(); + } + + private void verifyPartitionCount(String databaseName, String tableName, int expectedCount) throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException, UndefinedPartitionException { @@ -239,4 +503,40 @@ private void verifyPartitionCount(String databaseName, String tableName, int exp assertNotNull(partitions); assertEquals(partitions.size(), expectedCount); } + + private void dropPartitions(String databaseName, String tableName, List colums) + throws Exception { + String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); + List partitions = catalog.getPartitionsOfTable(databaseName, tableName); + + StringBuilder sb = new StringBuilder(); + for (CatalogProtos.PartitionDescProto partition : partitions) { + + sb.delete(0, sb.length()); + sb.append("ALTER TABLE ").append(canonicalTableName).append(" DROP PARTITION ("); + + String[] splitPartitionName = partition.getPartitionName().split(File.separator); + for(int i = 0; i < splitPartitionName.length; i++) { + String[] partitionColumnValue = splitPartitionName[i].split("="); + if (i > 0) { + sb.append(","); + } + + switch (colums.get(i).getDataType().getType()) { + case TEXT: + case TIME: + case TIMESTAMP: + case DATE: + sb.append(partitionColumnValue[0]).append("='").append(partitionColumnValue[1]).append("'"); + break; + default: + sb.append(partitionColumnValue[0]).append("=").append(partitionColumnValue[1]); + break; + } + } + sb.append(")"); + executeString(sb.toString()).close(); + } + } + } From c5bd637439694e28633100176eb732652f653823 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 1 Oct 2015 19:04:49 +0900 Subject: [PATCH 4/4] Check if there is no partitions in the partition path. --- .../apache/tajo/master/exec/DDLExecutor.java | 20 +++++++++++++++---- .../rules/PartitionedTableRewriter.java | 2 +- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index f626506309..57839445df 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -626,13 +626,25 @@ public void repairPartition(TajoMaster.MasterContext context, final QueryContext // Find missing partitions from CatalogStore List targetPartitions = TUtil.newList(); for(Path filteredPath : filteredPaths) { - PartitionDescProto targetPartition = getPartitionDesc(tablePath, filteredPath); - if (!existingPartitionNames.contains(targetPartition.getPartitionName())) { + + int startIdx = filteredPath.toString().indexOf(PartitionedTableRewriter.getColumnPartitionPathPrefix + (partitionColumns)); + + // if there is partition column in the path + if (startIdx > -1) { + PartitionDescProto targetPartition = getPartitionDesc(tablePath, filteredPath); + if (!existingPartitionNames.contains(targetPartition.getPartitionName())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Partitions not in CatalogStore:" + targetPartition.getPartitionName()); + } + targetPartitions.add(targetPartition); + } + } else { if (LOG.isDebugEnabled()) { - LOG.debug("Partitions not in CatalogStore:" + targetPartition.getPartitionName()); + LOG.debug("Invalid partition path:" + filteredPath.toString()); } - targetPartitions.add(targetPartition); } + } catalog.addPartitions(databaseName, simpleTableName, targetPartitions, true); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java index 5b1a1f179b..fc0b1bbeec 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java @@ -479,7 +479,7 @@ public static Tuple buildTupleFromPartitionPath(Schema partitionColumnSchema, Pa * @param partitionColumn the schema of column partition * @return The first part string of column partition path. */ - private static String getColumnPartitionPathPrefix(Schema partitionColumn) { + public static String getColumnPartitionPathPrefix(Schema partitionColumn) { StringBuilder sb = new StringBuilder(); sb.append(partitionColumn.getColumn(0).getSimpleName()).append("="); return sb.toString();