From c030fa7d2f4b9586c9768d797b44b963cea8448b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=ED=98=95=EC=A4=80?= Date: Tue, 17 Jun 2014 19:43:45 +0900 Subject: [PATCH] TAJO-880: NULL in CASE clause occurs Exception. --- .../engine/planner/physical/SeqScanExec.java | 4 +- .../tajo/master/querymaster/SubQuery.java | 1 + .../tajo/engine/query/TestJoinBroadcast.java | 51 +++++++++++++++---- 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java index 0a2b2792b2..a45cd7ba58 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java @@ -78,9 +78,9 @@ public SeqScanExec(TaskAttemptContext context, AbstractStorageManager sm, String pathNameKey = ""; if (fragments != null) { for (FragmentProto f : fragments) { - FileFragment fileFragement = (FileFragment) FragmentConvertor.convert( + FileFragment fileFragement = FragmentConvertor.convert( context.getConf(), plan.getTableDesc().getMeta().getStoreType(), f); - pathNameKey += fileFragement.getPath().getParent().getName(); + pathNameKey += fileFragement.getPath(); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java index 22817bd080..be0c62402a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java @@ -907,6 +907,7 @@ private static void scheduleFragmentsForLeafQuery(SubQuery subQuery) throws IOEx // Otherwise, it creates at least one fragments for a table, which may // span a number of blocks or possibly consists of a number of files. if (scan.getType() == NodeType.PARTITIONS_SCAN) { + // After calling this method, partition paths are removed from the physical plan. fragments = Repartitioner.getFragmentsFromPartitionedTable(subQuery.getStorageManager(), scan, table); } else { Path inputPath = table.getPath(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java index ae87c64439..4fe29a9efe 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java @@ -21,13 +21,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; -import org.apache.tajo.IntegrationTest; -import org.apache.tajo.QueryId; -import org.apache.tajo.QueryTestCaseBase; -import org.apache.tajo.TajoConstants; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.*; +import org.apache.tajo.catalog.*; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.Int4Datum; @@ -49,8 +44,9 @@ import java.io.File; import java.sql.ResultSet; -import static junit.framework.TestCase.*; -import static org.junit.Assert.assertNotNull; +import static junit.framework.TestCase.fail; +import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; +import static org.junit.Assert.*; @Category(IntegrationTest.class) public class TestJoinBroadcast extends QueryTestCaseBase { @@ -427,6 +423,43 @@ public Tuple createTuple(String[] columnDatas) { executeString("DROP TABLE orders_multifile PURGE"); } + @Test + public final void testBroadcastMultiColumnPartitionTable() throws Exception { + // TAJO-879: Some data is missing in the case of BROADCAST JOIN and multi-column partition. + String tableName = CatalogUtil.normalizeIdentifier("testBroadcastMultiColumnPartitionTable"); + ResultSet res = testBase.execute( + "create table " + tableName + " (col1 int4, col2 float4) partition by column(col3 text, col4 text) "); + res.close(); + TajoTestingCluster cluster = testBase.getTestingCluster(); + CatalogService catalog = cluster.getMaster().getCatalog(); + assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); + + /* + /tajo/warehouse/default/testbroadcastmulticolumnpartitiontable/col3=01/col4=1996 + /tajo/warehouse/default/testbroadcastmulticolumnpartitiontable/col3=10/col4=1993 + /tajo/warehouse/default/testbroadcastmulticolumnpartitiontable/col3=12/col4=1996 + */ + res = executeString("insert overwrite into " + tableName + + " select o_orderkey, o_totalprice, substr(o_orderdate, 6, 2), substr(o_orderdate, 1, 4) from orders"); + res.close(); + + + res = executeString( + "select distinct a.col3 from " + tableName + " as a " + + "left outer join lineitem_large b " + + "on a.col1 = b.l_orderkey" + ); + + String expected = "col3\n" + + "-------------------------------\n" + + "01\n" + + "10\n" + + "12\n"; + + assertEquals(expected, resultSetToString(res)); + cleanupQuery(res); + } + static interface TupleCreator { public Tuple createTuple(String[] columnDatas); }