From d4c60df86143834f9b95698262bb6c50da9b7527 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 5 Dec 2014 11:43:15 +0900 Subject: [PATCH] TAJO-1231: Implicit table properties in session is not stored in table property. --- .../apache/tajo/client/TestTajoClient.java | 37 ++++++++++++++++--- .../org/apache/tajo/plan/LogicalPlanner.java | 20 ++++++++-- .../apache/tajo/plan/util/PlannerUtil.java | 17 +++++++++ 3 files changed, 65 insertions(+), 9 deletions(-) diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java index 616799ff2b..21dd7d5706 100644 --- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -680,7 +680,7 @@ public final void testGetQueryStatusAndResultAfterFinish() throws Exception { } @Test - public void testSetCvsNull() throws Exception { + public void testNullCharSession() throws Exception { String sql = "select\n" + " c_custkey,\n" + @@ -692,17 +692,44 @@ public void testSetCvsNull() throws Exception { " c_custkey,\n" + " orders.o_orderkey;\n"; - TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); - Map variables = new HashMap(); variables.put(SessionVars.NULL_CHAR.keyname(), "\\\\T"); client.updateSessionVariables(variables); + TajoResultSet resultDesc = (TajoResultSet)client.executeQueryAndGetResult(sql); + resultDesc.close(); + assertNullCharSessionVar(resultDesc.getTableDesc()); + } + + @Test + public void testNullCharSessionInCTAS() throws Exception { + String sql = + "create table nullcharsession as select\n" + + " c_custkey,\n" + + " orders.o_orderkey,\n" + + " orders.o_orderstatus \n" + + "from\n" + + " orders full outer join customer on c_custkey = o_orderkey\n" + + "order by\n" + + " c_custkey,\n" + + " orders.o_orderkey;\n"; + Map variables = new HashMap(); + variables.put(SessionVars.NULL_CHAR.keyname(), "\\\\T"); + client.updateSessionVariables(variables); TajoResultSet res = (TajoResultSet)client.executeQueryAndGetResult(sql); + res.close(); + + TableDesc resultDesc = client.getTableDesc("nullcharsession"); + assertNullCharSessionVar(resultDesc); + } + + + public void assertNullCharSessionVar(TableDesc resultDesc) throws Exception { + TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); - assertEquals(res.getTableDesc().getMeta().getOption(StorageConstants.TEXT_NULL), "\\\\T"); + assertEquals(resultDesc.getMeta().getOption(StorageConstants.TEXT_NULL), "\\\\T"); - Path path = new Path(res.getTableDesc().getPath()); + Path path = new Path(resultDesc.getPath()); FileSystem fs = path.getFileSystem(tajoConf); FileStatus[] files = fs.listStatus(path); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 3b1b4e3110..69c0e4b5a0 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -1706,13 +1706,25 @@ public LogicalNode visitCreateTable(PlanContext context, Stack stack, Crea createTableNode.setStorageType(CatalogProtos.StoreType.CSV); } - // Set default storage properties to be created. - KeyValueSet keyValueSet = CatalogUtil.newPhysicalProperties(createTableNode.getStorageType()); + + + // Set default storage properties to table + KeyValueSet properties = CatalogUtil.newPhysicalProperties(createTableNode.getStorageType()); + + // Priority to apply table properties + // 1. Explicit table properties specified in WITH clause + // 2. Session variables + + // Set session variables to properties + PlannerUtil.applySessionToTableProperties(context.queryContext, createTableNode.getStorageType(), properties); + // Set table properties specified in WITH clause if (expr.hasParams()) { - keyValueSet.putAll(expr.getParams()); + properties.putAll(expr.getParams()); } - createTableNode.setOptions(keyValueSet); + createTableNode.setOptions(properties); + + if (expr.hasPartition()) { if (expr.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index 6868b6c0ea..c55c20315e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -21,6 +21,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.*; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.Column; @@ -35,10 +37,15 @@ import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import org.apache.tajo.plan.visitor.ExplainLogicalPlanVisitor; import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; +import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.TUtil; import java.util.*; +import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType.CSV; +import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType.TEXTFILE; + public class PlannerUtil { public static boolean checkIfDDLPlan(LogicalNode node) { @@ -776,4 +783,14 @@ public static String buildExplainString(LogicalNode node) { return explains.toString(); } + + public static void applySessionToTableProperties(OverridableConf sessionVars, + CatalogProtos.StoreType storeType, + KeyValueSet tableProperties) { + if (storeType == CSV || storeType == TEXTFILE) { + if (sessionVars.containsKey(SessionVars.NULL_CHAR)) { + tableProperties.set(StorageConstants.TEXT_NULL, sessionVars.get(SessionVars.NULL_CHAR)); + } + } + } }