From 8731ebc2f2a7dd57b41b3dc9a7c296718f49cc47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=ED=98=95=EC=A4=80?= Date: Sat, 24 May 2014 23:02:43 +0900 Subject: [PATCH 1/3] TAJO-842: NULL handling in JDBC. --- .../cli/DefaultTajoCliOutputFormatter.java | 13 +- .../java/org/apache/tajo/cli/SetCommand.java | 1 + .../java/org/apache/tajo/cli/TajoCli.java | 23 ++- .../apache/tajo/jdbc/TajoResultSetBase.java | 72 ++++++++- .../java/org/apache/tajo/conf/TajoConf.java | 7 +- .../planner/physical/StoreTableExec.java | 3 + .../apache/tajo/master/querymaster/Query.java | 4 + .../master/querymaster/QueryMasterTask.java | 27 +++- .../org/apache/tajo/QueryTestCaseBase.java | 5 +- .../org/apache/tajo/TajoTestingCluster.java | 6 +- .../java/org/apache/tajo/cli/TestTajoCli.java | 59 ++++++++ .../tajo/engine/query/TestInsertQuery.java | 3 +- .../tajo/engine/query/TestNullValues.java | 138 +++++++++++++++++- .../testSplitPartNested.result | 4 +- .../TestCaseByCases/testTAJO415Case.result | 4 +- .../testFullOuterJoin1.result | 4 +- .../testFullOuterJoinWithEmptyTable1.result | 10 +- .../testLeftOuterJoin1.result | 4 +- .../testLeftOuterJoinWithConstantExpr1.result | 4 +- .../testLeftOuterJoinWithConstantExpr2.result | 4 +- .../testLeftOuterJoinWithEmptyTable1.result | 10 +- .../testRightOuterJoin1.result | 4 +- .../testRightOuterJoinWithEmptyTable1.result | 10 +- .../TestJoinQuery/testFullOuterJoin1.result | 4 +- .../testFullOuterJoinWithEmptyTable1.result | 10 +- .../TestJoinQuery/testLeftOuterJoin1.result | 4 +- .../testLeftOuterJoinWithConstantExpr1.result | 4 +- .../testLeftOuterJoinWithConstantExpr2.result | 4 +- .../testLeftOuterJoinWithEmptyTable1.result | 10 +- .../testOuterJoinAndCaseWhen1.result | 10 +- .../TestJoinQuery/testRightOuterJoin1.result | 4 +- .../testRightOuterJoinWithEmptyTable1.result | 10 +- .../results/TestNetTypes/testJoin.result | 8 +- .../results/TestNetTypes/testSort2.result | 3 +- .../testCaseWhenWithoutElse.result | 2 +- .../results/TestTablePartitions/case1.result | 4 +- .../results/TestTablePartitions/case2.result | 8 +- .../testSelectResultWithNullFalse.result | 8 + .../testSelectResultWithNullTrue.result | 8 + 39 files changed, 421 insertions(+), 99 deletions(-) create mode 100644 tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullFalse.result create mode 100644 tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullTrue.result diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java index dd1f911985..8a41494f5a 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java @@ -24,6 +24,7 @@ import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.client.QueryStatus; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.util.FileUtil; import java.io.InputStream; @@ -36,6 +37,8 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter { private int printPauseRecords; private boolean printPause; private boolean printErrorTrace; + private boolean replaceNull; + private String nullKeyword; @Override public void init(TajoConf tajoConf) { @@ -44,6 +47,8 @@ public void init(TajoConf tajoConf) { this.printPause = tajoConf.getBoolVar(TajoConf.ConfVars.CLI_PRINT_PAUSE); this.printPauseRecords = tajoConf.getIntVar(TajoConf.ConfVars.CLI_PRINT_PAUSE_NUM_RECORDS); this.printErrorTrace = tajoConf.getBoolVar(TajoConf.ConfVars.CLI_PRINT_ERROR_TRACE); + this.replaceNull = tajoConf.getBoolVar(ConfVars.CLI_PRINT_NULL); + this.nullKeyword = tajoConf.getVar(ConfVars.CLI_PRINT_NULL_WORD); } @Override @@ -90,9 +95,13 @@ public void printResult(PrintWriter sout, InputStream sin, TableDesc tableDesc, while (res.next()) { for (int i = 1; i <= numOfColumns; i++) { if (i > 1) sout.print(", "); - String columnValue = res.getObject(i).toString(); + String columnValue = res.getString(i); if(res.wasNull()){ - sout.print("null"); + if (replaceNull) { + sout.print(nullKeyword); + } else { + sout.print(columnValue); + } } else { sout.print(columnValue); } diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java index 8b31d396c9..c694507534 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java @@ -44,6 +44,7 @@ public void invoke(String[] cmd) throws Exception { Map variables = new HashMap(); variables.put(cmd[1], cmd[2]); client.updateSessionVariables(variables); + context.setVariable(cmd[1], cmd[2]); } else { context.getOutput().println("usage: \\set [[NAME] VALUE]"); } diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java index e0ca62aaee..85787caa38 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java @@ -117,6 +117,15 @@ public PrintWriter getOutput() { public TajoConf getConf() { return conf; } + + public void setVariable(String key, String value) { + conf.set(key, value); + try { + initFormatter(); + } catch (Exception e) { + System.err.println(ERROR_PREFIX + e.getMessage()); + } + } } public TajoCli(TajoConf c, String [] args, InputStream in, OutputStream out) throws Exception { @@ -125,10 +134,7 @@ public TajoCli(TajoConf c, String [] args, InputStream in, OutputStream out) thr this.reader = new ConsoleReader(sin, out); this.reader.setExpandEvents(false); this.sout = new PrintWriter(reader.getOutput()); - Class formatterClass = conf.getClass(ConfVars.CLI_OUTPUT_FORMATTER_CLASS.varname, - DefaultTajoCliOutputFormatter.class); - this.outputFormatter = (TajoCliOutputFormatter)formatterClass.newInstance(); - this.outputFormatter.init(conf); + initFormatter(); CommandLineParser parser = new PosixParser(); CommandLine cmd = parser.parse(options, args); @@ -217,6 +223,15 @@ public TajoCli(TajoConf c, String [] args, InputStream in, OutputStream out) thr addShutdownHook(); } + private void initFormatter() throws Exception { + Class formatterClass = conf.getClass(ConfVars.CLI_OUTPUT_FORMATTER_CLASS.varname, + DefaultTajoCliOutputFormatter.class); + if (outputFormatter == null || !outputFormatter.getClass().equals(formatterClass)) { + outputFormatter = (TajoCliOutputFormatter)formatterClass.newInstance(); + } + outputFormatter.init(conf); + } + public TajoCliContext getContext() { return context; } diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java index 60eb2f1c20..27ffa22306 100644 --- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java +++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java @@ -21,10 +21,7 @@ import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.datum.TimeDatum; -import org.apache.tajo.datum.TimestampDatum; +import org.apache.tajo.datum.*; import org.apache.tajo.storage.Tuple; import java.io.IOException; @@ -67,6 +64,9 @@ public void beforeFirst() throws SQLException { public boolean getBoolean(int fieldId) throws SQLException { Datum datum = cur.get(fieldId - 1); handleNull(datum); + if (wasNull) { + return false; + } return datum.asBool(); } @@ -74,6 +74,9 @@ public boolean getBoolean(int fieldId) throws SQLException { public boolean getBoolean(String colName) throws SQLException { Datum datum = cur.get(findColumn(colName)); handleNull(datum); + if (wasNull) { + return false; + } return datum.asBool(); } @@ -81,6 +84,9 @@ public boolean getBoolean(String colName) throws SQLException { public byte getByte(int fieldId) throws SQLException { Datum datum = cur.get(fieldId - 1); handleNull(datum); + if (wasNull) { + return 0; + } return datum.asByte(); } @@ -88,6 +94,9 @@ public byte getByte(int fieldId) throws SQLException { public byte getByte(String name) throws SQLException { Datum datum = cur.get(findColumn(name)); handleNull(datum); + if (wasNull) { + return 0; + } return datum.asByte(); } @@ -95,6 +104,9 @@ public byte getByte(String name) throws SQLException { public byte[] getBytes(int fieldId) throws SQLException { Datum datum = cur.get(fieldId - 1); handleNull(datum); + if (wasNull) { + return null; + } return datum.asByteArray(); } @@ -102,6 +114,9 @@ public byte[] getBytes(int fieldId) throws SQLException { public byte[] getBytes(String name) throws SQLException { Datum datum = cur.get(findColumn(name)); handleNull(datum); + if (wasNull) { + return null; + } return datum.asByteArray(); } @@ -109,6 +124,9 @@ public byte[] getBytes(String name) throws SQLException { public double getDouble(int fieldId) throws SQLException { Datum datum = cur.get(fieldId - 1); handleNull(datum); + if (wasNull) { + return 0.0; + } return datum.asFloat8(); } @@ -116,6 +134,9 @@ public double getDouble(int fieldId) throws SQLException { public double getDouble(String name) throws SQLException { Datum datum = cur.get(findColumn(name)); handleNull(datum); + if (wasNull) { + return 0.0; + } return datum.asFloat8(); } @@ -123,6 +144,9 @@ public double getDouble(String name) throws SQLException { public float getFloat(int fieldId) throws SQLException { Datum datum = cur.get(fieldId - 1); handleNull(datum); + if (wasNull) { + return 0.0f; + } return datum.asFloat4(); } @@ -130,6 +154,9 @@ public float getFloat(int fieldId) throws SQLException { public float getFloat(String name) throws SQLException { Datum datum = cur.get(findColumn(name)); handleNull(datum); + if (wasNull) { + return 0.0f; + } return datum.asFloat4(); } @@ -137,6 +164,9 @@ public float getFloat(String name) throws SQLException { public int getInt(int fieldId) throws SQLException { Datum datum = cur.get(fieldId - 1); handleNull(datum); + if (wasNull) { + return 0; + } return datum.asInt4(); } @@ -144,6 +174,9 @@ public int getInt(int fieldId) throws SQLException { public int getInt(String name) throws SQLException { Datum datum = cur.get(findColumn(name)); handleNull(datum); + if (wasNull) { + return 0; + } return datum.asInt4(); } @@ -151,6 +184,9 @@ public int getInt(String name) throws SQLException { public long getLong(int fieldId) throws SQLException { Datum datum = cur.get(fieldId - 1); handleNull(datum); + if (wasNull) { + return 0; + } return datum.asInt8(); } @@ -158,6 +194,9 @@ public long getLong(int fieldId) throws SQLException { public long getLong(String name) throws SQLException { Datum datum = cur.get(findColumn(name)); handleNull(datum); + if (wasNull) { + return 0; + } return datum.asInt8(); } @@ -166,6 +205,9 @@ public Object getObject(int fieldId) throws SQLException { Datum d = cur.get(fieldId - 1); handleNull(d); + if (wasNull) { + return null; + } TajoDataTypes.Type dataType = schema.getColumn(fieldId - 1).getDataType().getType(); switch(dataType) { @@ -200,6 +242,9 @@ public Object getObject(String name) throws SQLException { public short getShort(int fieldId) throws SQLException { Datum datum = cur.get(fieldId - 1); handleNull(datum); + if (wasNull) { + return 0; + } return datum.asInt2(); } @@ -207,24 +252,37 @@ public short getShort(int fieldId) throws SQLException { public short getShort(String name) throws SQLException { Datum datum = cur.get(findColumn(name)); handleNull(datum); + if (wasNull) { + return 0; + } return datum.asInt2(); } @Override public String getString(int fieldId) throws SQLException { Datum datum = cur.get(fieldId - 1); - handleNull(datum); - return datum.asChars(); + return getString(datum, fieldId); } @Override public String getString(String name) throws SQLException { - Datum datum = cur.get(findColumn(name)); + int id = findColumn(name); + Datum datum = cur.get(id); + return getString(datum, id + 1); + } + + private String getString(Datum datum, int fieldId) throws SQLException { handleNull(datum); + if (wasNull) { + return null; + } + TajoDataTypes.Type dataType = datum.type(); switch(dataType) { + case BOOLEAN: + return String.valueOf(((BooleanDatum)datum).asBool()); case TIME: { return ((TimeDatum)datum).asChars(TajoConf.getCurrentTimeZone(), false); } diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index b62b24649c..8504ca725b 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -324,10 +324,15 @@ public static enum ConfVars { CLI_PRINT_PAUSE("tajo.cli.print.pause", true), CLI_PRINT_ERROR_TRACE("tajo.cli.print.error.trace", true), CLI_OUTPUT_FORMATTER_CLASS("tajo.cli.output.formatter", "org.apache.tajo.cli.DefaultTajoCliOutputFormatter"), + CLI_PRINT_NULL("tajo.cli.print.null", false), + CLI_PRINT_NULL_WORD("tajo.cli.print.null.word", "null"), //TIME & DATE TAJO_TIMEZONE("tajo.timezone", System.getProperty("user.timezone")), - TAJO_DATE_ORDER("tajo.date.order", "YMD") + TAJO_DATE_ORDER("tajo.date.order", "YMD"), + + // FILE FORMAT + CSVFILE_NULL("csvfile.null", "\\\\N") ; public final String varname; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index b0c3c3125a..f3f3204d0a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -20,9 +20,11 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.engine.planner.logical.InsertNode; import org.apache.tajo.engine.planner.logical.PersistentStoreNode; import org.apache.tajo.storage.Appender; +import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.storage.StorageManagerFactory; import org.apache.tajo.storage.Tuple; import org.apache.tajo.worker.TaskAttemptContext; @@ -57,6 +59,7 @@ public void init() throws IOException { appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, createTableNode.getTableSchema(), context.getOutputPath()); } else { + meta.putOption(StorageConstants.CSVFILE_NULL, context.getConf().getVar(ConfVars.CSVFILE_NULL)); appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, outSchema, context.getOutputPath()); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java index 04e82cabcb..e46aec6c67 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java @@ -37,6 +37,7 @@ import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.global.ExecutionBlock; import org.apache.tajo.engine.planner.global.ExecutionBlockCursor; @@ -47,6 +48,7 @@ import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.event.*; import org.apache.tajo.storage.AbstractStorageManager; +import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.TUtil; import java.io.IOException; @@ -481,6 +483,8 @@ public void execute(QueryMaster.QueryMasterContext context, QueryContext queryCo TableMeta meta = lastStage.getTableMeta(); TableStats stats = lastStage.getResultStats(); + meta.putOption(StorageConstants.CSVFILE_NULL, context.getConf().getVar(ConfVars.CSVFILE_NULL)); + TableDesc resultTableDesc = new TableDesc( query.getId().toString(), diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java index f81271521d..83201f5d10 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java @@ -102,7 +102,9 @@ public class QueryMasterTask extends CompositeService { private Map tableDescMap = new HashMap(); - private TajoConf systemConf; + //private TajoConf systemConf; + + private TajoConf queryConf; private AtomicLong lastClientHeartbeat = new AtomicLong(-1); @@ -133,11 +135,20 @@ public QueryMasterTask(QueryMaster.QueryMasterContext queryMasterContext, @Override public void init(Configuration conf) { - systemConf = (TajoConf)conf; + //systemConf = (TajoConf)conf; try { + queryConf = new TajoConf(conf); + if (session != null) { + Map sessionValues = session.getAllVariables(); + if (sessionValues != null) { + for (Map.Entry entry: sessionValues.entrySet()) { + queryConf.set(entry.getKey(), entry.getValue()); + } + } + } queryTaskContext = new QueryMasterTaskContext(); - String resourceManagerClassName = systemConf.getVar(TajoConf.ConfVars.RESOURCE_MANAGER_CLASS); + String resourceManagerClassName = queryConf.getVar(TajoConf.ConfVars.RESOURCE_MANAGER_CLASS); if(resourceManagerClassName.indexOf(TajoWorkerResourceManager.class.getName()) >= 0) { resourceAllocator = new TajoResourceAllocator(queryTaskContext); @@ -160,7 +171,7 @@ public void init(Configuration conf) { queryMetrics = new TajoMetrics(queryId.toString()); - super.init(systemConf); + super.init(queryConf); } catch (Throwable t) { LOG.error(t.getMessage(), t); initError = t; @@ -324,7 +335,7 @@ public synchronized void startQuery() { } CatalogService catalog = getQueryTaskContext().getQueryMasterContext().getWorkerContext().getCatalog(); LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(systemConf); + LogicalOptimizer optimizer = new LogicalOptimizer(queryConf); Expr expr = JsonHelper.fromJson(jsonExpr, Expr.class); LogicalPlan plan = planner.createPlan(session, expr); optimizer.optimize(plan); @@ -367,11 +378,11 @@ public synchronized void startQuery() { private void initStagingDir() throws IOException { Path stagingDir = null; Path outputDir = null; - FileSystem defaultFS = TajoConf.getWarehouseDir(systemConf).getFileSystem(systemConf); + FileSystem defaultFS = TajoConf.getWarehouseDir(queryConf).getFileSystem(queryConf); try { - stagingDir = initStagingDir(systemConf, defaultFS, queryId.toString()); + stagingDir = initStagingDir(queryConf, defaultFS, queryId.toString()); defaultFS.mkdirs(new Path(stagingDir, TajoConstants.RESULT_DIR_NAME)); // Create a subdirectories @@ -517,7 +528,7 @@ public QueryContext getQueryContext() { } public TajoConf getConf() { - return systemConf; + return queryConf; } public Clock getClock() { diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 1d6ee86038..d68eb5fe3e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -410,7 +410,10 @@ public String resultSetToString(ResultSet resultSet) throws SQLException { while (resultSet.next()) { for (int i = 1; i <= numOfColumns; i++) { if (i > 1) sb.append(","); - String columnValue = resultSet.getObject(i).toString(); + String columnValue = resultSet.getString(i); + if (resultSet.wasNull()) { + columnValue = "null"; + } sb.append(columnValue); } sb.append("\n"); diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java index 011ed07ac2..06d2bab2d4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -33,7 +33,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.tajo.catalog.*; -import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.client.TajoClient; import org.apache.tajo.conf.TajoConf; @@ -41,6 +40,7 @@ import org.apache.tajo.master.TajoMaster; import org.apache.tajo.master.rm.TajoWorkerResourceManager; import org.apache.tajo.util.CommonTestingUtil; +import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.NetUtils; import org.apache.tajo.worker.TajoWorker; @@ -560,7 +560,7 @@ public void shutdownMiniCluster() throws IOException { public static ResultSet run(String[] names, Schema[] schemas, - KeyValueSet option, + KeyValueSet tableOption, String[][] tables, String query) throws Exception { TpchTestBase instance = TpchTestBase.getInstance(); @@ -587,7 +587,7 @@ public static ResultSet run(String[] names, out.write((tables[i][j]+"\n").getBytes()); } out.close(); - TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV, option); + TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV, tableOption); client.createExternalTable(names[i], schemas[i], tablePath, meta); } Thread.sleep(1000); diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java index b3fbda49a3..1a9a2b9f65 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.TpchTestBase; +import org.apache.tajo.client.QueryStatus; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.storage.StorageUtil; @@ -36,6 +37,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.PrintWriter; import java.net.URL; import static org.junit.Assert.assertEquals; @@ -195,6 +197,59 @@ public void testDescTable() throws Exception { } } + @Test + public void testSelectResultWithNullFalse() throws Exception { + String sql = + "select\n" + + " c_custkey,\n" + + " orders.o_orderkey,\n" + + " orders.o_orderstatus \n" + + "from\n" + + " orders full outer join customer on c_custkey = o_orderkey\n" + + "order by\n" + + " c_custkey,\n" + + " orders.o_orderkey;\n"; + + TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); + tajoConf.setVar(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); + + tajoConf.setBoolVar(ConfVars.CLI_PRINT_NULL, false); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out); + tajoCli.executeScript(sql); + + String consoleResult = new String(out.toByteArray()); + assertOutputResult(consoleResult); + } + + @Test + public void testSelectResultWithNullTrue() throws Exception { + String sql = + "select\n" + + " c_custkey,\n" + + " orders.o_orderkey,\n" + + " orders.o_orderstatus \n" + + "from\n" + + " orders full outer join customer on c_custkey = o_orderkey\n" + + "order by\n" + + " c_custkey,\n" + + " orders.o_orderkey;\n"; + + TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); + tajoConf.setVar(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); + + tajoConf.setBoolVar(ConfVars.CLI_PRINT_NULL, true); + tajoConf.setVar(ConfVars.CLI_PRINT_NULL_WORD, "testnull"); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out); + tajoCli.executeScript(sql); + + String consoleResult = new String(out.toByteArray()); + assertOutputResult(consoleResult); + } + private void assertOutputResult(String actual) throws Exception { assertOutputResult(name.getMethodName() + ".result", actual); } @@ -226,5 +281,9 @@ public static class TajoCliOutputTestFormatter extends DefaultTajoCliOutputForma protected String getResponseTimeReadable(float responseTime) { return ""; } + @Override + public void printProgress(PrintWriter sout, QueryStatus status) { + //nothing to do + } } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java index 8453488332..3aacbff38a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java @@ -342,7 +342,8 @@ public final void testInsertOverwriteTableWithNonFromQuery2() throws Exception { assertEquals(3, res.getMetaData().getColumnCount()); assertEquals(1, res.getInt(1)); - assertEquals("", res.getString(2)); + assertNull(res.getString(2)); + assertEquals(0.0, res.getDouble(2), 10); assertEquals("test", res.getString(3)); res.close(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestNullValues.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestNullValues.java index e288066ad1..f41b0abc18 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestNullValues.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestNullValues.java @@ -20,14 +20,15 @@ import org.apache.tajo.IntegrationTest; import org.apache.tajo.TajoTestingCluster; -import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.util.KeyValueSet; import org.junit.Test; import org.junit.experimental.categories.Category; import java.sql.ResultSet; +import java.sql.SQLException; import static org.junit.Assert.*; @@ -56,6 +57,7 @@ public final void testIsNull() throws Exception { ResultSet res = TajoTestingCluster .run(table, schemas, opts, new String[][]{data}, "select * from nulltable1 where col3 is null"); + try { assertTrue(res.next()); assertEquals(2, res.getInt(1)); @@ -159,4 +161,138 @@ public final void testIsNotNull3() throws Exception { res.close(); } } + + @Test + public final void testResultSetNullSimpleQuery() throws Exception { + String tableName = "nulltable5"; + ResultSet res = runNullTableQuery(tableName, "select col1, col2, col3, col4 from " + tableName); + + try { + int numRows = 0; + + String expected = + "null|a|1.0|true\n" + + "2|null|2.0|false\n" + + "3|c|null|true\n" + + "4|d|4.0|null"; + + String result = ""; + + String prefix = ""; + while(res.next()) { + for (int i = 0; i < 4; i++) { + result += prefix + res.getObject(i + 1); + prefix = "|"; + } + prefix = "\n"; + + assertResultSetNull(res, numRows, false, new int[]{1,2,3,4}); + assertResultSetNull(res, numRows, true, new int[]{1,2,3,4}); + numRows++; + } + assertEquals(4, numRows); + assertEquals(expected, result); + } finally { + res.close(); + } + } + + @Test + public final void testResultSetNull() throws Exception { + String tableName = "nulltable6"; + String query = "select " + + "col1, coalesce(col1, 99999), " + + "col2, coalesce(col2, 'null_value'), " + + "col3, coalesce(col3, 99999.0)," + + "col4 " + + "from " + tableName; + + ResultSet res = runNullTableQuery(tableName, query); + + try { + int numRows = 0; + String expected = + "null|99999|a|a|1.0|1.0|true\n" + + "2|2|null|null_value|2.0|2.0|false\n" + + "3|3|c|c|null|99999.0|true\n" + + "4|4|d|d|4.0|4.0|null"; + + String result = ""; + + String prefix = ""; + while(res.next()) { + for (int i = 0; i < 7; i++) { + result += prefix + res.getObject(i + 1); + prefix = "|"; + } + prefix = "\n"; + + assertResultSetNull(res, numRows, false, new int[]{1,3,5,7}); + assertResultSetNull(res, numRows, true, new int[]{1,3,5,7}); + numRows++; + } + assertEquals(4, numRows); + assertEquals(expected, result); + } finally { + res.close(); + } + } + + private ResultSet runNullTableQuery(String tableName, String query) throws Exception { + String [] table = new String[] {tableName}; + Schema schema = new Schema(); + schema.addColumn("col1", Type.INT4); + schema.addColumn("col2", Type.TEXT); + schema.addColumn("col3", Type.FLOAT4); + schema.addColumn("col4", Type.BOOLEAN); + Schema [] schemas = new Schema[] {schema}; + String [] data = { + "\\N|a|1.0|t", + "2|\\N|2.0|f", + "3|c|\\N|t", + "4|d|4.0|\\N" + }; + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.put(StorageConstants.CSVFILE_NULL, "\\\\N"); + + ResultSet res = TajoTestingCluster + .run(table, schemas, tableOptions, new String[][]{data}, query); + + return res; + } + + private void assertResultSetNull(ResultSet res, int numRows, boolean useName, int[] nullIndex) throws SQLException { + if (numRows == 0) { + if (useName) { + assertEquals(0, res.getInt(res.getMetaData().getColumnName(nullIndex[numRows]))); + } else { + assertEquals(0, res.getInt(nullIndex[numRows])); + } + } + + if (numRows == 1) { + if (useName) { + assertNull(res.getString(res.getMetaData().getColumnName(nullIndex[numRows]))); + } else { + assertNull(res.getString(nullIndex[numRows])); + }; + } + + if (numRows == 2) { + if (useName) { + assertEquals(0.0, res.getDouble(res.getMetaData().getColumnName(nullIndex[numRows])), 10); + } else { + assertEquals(0.0, res.getDouble(nullIndex[numRows]), 10); + } + } + + if (numRows == 3) { + if (useName) { + assertEquals(false, res.getBoolean(res.getMetaData().getColumnName(nullIndex[numRows]))); + } else { + assertEquals(false, res.getBoolean(nullIndex[numRows])); + } + } + } } diff --git a/tajo-core/src/test/resources/results/TestBuiltinFunctions/testSplitPartNested.result b/tajo-core/src/test/resources/results/TestBuiltinFunctions/testSplitPartNested.result index 5dd41937b9..ec0f07b411 100644 --- a/tajo-core/src/test/resources/results/TestBuiltinFunctions/testSplitPartNested.result +++ b/tajo-core/src/test/resources/results/TestBuiltinFunctions/testSplitPartNested.result @@ -1,7 +1,7 @@ ?split_part ------------------------------- - +null KE KE - +null KE \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO415Case.result b/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO415Case.result index 675037bbec..6c527afd88 100644 --- a/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO415Case.result +++ b/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO415Case.result @@ -3,5 +3,5 @@ c_custkey,o_orderkey,cnt 1,1,1 2,2,1 3,3,1 -4,0,1 -5,0,1 \ No newline at end of file +4,null,1 +5,null,1 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinBroadcast/testFullOuterJoin1.result b/tajo-core/src/test/resources/results/TestJoinBroadcast/testFullOuterJoin1.result index 81dc055042..695a414850 100644 --- a/tajo-core/src/test/resources/results/TestJoinBroadcast/testFullOuterJoin1.result +++ b/tajo-core/src/test/resources/results/TestJoinBroadcast/testFullOuterJoin1.result @@ -3,5 +3,5 @@ c_custkey,o_orderkey 1,1 2,2 3,3 -4,0 -5,0 \ No newline at end of file +4,null +5,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinBroadcast/testFullOuterJoinWithEmptyTable1.result b/tajo-core/src/test/resources/results/TestJoinBroadcast/testFullOuterJoinWithEmptyTable1.result index 9124c6c974..5b849fcb37 100644 --- a/tajo-core/src/test/resources/results/TestJoinBroadcast/testFullOuterJoinWithEmptyTable1.result +++ b/tajo-core/src/test/resources/results/TestJoinBroadcast/testFullOuterJoinWithEmptyTable1.result @@ -1,7 +1,7 @@ c_custkey,o_orderkey ------------------------------- -1,0 -2,0 -3,0 -4,0 -5,0 \ No newline at end of file +1,null +2,null +3,null +4,null +5,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoin1.result b/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoin1.result index 8893e121e7..9e2a53a6a0 100644 --- a/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoin1.result +++ b/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoin1.result @@ -3,5 +3,5 @@ c_custkey,o_orderkey,o_orderstatus,o_orderdate 1,1,O,1996-01-02 2,2,O,1996-12-01 3,3,F,1993-10-14 -4,0,, -5,0,, \ No newline at end of file +4,null,null,null +5,null,null,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr1.result b/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr1.result index ed2817286a..670a069919 100644 --- a/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr1.result +++ b/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr1.result @@ -3,5 +3,5 @@ c_custkey,o_orderkey,val 1,1,val 2,2,val 3,3,val -4,0,val -5,0,val \ No newline at end of file +4,null,val +5,null,val \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr2.result b/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr2.result index ed2817286a..670a069919 100644 --- a/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr2.result +++ b/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr2.result @@ -3,5 +3,5 @@ c_custkey,o_orderkey,val 1,1,val 2,2,val 3,3,val -4,0,val -5,0,val \ No newline at end of file +4,null,val +5,null,val \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable1.result b/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable1.result index 5e85b28d81..e2f94f47cb 100644 --- a/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable1.result +++ b/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable1.result @@ -1,7 +1,7 @@ c_custkey,o_orderkey,o_orderstatus,o_orderdate ------------------------------- -1,0,, -2,0,, -3,0,, -4,0,, -5,0,, \ No newline at end of file +1,null,null,null +2,null,null,null +3,null,null,null +4,null,null,null +5,null,null,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinBroadcast/testRightOuterJoin1.result b/tajo-core/src/test/resources/results/TestJoinBroadcast/testRightOuterJoin1.result index 81dc055042..695a414850 100644 --- a/tajo-core/src/test/resources/results/TestJoinBroadcast/testRightOuterJoin1.result +++ b/tajo-core/src/test/resources/results/TestJoinBroadcast/testRightOuterJoin1.result @@ -3,5 +3,5 @@ c_custkey,o_orderkey 1,1 2,2 3,3 -4,0 -5,0 \ No newline at end of file +4,null +5,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinBroadcast/testRightOuterJoinWithEmptyTable1.result b/tajo-core/src/test/resources/results/TestJoinBroadcast/testRightOuterJoinWithEmptyTable1.result index 9124c6c974..5b849fcb37 100644 --- a/tajo-core/src/test/resources/results/TestJoinBroadcast/testRightOuterJoinWithEmptyTable1.result +++ b/tajo-core/src/test/resources/results/TestJoinBroadcast/testRightOuterJoinWithEmptyTable1.result @@ -1,7 +1,7 @@ c_custkey,o_orderkey ------------------------------- -1,0 -2,0 -3,0 -4,0 -5,0 \ No newline at end of file +1,null +2,null +3,null +4,null +5,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testFullOuterJoin1.result b/tajo-core/src/test/resources/results/TestJoinQuery/testFullOuterJoin1.result index 81dc055042..695a414850 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testFullOuterJoin1.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testFullOuterJoin1.result @@ -3,5 +3,5 @@ c_custkey,o_orderkey 1,1 2,2 3,3 -4,0 -5,0 \ No newline at end of file +4,null +5,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testFullOuterJoinWithEmptyTable1.result b/tajo-core/src/test/resources/results/TestJoinQuery/testFullOuterJoinWithEmptyTable1.result index 9124c6c974..5b849fcb37 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testFullOuterJoinWithEmptyTable1.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testFullOuterJoinWithEmptyTable1.result @@ -1,7 +1,7 @@ c_custkey,o_orderkey ------------------------------- -1,0 -2,0 -3,0 -4,0 -5,0 \ No newline at end of file +1,null +2,null +3,null +4,null +5,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoin1.result b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoin1.result index 8893e121e7..9e2a53a6a0 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoin1.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoin1.result @@ -3,5 +3,5 @@ c_custkey,o_orderkey,o_orderstatus,o_orderdate 1,1,O,1996-01-02 2,2,O,1996-12-01 3,3,F,1993-10-14 -4,0,, -5,0,, \ No newline at end of file +4,null,null,null +5,null,null,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithConstantExpr1.result b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithConstantExpr1.result index ed2817286a..670a069919 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithConstantExpr1.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithConstantExpr1.result @@ -3,5 +3,5 @@ c_custkey,o_orderkey,val 1,1,val 2,2,val 3,3,val -4,0,val -5,0,val \ No newline at end of file +4,null,val +5,null,val \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithConstantExpr2.result b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithConstantExpr2.result index ed2817286a..670a069919 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithConstantExpr2.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithConstantExpr2.result @@ -3,5 +3,5 @@ c_custkey,o_orderkey,val 1,1,val 2,2,val 3,3,val -4,0,val -5,0,val \ No newline at end of file +4,null,val +5,null,val \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable1.result b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable1.result index 5e85b28d81..e2f94f47cb 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable1.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable1.result @@ -1,7 +1,7 @@ c_custkey,o_orderkey,o_orderstatus,o_orderdate ------------------------------- -1,0,, -2,0,, -3,0,, -4,0,, -5,0,, \ No newline at end of file +1,null,null,null +2,null,null,null +3,null,null,null +4,null,null,null +5,null,null,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testOuterJoinAndCaseWhen1.result b/tajo-core/src/test/resources/results/TestJoinQuery/testOuterJoinAndCaseWhen1.result index f032d32801..8f204fb19e 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testOuterJoinAndCaseWhen1.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testOuterJoinAndCaseWhen1.result @@ -1,7 +1,7 @@ id,name,id2,name2,c1,c2 ------------------------------- -1,ooo,1,,9991231,ooo -2,ppp,2,,9991231,ppp -3,qqq,0,,9991231,9991231 -4,rrr,0,,9991231,9991231 -5,xxx,0,,9991231,9991231 \ No newline at end of file +1,ooo,1,null,9991231,ooo +2,ppp,2,null,9991231,ppp +3,qqq,null,null,9991231,9991231 +4,rrr,null,null,9991231,9991231 +5,xxx,null,null,9991231,9991231 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testRightOuterJoin1.result b/tajo-core/src/test/resources/results/TestJoinQuery/testRightOuterJoin1.result index 81dc055042..695a414850 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testRightOuterJoin1.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testRightOuterJoin1.result @@ -3,5 +3,5 @@ c_custkey,o_orderkey 1,1 2,2 3,3 -4,0 -5,0 \ No newline at end of file +4,null +5,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testRightOuterJoinWithEmptyTable1.result b/tajo-core/src/test/resources/results/TestJoinQuery/testRightOuterJoinWithEmptyTable1.result index 9124c6c974..5b849fcb37 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testRightOuterJoinWithEmptyTable1.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testRightOuterJoinWithEmptyTable1.result @@ -1,7 +1,7 @@ c_custkey,o_orderkey ------------------------------- -1,0 -2,0 -3,0 -4,0 -5,0 \ No newline at end of file +1,null +2,null +3,null +4,null +5,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestNetTypes/testJoin.result b/tajo-core/src/test/resources/results/TestNetTypes/testJoin.result index ea979acc9c..568b488112 100644 --- a/tajo-core/src/test/resources/results/TestNetTypes/testJoin.result +++ b/tajo-core/src/test/resources/results/TestNetTypes/testJoin.result @@ -1,6 +1,6 @@ id,name,score,type,addr,id,name,score,type,addr ------------------------------- -1,ooo,1.1,a,127.0.0.1,0,,20.0,d,127.0.0.1 -3,qqq,3.4,c,127.0.0.8,1,,0.0,a,127.0.0.8 -3,qqq,3.4,c,127.0.0.8,2,,0.0,b,127.0.0.8 -4,rrr,4.5,d,127.0.0.1,0,,20.0,d,127.0.0.1 \ No newline at end of file +1,ooo,1.1,a,127.0.0.1,null,null,20.0,d,127.0.0.1 +3,qqq,3.4,c,127.0.0.8,1,null,null,a,127.0.0.8 +3,qqq,3.4,c,127.0.0.8,2,null,null,b,127.0.0.8 +4,rrr,4.5,d,127.0.0.1,null,null,20.0,d,127.0.0.1 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestNetTypes/testSort2.result b/tajo-core/src/test/resources/results/TestNetTypes/testSort2.result index debbb987c1..9bd50384b3 100644 --- a/tajo-core/src/test/resources/results/TestNetTypes/testSort2.result +++ b/tajo-core/src/test/resources/results/TestNetTypes/testSort2.result @@ -3,4 +3,5 @@ addr 127.0.0.1 127.0.0.8 127.0.0.8 -255.255.255.255 \ No newline at end of file +255.255.255.255 +null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testCaseWhenWithoutElse.result b/tajo-core/src/test/resources/results/TestSelectQuery/testCaseWhenWithoutElse.result index 843b4e86c5..bde7df8b67 100644 --- a/tajo-core/src/test/resources/results/TestSelectQuery/testCaseWhenWithoutElse.result +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testCaseWhenWithoutElse.result @@ -1,6 +1,6 @@ r_regionkey,cond ------------------------------- -0,0 +0,null 1,11 2,12 3,13 diff --git a/tajo-core/src/test/resources/results/TestTablePartitions/case1.result b/tajo-core/src/test/resources/results/TestTablePartitions/case1.result index 4bfe8a269c..aece52e74e 100644 --- a/tajo-core/src/test/resources/results/TestTablePartitions/case1.result +++ b/tajo-core/src/test/resources/results/TestTablePartitions/case1.result @@ -1,4 +1,4 @@ col1,col2,null_col,key ------------------------------- -2,2,0,38.0 -3,2,0,45.0 \ No newline at end of file +2,2,null,38.0 +3,2,null,45.0 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestTablePartitions/case2.result b/tajo-core/src/test/resources/results/TestTablePartitions/case2.result index 45b1a44fdb..af566182a0 100644 --- a/tajo-core/src/test/resources/results/TestTablePartitions/case2.result +++ b/tajo-core/src/test/resources/results/TestTablePartitions/case2.result @@ -1,6 +1,6 @@ col1,col2,null_col,key ------------------------------- -2,2,0,38.0 -2,2,0,38.0 -3,2,0,45.0 -3,2,0,45.0 \ No newline at end of file +2,2,null,38.0 +2,2,null,38.0 +3,2,null,45.0 +3,2,null,45.0 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullFalse.result b/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullFalse.result new file mode 100644 index 0000000000..4bb8e962ab --- /dev/null +++ b/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullFalse.result @@ -0,0 +1,8 @@ +c_custkey, o_orderkey, o_orderstatus +------------------------------- +1, 1, O +2, 2, O +3, 3, F +4, null, null +5, null, null +(5 rows, , 30 B selected) diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullTrue.result b/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullTrue.result new file mode 100644 index 0000000000..566262e5fa --- /dev/null +++ b/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullTrue.result @@ -0,0 +1,8 @@ +c_custkey, o_orderkey, o_orderstatus +------------------------------- +1, 1, O +2, 2, O +3, 3, F +4, testnull, testnull +5, testnull, testnull +(5 rows, , 30 B selected) From f8d3452cf88f73858a0dd362a09c64025a7f0819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=ED=98=95=EC=A4=80?= Date: Tue, 27 May 2014 23:34:53 +0900 Subject: [PATCH 2/3] TAJO-842: NULL handling in JDBC. --- .../cli/DefaultTajoCliOutputFormatter.java | 12 +-- .../apache/tajo/jdbc/TajoResultSetBase.java | 6 +- .../java/org/apache/tajo/conf/TajoConf.java | 5 +- .../planner/physical/StoreTableExec.java | 3 +- .../org/apache/tajo/master/GlobalEngine.java | 7 +- .../apache/tajo/master/querymaster/Query.java | 6 +- .../master/querymaster/QueryMasterTask.java | 27 ++----- .../java/org/apache/tajo/worker/Task.java | 2 +- .../tajo/worker/TaskAttemptContext.java | 13 +++- .../java/org/apache/tajo/cli/TestTajoCli.java | 5 +- .../apache/tajo/client/TestTajoClient.java | 53 +++++++++++++- .../planner/physical/TestBNLJoinExec.java | 6 +- .../planner/physical/TestBSTIndexExec.java | 3 +- .../physical/TestExternalSortExec.java | 3 +- .../physical/TestFullOuterHashJoinExec.java | 10 ++- .../physical/TestFullOuterMergeJoinExec.java | 13 ++-- .../physical/TestHashAntiJoinExec.java | 3 +- .../planner/physical/TestHashJoinExec.java | 5 +- .../physical/TestHashSemiJoinExec.java | 3 +- .../physical/TestLeftOuterHashJoinExec.java | 11 +-- .../physical/TestLeftOuterNLJoinExec.java | 11 +-- .../planner/physical/TestMergeJoinExec.java | 3 +- .../planner/physical/TestNLJoinExec.java | 5 +- .../planner/physical/TestPhysicalPlanner.java | 73 +++++++++++++------ .../TestProgressExternalSortExec.java | 3 +- .../physical/TestRightOuterHashJoinExec.java | 7 +- .../physical/TestRightOuterMergeJoinExec.java | 13 ++-- .../engine/planner/physical/TestSortExec.java | 4 +- .../worker/TestRangeRetrieverHandler.java | 7 +- 29 files changed, 206 insertions(+), 116 deletions(-) diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java index 8a41494f5a..a619981d22 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java @@ -37,8 +37,7 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter { private int printPauseRecords; private boolean printPause; private boolean printErrorTrace; - private boolean replaceNull; - private String nullKeyword; + private String nullChar; @Override public void init(TajoConf tajoConf) { @@ -47,8 +46,7 @@ public void init(TajoConf tajoConf) { this.printPause = tajoConf.getBoolVar(TajoConf.ConfVars.CLI_PRINT_PAUSE); this.printPauseRecords = tajoConf.getIntVar(TajoConf.ConfVars.CLI_PRINT_PAUSE_NUM_RECORDS); this.printErrorTrace = tajoConf.getBoolVar(TajoConf.ConfVars.CLI_PRINT_ERROR_TRACE); - this.replaceNull = tajoConf.getBoolVar(ConfVars.CLI_PRINT_NULL); - this.nullKeyword = tajoConf.getVar(ConfVars.CLI_PRINT_NULL_WORD); + this.nullChar = tajoConf.getVar(ConfVars.CLI_NULL_CHAR); } @Override @@ -97,11 +95,7 @@ public void printResult(PrintWriter sout, InputStream sin, TableDesc tableDesc, if (i > 1) sout.print(", "); String columnValue = res.getString(i); if(res.wasNull()){ - if (replaceNull) { - sout.print(nullKeyword); - } else { - sout.print(columnValue); - } + sout.print(nullChar); } else { sout.print(columnValue); } diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java index 27ffa22306..bdcf216fc3 100644 --- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java +++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java @@ -125,7 +125,7 @@ public double getDouble(int fieldId) throws SQLException { Datum datum = cur.get(fieldId - 1); handleNull(datum); if (wasNull) { - return 0.0; + return 0.0d; } return datum.asFloat8(); } @@ -135,7 +135,7 @@ public double getDouble(String name) throws SQLException { Datum datum = cur.get(findColumn(name)); handleNull(datum); if (wasNull) { - return 0.0; + return 0.0d; } return datum.asFloat8(); } @@ -282,7 +282,7 @@ private String getString(Datum datum, int fieldId) throws SQLException { switch(dataType) { case BOOLEAN: - return String.valueOf(((BooleanDatum)datum).asBool()); + return String.valueOf(datum.asBool()); case TIME: { return ((TimeDatum)datum).asChars(TajoConf.getCurrentTimeZone(), false); } diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 8504ca725b..7ddeadc38e 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -324,15 +324,14 @@ public static enum ConfVars { CLI_PRINT_PAUSE("tajo.cli.print.pause", true), CLI_PRINT_ERROR_TRACE("tajo.cli.print.error.trace", true), CLI_OUTPUT_FORMATTER_CLASS("tajo.cli.output.formatter", "org.apache.tajo.cli.DefaultTajoCliOutputFormatter"), - CLI_PRINT_NULL("tajo.cli.print.null", false), - CLI_PRINT_NULL_WORD("tajo.cli.print.null.word", "null"), + CLI_NULL_CHAR("tajo.cli.nullchar", ""), //TIME & DATE TAJO_TIMEZONE("tajo.timezone", System.getProperty("user.timezone")), TAJO_DATE_ORDER("tajo.date.order", "YMD"), // FILE FORMAT - CSVFILE_NULL("csvfile.null", "\\\\N") + CSVFILE_NULL("tajo.csvfile.null", "\\\\N") ; public final String varname; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index f3f3204d0a..b1d0400abe 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -59,7 +59,8 @@ public void init() throws IOException { appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, createTableNode.getTableSchema(), context.getOutputPath()); } else { - meta.putOption(StorageConstants.CSVFILE_NULL, context.getConf().getVar(ConfVars.CSVFILE_NULL)); + String nullChar = context.getQueryContext().get(ConfVars.CSVFILE_NULL.varname, ConfVars.CSVFILE_NULL.defaultVal); + meta.putOption(StorageConstants.CSVFILE_NULL, nullChar); appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, outSchema, context.getOutputPath()); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 3b81ce2ea1..2b06a42b5b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -120,6 +120,7 @@ public void stop() { public SubmitQueryResponse executeQuery(Session session, String query, boolean isJson) { LOG.info("Query: " + query); QueryContext queryContext = new QueryContext(); + queryContext.putAll(session.getAllVariables()); Expr planningContext; try { @@ -257,7 +258,7 @@ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, boolean isInsert = rootNode.getChild() != null && rootNode.getChild().getType() == NodeType.INSERT; if (isInsert) { InsertNode insertNode = rootNode.getChild(); - insertNonFromQuery(insertNode, responseBuilder); + insertNonFromQuery(queryContext, insertNode, responseBuilder); } else { Schema schema = PlannerUtil.targetToSchema(targets); RowStoreUtil.RowStoreEncoder encoder = RowStoreUtil.createEncoder(schema); @@ -300,7 +301,7 @@ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, return response; } - private void insertNonFromQuery(InsertNode insertNode, SubmitQueryResponse.Builder responseBuilder) + private void insertNonFromQuery(QueryContext queryContext, InsertNode insertNode, SubmitQueryResponse.Builder responseBuilder) throws Exception { String nodeUniqName = insertNode.getTableName() == null ? insertNode.getPath().getName() : insertNode.getTableName(); String queryId = nodeUniqName + "_" + System.currentTimeMillis(); @@ -321,7 +322,7 @@ private void insertNonFromQuery(InsertNode insertNode, SubmitQueryResponse.Build } TaskAttemptContext taskAttemptContext = - new TaskAttemptContext(context.getConf(), null, (CatalogProtos.FragmentProto[]) null, stagingDir); + new TaskAttemptContext(context.getConf(), queryContext, null, (CatalogProtos.FragmentProto[]) null, stagingDir); taskAttemptContext.setOutputPath(new Path(stagingResultDir, "part-01-000000")); EvalExprExec evalExprExec = new EvalExprExec(taskAttemptContext, (EvalExprNode) insertNode.getChild()); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java index e46aec6c67..0ce6d7e1c2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java @@ -481,9 +481,11 @@ public void execute(QueryMaster.QueryMasterContext context, QueryContext queryCo Path finalOutputDir) throws Exception { SubQuery lastStage = query.getSubQuery(finalExecBlockId); TableMeta meta = lastStage.getTableMeta(); - TableStats stats = lastStage.getResultStats(); - meta.putOption(StorageConstants.CSVFILE_NULL, context.getConf().getVar(ConfVars.CSVFILE_NULL)); + String nullChar = queryContext.get(ConfVars.CSVFILE_NULL.varname, ConfVars.CSVFILE_NULL.defaultVal); + meta.putOption(StorageConstants.CSVFILE_NULL, nullChar); + + TableStats stats = lastStage.getResultStats(); TableDesc resultTableDesc = new TableDesc( diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java index 83201f5d10..f81271521d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java @@ -102,9 +102,7 @@ public class QueryMasterTask extends CompositeService { private Map tableDescMap = new HashMap(); - //private TajoConf systemConf; - - private TajoConf queryConf; + private TajoConf systemConf; private AtomicLong lastClientHeartbeat = new AtomicLong(-1); @@ -135,20 +133,11 @@ public QueryMasterTask(QueryMaster.QueryMasterContext queryMasterContext, @Override public void init(Configuration conf) { - //systemConf = (TajoConf)conf; + systemConf = (TajoConf)conf; try { - queryConf = new TajoConf(conf); - if (session != null) { - Map sessionValues = session.getAllVariables(); - if (sessionValues != null) { - for (Map.Entry entry: sessionValues.entrySet()) { - queryConf.set(entry.getKey(), entry.getValue()); - } - } - } queryTaskContext = new QueryMasterTaskContext(); - String resourceManagerClassName = queryConf.getVar(TajoConf.ConfVars.RESOURCE_MANAGER_CLASS); + String resourceManagerClassName = systemConf.getVar(TajoConf.ConfVars.RESOURCE_MANAGER_CLASS); if(resourceManagerClassName.indexOf(TajoWorkerResourceManager.class.getName()) >= 0) { resourceAllocator = new TajoResourceAllocator(queryTaskContext); @@ -171,7 +160,7 @@ public void init(Configuration conf) { queryMetrics = new TajoMetrics(queryId.toString()); - super.init(queryConf); + super.init(systemConf); } catch (Throwable t) { LOG.error(t.getMessage(), t); initError = t; @@ -335,7 +324,7 @@ public synchronized void startQuery() { } CatalogService catalog = getQueryTaskContext().getQueryMasterContext().getWorkerContext().getCatalog(); LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(queryConf); + LogicalOptimizer optimizer = new LogicalOptimizer(systemConf); Expr expr = JsonHelper.fromJson(jsonExpr, Expr.class); LogicalPlan plan = planner.createPlan(session, expr); optimizer.optimize(plan); @@ -378,11 +367,11 @@ public synchronized void startQuery() { private void initStagingDir() throws IOException { Path stagingDir = null; Path outputDir = null; - FileSystem defaultFS = TajoConf.getWarehouseDir(queryConf).getFileSystem(queryConf); + FileSystem defaultFS = TajoConf.getWarehouseDir(systemConf).getFileSystem(systemConf); try { - stagingDir = initStagingDir(queryConf, defaultFS, queryId.toString()); + stagingDir = initStagingDir(systemConf, defaultFS, queryId.toString()); defaultFS.mkdirs(new Path(stagingDir, TajoConstants.RESULT_DIR_NAME)); // Create a subdirectories @@ -528,7 +517,7 @@ public QueryContext getQueryContext() { } public TajoConf getConf() { - return queryConf; + return systemConf; } public Clock getClock() { diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index 4e4f5fc549..bed6a93785 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -145,7 +145,7 @@ public Task(QueryUnitAttemptId taskId, this.taskDir = StorageUtil.concatPath(taskRunnerContext.getBaseDir(), taskId.getQueryUnitId().getId() + "_" + taskId.getId()); - this.context = new TaskAttemptContext(systemConf, taskId, + this.context = new TaskAttemptContext(systemConf, queryContext, taskId, request.getFragments().toArray(new FragmentProto[request.getFragments().size()]), taskDir); this.context.setDataChannel(request.getDataChannel()); this.context.setEnforcer(request.getEnforcer()); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java index f42df1d84f..b1246ece1f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java @@ -30,6 +30,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.DataChannel; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; @@ -68,11 +69,13 @@ public class TaskAttemptContext { private Path outputPath; private DataChannel dataChannel; private Enforcer enforcer; + private QueryContext queryContext; - public TaskAttemptContext(TajoConf conf, final QueryUnitAttemptId queryId, + public TaskAttemptContext(TajoConf conf, QueryContext queryContext, final QueryUnitAttemptId queryId, final FragmentProto[] fragments, final Path workDir) { this.conf = conf; + this.queryContext = queryContext; this.queryId = queryId; if (fragments != null) { @@ -94,9 +97,9 @@ public TaskAttemptContext(TajoConf conf, final QueryUnitAttemptId queryId, } @VisibleForTesting - public TaskAttemptContext(TajoConf conf, final QueryUnitAttemptId queryId, + public TaskAttemptContext(TajoConf conf, QueryContext queryContext, final QueryUnitAttemptId queryId, final Fragment [] fragments, final Path workDir) { - this(conf, queryId, FragmentConvertor.toFragmentProtoArray(fragments), workDir); + this(conf, queryContext, queryId, FragmentConvertor.toFragmentProtoArray(fragments), workDir); } public TajoConf getConf() { @@ -269,4 +272,8 @@ public boolean equals(Object obj) { return false; } } + + public QueryContext getQueryContext() { + return queryContext; + } } \ No newline at end of file diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java index 1a9a2b9f65..a49f23b89f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java @@ -213,7 +213,7 @@ public void testSelectResultWithNullFalse() throws Exception { TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); tajoConf.setVar(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); - tajoConf.setBoolVar(ConfVars.CLI_PRINT_NULL, false); + tajoConf.setVar(ConfVars.CLI_NULL_CHAR, ""); ByteArrayOutputStream out = new ByteArrayOutputStream(); tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out); @@ -239,8 +239,7 @@ public void testSelectResultWithNullTrue() throws Exception { TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); tajoConf.setVar(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); - tajoConf.setBoolVar(ConfVars.CLI_PRINT_NULL, true); - tajoConf.setVar(ConfVars.CLI_PRINT_NULL_WORD, "testnull"); + tajoConf.setVar(ConfVars.CLI_NULL_CHAR, "testnull"); ByteArrayOutputStream out = new ByteArrayOutputStream(); tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out); diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java index 49e68742db..a28ae7ef67 100644 --- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -24,6 +24,7 @@ import com.google.protobuf.ServiceException; import com.sun.org.apache.commons.logging.Log; import com.sun.org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.*; @@ -32,8 +33,10 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.ipc.ClientProtos; import org.apache.tajo.jdbc.TajoResultSet; +import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.util.CommonTestingUtil; import org.junit.AfterClass; @@ -42,12 +45,12 @@ import org.junit.experimental.categories.Category; import java.io.IOException; +import java.io.InputStream; import java.sql.ResultSet; import java.sql.SQLException; import java.util.*; import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; @Category(IntegrationTest.class) public class TestTajoClient { @@ -663,4 +666,52 @@ public final void testGetQueryStatusAndResultAfterFinish() throws Exception { client.closeQuery(queryId); } } + + @Test + public void testSetCvsNull() throws Exception { + String sql = + "select\n" + + " c_custkey,\n" + + " orders.o_orderkey,\n" + + " orders.o_orderstatus \n" + + "from\n" + + " orders full outer join customer on c_custkey = o_orderkey\n" + + "order by\n" + + " c_custkey,\n" + + " orders.o_orderkey;\n"; + + TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); + + Map variables = new HashMap(); + variables.put(ConfVars.CSVFILE_NULL.varname, "\\\\T"); + client.updateSessionVariables(variables); + + TajoResultSet res = (TajoResultSet)client.executeQueryAndGetResult(sql); + + assertEquals(res.getTableDesc().getMeta().getOption(StorageConstants.CSVFILE_NULL), "\\\\T"); + + Path path = res.getTableDesc().getPath(); + FileSystem fs = path.getFileSystem(tajoConf); + + FileStatus[] files = fs.listStatus(path); + assertNotNull(files); + assertEquals(1, files.length); + + InputStream in = fs.open(files[0].getPath()); + byte[] buf = new byte[1024]; + + int readBytes = in.read(buf); + assertTrue(readBytes > 0); + + // text type field's value is replaced with \T + String expected = "1|1|O\n" + + "2|2|O\n" + + "3|3|F\n" + + "4||\\T\n" + + "5||\\T\n"; + + String resultDatas = new String(buf, 0, readBytes); + + assertEquals(expected, resultDatas); + } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java index c79796bbd3..d84796a2c8 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java @@ -34,6 +34,7 @@ import org.apache.tajo.engine.planner.logical.JoinNode; import org.apache.tajo.engine.planner.logical.LogicalNode; import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; @@ -151,7 +152,7 @@ public final void testBNLCrossJoin() throws IOException, PlanningException { Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testBNLCrossJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -188,7 +189,8 @@ public final void testBNLInnerJoin() throws IOException, PlanningException { enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.BLOCK_NESTED_LOOP_JOIN); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testBNLInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index a47bde3265..bfc3522770 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -38,6 +38,7 @@ import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.logical.LogicalNode; import org.apache.tajo.engine.planner.logical.ScanNode; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.FragmentConvertor; @@ -168,7 +169,7 @@ public void testEqual() throws Exception { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testEqual"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); Expr expr = analyzer.parse(QUERY); LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index ff3befe715..1ce5b5bef1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -33,6 +33,7 @@ import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; @@ -120,7 +121,7 @@ public final void testNext() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = new Path(testDir, TestExternalSortExec.class.getName()); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index b05688de86..1a8a90e503 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -34,6 +34,7 @@ import org.apache.tajo.engine.planner.logical.JoinNode; import org.apache.tajo.engine.planner.logical.LogicalNode; import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.session.Session; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; @@ -269,7 +270,7 @@ public final void testFullOuterHashJoinExec0() throws IOException, PlanningExcep FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -306,7 +307,7 @@ public final void testFullOuterHashJoinExec1() throws IOException, PlanningExcep FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuter_HashJoinExec1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -342,7 +343,7 @@ public final void testFullOuterHashJoinExec2() throws IOException, PlanningExcep FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -380,7 +381,8 @@ public final void testFullOuterHashJoinExec3() throws IOException, PlanningExcep FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec3"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java index 0386179a12..50e5906740 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java @@ -35,6 +35,7 @@ import org.apache.tajo.engine.planner.logical.JoinNode; import org.apache.tajo.engine.planner.logical.LogicalNode; import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.session.Session; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; @@ -315,7 +316,7 @@ public final void testFullOuterMergeJoin0() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -352,7 +353,7 @@ public final void testFullOuterMergeJoin1() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -389,7 +390,7 @@ public final void testFullOuterMergeJoin2() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -427,7 +428,7 @@ public final void testFullOuterMergeJoin3() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin3"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -468,7 +469,7 @@ public final void testFullOuterMergeJoin4() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin4"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -507,7 +508,7 @@ public final void testFullOuterMergeJoin5() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(phone3Frags,emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin5"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index 1dbbcf0b9e..794ca79654 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -32,6 +32,7 @@ import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; @@ -153,7 +154,7 @@ public final void testHashAntiJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashAntiJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java index 66222da7f5..0e0753618a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java @@ -35,6 +35,7 @@ import org.apache.tajo.engine.planner.logical.JoinNode; import org.apache.tajo.engine.planner.logical.LogicalNode; import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.session.Session; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; @@ -151,7 +152,7 @@ public final void testHashInnerJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -194,7 +195,7 @@ public final void testCheckIfInMemoryInnerJoinIsPossible() throws IOException, P FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index 4e5de98a8d..835260f9eb 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -32,6 +32,7 @@ import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; @@ -157,7 +158,7 @@ public final void testHashSemiJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashSemiJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java index de3d298198..bb25875004 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java @@ -34,6 +34,7 @@ import org.apache.tajo.engine.planner.logical.JoinNode; import org.apache.tajo.engine.planner.logical.LogicalNode; import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.session.Session; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; @@ -271,7 +272,7 @@ public final void testLeftOuterHashJoinExec0() throws IOException, PlanningExcep FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuterHashJoinExec0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -302,7 +303,7 @@ public final void testLeftOuter_HashJoinExec1() throws IOException, PlanningExce FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[1]); @@ -343,7 +344,7 @@ public final void testLeftOuter_HashJoinExec2() throws IOException, PlanningExce FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[2]); @@ -385,7 +386,7 @@ public final void testLeftOuter_HashJoinExec3() throws IOException, PlanningExce FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec3"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[3]); @@ -427,7 +428,7 @@ public final void testLeftOuter_HashJoinExec4() throws IOException, PlanningExce FileFragment[] merged = TUtil.concat(phone3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec4"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[4]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java index e806e55ad6..e935f578a9 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java @@ -35,6 +35,7 @@ import org.apache.tajo.engine.planner.PlanningException; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.session.Session; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; @@ -260,7 +261,7 @@ public final void testLeftOuterNLJoinExec0() throws IOException, PlanningExcepti FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuterNLJoinExec0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[0]); @@ -302,7 +303,7 @@ public final void testLeftOuterNLJoinExec1() throws IOException, PlanningExcepti Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[1]); @@ -346,7 +347,7 @@ public final void testLeftOuter_NLJoinExec2() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[2]); @@ -391,7 +392,7 @@ public final void testLeftOuter_NLJoinExec3() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec3"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[3]); @@ -435,7 +436,7 @@ public final void testLeftOuter_NLJoinExec4() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(phone3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec4"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[4]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java index 0e4fd9ac46..a4449fa653 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java @@ -36,6 +36,7 @@ import org.apache.tajo.engine.planner.logical.JoinNode; import org.apache.tajo.engine.planner.logical.LogicalNode; import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; @@ -165,7 +166,7 @@ public final void testMergeInnerJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testMergeInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java index 120113f30c..019929daab 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java @@ -36,6 +36,7 @@ import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; @@ -151,7 +152,7 @@ public final void testNLCrossJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testNLCrossJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[0]); @@ -180,7 +181,7 @@ public final void testNLInnerJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testNLInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[1]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 50a0f44abd..a823d2bb9b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -44,6 +44,7 @@ import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.ipc.TajoWorkerProtocol; import org.apache.tajo.master.TajoMaster; import org.apache.tajo.master.session.Session; @@ -200,7 +201,8 @@ public final void testCreateScanPlan() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateScanPlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); @@ -230,7 +232,8 @@ public final void testCreateScanWithFilterPlan() throws IOException, PlanningExc FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateScanWithFilterPlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[16]); @@ -258,7 +261,8 @@ public final void testGroupByPlan() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByPlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[7]); @@ -289,7 +293,8 @@ public final void testHashGroupByPlanWithALLField() throws IOException, Planning Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir( "target/test-data/testHashGroupByPlanWithALLField"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[15]); @@ -317,7 +322,8 @@ public final void testSortGroupByPlan() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testSortGroupByPlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[]{frags[0]}, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[7]); @@ -376,7 +382,8 @@ public final void testStorePlan() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); ctx.setOutputPath(new Path(workDir, "grouped1")); @@ -417,7 +424,8 @@ public final void testStorePlanWithRCFile() throws IOException, PlanningExceptio FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlanWithRCFile"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); ctx.setOutputPath(new Path(workDir, "grouped2")); @@ -457,7 +465,8 @@ public final void testEnforceForDefaultColumnPartitionStorePlan() throws IOExcep FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); ctx.setOutputPath(new Path(workDir, "grouped3")); @@ -483,7 +492,8 @@ public final void testEnforceForHashBasedColumnPartitionStorePlan() throws IOExc FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(enforcer); ctx.setOutputPath(new Path(workDir, "grouped4")); @@ -506,7 +516,8 @@ public final void testEnforceForSortBasedColumnPartitionStorePlan() throws IOExc FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(enforcer); ctx.setOutputPath(new Path(workDir, "grouped5")); @@ -522,7 +533,8 @@ public final void testPartitionedStorePlan() throws IOException, PlanningExcepti Integer.MAX_VALUE); QueryUnitAttemptId id = LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testPartitionedStorePlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, id, new FileFragment[] { frags[0] }, workDir); + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + id, new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[7]); LogicalPlan plan = planner.createPlan(session, context); @@ -582,7 +594,8 @@ public final void testPartitionedStorePlanWithEmptyGroupingSet() Path workDir = CommonTestingUtil.getTestDir( "target/test-data/testPartitionedStorePlanWithEmptyGroupingSet"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, id, new FileFragment[] { frags[0] }, workDir); + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + id, new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[14]); LogicalPlan plan = planner.createPlan(session, expr); @@ -635,7 +648,8 @@ public final void testAggregationFunction() throws IOException, PlanningExceptio FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testAggregationFunction"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[8]); @@ -665,7 +679,8 @@ public final void testCountFunction() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCountFunction"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[9]); @@ -692,7 +707,8 @@ public final void testGroupByWithNullValue() throws IOException, PlanningExcepti FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByWithNullValue"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[11]); @@ -716,7 +732,8 @@ public final void testUnionPlan() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testUnionPlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[0]); @@ -743,7 +760,8 @@ public final void testUnionPlan() throws IOException, PlanningException { @Test public final void testEvalExpr() throws IOException, PlanningException { Path workDir = CommonTestingUtil.getTestDir("target/test-data/testEvalExpr"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { }, workDir); Expr expr = analyzer.parse(QUERIES[12]); LogicalPlan plan = planner.createPlan(session, expr); @@ -779,7 +797,8 @@ public final void testCreateIndex() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); Expr context = analyzer.parse(createIndexStmt[0]); LogicalPlan plan = planner.createPlan(session, context); @@ -806,7 +825,8 @@ public final void testDuplicateEliminate() throws IOException, PlanningException Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testDuplicateEliminate"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(duplicateElimination[0]); @@ -839,7 +859,8 @@ public final void testIndexedStoreExec() throws IOException, PlanningException { employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testIndexedStoreExec"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(SORT_QUERY[0]); @@ -940,7 +961,8 @@ public final void testSortEnforcer() throws IOException, PlanningException { Enforcer enforcer = new Enforcer(); enforcer.enforceSortAlgorithm(sortNode.getPID(), SortAlgorithm.IN_MEMORY_SORT); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(enforcer); @@ -961,7 +983,8 @@ public final void testSortEnforcer() throws IOException, PlanningException { enforcer = new Enforcer(); enforcer.enforceSortAlgorithm(sortNode.getPID(), SortAlgorithm.MERGE_SORT); - ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(enforcer); @@ -988,7 +1011,8 @@ public final void testGroupByEnforcer() throws IOException, PlanningException { Enforcer enforcer = new Enforcer(); enforcer.enforceHashAggregation(groupByNode.getPID()); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(enforcer); @@ -1009,7 +1033,8 @@ public final void testGroupByEnforcer() throws IOException, PlanningException { enforcer = new Enforcer(); enforcer.enforceSortAggregation(groupByNode.getPID(), null); - ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java index c60e05ce39..ed6cb4efc3 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java @@ -38,6 +38,7 @@ import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; @@ -138,7 +139,7 @@ private void testProgress(int sortBufferBytesNum) throws Exception { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = new Path(testDir, TestExternalSortExec.class.getName()); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java index a45e39782f..de90f70f00 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java @@ -35,6 +35,7 @@ import org.apache.tajo.engine.planner.PlanningException; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.session.Session; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; @@ -236,7 +237,7 @@ public final void testRightOuter_HashJoinExec0() throws IOException, PlanningExc FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestRightOuter_HashJoinExec0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); @@ -277,7 +278,7 @@ public final void testRightOuter_HashJoinExec1() throws IOException, PlanningExc FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestRightOuter_HashJoinExec1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[1]); @@ -318,7 +319,7 @@ public final void testRightOuter_HashJoinExec2() throws IOException, PlanningExc FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestRightOuter_HashJoinExec2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[2]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java index 5b504b2a56..d971073222 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java @@ -34,6 +34,7 @@ import org.apache.tajo.engine.planner.logical.JoinNode; import org.apache.tajo.engine.planner.logical.LogicalNode; import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.session.Session; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; @@ -314,7 +315,7 @@ public final void testRightOuterMergeJoin0() throws IOException, PlanningExcepti FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -351,7 +352,7 @@ public final void testRightOuter_MergeJoin1() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -387,7 +388,7 @@ public final void testRightOuterMergeJoin2() throws IOException, PlanningExcepti FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -423,7 +424,7 @@ public final void testRightOuter_MergeJoin3() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuter_MergeJoin3"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -461,7 +462,7 @@ public final void testRightOuter_MergeJoin4() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuter_MergeJoin4"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -498,7 +499,7 @@ public final void testRightOuterMergeJoin5() throws IOException, PlanningExcepti Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin5"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index 87262e8c89..da6fb34538 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -34,6 +34,7 @@ import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; @@ -109,7 +110,8 @@ public static void setUp() throws Exception { public final void testNext() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employeeMeta, tablePath, Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestSortExec"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility .newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[0]); diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java index b904bb1f84..53757497e0 100644 --- a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java +++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java @@ -41,6 +41,7 @@ import org.apache.tajo.engine.planner.physical.PhysicalExec; import org.apache.tajo.engine.planner.physical.ProjectionExec; import org.apache.tajo.engine.planner.physical.RangeShuffleFileWriteExec; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder; import org.apache.tajo.storage.fragment.FileFragment; @@ -142,7 +143,8 @@ public void testGet() throws Exception { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employeeMeta, tableDir, Integer.MAX_VALUE); - TaskAttemptContext ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] {frags[0]}, testDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(SORT_QUERY[0]); @@ -265,7 +267,8 @@ public void testGetFromDescendingOrder() throws Exception { FileFragment[] frags = sm.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE); TaskAttemptContext - ctx = new TaskAttemptContext(conf, LocalTajoTestingUtility.newQueryUnitAttemptId(), + ctx = new TaskAttemptContext(conf, new QueryContext(), + LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] {frags[0]}, testDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(SORT_QUERY[1]); From 2016d43429de8d5bfe75523da1d75b55570e1b34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=ED=98=95=EC=A4=80?= Date: Wed, 28 May 2014 00:39:20 +0900 Subject: [PATCH 3/3] TAJO-842: NULL handling in JDBC. --- .../test/java/org/apache/tajo/client/TestTajoClient.java | 7 +++++++ .../TestTajoCli/testSelectResultWithNullFalse.result | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java index e17bcf0309..732f02c9f5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -165,6 +165,13 @@ public final void testDropCurrentDatabase() throws IOException, ServiceException public final void testSessionVariables() throws IOException, ServiceException, InterruptedException { String prefixName = "key_"; String prefixValue = "val_"; + + List unsetList = new ArrayList(); + for(Map.Entry entry: client.getAllSessionVariables().entrySet()) { + unsetList.add(entry.getKey()); + } + client.unsetSessionVariables(unsetList); + for (int i = 0; i < 10; i++) { String key = prefixName + i; String val = prefixValue + i; diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullFalse.result b/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullFalse.result index 4bb8e962ab..a5a60396df 100644 --- a/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullFalse.result +++ b/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullFalse.result @@ -3,6 +3,6 @@ c_custkey, o_orderkey, o_orderstatus 1, 1, O 2, 2, O 3, 3, F -4, null, null -5, null, null +4, , +5, , (5 rows, , 30 B selected)