Skip to content
This repository was archived by the owner on May 12, 2021. It is now read-only.

TAJO-1674: Validation of CTAS schema mismatch. #726

Closed
hyunsik wants to merge 7 commits into from
Closed

TAJO-1674: Validation of CTAS schema mismatch. #726

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
4e0b94e
TAJO-1674: Validation of CTAS schema mismatch.
hyunsik Sep 4, 2015
261a1e5
Fixed test failures.
hyunsik Sep 4, 2015
2561c45
Allow null type.
hyunsik Sep 4, 2015
672eafd
Fixed wrong import.
hyunsik Sep 4, 2015
190304f
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo…
hyunsik Sep 7, 2015
4bbb029
Fixed casting map and unnecessary format changes.
hyunsik Sep 7, 2015
8c53c57
Merge branch 'TAJO-1674' of github.com:hyunsik/tajo into TAJO-1674
hyunsik Sep 7, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -873,7 +873,11 @@ public static Pair<List<PartitionKeyProto>, String> getPartitionKeyNamePair(Stri
return pair;
}

/* It is the relationship graph of type conversions. */
/*
* It is the relationship graph of type conversions.
* It contains tuples, each of which (LHS type, RHS type, Result type).
*/

public static final Map<Type, Map<Type, Type>> OPERATION_CASTING_MAP = Maps.newHashMap();

static {
Expand Down Expand Up @@ -928,10 +932,22 @@ public static Pair<List<PartitionKeyProto>, String> getPartitionKeyNamePair(Stri
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.FLOAT8, Type.FLOAT8, Type.FLOAT8);
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.FLOAT8, Type.TEXT, Type.TEXT);


TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TEXT, Type.DATE, Type.DATE);
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TEXT, Type.TIME, Type.TIME);
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TEXT, Type.TIMESTAMP, Type.TIMESTAMP);
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TEXT, Type.TEXT, Type.TEXT);
Copy link
Contributor

@blrunner blrunner Sep 7, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We might need time type and date type.

TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TEXT, Type.VARCHAR, Type.TEXT);

TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.VARCHAR, Type.DATE, Type.DATE);
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.VARCHAR, Type.TIME, Type.TIME);
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.VARCHAR, Type.TIMESTAMP, Type.TIMESTAMP);
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.VARCHAR, Type.TEXT, Type.TEXT);
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.VARCHAR, Type.VARCHAR, Type.TEXT);

TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TIMESTAMP, Type.TIMESTAMP, Type.TIMESTAMP);
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TIMESTAMP, Type.TEXT, Type.TEXT);
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TEXT, Type.TEXT, Type.TEXT);
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TIMESTAMP, Type.VARCHAR, Type.TEXT);

TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TIME, Type.TIME, Type.TIME);
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.DATE, Type.DATE, Type.DATE);
Expand Down
22 changes: 0 additions & 22 deletions tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,11 @@
import org.apache.tajo.engine.planner.global.ExecutionBlock;
import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.ipc.ClientProtos;
import org.apache.tajo.jdbc.FetchResultSet;
import org.apache.tajo.jdbc.TajoMemoryResultSet;
import org.apache.tajo.plan.logical.NodeType;
import org.apache.tajo.querymaster.QueryMasterTask;
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.worker.TajoWorker;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
Expand Down Expand Up @@ -1081,25 +1078,6 @@ public final void testColumnPartitionedTableWithSmallerExpressions6() throws Exc
executeString("DROP TABLE " + tableName + " PURGE").close();
}

private MasterPlan getQueryPlan(ResultSet res) {
QueryId queryId;
if (res instanceof TajoMemoryResultSet) {
queryId = ((TajoMemoryResultSet) res).getQueryId();
} else {
queryId = ((FetchResultSet) res).getQueryId();
}

for (TajoWorker eachWorker : testingCluster.getTajoWorkers()) {
QueryMasterTask queryMasterTask = eachWorker.getWorkerContext().getQueryMaster().getQueryMasterTask(queryId, true);
if (queryMasterTask != null) {
return queryMasterTask.getQuery().getPlan();
}
}

fail("Can't find query from workers" + queryId);
return null;
}

@Test
public void testScatteredHashShuffle() throws Exception {
testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_TABLE_PARTITION_VOLUME.varname, "2");
Expand Down
2 changes: 2 additions & 0 deletions tajo-core-tests/src/test/resources/queries/TestCreateTable/negative/type_mismatch.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
CREATE TABLE MISMATCH1 (n_name text, n_comment text, n_nationkey int8, n_regionkey int8) AS SELECT * FROM default.nation;
CREATE TABLE MISMATCH2 (n_name text, n_comment text) PARTITION BY COLUMN (n_nationkey int8, n_regionkey int8) AS SELECT * FROM default.nation;
16 changes: 15 additions & 1 deletion tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.tajo.plan.verifier;

import com.google.common.base.Preconditions;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.common.TajoDataTypes.Type;
Expand All @@ -31,6 +32,7 @@
import org.apache.tajo.plan.logical.*;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
import org.apache.tajo.util.TUtil;

import java.util.Stack;

Expand Down Expand Up @@ -241,7 +243,15 @@ public LogicalNode visitInsert(Context context, LogicalPlan plan, LogicalPlan.Qu
private static void ensureDomains(VerificationState state, Schema targetTableScheme, Schema schema)
throws TajoException {
for (int i = 0; i < schema.size(); i++) {
if (!schema.getColumn(i).getDataType().equals(targetTableScheme.getColumn(i).getDataType())) {

// null can be used anywhere
if (schema.getColumn(i).getDataType().getType() == Type.NULL_TYPE) {
continue;
}

// checking castable between two data types
if (!TUtil.containsInNestedMap(CatalogUtil.OPERATION_CASTING_MAP,
schema.getColumn(i).getDataType().getType(), targetTableScheme.getColumn(i).getDataType().getType())) {
Column targetColumn = targetTableScheme.getColumn(i);
Column insertColumn = schema.getColumn(i);
state.addVerification(makeDataTypeMisMatch(insertColumn, targetColumn));
Expand All @@ -254,6 +264,10 @@ public LogicalNode visitCreateTable(Context context, LogicalPlan plan, LogicalPl
CreateTableNode node, Stack<LogicalNode> stack) throws TajoException {
super.visitCreateTable(context, plan, block, node, stack);
// here, we don't need check table existence because this check is performed in PreLogicalPlanVerifier.

if (node.hasSubQuery()) {
ensureDomains(context.state, node.getLogicalSchema(), node.getChild(0).getOutSchema());
}
return node;
}

Expand Down
5 changes: 3 additions & 2 deletions tajo-plan/src/main/java/org/apache/tajo/plan/verifier/SyntaxErrorUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.tajo.catalog.Column;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.error.Errors.ResultCode;
import org.apache.tajo.exception.DataTypeMismatchException;
import org.apache.tajo.plan.logical.NodeType;

public class SyntaxErrorUtil {
Expand All @@ -29,8 +30,8 @@ public static SyntaxErrorException makeSyntaxError(String message) {
return new SyntaxErrorException(ResultCode.SYNTAX_ERROR, message);
}

public static SyntaxErrorException makeDataTypeMisMatch(Column src, Column target) {
return new SyntaxErrorException(ResultCode.DATATYPE_MISMATCH,
public static DataTypeMismatchException makeDataTypeMisMatch(Column src, Column target) {
return new DataTypeMismatchException(
src.getSimpleName(), src.getDataType().getType().name(),
target.getSimpleName(), target.getDataType().getType().name());
}
Expand Down