Skip to content

Commit

Permalink
TEIID-2665: Added support for decimal, date and varchar types during …
Browse files Browse the repository at this point in the history 
…the metadata import, added support for analytic functions and rollup functionality for pushdown
  • Loading branch information
@rareddy
rareddy committed Oct 4, 2013
1 parent 864dc90 commit c6c07bf
Show file tree
Hide file tree
Showing 4 changed files with 79 additions and 33 deletions.
67 changes: 58 additions & 9 deletions connectors/translator-hive/src/main/java/org/teiid/translator/hive/HiveExecutionFactory.java
View file
Expand Up @@ -24,9 +24,11 @@
import static org.teiid.translator.TypeFacility.RUNTIME_NAMES.BIG_INTEGER;
import static org.teiid.translator.TypeFacility.RUNTIME_NAMES.DATE;
import static org.teiid.translator.TypeFacility.RUNTIME_NAMES.DOUBLE;
import static org.teiid.translator.TypeFacility.RUNTIME_NAMES.FLOAT;
import static org.teiid.translator.TypeFacility.RUNTIME_NAMES.INTEGER;
import static org.teiid.translator.TypeFacility.RUNTIME_NAMES.OBJECT;
import static org.teiid.translator.TypeFacility.RUNTIME_NAMES.STRING;
import static org.teiid.translator.TypeFacility.RUNTIME_NAMES.TIMESTAMP;

import java.sql.ResultSet;
import java.sql.SQLException;
Expand All @@ -39,6 +41,8 @@
import org.teiid.language.Command;
import org.teiid.language.Function;
import org.teiid.language.Limit;
import org.teiid.metadata.AggregateAttributes;
import org.teiid.metadata.FunctionMethod;
import org.teiid.translator.ExecutionContext;
import org.teiid.translator.SourceSystemFunctions;
import org.teiid.translator.Translator;
Expand All @@ -58,7 +62,6 @@ public class HiveExecutionFactory extends JDBCExecutionFactory {
public static String HIVE = "hive"; //$NON-NLS-1$
protected ConvertModifier convert = new ConvertModifier();


public HiveExecutionFactory() {
setSupportedJoinCriteria(SupportedJoinCriteria.EQUI);
}
Expand All @@ -71,15 +74,16 @@ public void start() throws TranslatorException {
this.convert.addTypeMapping("int", FunctionModifier.INTEGER); //$NON-NLS-1$
this.convert.addTypeMapping("bigint", FunctionModifier.BIGINTEGER, FunctionModifier.LONG); //$NON-NLS-1$
this.convert.addTypeMapping("boolean", FunctionModifier.BOOLEAN); //$NON-NLS-1$
this.convert.addTypeMapping("double", FunctionModifier.DOUBLE, FunctionModifier.BIGDECIMAL); //$NON-NLS-1$
this.convert.addTypeMapping("double", FunctionModifier.DOUBLE); //$NON-NLS-1$
this.convert.addTypeMapping("decimal", FunctionModifier.BIGDECIMAL); //$NON-NLS-1$
this.convert.addTypeMapping("float", FunctionModifier.FLOAT); //$NON-NLS-1$
this.convert.addTypeMapping("string", FunctionModifier.STRING); //$NON-NLS-1$
this.convert.addTypeMapping("timestamp", FunctionModifier.TIMESTAMP); //$NON-NLS-1$
this.convert.addTypeMapping("binary", FunctionModifier.BLOB); //$NON-NLS-1$
this.convert.addTypeMapping("decimal", FunctionModifier.BIGDECIMAL); //$NON-NLS-1$
this.convert.addTypeMapping("date", FunctionModifier.DATE); //$NON-NLS-1$

// unsupported types
//FunctionModifier.DATE,
//FunctionModifier.TIME,
//FunctionModifier.CHAR,
//FunctionModifier.CLOB,
Expand All @@ -89,8 +93,8 @@ public void start() throws TranslatorException {

registerFunctionModifier(SourceSystemFunctions.BITAND, new AliasModifier("&")); //$NON-NLS-1$
registerFunctionModifier(SourceSystemFunctions.BITNOT, new AliasModifier("~")); //$NON-NLS-1$
registerFunctionModifier(SourceSystemFunctions.BITOR, new AliasModifier("&")); //$NON-NLS-1$
registerFunctionModifier(SourceSystemFunctions.BITXOR, new AliasModifier("|")); //$NON-NLS-1$
registerFunctionModifier(SourceSystemFunctions.BITOR, new AliasModifier("|")); //$NON-NLS-1$
registerFunctionModifier(SourceSystemFunctions.BITXOR, new AliasModifier("^")); //$NON-NLS-1$
registerFunctionModifier(SourceSystemFunctions.CURDATE, new AliasModifier("unix_timestamp")); //$NON-NLS-1$
registerFunctionModifier(SourceSystemFunctions.IFNULL, new AliasModifier("coalesce")); //$NON-NLS-1$
registerFunctionModifier(SourceSystemFunctions.MOD, new ModFunctionModifier("%", getLanguageFactory(), Arrays.asList(TypeFacility.RUNTIME_TYPES.BIG_INTEGER, TypeFacility.RUNTIME_TYPES.BIG_DECIMAL))); //$NON-NLS-1$
Expand All @@ -102,7 +106,6 @@ public List<?> translate(Function function) {
}
});


addPushDownFunction(HIVE, "lower", STRING, STRING); //$NON-NLS-1$
addPushDownFunction(HIVE, "upper", STRING, STRING); //$NON-NLS-1$
addPushDownFunction(HIVE, "positive", INTEGER, DOUBLE); //$NON-NLS-1$
Expand All @@ -117,9 +120,29 @@ public List<?> translate(Function function) {
addPushDownFunction(HIVE, "unhex", STRING, STRING); //$NON-NLS-1$
addPushDownFunction(HIVE, "bin", STRING, BIG_INTEGER); //$NON-NLS-1$
addPushDownFunction(HIVE, "day", INTEGER, DATE); //$NON-NLS-1$
addPushDownFunction(HIVE, "datediff", INTEGER, DATE, DATE); //$NON-NLS-1$
addPushDownFunction(HIVE, "date_add", INTEGER, DATE, INTEGER); //$NON-NLS-1$
addPushDownFunction(HIVE, "date_sub", INTEGER, DATE, INTEGER); //$NON-NLS-1$
addPushDownFunction(HIVE, "datediff", INTEGER, STRING, STRING); //$NON-NLS-1$
addPushDownFunction(HIVE, "date_add", INTEGER, STRING, INTEGER); //$NON-NLS-1$
addPushDownFunction(HIVE, "date_sub", INTEGER, STRING, INTEGER); //$NON-NLS-1$
addPushDownFunction(HIVE, "from_unixtime", STRING, BIG_INTEGER); //$NON-NLS-1$
addPushDownFunction(HIVE, "from_unixtime", STRING, BIG_INTEGER, STRING); //$NON-NLS-1$
addPushDownFunction(HIVE, "unix_timestamp", BIG_INTEGER, STRING); //$NON-NLS-1$
addPushDownFunction(HIVE, "unix_timestamp", BIG_INTEGER, STRING, STRING); //$NON-NLS-1$
addPushDownFunction(HIVE, "to_date", STRING, STRING); //$NON-NLS-1$
addPushDownFunction(HIVE, "from_utc_timestamp", TIMESTAMP, TIMESTAMP, STRING); //$NON-NLS-1$
addPushDownFunction(HIVE, "to_utc_timestamp", TIMESTAMP, TIMESTAMP, STRING); //$NON-NLS-1$

addAggregatePushDownFunction(HIVE, "LEAD", OBJECT, OBJECT); //$NON-NLS-1$
addAggregatePushDownFunction(HIVE, "LEAD", OBJECT, OBJECT, INTEGER); //$NON-NLS-1$
addAggregatePushDownFunction(HIVE, "LEAD", OBJECT, OBJECT, INTEGER, OBJECT); //$NON-NLS-1$
addAggregatePushDownFunction(HIVE, "LAG", OBJECT, OBJECT); //$NON-NLS-1$
addAggregatePushDownFunction(HIVE, "LAG", OBJECT, OBJECT, INTEGER); //$NON-NLS-1$
addAggregatePushDownFunction(HIVE, "LAG", OBJECT, OBJECT, INTEGER, OBJECT); //$NON-NLS-1$
addAggregatePushDownFunction(HIVE, "FIRST_VALUE", OBJECT, OBJECT); //$NON-NLS-1$
addAggregatePushDownFunction(HIVE, "LAST_VALUE", OBJECT, OBJECT); //$NON-NLS-1$
addAggregatePushDownFunction(HIVE, "PERCENT_RANK", FLOAT); //$NON-NLS-1$
addAggregatePushDownFunction(HIVE, "CUME_DIST", FLOAT); //$NON-NLS-1$
addAggregatePushDownFunction(HIVE, "NTILE", BIG_INTEGER, INTEGER); //$NON-NLS-1$

}

@Override
Expand Down Expand Up @@ -246,6 +269,22 @@ public boolean supportsAggregatesEnhancedNumeric() {
public boolean supportsCommonTableExpressions() {
return false;
}

@Override
public boolean supportsElementaryOlapOperations() {
return true;
}

@Override
public boolean supportsRowLimit() {
return true;
}

@Override
public boolean supportsGroupByRollup() {
//https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation,+Cube,+Grouping+and+Rollup
return true;
}

@Override
public String translateLiteralBoolean(Boolean booleanValue) {
Expand Down Expand Up @@ -313,13 +352,15 @@ public List<String> getSupportedFunctions() {
supportedFunctions.add(SourceSystemFunctions.POWER);
supportedFunctions.add(SourceSystemFunctions.SECOND);
supportedFunctions.add(SourceSystemFunctions.SQRT);
supportedFunctions.add(SourceSystemFunctions.RADIANS);
supportedFunctions.add(SourceSystemFunctions.ROUND);
supportedFunctions.add(SourceSystemFunctions.RTRIM);
supportedFunctions.add(SourceSystemFunctions.RPAD);
supportedFunctions.add(SourceSystemFunctions.MONTH);
supportedFunctions.add(SourceSystemFunctions.PI);
supportedFunctions.add(SourceSystemFunctions.SIN);
supportedFunctions.add(SourceSystemFunctions.SUBSTRING);
supportedFunctions.add(SourceSystemFunctions.TAN);
supportedFunctions.add(SourceSystemFunctions.TRIM);
supportedFunctions.add(SourceSystemFunctions.UCASE);
supportedFunctions.add(SourceSystemFunctions.YEAR);
Expand All @@ -339,4 +380,12 @@ public Object retrieveValue(ResultSet results, int columnIndex, Class<?> expecte
}
return super.retrieveValue(results, columnIndex, expectedType);
}

protected FunctionMethod addAggregatePushDownFunction(String qualifier, String name, String returnType, String...paramTypes) {
FunctionMethod method = addPushDownFunction(qualifier, name, returnType, paramTypes);
AggregateAttributes attr = new AggregateAttributes();
attr.setAnalytic(true);
method.setAggregateAttributes(attr);
return method;
}
}
6 changes: 6 additions & 0 deletions ...ectors/translator-hive/src/main/java/org/teiid/translator/hive/HiveMetadataProcessor.java
View file
Expand Up @@ -88,9 +88,15 @@ else if (type.equalsIgnoreCase("decimal")) { //$NON-NLS-1$
else if (type.equalsIgnoreCase("timestamp")) { //$NON-NLS-1$
return TypeFacility.RUNTIME_NAMES.TIMESTAMP;
}
else if (type.equalsIgnoreCase("date")) { //$NON-NLS-1$
return TypeFacility.RUNTIME_NAMES.DATE;
}
else if (type.equalsIgnoreCase("BINARY")) { //$NON-NLS-1$
return TypeFacility.RUNTIME_NAMES.VARBINARY;
}
else if (type.equalsIgnoreCase("varchar")) { //$NON-NLS-1$
return TypeFacility.RUNTIME_NAMES.STRING;
}
return TypeFacility.RUNTIME_NAMES.STRING;
}

Expand Down
28 changes: 6 additions & 22 deletions ...ors/translator-hive/src/main/java/org/teiid/translator/hive/HiveSQLConversionVisitor.java
View file
Expand Up @@ -21,30 +21,12 @@
*/
package org.teiid.translator.hive;

import static org.teiid.language.SQLConstants.Reserved.ALL;
import static org.teiid.language.SQLConstants.Reserved.DISTINCT;
import static org.teiid.language.SQLConstants.Reserved.FROM;
import static org.teiid.language.SQLConstants.Reserved.FULL;
import static org.teiid.language.SQLConstants.Reserved.JOIN;
import static org.teiid.language.SQLConstants.Reserved.LEFT;
import static org.teiid.language.SQLConstants.Reserved.ON;
import static org.teiid.language.SQLConstants.Reserved.OUTER;
import static org.teiid.language.SQLConstants.Reserved.RIGHT;
import static org.teiid.language.SQLConstants.Reserved.SELECT;
import static org.teiid.language.SQLConstants.Reserved.*;

import java.util.List;

import org.teiid.language.ColumnReference;
import org.teiid.language.Condition;
import org.teiid.language.DerivedColumn;
import org.teiid.language.Expression;
import org.teiid.language.Join;
import org.teiid.language.Limit;
import org.teiid.language.OrderBy;
import org.teiid.language.*;
import org.teiid.language.SQLConstants.Tokens;
import org.teiid.language.Select;
import org.teiid.language.SetQuery;
import org.teiid.language.TableReference;
import org.teiid.translator.jdbc.SQLConversionVisitor;

public class HiveSQLConversionVisitor extends SQLConversionVisitor {
Expand All @@ -53,7 +35,8 @@ public HiveSQLConversionVisitor(HiveExecutionFactory hef) {
super(hef);
}

public void visit(Join obj) {
@Override
public void visit(Join obj) {
TableReference leftItem = obj.getLeftItem();
if(useParensForJoins() && leftItem instanceof Join) {
buffer.append(Tokens.LPAREN);
Expand All @@ -67,7 +50,8 @@ public void visit(Join obj) {
switch(obj.getJoinType()) {
case CROSS_JOIN:
// Hive just works with "JOIN" keyword no inner or cross
//buffer.append(CROSS);
// fixed in - https://issues.apache.org/jira/browse/HIVE-2549
buffer.append(CROSS);
break;
case FULL_OUTER_JOIN:
buffer.append(FULL)
Expand Down
11 changes: 9 additions & 2 deletions ...ors/translator-hive/src/test/java/org/teiid/translator/hive/TestHiveExecutionFactory.java
View file
Expand Up @@ -72,7 +72,14 @@ private void helpTestVisitor(QueryMetadataInterface metadata, String input, Stri
helpTest(LANG_FACTORY.createLiteral(new BigInteger("123451266182"), BigInteger.class), TypeFacility.RUNTIME_NAMES.BIG_INTEGER, "cast(123451266182 AS bigint)");
helpTest(LANG_FACTORY.createLiteral(new String("foo-bar"), String.class), TypeFacility.RUNTIME_NAMES.STRING, "cast('foo-bar' AS string)");
helpTest(LANG_FACTORY.createLiteral(Boolean.TRUE, Boolean.class), TypeFacility.RUNTIME_NAMES.STRING, "cast(true AS string)");
//helpTest(LANG_FACTORY.createLiteral(new Timestamp(1320883518391L), Timestamp.class), TypeFacility.RUNTIME_NAMES.STRING, "cast('2011-11-09 18:05:18.391' AS string)");
helpTest(LANG_FACTORY.createLiteral(new Integer(12345), Integer.class), TypeFacility.RUNTIME_NAMES.BOOLEAN, "cast(12345 AS boolean)");
}

@Test
public void testFunction() throws Exception {
String input = "SELECT MOD(A.intkey,2) FROM BQT1.SMALLA A";
String output = "SELECT (A.IntKey % 2) FROM SmallA A";
helpTestVisitor(bqt, input, output);
}

@Test
Expand All @@ -85,7 +92,7 @@ public void testEqualityJoinCriteria() throws Exception {
@Test
public void testCrossJoinCriteria() throws Exception {
String input = "SELECT A.intkey FROM BQT1.SMALLA A Cross join BQT1.SmallB B";
String output = "SELECT A.IntKey FROM SmallA A JOIN SmallB B";
String output = "SELECT A.IntKey FROM SmallA A CROSS JOIN SmallB B";
helpTestVisitor(bqt, input, output);
}

Expand Down

0 comments on commit c6c07bf

Please sign in to comment.