From c6c07bf208c32bf3b431c795a677111401545f44 Mon Sep 17 00:00:00 2001 From: Ramesh Reddy Date: Thu, 26 Sep 2013 11:19:35 -0500 Subject: [PATCH] TEIID-2665: Added support for decimal, date and varchar types during the metadata import, added support for analytic functions and rollup functionality for pushdown --- .../translator/hive/HiveExecutionFactory.java | 67 ++++++++++++++++--- .../hive/HiveMetadataProcessor.java | 6 ++ .../hive/HiveSQLConversionVisitor.java | 28 ++------ .../hive/TestHiveExecutionFactory.java | 11 ++- 4 files changed, 79 insertions(+), 33 deletions(-) diff --git a/connectors/translator-hive/src/main/java/org/teiid/translator/hive/HiveExecutionFactory.java b/connectors/translator-hive/src/main/java/org/teiid/translator/hive/HiveExecutionFactory.java index 39a1ac4c93..5bbe326d8c 100644 --- a/connectors/translator-hive/src/main/java/org/teiid/translator/hive/HiveExecutionFactory.java +++ b/connectors/translator-hive/src/main/java/org/teiid/translator/hive/HiveExecutionFactory.java @@ -24,9 +24,11 @@ import static org.teiid.translator.TypeFacility.RUNTIME_NAMES.BIG_INTEGER; import static org.teiid.translator.TypeFacility.RUNTIME_NAMES.DATE; import static org.teiid.translator.TypeFacility.RUNTIME_NAMES.DOUBLE; +import static org.teiid.translator.TypeFacility.RUNTIME_NAMES.FLOAT; import static org.teiid.translator.TypeFacility.RUNTIME_NAMES.INTEGER; import static org.teiid.translator.TypeFacility.RUNTIME_NAMES.OBJECT; import static org.teiid.translator.TypeFacility.RUNTIME_NAMES.STRING; +import static org.teiid.translator.TypeFacility.RUNTIME_NAMES.TIMESTAMP; import java.sql.ResultSet; import java.sql.SQLException; @@ -39,6 +41,8 @@ import org.teiid.language.Command; import org.teiid.language.Function; import org.teiid.language.Limit; +import org.teiid.metadata.AggregateAttributes; +import org.teiid.metadata.FunctionMethod; import org.teiid.translator.ExecutionContext; import org.teiid.translator.SourceSystemFunctions; import org.teiid.translator.Translator; @@ -58,7 +62,6 @@ public class HiveExecutionFactory extends JDBCExecutionFactory { public static String HIVE = "hive"; //$NON-NLS-1$ protected ConvertModifier convert = new ConvertModifier(); - public HiveExecutionFactory() { setSupportedJoinCriteria(SupportedJoinCriteria.EQUI); } @@ -71,15 +74,16 @@ public void start() throws TranslatorException { this.convert.addTypeMapping("int", FunctionModifier.INTEGER); //$NON-NLS-1$ this.convert.addTypeMapping("bigint", FunctionModifier.BIGINTEGER, FunctionModifier.LONG); //$NON-NLS-1$ this.convert.addTypeMapping("boolean", FunctionModifier.BOOLEAN); //$NON-NLS-1$ - this.convert.addTypeMapping("double", FunctionModifier.DOUBLE, FunctionModifier.BIGDECIMAL); //$NON-NLS-1$ + this.convert.addTypeMapping("double", FunctionModifier.DOUBLE); //$NON-NLS-1$ + this.convert.addTypeMapping("decimal", FunctionModifier.BIGDECIMAL); //$NON-NLS-1$ this.convert.addTypeMapping("float", FunctionModifier.FLOAT); //$NON-NLS-1$ this.convert.addTypeMapping("string", FunctionModifier.STRING); //$NON-NLS-1$ this.convert.addTypeMapping("timestamp", FunctionModifier.TIMESTAMP); //$NON-NLS-1$ this.convert.addTypeMapping("binary", FunctionModifier.BLOB); //$NON-NLS-1$ this.convert.addTypeMapping("decimal", FunctionModifier.BIGDECIMAL); //$NON-NLS-1$ + this.convert.addTypeMapping("date", FunctionModifier.DATE); //$NON-NLS-1$ // unsupported types - //FunctionModifier.DATE, //FunctionModifier.TIME, //FunctionModifier.CHAR, //FunctionModifier.CLOB, @@ -89,8 +93,8 @@ public void start() throws TranslatorException { registerFunctionModifier(SourceSystemFunctions.BITAND, new AliasModifier("&")); //$NON-NLS-1$ registerFunctionModifier(SourceSystemFunctions.BITNOT, new AliasModifier("~")); //$NON-NLS-1$ - registerFunctionModifier(SourceSystemFunctions.BITOR, new AliasModifier("&")); //$NON-NLS-1$ - registerFunctionModifier(SourceSystemFunctions.BITXOR, new AliasModifier("|")); //$NON-NLS-1$ + registerFunctionModifier(SourceSystemFunctions.BITOR, new AliasModifier("|")); //$NON-NLS-1$ + registerFunctionModifier(SourceSystemFunctions.BITXOR, new AliasModifier("^")); //$NON-NLS-1$ registerFunctionModifier(SourceSystemFunctions.CURDATE, new AliasModifier("unix_timestamp")); //$NON-NLS-1$ registerFunctionModifier(SourceSystemFunctions.IFNULL, new AliasModifier("coalesce")); //$NON-NLS-1$ registerFunctionModifier(SourceSystemFunctions.MOD, new ModFunctionModifier("%", getLanguageFactory(), Arrays.asList(TypeFacility.RUNTIME_TYPES.BIG_INTEGER, TypeFacility.RUNTIME_TYPES.BIG_DECIMAL))); //$NON-NLS-1$ @@ -102,7 +106,6 @@ public List translate(Function function) { } }); - addPushDownFunction(HIVE, "lower", STRING, STRING); //$NON-NLS-1$ addPushDownFunction(HIVE, "upper", STRING, STRING); //$NON-NLS-1$ addPushDownFunction(HIVE, "positive", INTEGER, DOUBLE); //$NON-NLS-1$ @@ -117,9 +120,29 @@ public List translate(Function function) { addPushDownFunction(HIVE, "unhex", STRING, STRING); //$NON-NLS-1$ addPushDownFunction(HIVE, "bin", STRING, BIG_INTEGER); //$NON-NLS-1$ addPushDownFunction(HIVE, "day", INTEGER, DATE); //$NON-NLS-1$ - addPushDownFunction(HIVE, "datediff", INTEGER, DATE, DATE); //$NON-NLS-1$ - addPushDownFunction(HIVE, "date_add", INTEGER, DATE, INTEGER); //$NON-NLS-1$ - addPushDownFunction(HIVE, "date_sub", INTEGER, DATE, INTEGER); //$NON-NLS-1$ + addPushDownFunction(HIVE, "datediff", INTEGER, STRING, STRING); //$NON-NLS-1$ + addPushDownFunction(HIVE, "date_add", INTEGER, STRING, INTEGER); //$NON-NLS-1$ + addPushDownFunction(HIVE, "date_sub", INTEGER, STRING, INTEGER); //$NON-NLS-1$ + addPushDownFunction(HIVE, "from_unixtime", STRING, BIG_INTEGER); //$NON-NLS-1$ + addPushDownFunction(HIVE, "from_unixtime", STRING, BIG_INTEGER, STRING); //$NON-NLS-1$ + addPushDownFunction(HIVE, "unix_timestamp", BIG_INTEGER, STRING); //$NON-NLS-1$ + addPushDownFunction(HIVE, "unix_timestamp", BIG_INTEGER, STRING, STRING); //$NON-NLS-1$ + addPushDownFunction(HIVE, "to_date", STRING, STRING); //$NON-NLS-1$ + addPushDownFunction(HIVE, "from_utc_timestamp", TIMESTAMP, TIMESTAMP, STRING); //$NON-NLS-1$ + addPushDownFunction(HIVE, "to_utc_timestamp", TIMESTAMP, TIMESTAMP, STRING); //$NON-NLS-1$ + + addAggregatePushDownFunction(HIVE, "LEAD", OBJECT, OBJECT); //$NON-NLS-1$ + addAggregatePushDownFunction(HIVE, "LEAD", OBJECT, OBJECT, INTEGER); //$NON-NLS-1$ + addAggregatePushDownFunction(HIVE, "LEAD", OBJECT, OBJECT, INTEGER, OBJECT); //$NON-NLS-1$ + addAggregatePushDownFunction(HIVE, "LAG", OBJECT, OBJECT); //$NON-NLS-1$ + addAggregatePushDownFunction(HIVE, "LAG", OBJECT, OBJECT, INTEGER); //$NON-NLS-1$ + addAggregatePushDownFunction(HIVE, "LAG", OBJECT, OBJECT, INTEGER, OBJECT); //$NON-NLS-1$ + addAggregatePushDownFunction(HIVE, "FIRST_VALUE", OBJECT, OBJECT); //$NON-NLS-1$ + addAggregatePushDownFunction(HIVE, "LAST_VALUE", OBJECT, OBJECT); //$NON-NLS-1$ + addAggregatePushDownFunction(HIVE, "PERCENT_RANK", FLOAT); //$NON-NLS-1$ + addAggregatePushDownFunction(HIVE, "CUME_DIST", FLOAT); //$NON-NLS-1$ + addAggregatePushDownFunction(HIVE, "NTILE", BIG_INTEGER, INTEGER); //$NON-NLS-1$ + } @Override @@ -246,6 +269,22 @@ public boolean supportsAggregatesEnhancedNumeric() { public boolean supportsCommonTableExpressions() { return false; } + + @Override + public boolean supportsElementaryOlapOperations() { + return true; + } + + @Override + public boolean supportsRowLimit() { + return true; + } + + @Override + public boolean supportsGroupByRollup() { + //https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation,+Cube,+Grouping+and+Rollup + return true; + } @Override public String translateLiteralBoolean(Boolean booleanValue) { @@ -313,6 +352,7 @@ public List getSupportedFunctions() { supportedFunctions.add(SourceSystemFunctions.POWER); supportedFunctions.add(SourceSystemFunctions.SECOND); supportedFunctions.add(SourceSystemFunctions.SQRT); + supportedFunctions.add(SourceSystemFunctions.RADIANS); supportedFunctions.add(SourceSystemFunctions.ROUND); supportedFunctions.add(SourceSystemFunctions.RTRIM); supportedFunctions.add(SourceSystemFunctions.RPAD); @@ -320,6 +360,7 @@ public List getSupportedFunctions() { supportedFunctions.add(SourceSystemFunctions.PI); supportedFunctions.add(SourceSystemFunctions.SIN); supportedFunctions.add(SourceSystemFunctions.SUBSTRING); + supportedFunctions.add(SourceSystemFunctions.TAN); supportedFunctions.add(SourceSystemFunctions.TRIM); supportedFunctions.add(SourceSystemFunctions.UCASE); supportedFunctions.add(SourceSystemFunctions.YEAR); @@ -339,4 +380,12 @@ public Object retrieveValue(ResultSet results, int columnIndex, Class expecte } return super.retrieveValue(results, columnIndex, expectedType); } + + protected FunctionMethod addAggregatePushDownFunction(String qualifier, String name, String returnType, String...paramTypes) { + FunctionMethod method = addPushDownFunction(qualifier, name, returnType, paramTypes); + AggregateAttributes attr = new AggregateAttributes(); + attr.setAnalytic(true); + method.setAggregateAttributes(attr); + return method; + } } diff --git a/connectors/translator-hive/src/main/java/org/teiid/translator/hive/HiveMetadataProcessor.java b/connectors/translator-hive/src/main/java/org/teiid/translator/hive/HiveMetadataProcessor.java index c816d69581..57a8839914 100644 --- a/connectors/translator-hive/src/main/java/org/teiid/translator/hive/HiveMetadataProcessor.java +++ b/connectors/translator-hive/src/main/java/org/teiid/translator/hive/HiveMetadataProcessor.java @@ -88,9 +88,15 @@ else if (type.equalsIgnoreCase("decimal")) { //$NON-NLS-1$ else if (type.equalsIgnoreCase("timestamp")) { //$NON-NLS-1$ return TypeFacility.RUNTIME_NAMES.TIMESTAMP; } + else if (type.equalsIgnoreCase("date")) { //$NON-NLS-1$ + return TypeFacility.RUNTIME_NAMES.DATE; + } else if (type.equalsIgnoreCase("BINARY")) { //$NON-NLS-1$ return TypeFacility.RUNTIME_NAMES.VARBINARY; } + else if (type.equalsIgnoreCase("varchar")) { //$NON-NLS-1$ + return TypeFacility.RUNTIME_NAMES.STRING; + } return TypeFacility.RUNTIME_NAMES.STRING; } diff --git a/connectors/translator-hive/src/main/java/org/teiid/translator/hive/HiveSQLConversionVisitor.java b/connectors/translator-hive/src/main/java/org/teiid/translator/hive/HiveSQLConversionVisitor.java index 19916984eb..8fdb009dbd 100644 --- a/connectors/translator-hive/src/main/java/org/teiid/translator/hive/HiveSQLConversionVisitor.java +++ b/connectors/translator-hive/src/main/java/org/teiid/translator/hive/HiveSQLConversionVisitor.java @@ -21,30 +21,12 @@ */ package org.teiid.translator.hive; -import static org.teiid.language.SQLConstants.Reserved.ALL; -import static org.teiid.language.SQLConstants.Reserved.DISTINCT; -import static org.teiid.language.SQLConstants.Reserved.FROM; -import static org.teiid.language.SQLConstants.Reserved.FULL; -import static org.teiid.language.SQLConstants.Reserved.JOIN; -import static org.teiid.language.SQLConstants.Reserved.LEFT; -import static org.teiid.language.SQLConstants.Reserved.ON; -import static org.teiid.language.SQLConstants.Reserved.OUTER; -import static org.teiid.language.SQLConstants.Reserved.RIGHT; -import static org.teiid.language.SQLConstants.Reserved.SELECT; +import static org.teiid.language.SQLConstants.Reserved.*; import java.util.List; -import org.teiid.language.ColumnReference; -import org.teiid.language.Condition; -import org.teiid.language.DerivedColumn; -import org.teiid.language.Expression; -import org.teiid.language.Join; -import org.teiid.language.Limit; -import org.teiid.language.OrderBy; +import org.teiid.language.*; import org.teiid.language.SQLConstants.Tokens; -import org.teiid.language.Select; -import org.teiid.language.SetQuery; -import org.teiid.language.TableReference; import org.teiid.translator.jdbc.SQLConversionVisitor; public class HiveSQLConversionVisitor extends SQLConversionVisitor { @@ -53,7 +35,8 @@ public HiveSQLConversionVisitor(HiveExecutionFactory hef) { super(hef); } - public void visit(Join obj) { + @Override + public void visit(Join obj) { TableReference leftItem = obj.getLeftItem(); if(useParensForJoins() && leftItem instanceof Join) { buffer.append(Tokens.LPAREN); @@ -67,7 +50,8 @@ public void visit(Join obj) { switch(obj.getJoinType()) { case CROSS_JOIN: // Hive just works with "JOIN" keyword no inner or cross - //buffer.append(CROSS); + // fixed in - https://issues.apache.org/jira/browse/HIVE-2549 + buffer.append(CROSS); break; case FULL_OUTER_JOIN: buffer.append(FULL) diff --git a/connectors/translator-hive/src/test/java/org/teiid/translator/hive/TestHiveExecutionFactory.java b/connectors/translator-hive/src/test/java/org/teiid/translator/hive/TestHiveExecutionFactory.java index c8cb99da31..cbfc55893b 100644 --- a/connectors/translator-hive/src/test/java/org/teiid/translator/hive/TestHiveExecutionFactory.java +++ b/connectors/translator-hive/src/test/java/org/teiid/translator/hive/TestHiveExecutionFactory.java @@ -72,7 +72,14 @@ private void helpTestVisitor(QueryMetadataInterface metadata, String input, Stri helpTest(LANG_FACTORY.createLiteral(new BigInteger("123451266182"), BigInteger.class), TypeFacility.RUNTIME_NAMES.BIG_INTEGER, "cast(123451266182 AS bigint)"); helpTest(LANG_FACTORY.createLiteral(new String("foo-bar"), String.class), TypeFacility.RUNTIME_NAMES.STRING, "cast('foo-bar' AS string)"); helpTest(LANG_FACTORY.createLiteral(Boolean.TRUE, Boolean.class), TypeFacility.RUNTIME_NAMES.STRING, "cast(true AS string)"); - //helpTest(LANG_FACTORY.createLiteral(new Timestamp(1320883518391L), Timestamp.class), TypeFacility.RUNTIME_NAMES.STRING, "cast('2011-11-09 18:05:18.391' AS string)"); + helpTest(LANG_FACTORY.createLiteral(new Integer(12345), Integer.class), TypeFacility.RUNTIME_NAMES.BOOLEAN, "cast(12345 AS boolean)"); + } + + @Test + public void testFunction() throws Exception { + String input = "SELECT MOD(A.intkey,2) FROM BQT1.SMALLA A"; + String output = "SELECT (A.IntKey % 2) FROM SmallA A"; + helpTestVisitor(bqt, input, output); } @Test @@ -85,7 +92,7 @@ public void testEqualityJoinCriteria() throws Exception { @Test public void testCrossJoinCriteria() throws Exception { String input = "SELECT A.intkey FROM BQT1.SMALLA A Cross join BQT1.SmallB B"; - String output = "SELECT A.IntKey FROM SmallA A JOIN SmallB B"; + String output = "SELECT A.IntKey FROM SmallA A CROSS JOIN SmallB B"; helpTestVisitor(bqt, input, output); }