HIVE-2249 When creating constant expression for numbers, try to infer…

… type from another comparison operand, instead of trying to use integer first, and then long and double (Zhiqiu Kong via Siying Dong)

git-svn-id: https://svn.apache.org/repos/asf/hive/trunk@1238175 13f79535-47bb-0310-9956-ffa450edef68

contrib/src/test/results/clientpositive/dboutput.q.out

@@ -141,7 +141,7 @@ STAGE PLANS:
             alias: src
             Filter Operator
               predicate:
-                  expr: (key < 10)
+                  expr: (key < 10.0)
                   type: boolean
               Select Operator
                 expressions:

contrib/src/test/results/clientpositive/serde_typedbytes4.q.out

@@ -48,7 +48,7 @@ STAGE PLANS:
             alias: src
             Filter Operator
               predicate:
-                  expr: (key < 100)
+                  expr: (key < 100.0)
                   type: boolean
               Select Operator
                 expressions:

data/files/infer_const_type.txt

@@ -0,0 +1,3 @@
+127
32767
12345
-12345
906
-307.0
1234
+126
32767
12345
-12345
906
-307.0
1234
+126
32767
12345
-12345
906
-307.0
1.57

ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java

@@ -21,9 +21,11 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.Stack;
 
 import org.apache.commons.lang.StringUtils;
@@ -51,6 +53,8 @@
 import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
 import org.apache.hadoop.hive.serde.Constants;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
@@ -681,6 +685,79 @@ static ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr,
           }
         }
 
+        // Try to infer the type of the constant only if there are two
+        // nodes, one of them is column and the other is numeric const
+        if (fi.getGenericUDF() instanceof GenericUDFBaseCompare
+            && children.size() == 2
+            && ((children.get(0) instanceof ExprNodeConstantDesc
+                && children.get(1) instanceof ExprNodeColumnDesc)
+                || (children.get(0) instanceof ExprNodeColumnDesc
+                    && children.get(1) instanceof ExprNodeConstantDesc))) {
+          int constIdx =
+              children.get(0) instanceof ExprNodeConstantDesc ? 0 : 1;
+
+          Set<String> inferTypes = new HashSet<String>(Arrays.asList(
+              Constants.TINYINT_TYPE_NAME.toLowerCase(),
+              Constants.SMALLINT_TYPE_NAME.toLowerCase(),
+              Constants.INT_TYPE_NAME.toLowerCase(),
+              Constants.BIGINT_TYPE_NAME.toLowerCase(),
+              Constants.FLOAT_TYPE_NAME.toLowerCase(),
+              Constants.DOUBLE_TYPE_NAME.toLowerCase(),
+              Constants.STRING_TYPE_NAME.toLowerCase()
+              ));
+
+          String constType = children.get(constIdx).getTypeString().toLowerCase();
+          String columnType = children.get(1 - constIdx).getTypeString().toLowerCase();
+
+          if (inferTypes.contains(constType) && inferTypes.contains(columnType)
+              && !columnType.equalsIgnoreCase(constType)) {
+            String constValue =
+                ((ExprNodeConstantDesc) children.get(constIdx)).getValue().toString();
+            boolean triedDouble = false;
+
+            Number value = null;
+            try {
+              if (columnType.equalsIgnoreCase(Constants.TINYINT_TYPE_NAME)) {
+                value = new Byte(constValue);
+              } else if (columnType.equalsIgnoreCase(Constants.SMALLINT_TYPE_NAME)) {
+                value = new Short(constValue);
+              } else if (columnType.equalsIgnoreCase(Constants.INT_TYPE_NAME)) {
+                value = new Integer(constValue);
+              } else if (columnType.equalsIgnoreCase(Constants.BIGINT_TYPE_NAME)) {
+                value = new Long(constValue);
+              } else if (columnType.equalsIgnoreCase(Constants.FLOAT_TYPE_NAME)) {
+                value = new Float(constValue);
+              } else if (columnType.equalsIgnoreCase(Constants.DOUBLE_TYPE_NAME)
+                  || (columnType.equalsIgnoreCase(Constants.STRING_TYPE_NAME)
+                     && !constType.equalsIgnoreCase(Constants.BIGINT_TYPE_NAME))) {
+                // no smart inference for queries like "str_col = bigint_const"
+                triedDouble = true;
+                value = new Double(constValue);
+              }
+            } catch (NumberFormatException nfe) {
+              // this exception suggests the precise type inference did not succeed
+              // we'll try again to convert it to double
+              // however, if we already tried this, or the column is NUMBER type and
+              // the operator is EQUAL, return false due to the type mismatch
+              if (triedDouble ||
+                  (fi.getGenericUDF() instanceof GenericUDFOPEqual
+                  && !columnType.equals(Constants.STRING_TYPE_NAME))) {
+                return new ExprNodeConstantDesc(false);
+              }
+
+              try {
+                value = new Double(constValue);
+              } catch (NumberFormatException ex) {
+                return new ExprNodeConstantDesc(false);
+              }
+            }
+
+            if (value != null) {
+              children.set(constIdx, new ExprNodeConstantDesc(value));
+            }
+          }
+        }
+
         desc = ExprNodeGenericFuncDesc.newInstance(fi.getGenericUDF(), children);
       }
       // UDFOPPositive is a no-op.

ql/src/test/queries/clientpositive/infer_const_type.q

@@ -0,0 +1,66 @@
+DROP TABLE infertypes;
+CREATE TABLE infertypes(ti TINYINT, si SMALLINT, i INT, bi BIGINT, fl FLOAT, db DOUBLE, str STRING);
+
+LOAD DATA LOCAL INPATH '../data/files/infer_const_type.txt' OVERWRITE INTO TABLE infertypes;
+
+SELECT * FROM infertypes;
+
+EXPLAIN SELECT * FROM infertypes WHERE
+  ti  = '127' AND
+  si  = 32767 AND
+  i   = '12345' AND
+  bi  = '-12345' AND
+  fl  = '0906' AND
+  db  = '-307' AND
+  str = 1234;
+
+SELECT * FROM infertypes WHERE
+  ti  = '127' AND
+  si  = 32767 AND
+  i   = '12345' AND
+  bi  = '-12345' AND
+  fl  = '0906' AND
+  db  = '-307' AND
+  str = 1234;
+
+-- all should return false as all numbers exceeed the largest number 
+-- which could be represented by the corresponding type
+-- and string_col = long_const should return false
+EXPLAIN SELECT * FROM infertypes WHERE
+  ti  = '128' OR
+  si  = 32768 OR
+  i   = '2147483648' OR
+  bi  = '9223372036854775808' OR
+  fl  = 'float' OR
+  db  = 'double';
+
+SELECT * FROM infertypes WHERE
+  ti  = '128' OR
+  si  = 32768 OR
+  i   = '2147483648' OR
+  bi  = '9223372036854775808' OR
+  fl  = 'float' OR
+  db  = 'double';
+
+-- for the query like: int_col = double, should return false 
+EXPLAIN SELECT * FROM infertypes WHERE
+  ti  = '127.0' OR
+  si  = 327.0 OR
+  i   = '-100.0';
+
+SELECT * FROM infertypes WHERE
+  ti  = '127.0' OR
+  si  = 327.0 OR
+  i   = '-100.0';
+
+EXPLAIN SELECT * FROM infertypes WHERE
+  ti < '127.0' AND
+  i > '100.0' AND
+  str = 1.57;
+
+SELECT * FROM infertypes WHERE
+  ti < '127.0' AND
+  i > '100.0' AND
+  str = 1.57;
+
+DROP TABLE infertypes;

ql/src/test/queries/clientpositive/insert1_overwrite_partitions.q

@@ -7,28 +7,28 @@ load data local inpath '../data/files/kv3.txt' INTO TABLE sourceTable partition(
 CREATE TABLE destinTable (one string,two string) PARTITIONED BY (ds string,hr string);
 
 EXPLAIN INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists
-SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one limit 5;
+SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one desc, two desc limit 5;
 
 INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists
-SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one limit 5;
+SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one desc, two desc limit 5;
 
-select one,two from destinTable;
+select one,two from destinTable order by one desc, two desc;
 
 EXPLAIN INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists
-SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='12' order by one limit 5;
+SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='12' order by one desc, two desc limit 5;
 
 INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists
-SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='12' order by one limit 5;
+SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='12' order by one desc, two desc limit 5;
 
-select one,two from destinTable;
+select one,two from destinTable order by one desc, two desc;
 
 drop table destinTable;
 
 CREATE TABLE destinTable (one string,two string);
 
-EXPLAIN INSERT OVERWRITE TABLE destinTable SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one limit 5;
+EXPLAIN INSERT OVERWRITE TABLE destinTable SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one desc, two desc limit 5;
 
-INSERT OVERWRITE TABLE destinTable SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one limit 5;
+INSERT OVERWRITE TABLE destinTable SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one desc, two desc limit 5;
 
 drop table destinTable;
 

ql/src/test/queries/clientpositive/insert2_overwrite_partitions.q

@@ -11,20 +11,20 @@ load data local inpath '../data/files/kv3.txt' INTO TABLE db1.sourceTable partit
 CREATE TABLE db2.destinTable (one string,two string) PARTITIONED BY (ds string);
 
 EXPLAIN INSERT OVERWRITE TABLE db2.destinTable PARTITION (ds='2011-11-11')
-SELECT one,two FROM db1.sourceTable WHERE ds='2011-11-11' order by one limit 5;
+SELECT one,two FROM db1.sourceTable WHERE ds='2011-11-11' order by one desc, two desc limit 5;
 
 INSERT OVERWRITE TABLE db2.destinTable PARTITION (ds='2011-11-11')
-SELECT one,two FROM db1.sourceTable WHERE ds='2011-11-11' order by one limit 5;
+SELECT one,two FROM db1.sourceTable WHERE ds='2011-11-11' order by one desc, two desc limit 5;
 
-select one,two from db2.destinTable;
+select one,two from db2.destinTable order by one desc, two desc;
 
 EXPLAIN INSERT OVERWRITE TABLE db2.destinTable PARTITION (ds='2011-11-11')
-SELECT one,two FROM db1.sourceTable WHERE ds='2011-11-11' order by one limit 5;
+SELECT one,two FROM db1.sourceTable WHERE ds='2011-11-11' order by one desc, two desc limit 5;
 
 INSERT OVERWRITE TABLE db2.destinTable PARTITION (ds='2011-11-11')
-SELECT one,two FROM db1.sourceTable WHERE ds='2011-11-11' order by one limit 5;
+SELECT one,two FROM db1.sourceTable WHERE ds='2011-11-11' order by one desc, two desc limit 5;
 
-select one,two from db2.destinTable;
+select one,two from db2.destinTable order by one desc, two desc;
 
 drop table db2.destinTable;
 

ql/src/test/queries/clientpositive/ppr_pushdown.q

@@ -19,15 +19,15 @@ insert overwrite table ppr_test partition(ds = '12%4') select * from (select '12
 insert overwrite table ppr_test partition(ds = '12*4') select * from (select '12*4' from src limit 1 union all select 'abcd' from src limit 1) s;
 
 
-select * from ppr_test where ds = '1234';
-select * from ppr_test where ds = '1224';
-select * from ppr_test where ds = '1214';
-select * from ppr_test where ds = '12.4';
-select * from ppr_test where ds = '12+4';
-select * from ppr_test where ds = '12:4';
-select * from ppr_test where ds = '12%4';
-select * from ppr_test where ds = '12*4';
-select * from ppr_test where ds = '12.*4';
+select * from ppr_test where ds = '1234' order by key;
+select * from ppr_test where ds = '1224' order by key;
+select * from ppr_test where ds = '1214' order by key;
+select * from ppr_test where ds = '12.4' order by key;
+select * from ppr_test where ds = '12+4' order by key;
+select * from ppr_test where ds = '12:4' order by key;
+select * from ppr_test where ds = '12%4' order by key;
+select * from ppr_test where ds = '12*4' order by key;
+select * from ppr_test where ds = '12.*4' order by key;
 
 select * from ppr_test where ds = '1234' and key = '1234';
 select * from ppr_test where ds = '1224' and key = '1224';

ql/src/test/results/clientpositive/auto_join0.q.out

@@ -50,7 +50,7 @@ STAGE PLANS:
             alias: src
             Filter Operator
               predicate:
-                  expr: (key < 10)
+                  expr: (key < 10.0)
                   type: boolean
               Select Operator
                 expressions:
@@ -77,7 +77,7 @@ STAGE PLANS:
             alias: src
             Filter Operator
               predicate:
-                  expr: (key < 10)
+                  expr: (key < 10.0)
                   type: boolean
               Select Operator
                 expressions:
@@ -210,7 +210,7 @@ STAGE PLANS:
             alias: src
             Filter Operator
               predicate:
-                  expr: (key < 10)
+                  expr: (key < 10.0)
                   type: boolean
               Select Operator
                 expressions:
@@ -237,7 +237,7 @@ STAGE PLANS:
             alias: src
             Filter Operator
               predicate:
-                  expr: (key < 10)
+                  expr: (key < 10.0)
                   type: boolean
               Select Operator
                 expressions:
@@ -286,7 +286,7 @@ STAGE PLANS:
             alias: src
             Filter Operator
               predicate:
-                  expr: (key < 10)
+                  expr: (key < 10.0)
                   type: boolean
               Select Operator
                 expressions:
@@ -308,7 +308,7 @@ STAGE PLANS:
             alias: src
             Filter Operator
               predicate:
-                  expr: (key < 10)
+                  expr: (key < 10.0)
                   type: boolean
               Select Operator
                 expressions:

ql/src/test/results/clientpositive/auto_join11.q.out

@@ -66,7 +66,7 @@ STAGE PLANS:
             alias: src
             Filter Operator
               predicate:
-                  expr: (key < 100)
+                  expr: (key < 100.0)
                   type: boolean
               Select Operator
                 expressions:
@@ -148,7 +148,7 @@ STAGE PLANS:
             alias: src
             Filter Operator
               predicate:
-                  expr: (key < 100)
+                  expr: (key < 100.0)
                   type: boolean
               Select Operator
                 expressions:
@@ -220,7 +220,7 @@ STAGE PLANS:
             alias: src
             Filter Operator
               predicate:
-                  expr: (key < 100)
+                  expr: (key < 100.0)
                   type: boolean
               Select Operator
                 expressions: