Support NULL literal as function argument

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/DataType.java

@@ -26,7 +26,7 @@
 @RequiredArgsConstructor
 public enum DataType {
   TYPE_ERROR(ExprCoreType.UNKNOWN),
-  NULL(ExprCoreType.UNKNOWN),
+  NULL(ExprCoreType.UNDEFINED),
 
   INTEGER(ExprCoreType.INTEGER),
   LONG(ExprCoreType.LONG),

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprNullValue.java

@@ -49,7 +49,7 @@ public Object value() {
 
   @Override
   public ExprType type() {
-    return ExprCoreType.UNKNOWN;
+    return ExprCoreType.UNDEFINED;
   }
 
   @Override

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java

@@ -19,6 +19,7 @@
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -29,14 +30,21 @@
  */
 public enum ExprCoreType implements ExprType {
   /**
-   * UNKNOWN.
+   * Unknown due to unsupported data type.
    */
   UNKNOWN,
 
+  /**
+   * Undefined type for special literal such as NULL.
+   * As the root of data type tree, it is compatible with any other type.
+   * In other word, undefined type is the "narrowest" type.
+   */
+  UNDEFINED,
+
   /**
    * Numbers.
    */
-  BYTE,
+  BYTE(UNDEFINED),
   SHORT(BYTE),
   INTEGER(SHORT),
   LONG(INTEGER),
@@ -46,38 +54,38 @@ public enum ExprCoreType implements ExprType {
   /**
    * Boolean.
    */
-  BOOLEAN,
+  BOOLEAN(UNDEFINED),
 
   /**
    * String.
    */
-  STRING,
+  STRING(UNDEFINED),
 
 
   /**
    * Date.
    * Todo. compatible relationship.
    */
-  TIMESTAMP,
-  DATE,
-  TIME,
-  DATETIME,
-  INTERVAL,
+  TIMESTAMP(UNDEFINED),
+  DATE(UNDEFINED),
+  TIME(UNDEFINED),
+  DATETIME(UNDEFINED),
+  INTERVAL(UNDEFINED),
 
   /**
    * Struct.
    */
-  STRUCT,
+  STRUCT(UNDEFINED),
 
   /**
    * Array.
    */
-  ARRAY;
+  ARRAY(UNDEFINED);
 
   /**
-   * Parent of current base type.
+   * Parents (wider/compatible types) of current base type.
    */
-  private ExprCoreType parent;
+  private final List<ExprType> parents = new ArrayList<>();
 
   /**
    * The mapping between Type and legacy JDBC type name.
@@ -91,13 +99,13 @@ public enum ExprCoreType implements ExprType {
 
   ExprCoreType(ExprCoreType... compatibleTypes) {
     for (ExprCoreType subType : compatibleTypes) {
-      subType.parent = this;
+      subType.parents.add(this);
     }
   }
 
   @Override
   public List<ExprType> getParent() {
-    return Arrays.asList(parent == null ? UNKNOWN : parent);
+    return parents.isEmpty() ? ExprType.super.getParent() : parents;
   }
 
   @Override
@@ -113,9 +121,11 @@ public String legacyTypeName() {
   /**
    * Return all the valid ExprCoreType.
    */
-  public static List<ExprType> coreTypes() {
-    return Arrays.stream(ExprCoreType.values()).filter(type -> type != UNKNOWN)
-        .collect(Collectors.toList());
+  public static List<ExprCoreType> coreTypes() {
+    return Arrays.stream(ExprCoreType.values())
+                 .filter(type -> type != UNKNOWN)
+                 .filter(type -> type != UNDEFINED)
+                 .collect(Collectors.toList());
   }
 
   public static List<ExprType> numberTypes() {

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/conditional/cases/CaseClause.java

@@ -16,6 +16,7 @@
 
 package com.amazon.opendistroforelasticsearch.sql.expression.conditional.cases;
 
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.UNDEFINED;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.UNKNOWN;
 
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprNullValue;
@@ -75,8 +76,8 @@ public ExprValue valueOf(Environment<Expression, ExprValue> valueEnv) {
   public ExprType type() {
     List<ExprType> types = allResultTypes();
 
-    // Return unknown if all WHEN/ELSE return NULL
-    return types.isEmpty() ? UNKNOWN : types.get(0);
+    // Return undefined if all WHEN/ELSE return NULL
+    return types.isEmpty() ? UNDEFINED : types.get(0);
   }
 
   @Override
@@ -98,7 +99,7 @@ public List<ExprType> allResultTypes() {
       types.add(defaultResult.type());
     }
 
-    types.removeIf(type -> (type == UNKNOWN));
+    types.removeIf(type -> (type == UNDEFINED));
     return types;
   }
 

core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprNullValueTest.java

@@ -42,7 +42,7 @@ public void getValue() {
 
   @Test
   public void getType() {
-    assertEquals(ExprCoreType.UNKNOWN, LITERAL_NULL.type());
+    assertEquals(ExprCoreType.UNDEFINED, LITERAL_NULL.type());
   }
 
   @Test

core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprTypeTest.java

@@ -25,6 +25,7 @@
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.SHORT;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT;
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.UNDEFINED;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.UNKNOWN;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -51,6 +52,12 @@ public void isCompatible() {
     assertFalse(INTEGER.isCompatible(UNKNOWN));
   }
 
+  @Test
+  public void isCompatibleWithUndefined() {
+    ExprCoreType.coreTypes().forEach(type -> assertTrue(type.isCompatible(UNDEFINED)));
+    ExprCoreType.coreTypes().forEach(type -> assertFalse(UNDEFINED.isCompatible(type)));
+  }
+
   @Test
   public void getParent() {
     assertThat(((ExprType) () -> "test").getParent(), Matchers.contains(UNKNOWN));

core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/conditional/cases/CaseClauseTest.java

@@ -77,7 +77,7 @@ void should_use_type_of_when_clause() {
 
   @Test
   void should_use_type_of_nonnull_when_or_else_clause() {
-    when(whenClause.type()).thenReturn(ExprCoreType.UNKNOWN);
+    when(whenClause.type()).thenReturn(ExprCoreType.UNDEFINED);
     Expression defaultResult = mock(Expression.class);
     when(defaultResult.type()).thenReturn(ExprCoreType.STRING);
 
@@ -87,12 +87,12 @@ void should_use_type_of_nonnull_when_or_else_clause() {
 
   @Test
   void should_use_unknown_type_of_if_all_when_and_else_return_null() {
-    when(whenClause.type()).thenReturn(ExprCoreType.UNKNOWN);
+    when(whenClause.type()).thenReturn(ExprCoreType.UNDEFINED);
     Expression defaultResult = mock(Expression.class);
-    when(defaultResult.type()).thenReturn(ExprCoreType.UNKNOWN);
+    when(defaultResult.type()).thenReturn(ExprCoreType.UNDEFINED);
 
     CaseClause caseClause = new CaseClause(ImmutableList.of(whenClause), defaultResult);
-    assertEquals(ExprCoreType.UNKNOWN, caseClause.type());
+    assertEquals(ExprCoreType.UNDEFINED, caseClause.type());
   }
 
   @Test
@@ -109,9 +109,9 @@ void should_return_all_result_types_including_default() {
 
   @Test
   void should_return_all_result_types_excluding_null_result() {
-    when(whenClause.type()).thenReturn(ExprCoreType.UNKNOWN);
+    when(whenClause.type()).thenReturn(ExprCoreType.UNDEFINED);
     Expression defaultResult = mock(Expression.class);
-    when(defaultResult.type()).thenReturn(ExprCoreType.UNKNOWN);
+    when(defaultResult.type()).thenReturn(ExprCoreType.UNDEFINED);
 
     CaseClause caseClause = new CaseClause(ImmutableList.of(whenClause), defaultResult);
     assertEquals(

core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/WideningTypeRuleTest.java

@@ -21,7 +21,6 @@
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.SHORT;
-import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.UNKNOWN;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.WideningTypeRule.IMPOSSIBLE_WIDENING;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.WideningTypeRule.TYPE_EQUAL;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -33,9 +32,7 @@
 import com.google.common.collect.ImmutableTable;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Table;
-import java.util.Arrays;
 import java.util.List;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
@@ -63,9 +60,7 @@ class WideningTypeRuleTest {
           .build();
 
   private static Stream<Arguments> distanceArguments() {
-    List<ExprCoreType> exprTypes =
-        Arrays.asList(ExprCoreType.values()).stream().filter(type -> type != UNKNOWN).collect(
-            Collectors.toList());
+    List<ExprCoreType> exprTypes = ExprCoreType.coreTypes();
     return Lists.cartesianProduct(exprTypes, exprTypes).stream()
         .map(list -> {
           ExprCoreType type1 = list.get(0);
@@ -81,9 +76,7 @@ private static Stream<Arguments> distanceArguments() {
   }
 
   private static Stream<Arguments> validMaxTypes() {
-    List<ExprCoreType> exprTypes =
-        Arrays.asList(ExprCoreType.values()).stream().filter(type -> type != UNKNOWN).collect(
-            Collectors.toList());
+    List<ExprCoreType> exprTypes = ExprCoreType.coreTypes();
     return Lists.cartesianProduct(exprTypes, exprTypes).stream()
         .map(list -> {
           ExprCoreType type1 = list.get(0);

docs/user/general/values.rst

@@ -11,7 +11,7 @@ Data Types
 
 NULL and MISSING Values
 =======================
-ODFE SQL has two ways to represent missing information. (1) The presence of the field with a NULL for its value. and (2) the absence of the filed.
+ODFE SQL has two ways to represent missing information. (1) The presence of the field with a NULL for its value. and (2) the absence of the field.
 
 Please note, when response is in table format, the MISSING value is translate to NULL value.
 
@@ -31,7 +31,7 @@ Here is an example, Nanette doesn't have email field and Dail has employer filed
 
 General NULL and MISSING Values Handling
 ----------------------------------------
-In general, if any operand evaluates to a MISSING value, the enclosing operator will return MISSING; if none of operands evaluates to a MISSING value but there is an operand evaluates to a NULL value, the enclosing operator will return NULL.
+In general, if any operand evaluates to a MISSING value, the enclosing operator will return MISSING; if none of operands evaluates to a MISSING value but there is an operand evaluates to a NULL value, the enclosing operator will return NULL. To handle null value properly, you can use special operators such as ``IS (NOT) NULL`` or conditional functions such as ``IFNULL``. Please find more details in their docs.
 
 Here is an example::
 
@@ -46,6 +46,21 @@ Here is an example::
     | Dale        | null                      | True                |
     +-------------+---------------------------+---------------------+
 
+
+NULL Literal Handling
+---------------------
+
+Because the type of a null literal is unknown, a special data type ``UNDEFINED`` is reserved to allow it be accepted as a valid function argument::
+
+    od> SELECT NULL, NULL = NULL, 1 + NULL, LENGTH(NULL);
+    fetched rows / total rows = 1/1
+    +--------+---------------+------------+----------------+
+    | NULL   | NULL = NULL   | 1 + NULL   | LENGTH(NULL)   |
+    |--------+---------------+------------+----------------|
+    | null   | null          | null       | null           |
+    +--------+---------------+------------+----------------+
+
+
 Special NULL and MISSING Values Handling
 ----------------------------------------
 THe AND, OR and NOT have special logic to handling NULL and MISSING value.

integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/NullLiteralIT.java

@@ -0,0 +1,63 @@
+/*
+ *    Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License").
+ *    You may not use this file except in compliance with the License.
+ *    A copy of the License is located at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    or in the "license" file accompanying this file. This file is distributed
+ *    on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *    express or implied. See the License for the specific language governing
+ *    permissions and limitations under the License.
+ *
+ */
+
+package com.amazon.opendistroforelasticsearch.sql.sql;
+
+import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows;
+import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema;
+import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows;
+import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema;
+
+import com.amazon.opendistroforelasticsearch.sql.legacy.SQLIntegTestCase;
+import org.json.JSONObject;
+import org.junit.Test;
+
+/**
+ * This manual IT for NULL literal cannot be replaced with comparison test because other database
+ * has different type for expression with NULL involved, such as NULL rather than concrete type
+ * inferred like what we do in core engine.
+ */
+public class NullLiteralIT extends SQLIntegTestCase {
+
+  @Test
+  public void testNullLiteralSchema() {
+    verifySchema(
+        query("SELECT NULL, ABS(NULL), 1 + NULL, NULL + 1.0"),
+        schema("NULL", "undefined"),
+        schema("ABS(NULL)", "byte"),
+        schema("1 + NULL", "integer"),
+        schema("NULL + 1.0", "double"));
+  }
+
+  @Test
+  public void testNullLiteralInOperator() {
+    verifyDataRows(
+        query("SELECT NULL = NULL, NULL AND TRUE"),
+        rows(null, null));
+  }
+
+  @Test
+  public void testNullLiteralInFunction() {
+    verifyDataRows(
+        query("SELECT ABS(NULL), POW(2, FLOOR(NULL))"),
+        rows(null, null));
+  }
+
+  private JSONObject query(String sql) {
+    return new JSONObject(executeQuery(sql, "jdbc"));
+  }
+
+}

integ-test/src/test/resources/correctness/expressions/literals.txt

@@ -1,3 +1,5 @@
+null
+NULL
 1
 true
 -4.567

sql-jdbc/src/main/java/com/amazon/opendistroforelasticsearch/jdbc/types/ElasticsearchType.java

@@ -77,14 +77,15 @@ public enum ElasticsearchType {
     TIMESTAMP(JDBCType.TIMESTAMP, Timestamp.class, 24, 24, false),
     BINARY(JDBCType.VARBINARY, String.class, Integer.MAX_VALUE, 0, false),
     NULL(JDBCType.NULL, null, 0, 0, false),
+    UNDEFINED(JDBCType.NULL, null, 0, 0, false),
     UNSUPPORTED(JDBCType.OTHER, null, 0, 0, false);
 
     private static final Map<JDBCType, ElasticsearchType> jdbcTypeToESTypeMap;
 
     static {
         // Map JDBCType to corresponding ElasticsearchType
         jdbcTypeToESTypeMap = new HashMap<>();
-        jdbcTypeToESTypeMap.put(JDBCType.NULL, NULL);
+        jdbcTypeToESTypeMap.put(JDBCType.NULL, UNDEFINED);
         jdbcTypeToESTypeMap.put(JDBCType.BOOLEAN, BOOLEAN);
         jdbcTypeToESTypeMap.put(JDBCType.TINYINT, BYTE);
         jdbcTypeToESTypeMap.put(JDBCType.SMALLINT, SHORT);