QL: Refactor DataType for pluggability (#51328)

Change DataType from enum to class
Break DataType enums into QL (default) and SQL types
Make data type conversion pluggable so that new types can be introduced

As part of the process:
- static type conversion in QL package (such as Literal) has been
removed
- several utility classes have been broken into base (QL) and extended
(SQL) parts based on type awareness
- operators (+,-,/,*) are
- due to extensibility, serialization of arithmetic operation has been
slightly changed and pushed down to the operator executor itself

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java

@@ -12,19 +12,25 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.mapper.IgnoredFieldMapper;
+import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
 import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
 
 import java.io.IOException;
 import java.time.ZoneId;
 import java.util.ArrayDeque;
 import java.util.Deque;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.StringJoiner;
 
+import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
+import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
+import static org.elasticsearch.xpack.ql.type.DataTypes.SCALED_FLOAT;
 /**
  * Extractor for ES fields. Works for both 'normal' fields but also nested ones (which require hitName to be set).
  * The latter is used as metadata in assembling the results in the tabular response.
@@ -83,15 +89,19 @@ protected AbstractFieldHitExtractor(StreamInput in) throws IOException {
         } else {
             fullFieldName = null;
         }
-        String esType = in.readOptionalString();
-        dataType = esType != null ? DataType.fromTypeName(esType) : null;
+        String typeName = in.readOptionalString();
+        dataType = typeName != null ? loadTypeFromName(typeName) : null;
         useDocValue = in.readBoolean();
         hitName = in.readOptionalString();
         arrayLeniency = in.readBoolean();
         path = sourcePath(fieldName, useDocValue, hitName);
         zoneId = readZoneId(in);
     }
 
+    protected DataType loadTypeFromName(String typeName) {
+        return DataTypes.fromTypeName(typeName);
+    }
+
     protected abstract ZoneId readZoneId(StreamInput in) throws IOException;
 
     @Override
@@ -118,7 +128,7 @@ public Object extract(SearchHit hit) {
             // if the field was ignored because it was malformed and ignore_malformed was turned on
             if (fullFieldName != null
                     && hit.getFields().containsKey(IgnoredFieldMapper.NAME)
-                    && dataType.isFromDocValuesOnly() == false
+                    && isFromDocValuesOnly(dataType) == false
                     && dataType.isNumeric()) {
                 /*
                  * ignore_malformed makes sense for extraction from _source for numeric fields only.
@@ -171,11 +181,11 @@ protected Object unwrapMultiValue(Object values) {
             if (dataType == null) {
                 return values;
             }
-            if (dataType.isNumeric() && dataType.isFromDocValuesOnly() == false) {
-                if (dataType == DataType.DOUBLE || dataType == DataType.FLOAT || dataType == DataType.HALF_FLOAT) {
+            if (dataType.isNumeric() && isFromDocValuesOnly(dataType) == false) {
+                if (dataType == DataTypes.DOUBLE || dataType == DataTypes.FLOAT || dataType == DataTypes.HALF_FLOAT) {
                     Number result = null;
                     try {
-                        result = dataType.numberType().parse(values, true);
+                        result = numberType(dataType).parse(values, true);
                     } catch(IllegalArgumentException iae) {
                         return null;
                     }
@@ -185,13 +195,13 @@ protected Object unwrapMultiValue(Object values) {
                 } else {
                     Number result = null;
                     try {
-                        result = dataType.numberType().parse(values, true);
+                        result = numberType(dataType).parse(values, true);
                     } catch(IllegalArgumentException iae) {
                         return null;
                     }
                     return result;
                 }
-            } else if (dataType.isString()) {
+            } else if (DataTypes.isString(dataType)) {
                 return values.toString();
             } else {
                 return values;
@@ -200,12 +210,22 @@ protected Object unwrapMultiValue(Object values) {
         throw new QlIllegalArgumentException("Type {} (returned by [{}]) is not supported", values.getClass().getSimpleName(), fieldName);
     }
 
+    protected boolean isFromDocValuesOnly(DataType dataType) {
+        return dataType == KEYWORD // because of ignore_above.
+                    || dataType == DATETIME
+                    || dataType == SCALED_FLOAT; // because of scaling_factor
+    }
+
+    private static NumberType numberType(DataType dataType) {
+        return NumberType.valueOf(dataType.esType().toUpperCase(Locale.ROOT));
+    }
+
     protected abstract Object unwrapCustomValue(Object values);
 
     protected abstract boolean isPrimitive(List<?> list);
 
     @SuppressWarnings({ "unchecked", "rawtypes" })
-    protected Object extractFromSource(Map<String, Object> map) {
+    public Object extractFromSource(Map<String, Object> map) {
         Object value = null;
 
         // Used to avoid recursive method calls
@@ -286,7 +306,7 @@ public ZoneId zoneId() {
         return zoneId;
     }
 
-    protected DataType dataType() {
+    public DataType dataType() {
         return dataType;
     }
 

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expression.java

@@ -46,12 +46,17 @@ public boolean unresolved() {
         }
 
         public boolean resolved() {
-            return !failed;
+            return failed == false;
         }
 
         public String message() {
             return message;
         }
+
+        @Override
+        public String toString() {
+            return resolved() ? "" : message;
+        }
     }
 
     private TypeResolution lazyTypeResolution = null;

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java

@@ -10,7 +10,6 @@
 import org.elasticsearch.xpack.ql.expression.gen.pipeline.AttributeInput;
 import org.elasticsearch.xpack.ql.expression.gen.pipeline.ConstantInput;
 import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
-import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.DataTypes;
 
 import java.util.ArrayList;
@@ -112,7 +111,7 @@ public static String name(Expression e) {
     }
 
     public static boolean isNull(Expression e) {
-        return e.dataType() == DataType.NULL || (e.foldable() && e.fold() == null);
+        return e.dataType() == DataTypes.NULL || (e.foldable() && e.fold() == null);
     }
 
     public static List<String> names(Collection<? extends Expression> e) {
@@ -165,7 +164,7 @@ public static List<Attribute> onlyPrimitiveFieldAttributes(Collection<Attribute>
         Set<Attribute> seenMultiFields = new LinkedHashSet<>();
 
         for (Attribute a : attributes) {
-            if (!DataTypes.isUnsupported(a.dataType()) && a.dataType().isPrimitive()) {
+            if (DataTypes.isUnsupported(a.dataType()) == false && DataTypes.isPrimitive(a.dataType())) {
                 if (a instanceof FieldAttribute) {
                     FieldAttribute fa = (FieldAttribute) a;
                     // skip nested fields and seen multi-fields

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/FieldAttribute.java

@@ -9,6 +9,7 @@
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
 import org.elasticsearch.xpack.ql.type.EsField;
 import org.elasticsearch.xpack.ql.util.StringUtils;
 
@@ -53,7 +54,7 @@ public FieldAttribute(Source source, FieldAttribute parent, String name, DataTyp
         FieldAttribute nestedPar = null;
         if (parent != null) {
             nestedPar = parent.nestedParent;
-            if (parent.dataType() == DataType.NESTED) {
+            if (parent.dataType() == DataTypes.NESTED) {
                 nestedPar = parent;
             }
         }
@@ -103,7 +104,7 @@ private FieldAttribute innerField(EsField type) {
     }
 
     @Override
-    protected Attribute clone(Source source, String name, DataType type, String qualifier, Nullability nullability, NameId id, 
+    protected Attribute clone(Source source, String name, DataType type, String qualifier, Nullability nullability, NameId id,
             boolean synthetic) {
         FieldAttribute qualifiedParent = parent != null ? (FieldAttribute) parent.withQualifier(qualifier) : null;
         return new FieldAttribute(source, qualifiedParent, name, field, qualifier, nullability, id, synthetic);

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Literal.java

@@ -18,9 +18,9 @@
  */
 public class Literal extends LeafExpression {
 
-    public static final Literal TRUE = new Literal(Source.EMPTY, Boolean.TRUE, DataType.BOOLEAN);
-    public static final Literal FALSE = new Literal(Source.EMPTY, Boolean.FALSE, DataType.BOOLEAN);
-    public static final Literal NULL = new Literal(Source.EMPTY, null, DataType.NULL);
+    public static final Literal TRUE = new Literal(Source.EMPTY, Boolean.TRUE, DataTypes.BOOLEAN);
+    public static final Literal FALSE = new Literal(Source.EMPTY, Boolean.FALSE, DataTypes.BOOLEAN);
+    public static final Literal NULL = new Literal(Source.EMPTY, null, DataTypes.NULL);
 
     private final Object value;
     private final DataType dataType;
@@ -93,16 +93,6 @@ public String nodeString() {
         return toString() + "[" + dataType + "]";
     }
 
-    /**
-     * Utility method for creating 'in-line' Literals (out of values instead of expressions).
-     */
-    public static Literal of(Source source, Object value) {
-        if (value instanceof Literal) {
-            return (Literal) value;
-        }
-        return new Literal(source, value, DataTypes.fromJava(value));
-    }
-
     /**
      * Utility method for creating a literal out of a foldable expression.
      * Throws an exception if the expression is not foldable.

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/TypeResolutions.java

@@ -8,6 +8,7 @@
 import org.elasticsearch.xpack.ql.expression.Expression.TypeResolution;
 import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal;
 import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
 import org.elasticsearch.xpack.ql.type.EsField;
 
 import java.util.Locale;
@@ -16,7 +17,8 @@
 
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.ql.expression.Expressions.name;
-import static org.elasticsearch.xpack.ql.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
+import static org.elasticsearch.xpack.ql.type.DataTypes.NULL;
 
 public final class TypeResolutions {
 
@@ -35,29 +37,7 @@ public static TypeResolution isNumeric(Expression e, String operationName, Param
     }
 
     public static TypeResolution isString(Expression e, String operationName, ParamOrdinal paramOrd) {
-        return isType(e, DataType::isString, operationName, paramOrd, "string");
-    }
-
-    public static TypeResolution isDate(Expression e, String operationName, ParamOrdinal paramOrd) {
-        return isType(e, DataType::isDateBased, operationName, paramOrd, "date", "datetime");
-    }
-
-    public static TypeResolution isDateOrTime(Expression e, String operationName, ParamOrdinal paramOrd) {
-        return isType(e, DataType::isDateOrTimeBased, operationName, paramOrd, "date", "time", "datetime");
-    }
-
-    public static TypeResolution isNumericOrDate(Expression e, String operationName, ParamOrdinal paramOrd) {
-        return isType(e, dt -> dt.isNumeric() || dt.isDateBased(), operationName, paramOrd,
-            "date", "datetime", "numeric");
-    }
-
-    public static TypeResolution isNumericOrDateOrTime(Expression e, String operationName, ParamOrdinal paramOrd) {
-        return isType(e, dt -> dt.isNumeric() || dt.isDateOrTimeBased(), operationName, paramOrd,
-            "date", "time", "datetime", "numeric");
-    }
-
-    public static TypeResolution isGeo(Expression e, String operationName, ParamOrdinal paramOrd) {
-        return isType(e, DataType::isGeo, operationName, paramOrd, "geo_point", "geo_shape");
+        return isType(e, DataTypes::isString, operationName, paramOrd, "string");
     }
 
     public static TypeResolution isExact(Expression e, String message) {
@@ -118,7 +98,7 @@ public static TypeResolution isType(Expression e,
                                         String operationName,
                                         ParamOrdinal paramOrd,
                                         String... acceptedTypes) {
-        return predicate.test(e.dataType()) || e.dataType().isNull() ?
+        return predicate.test(e.dataType()) || e.dataType() == NULL ?
             TypeResolution.TYPE_RESOLVED :
             new TypeResolution(format(null, "{}argument of [{}] must be [{}], found value [{}] type [{}]",
                 paramOrd == null || paramOrd == ParamOrdinal.DEFAULT ? "" : paramOrd.name().toLowerCase(Locale.ROOT) + " ",

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/DefaultFunctionTypeRegistry.java

@@ -7,7 +7,6 @@
 
 import org.elasticsearch.xpack.ql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
-import org.elasticsearch.xpack.ql.expression.predicate.conditional.ConditionalFunction;
 
 
 public class DefaultFunctionTypeRegistry implements FunctionTypeRegistry {
@@ -16,7 +15,6 @@ public class DefaultFunctionTypeRegistry implements FunctionTypeRegistry {
 
     private enum Types {
         AGGREGATE(AggregateFunction.class),
-        CONDITIONAL(ConditionalFunction.class),
         SCALAR(ScalarFunction.class);
 
         private Class<? extends Function> baseClass;

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/UnresolvedFunction.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
 import org.elasticsearch.xpack.ql.util.StringUtils;
 
 import java.util.LinkedHashSet;
@@ -195,7 +196,7 @@ public UnresolvedFunction preprocessStar(UnresolvedFunction uf) {
                 // dedicated count optimization
                 if (uf.name.toUpperCase(Locale.ROOT).equals("COUNT")) {
                     return new UnresolvedFunction(uf.source(), uf.name(), uf.resolutionType,
-                            singletonList(new Literal(uf.arguments().get(0).source(), Integer.valueOf(1), DataType.INTEGER)));
+                            singletonList(new Literal(uf.arguments().get(0).source(), Integer.valueOf(1), DataTypes.INTEGER)));
                 }
                 return uf;
             }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/aggregate/Count.java

@@ -9,6 +9,7 @@
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
 
 import java.util.List;
 import java.util.Objects;
@@ -46,7 +47,7 @@ public boolean distinct() {
 
     @Override
     public DataType dataType() {
-        return DataType.LONG;
+        return DataTypes.LONG;
     }
 
     @Override

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/IntervalScripting.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ql.expression.function.scalar;
+
+// FIXME: accessor interface until making script generation pluggable
+public interface IntervalScripting {
+
+    String script();
+
+    String value();
+
+    String typeName();
+
+}