always use source lookup in genericFunctionQuery

-> where substr(analyzed_text, 1,1) = 'x'

threw GroupByOnArrayUnsupported exception, now it works
correctly using source lookup

core/src/main/java/io/crate/types/DataTypes.java

@@ -28,12 +28,16 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.Loggers;
 
 import java.io.IOException;
 import java.util.*;
 
 public class DataTypes {
 
+    private final static ESLogger logger = Loggers.getLogger(DataTypes.class);
+
     public final static UndefinedType UNDEFINED = UndefinedType.INSTANCE;
     public final static NotSupportedType NOT_SUPPORTED = NotSupportedType.INSTANCE;
 
@@ -147,9 +151,14 @@ public static boolean isCollectionType(DataType type) {
 
     public static DataType fromStream(StreamInput in) throws IOException {
         int i = in.readVInt();
-        DataType type = typeRegistry.get(i).create();
-        type.readFrom(in);
-        return type;
+        try {
+            DataType type = typeRegistry.get(i).create();
+            type.readFrom(in);
+            return type;
+        } catch (NullPointerException e) {
+            logger.error(String.format(Locale.ENGLISH, "%d is missing in typeRegistry", i), e);
+            throw e;
+        }
     }
 
     public static void toStream(DataType type, StreamOutput out) throws IOException {

core/src/main/java/io/crate/types/DoubleType.java

@@ -55,15 +55,15 @@ public Double value(Object value) {
         if (value == null) {
             return null;
         }
+        if (value instanceof Double) {
+            return (Double) value;
+        }
         if (value instanceof String) {
             return Double.valueOf((String)value);
         }
         if (value instanceof BytesRef) {
             return Double.valueOf(((BytesRef)value).utf8ToString());
         }
-        if (value instanceof Double) {
-            return (Double) value;
-        }
         return ((Number)value).doubleValue();
     }
 

core/src/main/java/io/crate/types/FloatType.java

@@ -55,6 +55,9 @@ public Float value(Object value) {
         if (value == null) {
             return null;
         }
+        if (value instanceof Float) {
+            return (Float) value;
+        }
         if (value instanceof String) {
             return Float.parseFloat((String)value);
         }

core/src/main/java/io/crate/types/IntegerType.java

@@ -55,6 +55,9 @@ public Integer value(Object value) {
         if (value == null) {
             return null;
         }
+        if (value instanceof Integer) {
+            return (Integer) value;
+        }
         if (value instanceof String) {
             return Integer.parseInt((String)value);
         }

core/src/main/java/io/crate/types/LongType.java

@@ -53,6 +53,9 @@ public Long value(Object value) {
         if (value == null) {
             return null;
         }
+        if (value instanceof Long) {
+            return (Long) value;
+        }
         if (value instanceof String) {
             return Long.valueOf((String)value);
         }

core/src/main/java/io/crate/types/ShortType.java

@@ -55,6 +55,9 @@ public Short value(Object value) {
         if (value == null) {
             return null;
         }
+        if (value instanceof Short) {
+            return (Short) value;
+        }
         if (value instanceof String) {
             return Short.valueOf((String)value);
         }

sql/src/main/java/io/crate/analyze/DataStatementAnalyzer.java

@@ -185,7 +185,7 @@ protected Symbol visitIsNotNullPredicate(IsNotNullPredicate node, T context) {
 
         return context.allocateFunction(
                 NotPredicate.INFO,
-                ImmutableList.<Symbol>of(context.allocateFunction(isNullInfo, ImmutableList.of(argument))));
+                Arrays.<Symbol>asList(context.allocateFunction(isNullInfo, Arrays.asList(argument))));
     }
 
     @Override
@@ -458,7 +458,7 @@ protected Symbol visitIsNullPredicate(IsNullPredicate node, T context) {
                 new FunctionIdent(io.crate.operation.predicate.IsNullPredicate.NAME,
                         ImmutableList.of(DataTypeVisitor.fromSymbol((value))));
         FunctionInfo functionInfo = context.getFunctionInfo(functionIdent);
-        return context.allocateFunction(functionInfo, ImmutableList.of(value));
+        return context.allocateFunction(functionInfo, Arrays.asList(value));
     }
 
     @Override
@@ -885,7 +885,7 @@ private void castTypes() {
                 left = DataTypeSymbol.toDataTypeSymbol(left, leftType);
                 if (rightType.isConvertableTo(leftType)) {
                     FunctionInfo functionInfo = CastFunctionResolver.functionInfo(rightType, leftType);
-                    ImmutableList<Symbol> arguments = ImmutableList.of(right);
+                    List<Symbol> arguments = Arrays.asList(right);
                     right = new Function(functionInfo, arguments);
                 } else {
                     throw new IllegalArgumentException(SymbolFormatter.format(

sql/src/main/java/io/crate/lucene/LuceneQueryBuilder.java

@@ -21,6 +21,8 @@
 
 package io.crate.lucene;
 
+import com.google.common.base.Predicates;
+import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableMap;
 import com.spatial4j.core.context.jts.JtsSpatialContext;
 import com.spatial4j.core.shape.Rectangle;
@@ -30,9 +32,11 @@
 import io.crate.analyze.WhereClause;
 import io.crate.lucene.match.MatchQueryBuilder;
 import io.crate.lucene.match.MultiMatchQueryBuilder;
+import io.crate.metadata.DocReferenceConverter;
 import io.crate.metadata.Functions;
 import io.crate.operation.Input;
 import io.crate.operation.collect.CollectInputSymbolVisitor;
+import io.crate.operation.collect.LuceneDocCollector;
 import io.crate.operation.operator.*;
 import io.crate.operation.operator.any.*;
 import io.crate.operation.predicate.IsNullPredicate;
@@ -45,6 +49,7 @@
 import io.crate.operation.scalar.geo.WithinFunction;
 import io.crate.planner.symbol.*;
 import io.crate.types.DataTypes;
+import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.*;
@@ -736,15 +741,22 @@ private Query genericFunctionQuery(Function function) {
             if (function.valueType() != DataTypes.BOOLEAN) {
                 raiseUnsupported(function);
             }
+            // avoid field-cache
+            // reason1: analyzed columns or columns with index off wouldn't work
+            //   substr(n, 1, 1) in the case of n => analyzed would throw an error because n would be an array
+            // reason2: would have to load each value into the field cache
+            DocReferenceConverter.convertIf(function, Predicates.<Reference>alwaysTrue());
 
             final CollectInputSymbolVisitor.Context ctx = inputSymbolVisitor.process(function);
             assert ctx.topLevelInputs().size() == 1;
             @SuppressWarnings("unchecked")
             final Input<Boolean> condition = (Input<Boolean>) ctx.topLevelInputs().get(0);
             @SuppressWarnings("unchecked")
             final List<LuceneCollectorExpression> expressions = ctx.docLevelExpressions();
-            CollectorContext collectorContext = new CollectorContext();
+            final CollectorContext collectorContext = new CollectorContext();
             collectorContext.searchContext(searchContext);
+            collectorContext.visitor(new LuceneDocCollector.CollectorFieldsVisitor(expressions.size()));
+
             for (LuceneCollectorExpression expression : expressions) {
                 expression.startCollect(collectorContext);
             }
@@ -756,6 +768,8 @@ public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws
                     }
                     return BitsFilteredDocIdSet.wrap(
                             new FunctionDocSet(
+                                    context.reader(),
+                                    collectorContext.visitor(),
                                     condition,
                                     expressions,
                                     context.reader().maxDoc(),
@@ -770,20 +784,37 @@ public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws
 
         static class FunctionDocSet extends MatchDocIdSet {
 
+            private final AtomicReader reader;
+            private final LuceneDocCollector.CollectorFieldsVisitor fieldsVisitor;
             private final Input<Boolean> condition;
             private final List<LuceneCollectorExpression> expressions;
+            private final boolean fieldsVisitorEnabled;
 
-            protected FunctionDocSet(Input<Boolean> condition,
+            protected FunctionDocSet(AtomicReader reader,
+                                     @Nullable LuceneDocCollector.CollectorFieldsVisitor fieldsVisitor,
+                                     Input<Boolean> condition,
                                      List<LuceneCollectorExpression> expressions,
                                      int maxDoc,
                                      @Nullable Bits acceptDocs) {
                 super(maxDoc, acceptDocs);
+                this.reader = reader;
+                this.fieldsVisitor = fieldsVisitor;
+                //noinspection SimplifiableConditionalExpression
+                this.fieldsVisitorEnabled = fieldsVisitor == null ? false : fieldsVisitor.required();
                 this.condition = condition;
                 this.expressions = expressions;
             }
 
             @Override
             protected boolean matchDoc(int doc) {
+                if (fieldsVisitorEnabled) {
+                    fieldsVisitor.reset();
+                    try {
+                        reader.document(doc, fieldsVisitor);
+                    } catch (IOException e) {
+                        throw Throwables.propagate(e);
+                    }
+                }
                 for (LuceneCollectorExpression expression : expressions) {
                     expression.setNextDocId(doc);
                 }