Add the ability to retrieve fields from field data

Adds a new FetchSubPhase, FieldDataFieldsFetchSubPhase, which loads the
field data cache for a field and returns an array of values for the
field.

Also removes `doc['<field>']` and `_source.<field>` workaround no longer
needed in field name resolving.

Closes elastic#4492

docs/reference/search/request-body.asciidoc

@@ -31,7 +31,7 @@ And here is a sample response:
             {
                 "_index" : "twitter",
                 "_type" : "tweet",
-                "_id" : "1", 
+                "_id" : "1",
                 "_source" : {
                     "user" : "kimchy",
                     "postDate" : "2009-11-15T14:12:12",
@@ -87,6 +87,8 @@ include::request/fields.asciidoc[]
 
 include::request/script-fields.asciidoc[]
 
+include::request/fielddata-fields.asciidoc[]
+
 include::request/post-filter.asciidoc[]
 
 include::request/highlighting.asciidoc[]
@@ -108,4 +110,3 @@ include::request/index-boost.asciidoc[]
 include::request/min-score.asciidoc[]
 
 include::request/named-queries-and-filters.asciidoc[]
-

docs/reference/search/request/fielddata-fields.asciidoc

@@ -0,0 +1,21 @@
+[[search-request-fielddata-fields]]
+=== Field Data Fields
+
+Allows to return the field data representiation of a field for each hit, for
+example:
+
+[source,js]
+--------------------------------------------------
+{
+    "query" : {
+        ...
+    },
+    "fielddata_fields" : ["test1", "test2"]
+}
+--------------------------------------------------
+
+Field data fields can work on fields that are not stored.
+
+It's important to understand that using the `fielddata_fields` parameter will
+cause the terms for that field to be loaded to memory (cached), which will
+result in more memory consumption.

src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java

@@ -417,6 +417,17 @@ public SearchRequestBuilder addField(String field) {
         return this;
     }
 
+    /**
+     * Adds a field data based field to load and return. The field does not have to be stored,
+     * but its recommended to use non analyzed or numeric fields.
+     *
+     * @param name The field to get from the field data cache
+     */
+    public SearchRequestBuilder addFieldDataField(String name) {
+        sourceBuilder().fieldDataField(name);
+        return this;
+    }
+
     /**
      * Adds a script based field to load and return. The field does not have to be stored,
      * but its recommended to use non analyzed or numeric fields.

src/main/java/org/elasticsearch/index/get/ShardGetService.java

@@ -239,49 +239,24 @@ public GetResult innerGet(String type, String id, String[] gFields, boolean real
                         } else if (field.equals(SizeFieldMapper.NAME) && docMapper.rootMapper(SizeFieldMapper.class).fieldType().stored()) {
                             value = source.source.length();
                         } else {
-                            if (field.contains("_source.")) {
-                                if (searchLookup == null) {
-                                    searchLookup = new SearchLookup(mapperService, fieldDataService, new String[]{type});
-                                }
-                                if (sourceAsMap == null) {
-                                    sourceAsMap = SourceLookup.sourceAsMap(source.source);
-                                }
-                                SearchScript searchScript = scriptService.search(searchLookup, "mvel", field, null);
-                                // we can't do this, only allow to run scripts against the source
-                                //searchScript.setNextReader(docIdAndVersion.reader);
-                                //searchScript.setNextDocId(docIdAndVersion.docId);
-
-                                // but, we need to inject the parsed source into the script, so it will be used...
-                                searchScript.setNextSource(sourceAsMap);
-
-                                try {
-                                    value = searchScript.run();
-                                } catch (RuntimeException e) {
-                                    if (logger.isTraceEnabled()) {
-                                        logger.trace("failed to execute get request script field [{}]", e, field);
-                                    }
-                                    // ignore
-                                }
-                            } else {
-                                if (searchLookup == null) {
-                                    searchLookup = new SearchLookup(mapperService, fieldDataService, new String[]{type});
-                                    searchLookup.source().setNextSource(source.source);
-                                }
+                            if (searchLookup == null) {
+                                searchLookup = new SearchLookup(mapperService, fieldDataService, new String[]{type});
+                                searchLookup.source().setNextSource(source.source);
+                            }
 
-                                FieldMapper<?> x = docMapper.mappers().smartNameFieldMapper(field);
-                                if (x == null) {
-                                    if (docMapper.objectMappers().get(field) != null) {
-                                        // Only fail if we know it is a object field, missing paths / fields shouldn't fail.
-                                        throw new ElasticsearchIllegalArgumentException("field [" + field + "] isn't a leaf field");
-                                    }
-                                } else if (docMapper.sourceMapper().enabled() || x.fieldType().stored()) {
-                                    List<Object> values = searchLookup.source().extractRawValues(field);
-                                    if (!values.isEmpty()) {
-                                        for (int i = 0; i < values.size(); i++) {
-                                            values.set(i, x.valueForSearch(values.get(i)));
-                                        }
-                                        value = values;
+                            FieldMapper<?> x = docMapper.mappers().smartNameFieldMapper(field);
+                            if (x == null) {
+                                if (docMapper.objectMappers().get(field) != null) {
+                                    // Only fail if we know it is a object field, missing paths / fields shouldn't fail.
+                                    throw new ElasticsearchIllegalArgumentException("field [" + field + "] isn't a leaf field");
+                                }
+                            } else if (docMapper.sourceMapper().enabled() || x.fieldType().stored()) {
+                                List<Object> values = searchLookup.source().extractRawValues(field);
+                                if (!values.isEmpty()) {
+                                    for (int i = 0; i < values.size(); i++) {
+                                        values.set(i, x.valueForSearch(values.get(i)));
                                     }
+                                    value = values;
                                 }
                             }
                         }
@@ -368,46 +343,26 @@ private GetResult innerGetLoadFromStoredFields(String type, String id, String[]
             SearchLookup searchLookup = null;
             for (String field : gFields) {
                 Object value = null;
-                if (field.contains("_source.") || field.contains("doc[")) {
+                FieldMappers x = docMapper.mappers().smartName(field);
+                if (x == null) {
+                    if (docMapper.objectMappers().get(field) != null) {
+                        // Only fail if we know it is a object field, missing paths / fields shouldn't fail.
+                        throw new ElasticsearchIllegalArgumentException("field [" + field + "] isn't a leaf field");
+                    }
+                } else if (!x.mapper().fieldType().stored()) {
                     if (searchLookup == null) {
                         searchLookup = new SearchLookup(mapperService, fieldDataService, new String[]{type});
-                        searchLookup.source().setNextSource(source);
                         searchLookup.setNextReader(docIdAndVersion.context);
+                        searchLookup.source().setNextSource(source);
                         searchLookup.setNextDocId(docIdAndVersion.docId);
                     }
-                    SearchScript searchScript = scriptService.search(searchLookup, "mvel", field, null);
-                    searchScript.setNextReader(docIdAndVersion.context);
-                    searchScript.setNextDocId(docIdAndVersion.docId);
-                    try {
-                        value = searchScript.run();
-                    } catch (RuntimeException e) {
-                        if (logger.isTraceEnabled()) {
-                            logger.trace("failed to execute get request script field [{}]", e, field);
-                        }
-                        // ignore
-                    }
-                } else {
-                    FieldMappers x = docMapper.mappers().smartName(field);
-                    if (x == null) {
-                        if (docMapper.objectMappers().get(field) != null) {
-                            // Only fail if we know it is a object field, missing paths / fields shouldn't fail.
-                            throw new ElasticsearchIllegalArgumentException("field [" + field + "] isn't a leaf field");
-                        }
-                    } else if (!x.mapper().fieldType().stored()) {
-                        if (searchLookup == null) {
-                            searchLookup = new SearchLookup(mapperService, fieldDataService, new String[]{type});
-                            searchLookup.setNextReader(docIdAndVersion.context);
-                            searchLookup.source().setNextSource(source);
-                            searchLookup.setNextDocId(docIdAndVersion.docId);
-                        }
 
-                        List<Object> values = searchLookup.source().extractRawValues(field);
-                        if (!values.isEmpty()) {
-                            for (int i = 0; i < values.size(); i++) {
-                                values.set(i, x.mapper().valueForSearch(values.get(i)));
-                            }
-                            value = values;
+                    List<Object> values = searchLookup.source().extractRawValues(field);
+                    if (!values.isEmpty()) {
+                        for (int i = 0; i < values.size(); i++) {
+                            values.set(i, x.mapper().valueForSearch(values.get(i)));
                         }
+                        value = values;
                     }
                 }
 

src/main/java/org/elasticsearch/percolator/PercolateContext.java

@@ -61,6 +61,7 @@
 import org.elasticsearch.search.facet.SearchContextFacets;
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.FetchSubPhase;
+import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext;
 import org.elasticsearch.search.fetch.partial.PartialFieldsContext;
 import org.elasticsearch.search.fetch.script.ScriptFieldsContext;
 import org.elasticsearch.search.fetch.source.FetchSourceContext;
@@ -407,6 +408,16 @@ public void rescore(RescoreSearchContext rescore) {
         throw new UnsupportedOperationException();
     }
 
+    @Override
+    public boolean hasFieldDataFields() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public FieldDataFieldsContext fieldDataFields() {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     public boolean hasScriptFields() {
         throw new UnsupportedOperationException();

src/main/java/org/elasticsearch/search/SearchModule.java

@@ -24,15 +24,14 @@
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.inject.SpawnModules;
 import org.elasticsearch.index.query.functionscore.FunctionScoreModule;
-import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
-import org.elasticsearch.indices.fielddata.breaker.InternalCircuitBreakerService;
 import org.elasticsearch.search.action.SearchServiceTransportAction;
 import org.elasticsearch.search.aggregations.AggregationModule;
 import org.elasticsearch.search.controller.SearchPhaseController;
 import org.elasticsearch.search.dfs.DfsPhase;
 import org.elasticsearch.search.facet.FacetModule;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.explain.ExplainFetchSubPhase;
+import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.matchedqueries.MatchedQueriesFetchSubPhase;
 import org.elasticsearch.search.fetch.partial.PartialFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.script.ScriptFieldsFetchSubPhase;
@@ -62,6 +61,7 @@ protected void configure() {
 
         bind(FetchPhase.class).asEagerSingleton();
         bind(ExplainFetchSubPhase.class).asEagerSingleton();
+        bind(FieldDataFieldsFetchSubPhase.class).asEagerSingleton();
         bind(ScriptFieldsFetchSubPhase.class).asEagerSingleton();
         bind(PartialFieldsFetchSubPhase.class).asEagerSingleton();
         bind(FetchSourceSubPhase.class).asEagerSingleton();

src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java

@@ -98,6 +98,7 @@ public static HighlightBuilder highlight() {
     private long timeoutInMillis = -1;
 
     private List<String> fieldNames;
+    private List<String> fieldDataFields;
     private List<ScriptField> scriptFields;
     private List<PartialField> partialFields;
     private FetchSourceContext fetchSourceContext;
@@ -568,6 +569,17 @@ public SearchSourceBuilder field(String name) {
         return this;
     }
 
+    /**
+     * Adds a field to load from the field data cache and return as part of the search request.
+     */
+    public SearchSourceBuilder fieldDataField(String name) {
+        if (fieldDataFields == null) {
+            fieldDataFields = new ArrayList<String>();
+        }
+        fieldDataFields.add(name);
+        return this;
+    }
+
     /**
      * Adds a script field under the given name with the provided script.
      *
@@ -769,6 +781,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             }
         }
 
+        if (fieldDataFields != null) {
+            builder.startArray("fielddata_fields");
+            for (String fieldName : fieldDataFields) {
+                builder.value(fieldName);
+            }
+            builder.endArray();
+        }
+
         if (partialFields != null) {
             builder.startObject("partial_fields");
             for (PartialField partialField : partialFields) {

src/main/java/org/elasticsearch/search/fetch/FetchPhase.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.fetch.explain.ExplainFetchSubPhase;
+import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.matchedqueries.MatchedQueriesFetchSubPhase;
 import org.elasticsearch.search.fetch.partial.PartialFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.script.ScriptFieldsFetchSubPhase;
@@ -61,9 +62,9 @@ public class FetchPhase implements SearchPhase {
     @Inject
     public FetchPhase(HighlightPhase highlightPhase, ScriptFieldsFetchSubPhase scriptFieldsPhase, PartialFieldsFetchSubPhase partialFieldsPhase,
                       MatchedQueriesFetchSubPhase matchedQueriesPhase, ExplainFetchSubPhase explainPhase, VersionFetchSubPhase versionPhase,
-                      FetchSourceSubPhase fetchSourceSubPhase) {
+                      FetchSourceSubPhase fetchSourceSubPhase, FieldDataFieldsFetchSubPhase fieldDataFieldsFetchSubPhase) {
         this.fetchSubPhases = new FetchSubPhase[]{scriptFieldsPhase, partialFieldsPhase, matchedQueriesPhase, explainPhase, highlightPhase,
-                fetchSourceSubPhase, versionPhase};
+                fetchSourceSubPhase, versionPhase, fieldDataFieldsFetchSubPhase};
     }
 
     @Override

src/main/java/org/elasticsearch/search/fetch/FieldsParseElement.java

@@ -37,27 +37,16 @@ public void parse(XContentParser parser, SearchContext context) throws Exception
             boolean added = false;
             while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                 String name = parser.text();
-                if (name.contains("_source.") || name.contains("doc[")) {
-                    // script field to load from source
-                    SearchScript searchScript = context.scriptService().search(context.lookup(), "mvel", name, null);
-                    context.scriptFields().add(new ScriptFieldsContext.ScriptField(name, searchScript, true));
-                } else {
-                    added = true;
-                    context.fieldNames().add(name);
-                }
+                added = true;
+                context.fieldNames().add(name);
             }
             if (!added) {
                 context.emptyFieldNames();
             }
         } else if (token == XContentParser.Token.VALUE_STRING) {
             String name = parser.text();
-            if (name.contains("_source.") || name.contains("doc[")) {
-                // script field to load from source
-                SearchScript searchScript = context.scriptService().search(context.lookup(), "mvel", name, null);
-                context.scriptFields().add(new ScriptFieldsContext.ScriptField(name, searchScript, true));
-            } else {
-                context.fieldNames().add(name);
-            }
+            context.fieldNames().add(name);
+
         }
     }
 }

src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsContext.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.fetch.fielddata;
+
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**
+ * All the required context to pull a field from the field data cache.
+ */
+public class FieldDataFieldsContext {
+
+    public static class FieldDataField {
+        private final String name;
+
+        public FieldDataField(String name) {
+            this.name = name;
+        }
+
+        public String name() {
+            return name;
+        }
+    }
+
+    private List<FieldDataField> fields = Lists.newArrayList();
+
+    public FieldDataFieldsContext() {
+    }
+
+    public void add(FieldDataField field) {
+        this.fields.add(field);
+    }
+
+    public List<FieldDataField> fields() {
+        return this.fields;
+    }
+}