Add ability to retrieve document metadata via fields option (#78939)

Currently we exclude metadata fields from being looked up using the fields option in search. However, as issue like #75836 show, they can still be retrieved e.g. via aliases and then fetching their values causes errors. With this change, we enable retrieval of metadata fields (like `_id`, `_ignored` etc.) using the fields option when the field is explicitely requested. We still continue to exclude any metadata field from matching wildcard patterns, but they should be retrievable via an exact name or if there is an alias definition with a path to a metadata field. This change adds support for the `_id`, `_routing`, `_ignored`, `_index` and `_version` field in particular. Backport of #78828, #78981 and #79042
elastic · Oct 14, 2021 · e7c6ed8 · e7c6ed8
1 parent 5d391f0
commit e7c6ed8
Show file tree

Hide file tree

Showing 15 changed files with 454 additions and 36 deletions.
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml
@@ -1007,3 +1007,60 @@ error for flattened includes whole path:
           fields:
             - field: flattened.bar
               format: "yyyy/MM/dd"
+
+---
+test fetching metadata fields:
+  - skip:
+      version: ' - 7.15.99'
+      reason:  'fetching metadata via fields was introduced in 7.16'
+
+  - do:
+      indices.create:
+        index:  test
+        body:
+          settings:
+            index.number_of_shards: 1
+          mappings:
+            properties:
+              field:
+                type: keyword
+              idAlias:
+                type: alias
+                path: _id
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        refresh: true
+        body:
+          field: foo
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "*" ]
+
+  - length: { hits.hits.0.fields : 2 }
+  - match: { hits.hits.0.fields.field.0: "foo" }
+  - match: { hits.hits.0.fields.idAlias.0: "1" }
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "_*" ]
+
+  - is_false: hits.hits.0.fields
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "_id", "_index", "_version" ]
+
+  - length: { hits.hits.0.fields : 3 }
+  - match: { hits.hits.0.fields._id.0: "1" }
+  - match: { hits.hits.0.fields._index.0: "test" }
+  - match: { hits.hits.0.fields._version.0: 1 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
@@ -109,7 +109,7 @@ public boolean isSearchable() {
 
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-            throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "].");
+            return new StoredValueFetcher(context.lookup(), NAME);
         }
 
         @Override

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java
@@ -68,7 +68,7 @@ public Query existsQuery(SearchExecutionContext context) {
 
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-            throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "].");
+            return new StoredValueFetcher(context.lookup(), NAME);
         }
     }
 

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java
@@ -16,8 +16,11 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.lookup.SourceLookup;
 
+import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.function.Supplier;
 
 public class IndexFieldMapper extends MetadataFieldMapper {
@@ -65,7 +68,15 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S
 
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-            throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "].");
+            return new ValueFetcher() {
+
+                private final List<Object> indexName = Collections.singletonList(context.getFullyQualifiedIndex().getName());
+
+                @Override
+                public List<Object> fetchValues(SourceLookup lookup, List<Object> ignoredValues) throws IOException {
+                    return indexName;
+                }
+            };
         }
     }
 

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java
@@ -84,7 +84,7 @@ public String typeName() {
 
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-            throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "].");
+            return new StoredValueFetcher(context.lookup(), NAME);
         }
     }
 

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StoredValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/StoredValueFetcher.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.search.lookup.LeafSearchLookup;
+import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.lookup.SourceLookup;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Value fetcher that loads from stored values.
+ */
+public final class StoredValueFetcher implements ValueFetcher {
+
+    private final SearchLookup lookup;
+    private LeafSearchLookup leafSearchLookup;
+    private final String fieldname;
+
+    public StoredValueFetcher(SearchLookup lookup, String fieldname) {
+        this.lookup = lookup;
+        this.fieldname = fieldname;
+    }
+
+    @Override
+    public void setNextReader(LeafReaderContext context) {
+        this.leafSearchLookup = lookup.getLeafSearchLookup(context);
+    }
+
+    @Override
+    public List<Object> fetchValues(SourceLookup lookup, List<Object> ignoredValues) throws IOException {
+        leafSearchLookup.setDocument(lookup.docId());
+        return leafSearchLookup.fields().get(fieldname).getValues();
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java
@@ -11,10 +11,15 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
+import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
 import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.util.Collections;
+import java.util.function.Supplier;
 
 /** Mapper for the _version field. */
 public class VersionFieldMapper extends MetadataFieldMapper {
@@ -46,7 +51,13 @@ public Query termQuery(Object value, SearchExecutionContext context) {
 
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-            throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "].");
+            return new DocValueFetcher(docValueFormat(format, null), context.getForField(this));
+        }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
+            failIfNoDocValues();
+            return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG);
         }
     }
 

diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java
@@ -10,12 +10,20 @@
 
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.IndexSearcher;
+import org.elasticsearch.core.List;
+import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
+import java.util.Collections;
 
 import static org.elasticsearch.index.mapper.IdFieldMapper.ID_FIELD_DATA_DEPRECATION_MESSAGE;
 import static org.hamcrest.Matchers.containsString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class IdFieldMapperTests extends MapperServiceTestCase {
 
@@ -60,4 +68,25 @@ public void testEnableFieldData() throws IOException {
         assertWarnings(ID_FIELD_DATA_DEPRECATION_MESSAGE);
         assertTrue(ft.isAggregatable());
     }
+
+    public void testFetchIdFieldValue() throws IOException {
+        MapperService mapperService = createMapperService(
+            fieldMapping(b -> b.field("type", "keyword"))
+        );
+        String id = randomAlphaOfLength(12);
+        withLuceneIndex(mapperService, iw -> {
+            iw.addDocument(mapperService.documentMapper().parse(source(id, b -> b.field("field", "value"), null)).rootDoc());
+        }, iw -> {
+            SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup());
+            SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class);
+            when(searchExecutionContext.lookup()).thenReturn(lookup);
+            IdFieldMapper.IdFieldType ft = (IdFieldMapper.IdFieldType) mapperService.fieldType("_id");
+            ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null);
+            IndexSearcher searcher = newSearcher(iw);
+            LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
+            lookup.source().setSegmentAndDocument(context, 0);
+            valueFetcher.setNextReader(context);
+            assertEquals(Collections.singletonList(id), valueFetcher.fetchValues(lookup.source(), List.of()));
+        });
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.IndexSearcher;
+import org.elasticsearch.core.List;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.search.lookup.SearchLookup;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class IgnoredFieldMapperTests extends MapperServiceTestCase {
+
+    public void testIncludeInObjectNotAllowed() throws Exception {
+        DocumentMapper docMapper = createDocumentMapper(mapping(b -> {}));
+
+        Exception e = expectThrows(MapperParsingException.class,
+            () -> docMapper.parse(source(b -> b.field("_ignored", 1))));
+
+        assertThat(e.getCause().getMessage(),
+            containsString("Field [_ignored] is a metadata field and cannot be added inside a document"));
+    }
+
+    public void testDefaults() throws IOException {
+        DocumentMapper mapper = createDocumentMapper(
+            mapping(b -> b.startObject("field").field("type", "keyword").field("ignore_above", 3).endObject())
+        );
+        ParsedDocument document = mapper.parse(source(b -> b.field("field", "value")));
+        IndexableField[] fields = document.rootDoc().getFields(IgnoredFieldMapper.NAME);
+        assertEquals(1, fields.length);
+        assertEquals(IndexOptions.DOCS, fields[0].fieldType().indexOptions());
+        assertTrue(fields[0].fieldType().stored());
+    }
+
+    public void testFetchIgnoredFieldValue() throws IOException {
+        MapperService mapperService = createMapperService(
+            fieldMapping(b -> b.field("type", "keyword").field("ignore_above", 3))
+        );
+        withLuceneIndex(mapperService, iw -> {
+            iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field("field", "value"))).rootDoc());
+        }, iw -> {
+            SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup());
+            SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class);
+            when(searchExecutionContext.lookup()).thenReturn(lookup);
+            IgnoredFieldMapper.IgnoredFieldType ft = (IgnoredFieldMapper.IgnoredFieldType) mapperService.fieldType("_ignored");
+            ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null);
+            IndexSearcher searcher = newSearcher(iw);
+            LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
+            lookup.source().setSegmentAndDocument(context, 0);
+            valueFetcher.setNextReader(context);
+            assertEquals(List.of("field"), valueFetcher.fetchValues(lookup.source(), List.of()));
+        });
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java
@@ -8,9 +8,22 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.IndexSearcher;
+import org.elasticsearch.core.List;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.search.lookup.SearchLookup;
+
+import java.io.IOException;
+import java.util.Collections;
+
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.when;
 
 public class IndexFieldMapperTests extends MapperServiceTestCase {
 
@@ -27,4 +40,29 @@ public void testIndexNotConfigurable() {
         assertThat(e.getMessage(), containsString("_index is not configurable"));
     }
 
+    public void testFetchFieldValue() throws IOException {
+        MapperService mapperService = createMapperService(
+            fieldMapping(b -> b.field("type", "keyword"))
+        );
+        String index = randomAlphaOfLength(12);
+        withLuceneIndex(mapperService, iw -> {
+            SourceToParse source = source(index, "id", b -> b.field("field", "value"), "", org.elasticsearch.core.Map.of());
+            iw.addDocument(mapperService.documentMapper().parse(source).rootDoc());
+        }, iw -> {
+            IndexFieldMapper.IndexFieldType ft = (IndexFieldMapper.IndexFieldType) mapperService.fieldType("_index");
+            SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup());
+            SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService);
+            when(searchExecutionContext.getForField(ft)).thenReturn(
+                ft.fielddataBuilder(index, () -> lookup).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService())
+            );
+            when(searchExecutionContext.getFullyQualifiedIndex()).thenReturn(new Index(index, "indexUUid"));
+            ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null);
+            IndexSearcher searcher = newSearcher(iw);
+            LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
+            lookup.source().setSegmentAndDocument(context, 0);
+            valueFetcher.setNextReader(context);
+            assertEquals(List.of(index), valueFetcher.fetchValues(lookup.source(), Collections.emptyList()));
+        });
+    }
+
 }