Skip to content

Commit

Permalink
Add nested and object fields to field capabilities response (#33803)
Browse files Browse the repository at this point in the history 
This commit adds nested and object fields to the field capabilities response.

Closes #33237
  • Loading branch information
@jimczi
jimczi committed Sep 26, 2018
1 parent 96b3417 commit a255880
Show file tree
Hide file tree
Showing 3 changed files with 115 additions and 12 deletions.
73 changes: 71 additions & 2 deletions rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,16 @@ setup:
nested2:
type: float
doc_values: false
level1:
type: nested
properties:
level2:
type: object
properties:
leaf1:
type: text
index: false

- do:
indices.create:
index: test2
Expand All @@ -48,6 +58,15 @@ setup:
nested2:
type: float
doc_values: true
level1:
type: nested
properties:
level2:
type: object
properties:
leaf1:
type: text
index: false
- do:
indices.create:
index: test3
Expand All @@ -64,14 +83,23 @@ setup:
geo:
type: keyword
object:
type: object
type: nested
properties:
nested1 :
type : long
index: false
nested2:
type: keyword
doc_values: false
level1:
type: object
properties:
level2:
type: object
properties:
leaf1:
type: text
index: false

---
"Get simple field caps":
Expand Down Expand Up @@ -112,7 +140,7 @@ setup:
- is_false: fields.geo.keyword.non_searchable_indices
- is_false: fields.geo.keyword.on_aggregatable_indices
---
"Get nested field caps":
"Get leaves field caps":

- do:
field_caps:
Expand Down Expand Up @@ -140,6 +168,47 @@ setup:
- is_false: fields.object\.nested2.keyword.non_aggregatable_indices
- is_false: fields.object\.nested2.keyword.non_searchable_indices
---
"Get object and nested field caps":
- skip:
version: " - 6.99.99"
reason: object and nested fields are returned since 7.0

- do:
field_caps:
index: 'test1,test2,test3'
fields: object*,level1*

- match: {fields.object.object.indices: ["test1", "test2"]}
- match: {fields.object.object.searchable: false}
- match: {fields.object.object.aggregatable: false}
- is_false: fields.object.object.non_aggregatable_indices
- is_false: fields.object.object.non_searchable_indices
- match: {fields.object.nested.indices: ["test3"]}
- match: {fields.object.nested.searchable: false}
- match: {fields.object.nested.aggregatable: false}
- is_false: fields.object.nested.non_aggregatable_indices
- is_false: fields.object.nested.non_searchable_indices
- match: {fields.level1.nested.indices: ["test1", "test2"]}
- match: {fields.level1.nested.searchable: false}
- match: {fields.level1.nested.aggregatable: false}
- is_false: fields.level1.nested.non_aggregatable_indices
- is_false: fields.level1.nested.non_searchable_indices
- match: {fields.level1.object.indices: ["test3"]}
- match: {fields.level1.object.searchable: false}
- match: {fields.level1.object.aggregatable: false}
- is_false: fields.level1.object.non_aggregatable_indices
- is_false: fields.level1.object.non_searchable_indices
- match: {fields.level1\.level2.object.searchable: false}
- match: {fields.level1\.level2.object.aggregatable: false}
- is_false: fields.level1\.level2.object.indices
- is_false: fields.level1\.level2.object.non_aggregatable_indices
- is_false: fields.level1\.level2.object.non_searchable_indices
- match: {fields.level1\.level2\.leaf1.text.searchable: false}
- match: {fields.level1\.level2\.leaf1.text.aggregatable: false}
- is_false: fields.level1\.level2\.leaf1.text.indices
- is_false: fields.level1\.level2\.leaf1.text.non_aggregatable_indices
- is_false: fields.level1\.level2\.leaf1.text..non_searchable_indices
---
"Get prefix field caps":

- do:
Expand Down
21 changes: 21 additions & 0 deletions ...c/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.ObjectMapper;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.threadpool.ThreadPool;
Expand Down Expand Up @@ -86,6 +87,26 @@ protected FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesI
if (indicesService.isMetaDataField(field) || fieldPredicate.test(ft.name())) {
FieldCapabilities fieldCap = new FieldCapabilities(field, ft.typeName(), ft.isSearchable(), ft.isAggregatable());
responseMap.put(field, fieldCap);
} else {
continue;
}
// add nested and object fields
int dotIndex = ft.name().lastIndexOf('.');
while (dotIndex > -1) {
String parentField = ft.name().substring(0, dotIndex);
if (responseMap.containsKey(parentField)) {
// we added this path on another field already
break;
}
// checks if the parent field contains sub-fields
if (mapperService.fullName(parentField) == null) {
// no field type, it must be an object field
ObjectMapper mapper = mapperService.getObjectMapper(parentField);
String type = mapper.nested().isNested() ? "nested" : "object";
FieldCapabilities fieldCap = new FieldCapabilities(parentField, type, false, false);
responseMap.put(parentField, fieldCap);
}
dotIndex = parentField.lastIndexOf('.');
}
}
}
Expand Down
33 changes: 23 additions & 10 deletions server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,12 @@
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.junit.Before;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
Expand Down Expand Up @@ -90,20 +93,26 @@ public void testGetFieldMappings() {
}

public void testFieldCapabilities() {
List<String> allFields = new ArrayList<>(ALL_FLAT_FIELDS);
allFields.addAll(ALL_OBJECT_FIELDS);
FieldCapabilitiesResponse index1 = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("index1")).actionGet();
assertFieldCaps(index1, ALL_FLAT_FIELDS);
assertFieldCaps(index1, allFields);
FieldCapabilitiesResponse filtered = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("filtered")).actionGet();
assertFieldCaps(filtered, FILTERED_FLAT_FIELDS);
List<String> filteredFields = new ArrayList<>(FILTERED_FLAT_FIELDS);
filteredFields.addAll(ALL_OBJECT_FIELDS);
assertFieldCaps(filtered, filteredFields);
//double check that submitting the filtered mappings to an unfiltered index leads to the same field_caps output
//as the one coming from a filtered index with same mappings
GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("filtered").get();
ImmutableOpenMap<String, MappingMetaData> filteredMapping = getMappingsResponse.getMappings().get("filtered");
assertAcked(client().admin().indices().prepareCreate("test").addMapping("_doc", filteredMapping.get("_doc").getSourceAsMap()));
FieldCapabilitiesResponse test = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("test")).actionGet();
assertFieldCaps(test, FILTERED_FLAT_FIELDS);
// properties.value is an object field in the new mapping
filteredFields.add("properties.value");
assertFieldCaps(test, filteredFields);
}

private static void assertFieldCaps(FieldCapabilitiesResponse fieldCapabilitiesResponse, String[] expectedFields) {
private static void assertFieldCaps(FieldCapabilitiesResponse fieldCapabilitiesResponse, Collection<String> expectedFields) {
Map<String, Map<String, FieldCapabilities>> responseMap = fieldCapabilitiesResponse.get();
Set<String> builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields();
for (String field : builtInMetaDataFields) {
Expand All @@ -118,7 +127,7 @@ private static void assertFieldCaps(FieldCapabilitiesResponse fieldCapabilitiesR
}

private static void assertFieldMappings(Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetaData>> mappings,
String[] expectedFields) {
Collection<String> expectedFields) {
assertEquals(1, mappings.size());
Map<String, GetFieldMappingsResponse.FieldMappingMetaData> fields = new HashMap<>(mappings.get("_doc"));
Set<String> builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields();
Expand Down Expand Up @@ -245,14 +254,18 @@ public Function<String, Predicate<String>> getFieldFilter() {
}
}

private static final String[] ALL_FLAT_FIELDS = new String[]{
private static final Collection<String> ALL_FLAT_FIELDS = Arrays.asList(
"name.first", "name.last_visible", "birth", "age_visible", "address.street", "address.location", "address.area_visible",
"properties.key_visible", "properties.key_visible.keyword", "properties.value", "properties.value.keyword_visible"
};
);

private static final String[] FILTERED_FLAT_FIELDS = new String[]{
"name.last_visible", "age_visible", "address.area_visible", "properties.key_visible", "properties.value.keyword_visible"
};
private static final Collection<String> ALL_OBJECT_FIELDS = Arrays.asList(
"name", "address", "properties"
);

private static final Collection<String> FILTERED_FLAT_FIELDS = Arrays.asList(
"name.last_visible", "age_visible", "address.area_visible", "properties.key_visible", "properties.value.keyword_visible"
);

private static final String TEST_ITEM = "{\n" +
" \"_doc\": {\n" +
Expand Down

0 comments on commit a255880

Please sign in to comment.