Extra testing and some cleanups for filtering on field caps (#85068)

* adds a test for mixed cluster requests
* fixes a bad stream version check (above test will fail if this isn't included)
* replaces private FieldCapsFilter interface with Predicate
* renames 'allowedTypes' to 'types' to maintain consistency with external API
* adds javadoc to ResponseRewriter
* removes isRuntimeField from FieldTypeLookup

Relates to #83636

docs/reference/search/field-caps.asciidoc

@@ -78,13 +78,28 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailab
   Defaults to `false`.
 
 `filters`::
-(Optional, string) Comma-separated list of filters to apply to the response.  The
-following filters are supported: +metadata,-metadata,-parent,-nested,-multifield
+(Optional, string) Comma-separated list of filters to apply to the response.
++
+.Valid values for `filters`
+[%collapsible%open]
+====
+`+metadata`::
+Only include metadata fields
+`-metadata`::
+Exclude metadata fields
+`-parent`::
+Exclude parent fields
+`-nested`::
+Exclude nested fields
+`-multifield`::
+Exclude multifields
+====
 
 `types`::
 (Optional, string) Comma-separated list of field types to include. Any fields that
 do not match one of these types will be excluded from the results.  Defaults to empty,
-meaning that all field types are returned.
+meaning that all field types are returned.  See <<field-caps-field-types,here>> for
+more information about field types in field capabilities requests and responses.
 
 [[search-field-caps-api-request-body]]
 ==== {api-request-body-title}
@@ -103,7 +118,7 @@ same name in the index mappings.
 [[search-field-caps-api-response-body]]
 ==== {api-response-body-title}
 
-
+[[field-caps-field-types]]
 The types used in the response describe _families_ of field types.
 Normally a type family is the same as the field type declared in the mapping,
 but to simplify matters certain field types that behave identically are

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java

@@ -8,16 +8,21 @@
 
 package org.elasticsearch.upgrades;
 
+import org.apache.http.HttpHost;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.xcontent.json.JsonXContent;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
@@ -27,6 +32,9 @@
  * reduce the transport message size between nodes and clusters, and the memory usage to hold these internal responses.
  * As the optimization is applied for only field-caps requests without index-filter and nodes on 8.2 or later,
  * these BWC tests verify these combinations of field-caps requests: (old|new|mixed indices) and (with|without index filter)
+ *
+ * In 8.2 we also added the ability to filter fields by type and metadata, with some post-hoc filtering applied on
+ * the co-ordinating node if older nodes were included in the system
  */
 public class FieldCapsIT extends AbstractRollingTestCase {
     private static boolean indicesCreated = false;
@@ -42,6 +50,7 @@ public void setupIndices() throws Exception {
                "red_field": { "type": "keyword" },
                "yellow_field": { "type": "integer" },
                "blue_field": { "type": "keyword" },
+               "multi_field" : { "type" : "ip", "fields" : { "keyword" : { "type" : "keyword" } } },
                "timestamp": {"type": "date"}
              }
             """;
@@ -50,6 +59,7 @@ public void setupIndices() throws Exception {
                "green_field": { "type": "keyword" },
                "yellow_field": { "type": "long" },
                "blue_field": { "type": "keyword" },
+               "multi_field" : { "type" : "ip", "fields" : { "keyword" : { "type" : "keyword" } } },
                "timestamp": {"type": "date"}
              }
             """;
@@ -88,7 +98,7 @@ public void setupIndices() throws Exception {
 
     public void testOldIndicesOnly() throws Exception {
         {
-            FieldCapabilitiesResponse resp = fieldCaps(List.of("old_red_*"), List.of("*"), null);
+            FieldCapabilitiesResponse resp = fieldCaps(List.of("old_red_*"), List.of("*"), null, null, null);
             assertThat(resp.getIndices(), equalTo(new String[] { "old_red_1", "old_red_2", "old_red_empty" }));
             assertThat(resp.getField("red_field").keySet(), contains("keyword"));
             assertTrue(resp.getField("red_field").get("keyword").isSearchable());
@@ -98,7 +108,7 @@ public void testOldIndicesOnly() throws Exception {
             assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
         }
         {
-            FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*"), List.of("*"), null);
+            FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*"), List.of("*"), null, null, null);
             assertThat(
                 resp.getIndices(),
                 equalTo(new String[] { "old_green_1", "old_green_2", "old_green_empty", "old_red_1", "old_red_2", "old_red_empty" })
@@ -116,7 +126,7 @@ public void testOldIndicesOnly() throws Exception {
     public void testOldIndicesWithIndexFilter() throws Exception {
         final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12");
         {
-            FieldCapabilitiesResponse resp = fieldCaps(List.of("old_red_*"), List.of("*"), indexFilter);
+            FieldCapabilitiesResponse resp = fieldCaps(List.of("old_red_*"), List.of("*"), indexFilter, null, null);
             assertThat(resp.getIndices(), equalTo(new String[] { "old_red_1", "old_red_2" }));
             assertThat(resp.getField("red_field").keySet(), contains("keyword"));
             assertTrue(resp.getField("red_field").get("keyword").isSearchable());
@@ -126,7 +136,7 @@ public void testOldIndicesWithIndexFilter() throws Exception {
             assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
         }
         {
-            FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*"), List.of("*"), indexFilter);
+            FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*"), List.of("*"), indexFilter, null, null);
             assertThat(resp.getIndices(), equalTo(new String[] { "old_green_1", "old_green_2", "old_red_1", "old_red_2" }));
             assertThat(resp.getField("red_field").keySet(), contains("keyword"));
             assertTrue(resp.getField("red_field").get("keyword").isSearchable());
@@ -141,7 +151,7 @@ public void testOldIndicesWithIndexFilter() throws Exception {
     public void testNewIndicesOnly() throws Exception {
         assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
         {
-            FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), null);
+            FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), null, null, null);
             assertThat(resp.getIndices(), equalTo(new String[] { "new_red_1", "new_red_2", "new_red_empty" }));
             assertThat(resp.getField("red_field").keySet(), contains("keyword"));
             assertTrue(resp.getField("red_field").get("keyword").isSearchable());
@@ -151,7 +161,7 @@ public void testNewIndicesOnly() throws Exception {
             assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
         }
         {
-            FieldCapabilitiesResponse resp = fieldCaps(List.of("new_*"), List.of("*"), null);
+            FieldCapabilitiesResponse resp = fieldCaps(List.of("new_*"), List.of("*"), null, null, null);
             assertThat(
                 resp.getIndices(),
                 equalTo(new String[] { "new_green_1", "new_green_2", "new_green_empty", "new_red_1", "new_red_2", "new_red_empty" })
@@ -170,7 +180,7 @@ public void testNewIndicesOnlyWithIndexFilter() throws Exception {
         assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
         final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12");
         {
-            FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), indexFilter);
+            FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), indexFilter, null, null);
             assertThat(resp.getIndices(), equalTo(new String[] { "new_red_1", "new_red_2" }));
             assertThat(resp.getField("red_field").keySet(), contains("keyword"));
             assertTrue(resp.getField("red_field").get("keyword").isSearchable());
@@ -180,7 +190,7 @@ public void testNewIndicesOnlyWithIndexFilter() throws Exception {
             assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
         }
         {
-            FieldCapabilitiesResponse resp = fieldCaps(List.of("new_*"), List.of("*"), indexFilter);
+            FieldCapabilitiesResponse resp = fieldCaps(List.of("new_*"), List.of("*"), indexFilter, null, null);
             assertThat(resp.getIndices(), equalTo(new String[] { "new_green_1", "new_green_2", "new_red_1", "new_red_2" }));
             assertThat(resp.getField("red_field").keySet(), contains("keyword"));
             assertTrue(resp.getField("red_field").get("keyword").isSearchable());
@@ -194,7 +204,7 @@ public void testNewIndicesOnlyWithIndexFilter() throws Exception {
 
     public void testAllIndices() throws Exception {
         assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
-        FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), null);
+        FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), null, null, null);
         assertThat(
             resp.getIndices(),
             equalTo(
@@ -227,7 +237,7 @@ public void testAllIndices() throws Exception {
     public void testAllIndicesWithIndexFilter() throws Exception {
         assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
         final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12");
-        FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), indexFilter);
+        FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), indexFilter, null, null);
         assertThat(
             resp.getIndices(),
             equalTo(
@@ -252,4 +262,53 @@ public void testAllIndicesWithIndexFilter() throws Exception {
         assertThat(resp.getField("blue_field").keySet(), contains("keyword"));
         assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
     }
+
+    @SuppressWarnings("unchecked")
+    // Returns a client connected to one of the upgraded nodes.
+    private RestClient getUpgradedNodeClient() throws IOException {
+        for (HttpHost host : getClusterHosts()) {
+            RestClient client = RestClient.builder(host).build();
+            Request nodesRequest = new Request("GET", "_nodes/_local/_none");
+            Map<String, ?> nodeMap = (Map<String, ?>) entityAsMap(client.performRequest(nodesRequest)).get("nodes");
+            Map<String, ?> nameMap = (Map<String, ?>) nodeMap.values().iterator().next();
+            String version = (String) nameMap.get("version");
+            if (version.equals(Version.CURRENT.toString())) {
+                return client;
+            }
+            client.close();
+        }
+        throw new IllegalStateException("Couldn't find node on version " + Version.CURRENT);
+    }
+
+    // Test field type filtering on mixed cluster
+    // We need to use a client that is connected to one of the upgraded nodes,
+    // because we are testing that the upgraded node will correctly apply filtering
+    // to responses from older nodes that don't understand the filter parameters
+    public void testAllIndicesWithFieldTypeFilter() throws Exception {
+        assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
+        RestClient restClient = getUpgradedNodeClient();
+        FieldCapabilitiesResponse resp = fieldCaps(restClient, List.of("old_*", "new_*"), List.of("*"), null, "keyword", null);
+        assertThat(resp.getField("red_field").keySet(), contains("keyword"));
+        assertNull(resp.getField("yellow_field"));
+        restClient.close();
+    }
+
+    // Test multifield exclusion on mixed cluster
+    // We need to use a client that is connected to one of the upgraded nodes,
+    // because we are testing that the upgraded node will correctly apply filtering
+    // to responses from older nodes that don't understand the filter parameters
+    public void testAllIndicesWithExclusionFilter() throws Exception {
+        assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
+        RestClient client = getUpgradedNodeClient();
+        {
+            FieldCapabilitiesResponse resp = fieldCaps(client, List.of("old_*", "new_*"), List.of("*"), null, null, null);
+            assertThat(resp.getField("multi_field.keyword").keySet(), contains("keyword"));
+        }
+        {
+            FieldCapabilitiesResponse resp = fieldCaps(client, List.of("old_*", "new_*"), List.of("*"), null, null, "-multifield");
+            assertThat(resp.getField("multi_field").keySet(), contains("ip"));
+            assertNull(resp.getField("multi_field.keyword"));
+        }
+        client.close();
+    }
 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/50_fieldtype_filter.yml

@@ -186,6 +186,21 @@ setup:
   - is_false: fields.text\\.keyword
   - is_true: fields.misc\\.keyword
 
+---
+"Exclude multifields but include runtime fields":
+  - do:
+      field_caps:
+        index: 'test1,test2,test3'
+        fields: '*'
+        filters: '-multifield'
+        body:
+          runtime_mappings:
+            text.keyword:
+              type: keyword
+
+  - is_true: fields.text\\.keyword
+  - is_true: fields.misc\\.keyword
+
 ---
 "Field type filters":
   - do:

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java

@@ -109,11 +109,11 @@ static Map<String, IndexFieldCapabilities> retrieveFieldCaps(
 
         boolean includeParentObjects = checkIncludeParents(filters);
 
-        FieldCapsFilter filter = buildFilter(indexFieldfilter, filters, types);
+        Predicate<MappedFieldType> filter = buildFilter(indexFieldfilter, filters, types, context);
         Map<String, IndexFieldCapabilities> responseMap = new HashMap<>();
         for (String field : fieldNames) {
             MappedFieldType ft = context.getFieldType(field);
-            if (filter.matches(ft, context)) {
+            if (filter.test(ft)) {
                 IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(
                     field,
                     ft.familyTypeName(),
@@ -168,9 +168,6 @@ private static boolean checkIncludeParents(String[] filters) {
             if ("-parent".equals(filter)) {
                 return false;
             }
-            if ("parent".equals(filter)) {
-                return true;
-            }
         }
         return true;
     }
@@ -190,30 +187,27 @@ private boolean canMatchShard(
         return SearchService.queryStillMatchesAfterRewrite(searchRequest, searchExecutionContext);
     }
 
-    private interface FieldCapsFilter {
-        boolean matches(MappedFieldType fieldType, SearchExecutionContext context);
-
-        default FieldCapsFilter and(FieldCapsFilter other) {
-            return (ft, context) -> matches(ft, context) && other.matches(ft, context);
-        }
-    }
-
-    private static FieldCapsFilter buildFilter(Predicate<String> fieldFilter, String[] filters, String[] fieldTypes) {
+    private static Predicate<MappedFieldType> buildFilter(
+        Predicate<String> fieldFilter,
+        String[] filters,
+        String[] fieldTypes,
+        SearchExecutionContext context
+    ) {
         // security filters don't exclude metadata fields
-        FieldCapsFilter fcf = (ft, c) -> fieldFilter.test(ft.name()) || c.isMetadataField(ft.name());
+        Predicate<MappedFieldType> fcf = ft -> fieldFilter.test(ft.name()) || context.isMetadataField(ft.name());
         if (fieldTypes.length > 0) {
             Set<String> acceptedTypes = Set.of(fieldTypes);
-            fcf = fcf.and((ft, c) -> acceptedTypes.contains(ft.familyTypeName()));
+            fcf = fcf.and(ft -> acceptedTypes.contains(ft.familyTypeName()));
         }
         for (String filter : filters) {
             if ("parent".equals(filter) || "-parent".equals(filter)) {
                 continue;
             }
-            FieldCapsFilter next = switch (filter) {
-                case "+metadata" -> (ft, c) -> c.isMetadataField(ft.name());
-                case "-metadata" -> (ft, c) -> c.isMetadataField(ft.name()) == false;
-                case "-nested" -> (ft, c) -> c.nestedLookup().getNestedParent(ft.name()) == null;
-                case "-multifield" -> (ft, c) -> c.isMultiField(ft.name()) == false;
+            Predicate<MappedFieldType> next = switch (filter) {
+                case "+metadata" -> ft -> context.isMetadataField(ft.name());
+                case "-metadata" -> ft -> context.isMetadataField(ft.name()) == false;
+                case "-nested" -> ft -> context.nestedLookup().getNestedParent(ft.name()) == null;
+                case "-multifield" -> ft -> context.isMultiField(ft.name()) == false;
                 default -> throw new IllegalArgumentException("Unknown field caps filter [" + filter + "]");
             };
             fcf = fcf.and(next);

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java

@@ -37,7 +37,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
     private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS;
     private String[] fields = Strings.EMPTY_ARRAY;
     private String[] filters = Strings.EMPTY_ARRAY;
-    private String[] allowedTypes = Strings.EMPTY_ARRAY;
+    private String[] types = Strings.EMPTY_ARRAY;
     private boolean includeUnmapped = false;
     // pkg private API mainly for cross cluster search to signal that we do multiple reductions ie. the results should not be merged
     private boolean mergeResults = true;
@@ -57,7 +57,7 @@ public FieldCapabilitiesRequest(StreamInput in) throws IOException {
         runtimeFields = in.readMap();
         if (in.getVersion().onOrAfter(Version.V_8_2_0)) {
             filters = in.readStringArray();
-            allowedTypes = in.readStringArray();
+            types = in.readStringArray();
         }
     }
 
@@ -95,7 +95,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeGenericMap(runtimeFields);
         if (out.getVersion().onOrAfter(Version.V_8_2_0)) {
             out.writeStringArray(filters);
-            out.writeStringArray(allowedTypes);
+            out.writeStringArray(types);
         }
     }
 
@@ -137,13 +137,13 @@ public String[] filters() {
         return filters;
     }
 
-    public FieldCapabilitiesRequest allowedTypes(String... types) {
-        this.allowedTypes = types;
+    public FieldCapabilitiesRequest types(String... types) {
+        this.types = types;
         return this;
     }
 
-    public String[] allowedTypes() {
-        return allowedTypes;
+    public String[] types() {
+        return types;
     }
 
     /**
@@ -243,7 +243,7 @@ public boolean equals(Object o) {
             && Objects.equals(indexFilter, that.indexFilter)
             && Objects.equals(nowInMillis, that.nowInMillis)
             && Arrays.equals(filters, that.filters)
-            && Arrays.equals(allowedTypes, that.allowedTypes)
+            && Arrays.equals(types, that.types)
             && Objects.equals(runtimeFields, that.runtimeFields);
     }
 
@@ -253,7 +253,7 @@ public int hashCode() {
         result = 31 * result + Arrays.hashCode(indices);
         result = 31 * result + Arrays.hashCode(fields);
         result = 31 * result + Arrays.hashCode(filters);
-        result = 31 * result + Arrays.hashCode(allowedTypes);
+        result = 31 * result + Arrays.hashCode(types);
         return result;
     }
 
@@ -266,7 +266,7 @@ public String getDescription() {
         stringBuilder.append("], filters[");
         stringBuilder.append(Strings.collectionToDelimitedString(Arrays.asList(filters), ","));
         stringBuilder.append("], types[");
-        stringBuilder.append(Strings.collectionToDelimitedString(Arrays.asList(allowedTypes), ","));
+        stringBuilder.append(Strings.collectionToDelimitedString(Arrays.asList(types), ","));
         stringBuilder.append("]");
         return stringBuilder.toString();
     }