Add filtering to fieldcaps endpoint (#83636)

Many consumers of the field caps API need to do some post-processing of the
results before they can use them; for instance, Kibana would like to exclude
multifields from certain field selections, or would like to display only geo_point
fields in Maps. ML and QL consumers exclude nested fields in certain
circumstances. This post-processing is possible at the moment, but can be
hacky; and in all cases it involves sending the whole (possibly very large) field
caps response over the wire and then whittling it down in the client. It is also not
guaranteed to be accurate - runtime fields may be incorrectly classified as multifields,
for example.

This commit pushes filtering into elasticsearch itself, reducing the amount of data
that needs to be transported and ensuring better accuracy. The field caps API gets
two new parameters:

* filters - a comma-delimited list that may contain any combination of: `+metadata`,
  `-metadata`, `-nested`, `-parent`, `-multifield`
* types - a comma-delimited list of field types; only fields that have a type in this set
  will be returned

The API will make best-effort attempts to apply the filters post-hoc to responses from
older nodes, so this should still work in a mixed-cluster or cross-cluster situation.

Fixes #82966, #72174

docs/changelog/83636.yaml

@@ -0,0 +1,6 @@
+pr: 83636
+summary: Add filtering to fieldcaps endpoint
+area: Search
+type: enhancement
+issues:
+ - 82966

docs/reference/search/field-caps.asciidoc

@@ -77,6 +77,15 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailab
   (Optional, Boolean) If `true`, unmapped fields are included in the response.
   Defaults to `false`.
 
+`filters`::
+(Optional, string) Comma-separated list of filters to apply to the response.  The
+following filters are supported: +metadata,-metadata,-parent,-nested,-multifield
+
+`types`::
+(Optional, string) Comma-separated list of field types to include. Any fields that
+do not match one of these types will be excluded from the results.  Defaults to empty,
+meaning that all field types are returned.
+
 [[search-field-caps-api-request-body]]
 ==== {api-request-body-title}
 

rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json

@@ -63,6 +63,14 @@
         "type":"boolean",
         "default":false,
         "description":"Indicates whether unmapped fields should be included in the response."
+      },
+      "filters": {
+        "type":"list",
+        "description":"An optional set of filters: can include +metadata,-metadata,-nested,-multifield,-parent"
+      },
+      "types": {
+        "type": "list",
+        "description":"Only return results for fields that have one of the types in the list"
       }
     },
     "body":{

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/50_fieldtype_filter.yml

@@ -0,0 +1,213 @@
+---
+setup:
+  - skip:
+      version: "- 8.1.99"
+      reason: Field type filters were added in 8.2
+  - do:
+      indices.create:
+        index: test1
+        body:
+          mappings:
+            properties:
+              text:
+                type:     text
+                fields:
+                  keyword:
+                    type: keyword
+              keyword:
+                type:     keyword
+              number:
+                type:     double
+              geo:
+                type:     geo_point
+              misc:
+                type:     text
+              object:
+                type: object
+                properties:
+                  nested1 :
+                    type : text
+                    index: false
+                  nested2:
+                    type: float
+                    doc_values: false
+              level1:
+                type: nested
+                properties:
+                  level2:
+                    type: object
+                    properties:
+                      leaf1:
+                        type: text
+                        index: false
+            runtime:
+              misc.keyword:
+                type: keyword
+
+  - do:
+      indices.create:
+        index: test2
+        body:
+          mappings:
+            properties:
+              text:
+                type:     text
+              keyword:
+                type:     keyword
+              number:
+                type:     double
+              date:
+                type:     date
+              geo:
+                type:     geo_point
+              object:
+                type: object
+                properties:
+                  nested1 :
+                    type : text
+                    index: true
+                  nested2:
+                    type: float
+                    doc_values: true
+              level1:
+                type: nested
+                properties:
+                  level2:
+                    type: object
+                    properties:
+                      leaf1:
+                        type: text
+                        index: false
+  - do:
+      indices.create:
+        index: test3
+        body:
+          mappings:
+            properties:
+              text:
+                type:     text
+              keyword:
+                type:     keyword
+              number:
+                type:     long
+              date:
+                type:     date
+              non_indexed_date:
+                type:     date
+                index:    false
+              non_indexed_keyword:
+                type:     keyword
+                index:    false
+              non_indexed_boolean:
+                type:     boolean
+                index:    false
+              non_indexed_ip:
+                type:     ip
+                index:    false
+              non_indexed_geo_point:
+                type:     geo_point
+                index:    false
+              geo:
+                type:     keyword
+              object:
+                type: nested
+                properties:
+                  nested1 :
+                    type : long
+                    index: false
+                  nested2:
+                    type: keyword
+                    doc_values: false
+---
+"No filters includes all the following fields":
+  - do:
+      field_caps:
+        index: 'test1,test2,test3'
+        fields: '*'
+
+  - is_true: fields.object
+  - is_true: fields.text
+  - is_true: fields.text\\.keyword
+  - is_true: fields._seq_no
+  - is_true: fields.level1\\.level2\\.leaf1
+  - is_true: fields.level1
+
+---
+"Exclude parent objects":
+  - do:
+      field_caps:
+        index: 'test1,test2,test3'
+        fields: '*'
+        filters: '-parent'
+
+  - is_true: fields.object\\.nested1
+  - is_false: fields.object
+
+---
+"Exclude metadata fields":
+  - do:
+      field_caps:
+        index: 'test1,test2,test3'
+        fields: '*'
+        filters: '-metadata'
+
+  - is_false: fields._seq_no
+
+---
+"Exclude non-metadata fields":
+  - do:
+      field_caps:
+        index: 'test1,test2,test3'
+        fields: '*'
+        filters: '+metadata'
+
+  - is_true: fields._seq_no
+  - is_false: fields.text
+
+---
+"Exclude nested fields":
+  - do:
+      field_caps:
+        index: 'test1,test2,test3'
+        fields: '*'
+        filters: '-nested'
+
+  - is_false: fields.level1
+  - is_false: fields.level1\\.level2\\.leaf1
+
+---
+"Exclude multifields":
+  - do:
+      field_caps:
+        index: 'test1,test2,test3'
+        fields: '*'
+        filters: '-multifield'
+
+  - is_false: fields.text\\.keyword
+  - is_true: fields.misc\\.keyword
+
+---
+"Field type filters":
+  - do:
+      field_caps:
+        index: 'test1,test2,test3'
+        fields: '*'
+        types: 'text,keyword,long'
+
+  - is_false: fields.date
+  - is_false: fields.non_indexed_boolean
+  - is_true: fields.non_indexed_keyword
+  - is_true: fields.misc
+
+---
+"Field type filters with field name restrictions":
+  - do:
+      field_caps:
+        index: 'test1,test2,test3'
+        fields: 'non_*,text'
+        types: 'text,keyword,long'
+
+  - is_false: fields.non_indexed_boolean
+  - is_true: fields.non_indexed_keyword
+  - is_false: fields.misc
+  - is_true: fields.text