Synthetic _source: ignore_malformed for ip (#90038)

This adds synthetic `_source` support for `ip` fields with `ignore_malfored` set to `true`. We save the field values in hidden stored field, just like we do for `ignore_above` keyword fields. Then we load them at load time.
elastic · Sep 26, 2022 · d0cf9f5 · d0cf9f5
1 parent 5a9a457
commit d0cf9f5
Show file tree

Hide file tree

Showing 12 changed files with 602 additions and 31 deletions.
diff --git a/docs/changelog/90038.yaml b/docs/changelog/90038.yaml
@@ -0,0 +1,5 @@
+pr: 90038
+summary: "Synthetic `_source`: `ignore_malformed` for `ip`"
+area: TSDB
+type: enhancement
+issues: []
diff --git a/docs/reference/mapping/types/ip.asciidoc b/docs/reference/mapping/types/ip.asciidoc
@@ -155,8 +155,7 @@ GET my-index-000001/_search
 ==== Synthetic source
 `ip` fields support <<synthetic-source,synthetic `_source`>> in their default
 configuration. Synthetic `_source` cannot be used together with
-<<ignore-malformed,`ignore_malformed`>>, <<copy-to,`copy_to`>>, or with
-<<doc-values,`doc_values`>> disabled.
+<<copy-to,`copy_to`>> or with <<doc-values,`doc_values`>> disabled.
 
 Synthetic source always sorts `ip` fields and removes duplicates. For example:
 [source,console,id=synthetic-source-ip-example]

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml
@@ -583,3 +583,99 @@ _source filtering:
       _source:
         kwd: foo
   - is_false: fields
+
+---
+ip with ignore_malformed:
+  - skip:
+      version: " - 8.5.99"
+      reason: introduced in 8.6.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              ip:
+                type: ip
+                ignore_malformed: true
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        refresh: true
+        body:
+          ip: 192.168.0.1
+  - do:
+      index:
+        index:   test
+        id:      2
+        refresh: true
+        body:
+          ip: garbage
+  - do:
+      index:
+        index:   test
+        id:      3
+        refresh: true
+        body:
+          ip:
+            - 10.10.1.1
+            - 192.8.1.2
+            - hot garbage
+            - 7
+  - do:
+      catch: "/failed to parse field \\[ip\\] of type \\[ip\\] in document with id '4'. Preview of field's value: '\\{object=wow\\}'/"
+      index:
+        index:   test
+        id:      4
+        refresh: true
+        body:
+          ip:
+            object: wow
+
+  - do:
+      get:
+        index: test
+        id:    1
+  - match: {_index: "test"}
+  - match: {_id: "1"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        ip: 192.168.0.1
+  - is_false: fields
+
+  - do:
+      get:
+        index: test
+        id:    2
+  - match: {_index: "test"}
+  - match: {_id: "2"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        ip: garbage
+  - is_false: fields
+
+  - do:
+      get:
+        index: test
+        id:    3
+  - match: {_index: "test"}
+  - match: {_id: "3"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        ip:
+          - 10.10.1.1
+          - 192.8.1.2
+          - hot garbage # fields saved by ignore_malformed are sorted after doc values
+          - 7
+  - is_false: fields
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/110_ignore_malformed.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/110_ignore_malformed.yml
@@ -0,0 +1,93 @@
+---
+ip:
+  - skip:
+      version: " - 8.4.99"
+      reason: introduced in 8.5.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              ip:
+                type: ip
+                ignore_malformed: true
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        refresh: true
+        body:
+          ip: 192.168.0.1
+  - do:
+      index:
+        index:   test
+        id:      2
+        refresh: true
+        body:
+          ip: garbage
+  - do:
+      index:
+        index:   test
+        id:      3
+        refresh: true
+        body:
+          ip:
+            - 10.10.1.1
+            - 192.8.1.2
+            - hot garbage
+            - 7
+  - do:
+      catch: "/failed to parse field \\[ip\\] of type \\[ip\\] in document with id '4'. Preview of field's value: '\\{object=wow\\}'/"
+      index:
+        index:   test
+        id:      4
+        refresh: true
+        body:
+          ip:
+            object: wow
+
+  - do:
+      get:
+        index: test
+        id:    1
+  - match: {_index: "test"}
+  - match: {_id: "1"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        ip: 192.168.0.1
+  - is_false: fields
+
+  - do:
+      get:
+        index: test
+        id:    2
+  - match: {_index: "test"}
+  - match: {_id: "2"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        ip: garbage
+  - is_false: fields
+
+  - do:
+      get:
+        index: test
+        id:    3
+  - match: {_index: "test"}
+  - match: {_id: "3"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        ip:
+          - 10.10.1.1
+          - 192.8.1.2
+          - hot garbage # fields saved by ignore_malformed are sorted after doc values
+          - 7
+  - is_false: fields