Synthetic _source: support ignore_above (#89466)

This allows you to use `ignore_above` with `keyword` fields in synthetic source. Ignored values are stored in a "backup" stored field and added to the end of the list of results. This makes `ignore_above` work pretty much the same way as it does when you don't have synthetic source. The only difference is the order of the results. But synthetic source changes the order of results anyway. That should be fine.
elastic · Sep 1, 2022 · 703571a · 703571a
1 parent 4bbe4ea
commit 703571a
Show file tree

Hide file tree

Showing 15 changed files with 524 additions and 89 deletions.
diff --git a/docs/changelog/89466.yaml b/docs/changelog/89466.yaml
@@ -0,0 +1,5 @@
+pr: 89466
+summary: "Synthetic _source: support `ignore_above`"
+area: "TSDB"
+type: feature
+issues: []
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml
@@ -33,6 +33,7 @@ keyword:
   - match:
       _source:
         kwd: foo
+#  - is_false: fields  TODO fix me
 
 ---
 fetch without refresh also produces synthetic source:
@@ -63,6 +64,7 @@ fetch without refresh also produces synthetic source:
         refresh: false # no refreshing!
         body:
           obj.kwd: foo
+#  - is_false: fields  TODO fix me
 
   - do:
       get:
@@ -76,6 +78,7 @@ fetch without refresh also produces synthetic source:
       _source: # synthetic source will convert the dotted field names into an object, even when loading from the translog
         obj:
           kwd: foo
+#  - is_false: fields  TODO fix me
 
 ---
 force_synthetic_source_ok:
@@ -103,6 +106,7 @@ force_synthetic_source_ok:
         refresh: true
         body:
           obj.kwd: foo
+#  - is_false: fields  TODO fix me
 
   # When _source is used in the fetch the original _source is perfect
   - do:
@@ -123,6 +127,7 @@ force_synthetic_source_ok:
       _source:
         obj:
           kwd: foo
+#  - is_false: fields  TODO fix me
 
 ---
 force_synthetic_source_bad_mapping:
@@ -185,6 +190,7 @@ stored text:
               text:
                 type: text
                 store: true
+#  - is_false: fields  TODO fix me
 
   - do:
       index:
@@ -193,6 +199,7 @@ stored text:
         refresh: true
         body:
           text: the quick brown fox
+#  - is_false: fields  TODO fix me
 
   - do:
       get:
@@ -205,6 +212,7 @@ stored text:
   - match:
       _source:
         text: the quick brown fox
+#  - is_false: fields  TODO fix me
 
 ---
 stored keyword:
@@ -223,6 +231,7 @@ stored keyword:
               kwd:
                 type: keyword
                 store: true
+#  - is_false: fields  TODO fix me
 
   - do:
       index:
@@ -231,6 +240,7 @@ stored keyword:
         refresh: true
         body:
           kwd: the quick brown fox
+#  - is_false: fields  TODO fix me
 
   - do:
       get:
@@ -243,3 +253,172 @@ stored keyword:
   - match:
       _source:
         kwd: the quick brown fox
+#  - is_false: fields  TODO fix me
+
+---
+doc values keyword with ignore_above:
+  - skip:
+      version: " - 8.4.99"
+      reason: introduced in 8.5.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              kwd:
+                type: keyword
+                ignore_above: 10
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        refresh: true
+        body:
+          kwd: the quick brown fox
+  - do:
+      index:
+        index:   test
+        id:      2
+        refresh: true
+        body:
+          kwd: short
+  - do:
+      index:
+        index:   test
+        id:      3
+        refresh: true
+        body:
+          kwd:
+            - jumped over the lazy dog
+            - short
+
+  - do:
+      get:
+        index: test
+        id:    1
+  - match: {_index: "test"}
+  - match: {_id: "1"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        kwd: the quick brown fox
+#  - is_false: fields  TODO fix me
+
+  - do:
+      get:
+        index: test
+        id:    2
+  - match: {_index: "test"}
+  - match: {_id: "2"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        kwd: short
+#  - is_false: fields  TODO fix me
+
+  - do:
+      get:
+        index: test
+        id:    3
+  - match: {_index: "test"}
+  - match: {_id: "3"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        kwd:
+          - short
+          - jumped over the lazy dog # fields saved by ignore_above are returned after doc values fields
+#  - is_false: fields  TODO fix me
+
+---
+stored keyword with ignore_above:
+  - skip:
+      version: " - 8.4.99"
+      reason: introduced in 8.5.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              kwd:
+                type: keyword
+                doc_values: false
+                store: true
+                ignore_above: 10
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        refresh: true
+        body:
+          kwd: the quick brown fox
+  - do:
+      index:
+        index:   test
+        id:      2
+        refresh: true
+        body:
+          kwd: short
+  - do:
+      index:
+        index:   test
+        id:      3
+        refresh: true
+        body:
+          kwd:
+            - jumped over the lazy dog
+            - short
+
+  - do:
+      get:
+        index: test
+        id:    1
+  - match: {_index: "test"}
+  - match: {_id: "1"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        kwd: the quick brown fox
+#  - is_false: fields  TODO fix me
+
+  - do:
+      get:
+        index: test
+        id:    2
+  - match: {_index: "test"}
+  - match: {_id: "2"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        kwd: short
+#  - is_false: fields  TODO fix me
+
+  - do:
+      get:
+        index: test
+        id:    3
+  - match: {_index: "test"}
+  - match: {_id: "3"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        kwd:
+          - short
+          - jumped over the lazy dog # fields saved by ignore_above are returned after doc values fields
+#  - is_false: fields  TODO fix me
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml
@@ -212,5 +212,5 @@ force_synthetic_source_bad_mapping:
         force_synthetic_source: true
         body:
           ids: [ 1, 2 ]
-  - match: {docs.0.error.reason: "field [text] of type [text] doesn't support synthetic source unless it is stored or has a sub-field of type [keyword] with doc values or stored and without ignore_above or a normalizer"}
-  - match: {docs.1.error.reason: "field [text] of type [text] doesn't support synthetic source unless it is stored or has a sub-field of type [keyword] with doc values or stored and without ignore_above or a normalizer"}
+  - match: {docs.0.error.reason: "field [text] of type [text] doesn't support synthetic source unless it is stored or has a sub-field of type [keyword] with doc values or stored and without a normalizer"}
+  - match: {docs.1.error.reason: "field [text] of type [text] doesn't support synthetic source unless it is stored or has a sub-field of type [keyword] with doc values or stored and without a normalizer"}