Add 'mode' option to _source field mapper (#88211)

Currently we have two parameters that control how the source of a document is stored, `enabled` and `synthetic`, both booleans. However, there are only three possible combinations of these, with `enabled:false` and `synthetic:true` being disallowed. To make this easier to reason about, this commit replaces the `enabled` parameter with a new `mode` parameter, which can take the values `stored`, `synthetic` and `disabled`. The `mode` parameter cannot be set in combination with `enabled`, and we will subsequently move towards deprecating `enabled` entirely.
elastic · Jul 18, 2022 · 5c11a81 · 5c11a81
1 parent 7274042
commit 5c11a81
Show file tree

Hide file tree

Showing 25 changed files with 183 additions and 95 deletions.
diff --git a/docs/changelog/88211.yaml b/docs/changelog/88211.yaml
@@ -0,0 +1,5 @@
+pr: 88211
+summary: Add 'mode' option to `_source` field mapper
+area: Search
+type: feature
+issues: []
diff --git a/docs/reference/mapping/fields/synthetic-source.asciidoc b/docs/reference/mapping/fields/synthetic-source.asciidoc
@@ -4,15 +4,15 @@
 Though very handy to have around, the source field takes up a significant amount
 of space on disk. Instead of storing source documents on disk exactly as you
 send them, Elasticsearch can reconstruct source content on the fly upon retrieval.
-Enable this by setting `synthetic: true` in `_source`:
+Enable this by setting `mode: synthetic` in `_source`:
 
 [source,console,id=enable-synthetic-source-example]
 ----
 PUT idx
 {
   "mappings": {
     "_source": {
-      "synthetic": true
+      "mode": "synthetic"
     }
   }
 }

diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc
@@ -228,7 +228,7 @@ Synthetic source always sorts `boolean` fields. For example:
 PUT idx
 {
   "mappings": {
-    "_source": { "synthetic": true },
+    "_source": { "mode": "synthetic" },
     "properties": {
       "bool": { "type": "boolean" }
     }

diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc
@@ -208,7 +208,7 @@ ifeval::["{release-state}"=="unreleased"]
 [[geo-point-synthetic-source]]
 ==== Synthetic source
 `geo_point` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with 
+default configuration. Synthetic `_source` cannot be used together with
 <<ignore-malformed,`ignore_malformed`>>, <<copy-to,`copy_to`>>, or with
 <<doc-values,`doc_values`>> disabled.
 
@@ -219,7 +219,7 @@ longitude) and reduces them to their stored precision. For example:
 PUT idx
 {
   "mappings": {
-    "_source": { "synthetic": true },
+    "_source": { "mode": "synthetic" },
     "properties": {
       "point": { "type": "geo_point" }
     }

diff --git a/docs/reference/mapping/types/ip.asciidoc b/docs/reference/mapping/types/ip.asciidoc
@@ -165,7 +165,7 @@ Synthetic source always sorts `ip` fields and removes duplicates. For example:
 PUT idx
 {
   "mappings": {
-    "_source": { "synthetic": true },
+    "_source": { "mode": "synthetic" },
     "properties": {
       "ip": { "type": "ip" }
     }

diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc
@@ -189,7 +189,7 @@ example:
 PUT idx
 {
   "mappings": {
-    "_source": { "synthetic": true },
+    "_source": { "mode": "synthetic" },
     "properties": {
       "kwd": { "type": "keyword" }
     }

diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc
@@ -243,7 +243,7 @@ Synthetic source always sorts numeric fields and removes duplicates. For example
 PUT idx
 {
   "mappings": {
-    "_source": { "synthetic": true },
+    "_source": { "mode": "synthetic" },
     "properties": {
       "long": { "type": "long" }
     }
@@ -271,7 +271,7 @@ Scaled floats will always apply their scaling factor so:
 PUT idx
 {
   "mappings": {
-    "_source": { "synthetic": true },
+    "_source": { "mode": "synthetic" },
     "properties": {
       "f": { "type": "scaled_float", "scaling_factor": 0.01 }
     }

diff --git a/docs/reference/mapping/types/text.asciidoc b/docs/reference/mapping/types/text.asciidoc
@@ -173,7 +173,7 @@ Synthetic source always sorts `keyword` fields and removes duplicates, so
 PUT idx
 {
   "mappings": {
-    "_source": { "synthetic": true },
+    "_source": { "mode": "synthetic" },
     "properties": {
       "text": {
         "type": "text",

diff --git a/...es/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/110_synthetic_source.yml b/...es/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/110_synthetic_source.yml
@@ -5,7 +5,7 @@ setup:
         body:
           mappings:
             _source:
-              synthetic: true
+              mode: synthetic
             properties:
               kwd:
                 type: keyword

diff --git a/...ex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/100_synthetic_source.yml b/...ex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/100_synthetic_source.yml
@@ -5,7 +5,7 @@ update:
         body:
           mappings:
             _source:
-              synthetic: true
+              mode: synthetic
             properties:
               kwd:
                 type: keyword

diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java
@@ -361,15 +361,15 @@ private void assertTsdbAgg(Matcher<?>... expected) throws IOException {
     }
 
     public void testSyntheticSource() throws IOException {
-        assumeTrue("added in 8.3.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_3_0));
+        assumeTrue("added in 8.4.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_4_0));
 
         switch (CLUSTER_TYPE) {
             case OLD -> {
                 Request createIndex = new Request("PUT", "/synthetic");
                 XContentBuilder indexSpec = XContentBuilder.builder(XContentType.JSON.xContent()).startObject();
                 indexSpec.startObject("mappings");
                 {
-                    indexSpec.startObject("_source").field("synthetic", true).endObject();
+                    indexSpec.startObject("_source").field("mode", "synthetic").endObject();
                     indexSpec.startObject("properties").startObject("kwd").field("type", "keyword").endObject().endObject();
                 }
                 indexSpec.endObject();

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml
@@ -1,15 +1,15 @@
 keyword:
   - skip:
-      version: " - 8.2.99"
-      reason: introduced in 8.3.0
+      version: " - 8.3.99"
+      reason: introduced in 8.4.0
 
   - do:
       indices.create:
         index: test
         body:
           mappings:
             _source:
-              synthetic: true
+              mode: synthetic
             properties:
               kwd:
                 type: keyword
@@ -37,8 +37,8 @@ keyword:
 ---
 fetch without refresh also produces synthetic source:
   - skip:
-      version: " - 8.2.99"
-      reason: introduced in 8.3.0
+      version: " - 8.3.99"
+      reason: introduced in 8.4.0
 
   - do:
       indices.create:
@@ -49,7 +49,7 @@ fetch without refresh also produces synthetic source:
               refresh_interval: -1
           mappings:
             _source:
-              synthetic: true
+              mode: synthetic
             properties:
               obj:
                 properties:
@@ -89,7 +89,7 @@ force_synthetic_source_ok:
         body:
           mappings:
             _source:
-              synthetic: false
+              mode: stored
             properties:
               obj:
                 properties:
@@ -138,7 +138,7 @@ force_synthetic_source_bad_mapping:
             number_of_shards: 1 # Use a single shard to get consistent error messages
           mappings:
             _source:
-              synthetic: false
+              mode: stored
             properties:
               text:
                 type: text

diff --git a/...api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/91_metrics_no_subobjects.yml b/...api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/91_metrics_no_subobjects.yml
@@ -125,8 +125,8 @@
 "Metrics object indexing with synthetic source":
   - skip:
       features: allowed_warnings_regex
-      version: " - 8.2.99"
-      reason: added in 8.3.0
+      version: " - 8.3.99"
+      reason: added in 8.4.0
 
   - do:
       indices.put_template:
@@ -135,7 +135,7 @@
           index_patterns: test-*
           mappings:
             _source:
-              synthetic: true
+              mode: synthetic
             dynamic_templates:
               - no_subobjects:
                   match: metrics
@@ -192,8 +192,8 @@
 "Root without subobjects with synthetic source":
   - skip:
       features: allowed_warnings_regex
-      version: " - 8.2.99"
-      reason: added in 8.3.0
+      version: " - 8.3.99"
+      reason: added in 8.4.0
 
   - do:
       indices.put_template:
@@ -202,7 +202,7 @@
           index_patterns: test-*
           mappings:
             _source:
-              synthetic: true
+              mode: synthetic
             subobjects: false
             properties:
               host.name:

diff --git a/...spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/...spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
@@ -10,7 +10,7 @@ invalid:
         body:
           mappings:
             _source:
-              synthetic: true
+              mode: synthetic
             properties:
               kwd:
                 type: keyword
@@ -29,7 +29,7 @@ nested is disabled:
         body:
           mappings:
             _source:
-              synthetic: true
+              mode: synthetic
             properties:
               n:
                 type: nested

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml
@@ -156,18 +156,18 @@
         body:
           mappings:
             _source:
-              synthetic: true
+              mode: synthetic
 
   - do:
-      catch: /Cannot update parameter \[synthetic\] from \[true\] to \[false\]/
+      catch: /Cannot update parameter \[mode\] from \[synthetic\] to \[stored\]/
       indices.put_mapping:
         index: test_index
         body:
           _source:
-            synthetic: false
+            mode: stored
 
 ---
-"enabling synthetic source from explicit fails":
+"enabling synthetic source from explicit succeeds":
   - skip:
       version:     " - 8.3.99"
       reason:      "Added in 8.4.0"
@@ -178,18 +178,17 @@
         body:
           mappings:
             _source:
-              synthetic: false
+              mode: stored
 
   - do:
-      catch: /Cannot update parameter \[synthetic\] from \[false\] to \[true\]/
       indices.put_mapping:
         index: test_index
         body:
           _source:
-            synthetic: true
+            mode: synthetic
 
 ---
-"enabling synthetic source fails":
+"enabling synthetic source succeeds":
   - skip:
       version:     " - 8.3.99"
       reason:      "Added in 8.4.0"
@@ -205,15 +204,14 @@
         id:      1
         refresh: true
         body:
-          kwd: foo
+          value: 4
 
   - do:
-      catch: /Cannot update parameter \[synthetic\] from \[false\] to \[true\]/
       indices.put_mapping:
         index: test_index
         body:
           _source:
-            synthetic: true
+            mode: synthetic
 
 ---
 "enabling synthetic source when no mapping succeeds":
@@ -233,4 +231,4 @@
         index: test_index
         body:
           _source:
-            synthetic: true
+            mode: synthetic
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml
@@ -1,15 +1,15 @@
 keyword:
   - skip:
-      version: " - 8.2.99"
-      reason: introduced in 8.3.0
+      version: " - 8.3.99"
+      reason: introduced in 8.4.0
 
   - do:
       indices.create:
         index: test
         body:
           mappings:
             _source:
-              synthetic: true
+              mode: synthetic
             properties:
               kwd:
                 type: keyword
@@ -58,7 +58,7 @@ force_synthetic_source_ok:
         body:
           mappings:
             _source:
-              synthetic: false
+              mode: stored
             properties:
               kwd:
                 type: keyword
@@ -118,7 +118,7 @@ force_synthetic_source_bad_mapping:
             number_of_shards: 1 # Use a single shard to get consistent error messages
           mappings:
             _source:
-              synthetic: false
+              mode: stored
             properties:
               text:
                 type: text

diff --git a/...ec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/50_synthetic_source.yml b/...ec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/50_synthetic_source.yml
@@ -11,7 +11,7 @@ setup:
             number_of_shards: 1
           mappings:
             _source:
-              synthetic: true
+              mode: synthetic
             properties:
               foo:
                 type: keyword

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml
@@ -1,15 +1,15 @@
 keyword:
   - skip:
-      version: " - 8.2.99"
-      reason: introduced in 8.3.0
+      version: " - 8.3.99"
+      reason: introduced in 8.4.0
 
   - do:
       indices.create:
         index: test
         body:
           mappings:
             _source:
-              synthetic: true
+              mode: synthetic
             properties:
               kwd:
                 type: keyword
@@ -45,7 +45,7 @@ force_synthetic_source_ok:
         body:
           mappings:
             _source:
-              synthetic: false
+              mode: stored
             properties:
               obj:
                 properties:
@@ -100,7 +100,7 @@ force_synthetic_source_bad_mapping:
             number_of_shards: 1 # Use a single shard to get consistent error messages
           mappings:
             _source:
-              synthetic: false
+              mode: stored
             properties:
               text:
                 type: text