The dynamic mapping parameter supports strict_allow_templates

Signed-off-by: Gao Binlong <gbinlong@amazon.com>
opensearch-project · Jun 26, 2024 · aa9ab72 · aa9ab72
1 parent 0eb39ae
commit aa9ab72
Show file tree

Hide file tree

Showing 9 changed files with 650 additions and 34 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Apply the date histogram rewrite optimization to range aggregation ([#13865](https://github.com/opensearch-project/OpenSearch/pull/13865))
 - [Writable Warm] Add composite directory implementation and integrate it with FileCache ([12782](https://github.com/opensearch-project/OpenSearch/pull/12782))
 - Fix race condition while parsing derived fields from search definition ([14445](https://github.com/opensearch-project/OpenSearch/pull/14445))
+- The dynamic mapping parameter supports strict_allow_templates
 
 ### Dependencies
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml
@@ -0,0 +1,155 @@
+---
+"Index documents with setting dynamic parameter to strict_allow_templates in the mapping of the index":
+  - skip:
+      version: " - 2.15.99"
+      reason: "introduced in 2.16.0"
+
+  - do:
+      indices.create:
+        index:    test_1
+        body:
+          mappings:
+            dynamic: strict_allow_templates
+            dynamic_templates: [
+              {
+                strings: {
+                  "match": "stringField*",
+                  "match_mapping_type": "string",
+                  "mapping": {
+                    "type": "keyword"
+                  }
+                }
+              },
+              {
+                object: {
+                  "match": "objectField*",
+                  "match_mapping_type": "object",
+                  "mapping": {
+                    "type": "object",
+                    "properties": {
+                      "bar1": {
+                        "type": "keyword"
+                      },
+                      "bar2": {
+                        "type": "text"
+                      }
+                    }
+                  }
+                }
+              },
+              {
+                boolean: {
+                  "match": "booleanField*",
+                  "match_mapping_type": "boolean",
+                  "mapping": {
+                    "type": "boolean"
+                  }
+                }
+              },
+              {
+                double: {
+                  "match": "doubleField*",
+                  "match_mapping_type": "double",
+                  "mapping": {
+                    "type": "double"
+                  }
+                }
+              },
+              {
+                long: {
+                  "match": "longField*",
+                  "match_mapping_type": "long",
+                  "mapping": {
+                    "type": "long"
+                  }
+                }
+              },
+              {
+                array: {
+                  "match": "arrayField*",
+                  "mapping": {
+                    "type": "keyword"
+                  }
+                }
+              },
+              {
+                date: {
+                  "match": "dateField*",
+                  "match_mapping_type": "date",
+                  "mapping": {
+                    "type": "date"
+                  }
+                }
+              }
+            ]
+            properties:
+              test1:
+                type: text
+
+  - do:
+      catch: /mapping set to strict_allow_templates, dynamic introduction of \[test2\] within \[\_doc\] is not allowed/
+      index:
+        index:   test_1
+        id:      1
+        body:    {
+          stringField: bar,
+          objectField: {
+            bar1: "bar1",
+            bar2: "bar2"
+          },
+          test1: test1,
+          test2: test2
+        }
+
+  - do:
+      index:
+        index:   test_1
+        id:      1
+        body:    {
+          stringField: bar,
+          objectField: {
+            bar1: "bar1",
+            bar2: "bar2"
+          },
+          booleanField: true,
+          doubleField: 1.0,
+          longField: 100,
+          arrayField: ["1","2"],
+          dateField: "2024-06-25T05:11:51.243Z",
+          test1: test1
+        }
+
+  - do:
+      get:
+        index: test_1
+        id: 1
+  - match: { _source: {
+      stringField: bar,
+      objectField: {
+        bar1: "bar1",
+        bar2: "bar2"
+      },
+      booleanField: true,
+      doubleField: 1.0,
+      longField: 100,
+      arrayField: [ "1","2" ],
+      dateField: "2024-06-25T05:11:51.243Z",
+      test1: test1
+    }
+  }
+
+  - do:
+      indices.get_mapping: {
+        index: test_1
+      }
+
+  - match: {test_1.mappings.dynamic:     strict_allow_templates}
+  - match: {test_1.mappings.properties.stringField.type:     keyword}
+  - match: {test_1.mappings.properties.objectField.properties.bar1.type:     keyword}
+  - match: {test_1.mappings.properties.objectField.properties.bar2.type:     text}
+  - match: {test_1.mappings.properties.booleanField.type:     boolean}
+  - match: {test_1.mappings.properties.doubleField.type:     double}
+  - match: {test_1.mappings.properties.longField.type:     long}
+  - match: {test_1.mappings.properties.arrayField.type:     keyword}
+  - match: {test_1.mappings.properties.dateField.type:     date}
+  - match: {test_1.mappings.properties.test1.type:     text}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml
@@ -159,3 +159,34 @@ setup:
       indices.get_mapping: {}
 
   - match: {test_index1.mappings.properties.text.type:     text}
+
+---
+"post a mapping with setting dynamic to strict_allow_templates":
+  - skip:
+      version: " - 2.15.99"
+      reason: "introduced in 2.16.0"
+  - do:
+      indices.put_mapping:
+        index: test_index1
+        body:
+          dynamic: strict_allow_templates
+          dynamic_templates: [
+            {
+              strings: {
+                "match": "foo*",
+                "match_mapping_type": "string",
+                "mapping": {
+                  "type": "keyword"
+                }
+              }
+            }
+          ]
+          properties:
+            test1:
+              type: text
+
+  - do:
+      indices.get_mapping: {}
+
+  - match: {test_index1.mappings.dynamic:     strict_allow_templates}
+  - match: {test_index1.mappings.properties.test1.type:     text}
diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java
@@ -546,9 +546,11 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper,
             ObjectMapper parentMapper = parentMapperTuple.v2();
             ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context);
             if (dynamic == ObjectMapper.Dynamic.STRICT) {
-                throw new StrictDynamicMappingException(mapper.fullPath(), currentFieldName);
-            } else if (dynamic == ObjectMapper.Dynamic.TRUE) {
-                Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.OBJECT);
+                throw new StrictDynamicMappingException(dynamic.toString(), mapper.fullPath(), currentFieldName);
+            } else if (dynamic == ObjectMapper.Dynamic.TRUE || dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) {
+                Mapper.Builder builder = context.root()
+                    .findTemplateBuilder(context, currentFieldName, XContentFieldType.OBJECT, dynamic, mapper.fullPath());
+
                 if (builder == null) {
                     builder = new ObjectMapper.Builder(currentFieldName).enabled(true);
                 }
@@ -592,9 +594,10 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper,
                 parentMapper = parentMapperTuple.v2();
                 ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context);
                 if (dynamic == ObjectMapper.Dynamic.STRICT) {
-                    throw new StrictDynamicMappingException(parentMapper.fullPath(), arrayFieldName);
-                } else if (dynamic == ObjectMapper.Dynamic.TRUE) {
-                    Mapper.Builder builder = context.root().findTemplateBuilder(context, arrayFieldName, XContentFieldType.OBJECT);
+                    throw new StrictDynamicMappingException(dynamic.toString(), parentMapper.fullPath(), arrayFieldName);
+                } else if (dynamic == ObjectMapper.Dynamic.TRUE || dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) {
+                    Mapper.Builder builder = context.root()
+                        .findTemplateBuilder(context, arrayFieldName, XContentFieldType.OBJECT, dynamic, parentMapper.fullPath());
                     if (builder == null) {
                         parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
                     } else {
@@ -695,9 +698,10 @@ private static void parseNullValue(ParseContext context, ObjectMapper parentMapp
         if (mapper != null) {
             // TODO: passing null to an object seems bogus?
             parseObjectOrField(context, mapper);
-        } else if (parentMapper.dynamic() == ObjectMapper.Dynamic.STRICT) {
-            throw new StrictDynamicMappingException(parentMapper.fullPath(), lastFieldName);
-        }
+        } else if (parentMapper.dynamic() == ObjectMapper.Dynamic.STRICT
+            || parentMapper.dynamic() == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) {
+                throw new StrictDynamicMappingException(parentMapper.dynamic().toString(), parentMapper.fullPath(), lastFieldName);
+            }
     }
 
     private static Mapper.Builder<?> newLongBuilder(String name, Settings settings) {
@@ -711,7 +715,9 @@ private static Mapper.Builder<?> newFloatBuilder(String name, Settings settings)
     private static Mapper.Builder<?> createBuilderFromDynamicValue(
         final ParseContext context,
         XContentParser.Token token,
-        String currentFieldName
+        String currentFieldName,
+        ObjectMapper.Dynamic dynamic,
+        String fullPath
     ) throws IOException {
         if (token == XContentParser.Token.VALUE_STRING) {
             String text = context.parser().text();
@@ -733,13 +739,15 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
             }
 
             if (parseableAsLong && context.root().numericDetection()) {
-                Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG);
+                Mapper.Builder builder = context.root()
+                    .findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath);
                 if (builder == null) {
                     builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings());
                 }
                 return builder;
             } else if (parseableAsDouble && context.root().numericDetection()) {
-                Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE);
+                Mapper.Builder builder = context.root()
+                    .findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE, dynamic, fullPath);
                 if (builder == null) {
                     builder = newFloatBuilder(currentFieldName, context.indexSettings().getSettings());
                 }
@@ -755,7 +763,8 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
                         // failure to parse this, continue
                         continue;
                     }
-                    Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, dateTimeFormatter);
+                    Mapper.Builder builder = context.root()
+                        .findTemplateBuilder(context, currentFieldName, dateTimeFormatter, dynamic, fullPath);
                     if (builder == null) {
                         boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings());
                         builder = new DateFieldMapper.Builder(
@@ -771,7 +780,8 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
                 }
             }
 
-            Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING);
+            Mapper.Builder builder = context.root()
+                .findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath);
             if (builder == null) {
                 builder = new TextFieldMapper.Builder(currentFieldName, context.mapperService().getIndexAnalyzers()).addMultiField(
                     new KeywordFieldMapper.Builder("keyword").ignoreAbove(256)
@@ -783,15 +793,17 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
             if (numberType == XContentParser.NumberType.INT
                 || numberType == XContentParser.NumberType.LONG
                 || numberType == XContentParser.NumberType.BIG_INTEGER) {
-                Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG);
+                Mapper.Builder builder = context.root()
+                    .findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath);
                 if (builder == null) {
                     builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings());
                 }
                 return builder;
             } else if (numberType == XContentParser.NumberType.FLOAT
                 || numberType == XContentParser.NumberType.DOUBLE
                 || numberType == XContentParser.NumberType.BIG_DECIMAL) {
-                    Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE);
+                    Mapper.Builder builder = context.root()
+                        .findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE, dynamic, fullPath);
                     if (builder == null) {
                         // no templates are defined, we use float by default instead of double
                         // since this is much more space-efficient and should be enough most of
@@ -801,19 +813,22 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
                     return builder;
                 }
         } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
-            Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.BOOLEAN);
+            Mapper.Builder builder = context.root()
+                .findTemplateBuilder(context, currentFieldName, XContentFieldType.BOOLEAN, dynamic, fullPath);
             if (builder == null) {
                 builder = new BooleanFieldMapper.Builder(currentFieldName);
             }
             return builder;
         } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
-            Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.BINARY);
+            Mapper.Builder builder = context.root()
+                .findTemplateBuilder(context, currentFieldName, XContentFieldType.BINARY, dynamic, fullPath);
             if (builder == null) {
                 builder = new BinaryFieldMapper.Builder(currentFieldName);
             }
             return builder;
         } else {
-            Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING);
+            Mapper.Builder builder = context.root()
+                .findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath);
             if (builder != null) {
                 return builder;
             }
@@ -832,13 +847,13 @@ private static void parseDynamicValue(
     ) throws IOException {
         ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context);
         if (dynamic == ObjectMapper.Dynamic.STRICT) {
-            throw new StrictDynamicMappingException(parentMapper.fullPath(), currentFieldName);
+            throw new StrictDynamicMappingException(dynamic.toString(), parentMapper.fullPath(), currentFieldName);
         }
         if (dynamic == ObjectMapper.Dynamic.FALSE) {
             return;
         }
         final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
-        final Mapper.Builder<?> builder = createBuilderFromDynamicValue(context, token, currentFieldName);
+        final Mapper.Builder<?> builder = createBuilderFromDynamicValue(context, token, currentFieldName, dynamic, parentMapper.fullPath());
         Mapper mapper = builder.build(builderContext);
         context.addDynamicMapper(mapper);
 
@@ -926,9 +941,11 @@ private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(
 
                 switch (dynamic) {
                     case STRICT:
-                        throw new StrictDynamicMappingException(parent.fullPath(), paths[i]);
+                        throw new StrictDynamicMappingException(dynamic.toString(), parent.fullPath(), paths[i]);
+                    case STRICT_ALLOW_TEMPLATES:
                     case TRUE:
-                        Mapper.Builder builder = context.root().findTemplateBuilder(context, paths[i], XContentFieldType.OBJECT);
+                        Mapper.Builder builder = context.root()
+                            .findTemplateBuilder(context, paths[i], XContentFieldType.OBJECT, dynamic, parent.fullPath());
                         if (builder == null) {
                             builder = new ObjectMapper.Builder(paths[i]).enabled(true);
                         }