Skip to content

Commit

Permalink
Deprecate multifield mappings templates (#81329)
Browse files Browse the repository at this point in the history 
Using fields within fields in mappings is deprecated, but previously would not be listed in the 
deprecation API if the mapping was part of an index template, a component template, or within 
mapping dynamic_template sections which can be present in either of the locations. This change 
checks all six combinations of locations for the deprecated mapping structure.
  • Loading branch information
@jbaiera
jbaiera committed Dec 8, 2021
1 parent 7d8f0b3 commit 48cbeda
Show file tree
Hide file tree
Showing 5 changed files with 938 additions and 28 deletions.
193 changes: 193 additions & 0 deletions ...precation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
Expand Down Expand Up @@ -124,6 +125,198 @@ static DeprecationIssue checkTemplatesWithTooManyFields(ClusterState state) {
return null;
}

/**
* Check templates that use `fields` within `fields` blocks
*/
@SuppressWarnings("unchecked")
static DeprecationIssue checkTemplatesWithChainedMultiFields(ClusterState state) {
Map<String, List<String>> templatesContainingChainedMultiFields = new HashMap<>();
state.getMetadata().getTemplates().forEach((templateCursor) -> {
String templateName = templateCursor.key;
templateCursor.value.getMappings().forEach((mappingCursor) -> {
String type = mappingCursor.key;
// There should be the type name at this level, but there was a bug where mappings could be stored without a type (#45120)
// to make sure, we try to detect this like we try to do in MappingMetadata#sourceAsMap()
Map<String, Object> mapping = XContentHelper.convertToMap(mappingCursor.value.compressedReference(), true).v2();
if (mapping.size() == 1 && mapping.containsKey(type)) {
// the type name is the root value, reduce it
mapping = (Map<String, Object>) mapping.get(type);
}
List<String> mappingIssues = IndexDeprecationChecks.findInPropertiesRecursively(
type,
mapping,
IndexDeprecationChecks::containsChainedMultiFields,
IndexDeprecationChecks::formatField,
"",
""
);
if (mappingIssues.size() > 0) {
templatesContainingChainedMultiFields.put(templateName, mappingIssues);
}
});
});
if (templatesContainingChainedMultiFields.isEmpty() == false) {
return new DeprecationIssue(
DeprecationIssue.Level.WARNING,
"Defining multi-fields within multi-fields on index template mappings is deprecated",
"https://ela.st/es-deprecation-7-chained-multi-fields",
String.format(
Locale.ROOT,
"Remove chained multi-fields from the \"%s\" template%s. Multi-fields within multi-fields "
+ "are not supported in 8.0.",
String.join(",", templatesContainingChainedMultiFields.keySet()),
templatesContainingChainedMultiFields.size() > 1 ? "s" : ""
),
false,
null
);
}
return null;
}

/**
* Check templates that use `fields` within `fields` blocks of dynamic templates
*/
@SuppressWarnings("unchecked")
static DeprecationIssue checkTemplatesWithChainedMultiFieldsInDynamicTemplates(ClusterState state) {
Map<String, List<String>> templatesContainingChainedMultiFields = new HashMap<>();
state.getMetadata().getTemplates().forEach((templateCursor) -> {
String templateName = templateCursor.key;
templateCursor.value.getMappings().forEach((mappingCursor) -> {
String type = mappingCursor.key;
// There should be the type name at this level, but there was a bug where mappings could be stored without a type (#45120)
// to make sure, we try to detect this like we try to do in MappingMetadata#sourceAsMap()
Map<String, Object> mapping = XContentHelper.convertToMap(mappingCursor.value.compressedReference(), true).v2();
if (mapping.size() == 1 && mapping.containsKey(type)) {
// the type name is the root value, reduce it
mapping = (Map<String, Object>) mapping.get(type);
}
List<String> mappingIssues = IndexDeprecationChecks.findInDynamicTemplates(
type,
mapping,
IndexDeprecationChecks::containsMappingWithChainedMultiFields,
IndexDeprecationChecks::formatField,
"",
""
);
if (mappingIssues.size() > 0) {
templatesContainingChainedMultiFields.put(templateName, mappingIssues);
}
});
});
if (templatesContainingChainedMultiFields.isEmpty() == false) {
return new DeprecationIssue(
DeprecationIssue.Level.WARNING,
"Defining multi-fields within multi-fields on index template dynamic_templates is deprecated",
"https://ela.st/es-deprecation-7-chained-multi-fields",
String.format(
Locale.ROOT,
"Remove chained multi-fields from the \"%s\" template%s. Multi-fields within multi-fields "
+ "are not supported in 8.0.",
String.join(",", templatesContainingChainedMultiFields.keySet()),
templatesContainingChainedMultiFields.size() > 1 ? "s" : ""
),
false,
null
);
}
return null;
}

/**
* Check component templates that use `fields` within `fields` blocks
*/
@SuppressWarnings("unchecked")
static DeprecationIssue checkComponentTemplatesWithChainedMultiFields(ClusterState state) {
Map<String, List<String>> templatesContainingChainedMultiFields = new HashMap<>();
state.getMetadata().componentTemplates().forEach((templateName, componentTemplate) -> {
CompressedXContent mappings = componentTemplate.template().mappings();
if (mappings != null) {
// Component templates root their mapping data under the "_doc" mapping type. Unpack it if that is the case.
Map<String, Object> mapping = XContentHelper.convertToMap(mappings.compressedReference(), true).v2();
if (mapping.size() == 1 && mapping.containsKey("_doc")) {
// the type name is the root value, reduce it
mapping = (Map<String, Object>) mapping.get("_doc");
}
List<String> mappingIssues = IndexDeprecationChecks.findInPropertiesRecursively(
"_doc",
mapping,
IndexDeprecationChecks::containsChainedMultiFields,
IndexDeprecationChecks::formatField,
"",
""
);
if (mappingIssues.size() > 0) {
templatesContainingChainedMultiFields.put(templateName, mappingIssues);
}
}
});
if (templatesContainingChainedMultiFields.isEmpty() == false) {
return new DeprecationIssue(
DeprecationIssue.Level.WARNING,
"Defining multi-fields within multi-fields on component templates is deprecated",
"https://ela.st/es-deprecation-7-chained-multi-fields",
String.format(
Locale.ROOT,
"Remove chained multi-fields from the \"%s\" component template%s. Multi-fields within multi-fields "
+ "are not supported in 8.0.",
String.join(",", templatesContainingChainedMultiFields.keySet()),
templatesContainingChainedMultiFields.size() > 1 ? "s" : ""
),
false,
null
);
}
return null;
}

/**
* Check component templates that use `fields` within `fields` blocks of dynamic templates
*/
@SuppressWarnings("unchecked")
static DeprecationIssue checkComponentTemplatesWithChainedMultiFieldsInDynamicTemplates(ClusterState state) {
Map<String, List<String>> templatesContainingChainedMultiFields = new HashMap<>();
state.getMetadata().componentTemplates().forEach((templateName, componentTemplate) -> {
CompressedXContent mappings = componentTemplate.template().mappings();
if (mappings != null) {
// Component templates root their mapping data under the "_doc" mapping type. Unpack it if that is the case.
Map<String, Object> mapping = XContentHelper.convertToMap(mappings.compressedReference(), true).v2();
if (mapping.size() == 1 && mapping.containsKey("_doc")) {
// the type name is the root value, reduce it
mapping = (Map<String, Object>) mapping.get("_doc");
}
List<String> mappingIssues = IndexDeprecationChecks.findInDynamicTemplates(
"_doc",
mapping,
IndexDeprecationChecks::containsMappingWithChainedMultiFields,
IndexDeprecationChecks::formatField,
"",
""
);
if (mappingIssues.size() > 0) {
templatesContainingChainedMultiFields.put(templateName, mappingIssues);
}
}
});
if (templatesContainingChainedMultiFields.isEmpty() == false) {
return new DeprecationIssue(
DeprecationIssue.Level.WARNING,
"Defining multi-fields within multi-fields on component template dynamic_templates is deprecated",
"https://ela.st/es-deprecation-7-chained-multi-fields",
String.format(
Locale.ROOT,
"Remove chained multi-fields from the \"%s\" component template%s. Multi-fields within multi-fields "
+ "are not supported in 8.0.",
String.join(",", templatesContainingChainedMultiFields.keySet()),
templatesContainingChainedMultiFields.size() > 1 ? "s" : ""
),
false,
null
);
}
return null;
}

/**
* Check templates that use `_field_names` explicitly, which was deprecated in https://github.com/elastic/elasticsearch/pull/42854
* and will throw an error on new indices in 8.0
Expand Down
5 changes: 5 additions & 0 deletions ...ugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ private DeprecationChecks() {}
ClusterDeprecationChecks::checkPollIntervalTooLow,
ClusterDeprecationChecks::checkTemplatesWithFieldNamesDisabled,
ClusterDeprecationChecks::checkTemplatesWithCustomAndMultipleTypes,
ClusterDeprecationChecks::checkTemplatesWithChainedMultiFields,
ClusterDeprecationChecks::checkComponentTemplatesWithChainedMultiFields,
ClusterDeprecationChecks::checkTemplatesWithChainedMultiFieldsInDynamicTemplates,
ClusterDeprecationChecks::checkComponentTemplatesWithChainedMultiFieldsInDynamicTemplates,
ClusterDeprecationChecks::checkClusterRoutingAllocationIncludeRelocationsSetting,
ClusterDeprecationChecks::checkGeoShapeTemplates,
ClusterDeprecationChecks::checkSparseVectorTemplates,
Expand Down Expand Up @@ -238,6 +242,7 @@ private static Set<Setting<Boolean>> getAllDeprecatedNodeRolesSettings() {
(clusterState, indexMetadata) -> IndexDeprecationChecks.oldIndicesCheck(indexMetadata),
(clusterState, indexMetadata) -> IndexDeprecationChecks.tooManyFieldsCheck(indexMetadata),
(clusterState, indexMetadata) -> IndexDeprecationChecks.chainedMultiFieldsCheck(indexMetadata),
(clusterState, indexMetadata) -> IndexDeprecationChecks.chainedMultiFieldsDynamicTemplateCheck(indexMetadata),
(clusterState, indexMetadata) -> IndexDeprecationChecks.deprecatedDateTimeFormat(indexMetadata),
(clusterState, indexMetadata) -> IndexDeprecationChecks.translogRetentionSettingCheck(indexMetadata),
(clusterState, indexMetadata) -> IndexDeprecationChecks.fieldNamesDisabledCheck(indexMetadata),
Expand Down
94 changes: 92 additions & 2 deletions ...deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
*/
package org.elasticsearch.xpack.deprecation;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetadata;
Expand All @@ -29,6 +31,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
Expand All @@ -47,6 +50,7 @@
* Index-specific deprecation checks
*/
public class IndexDeprecationChecks {
private static final Logger logger = LogManager.getLogger(IndexDeprecationChecks.class);

static final String JODA_TIME_DEPRECATION_DETAILS_SUFFIX = " See https://www.elastic.co/guide/en/elasticsearch/reference/master"
+ "/migrate-to-java-time.html for details. Failure to update custom data formats to java.time could cause inconsistentencies in "
Expand Down Expand Up @@ -124,12 +128,52 @@ static List<String> findInPropertiesRecursively(
return issues;
}

/**
* iterates through the "dynamicProperties" field of mappings and returns any predicates that match in the
* form of issue-strings.
*
* @param type the document type
* @param parentMap the mapping to read properties from
* @param predicate the predicate to check against for issues, issue is returned if predicate evaluates to true
* @param fieldFormatter a function that takes a type and template entry and returns a formatted field representation
* @return a list of issues found in fields
*/
@SuppressWarnings("unchecked")
static List<String> findInDynamicTemplates(
String type,
Map<String, Object> parentMap,
Function<Map<?, ?>, Boolean> predicate,
BiFunction<String, Map.Entry<?, ?>, String> fieldFormatter,
String fieldBeginMarker,
String fieldEndMarker
) {
List<String> issues = new ArrayList<>();
List<?> dynamicTemplates = (List<?>) parentMap.get("dynamic_templates");
if (dynamicTemplates == null) {
return issues;
}
// List of named object
for (Object entry : dynamicTemplates) {
Map<String, Object> topLevelTemplate = ((Map<String, Object>) entry);
Iterator<Map.Entry<String, Object>> templateCursor = topLevelTemplate.entrySet().iterator();
if (templateCursor.hasNext()) {
Map.Entry<String, Object> templateEntry = templateCursor.next();
Map<String, Object> templateBody = ((Map<String, Object>) templateEntry.getValue());
if (predicate.apply(templateBody)) {
issues.add(fieldBeginMarker + fieldFormatter.apply(type, templateEntry) + fieldEndMarker);
}
}
}

return issues;
}

private static String formatDateField(String type, Map.Entry<?, ?> entry) {
Map<?, ?> value = (Map<?, ?>) entry.getValue();
return String.format(Locale.ROOT, "Convert [%s] format %s to java.time.", entry.getKey(), value.get("format"));
}

private static String formatField(String type, Map.Entry<?, ?> entry) {
static String formatField(String type, Map.Entry<?, ?> entry) {
return entry.getKey().toString();
}

Expand Down Expand Up @@ -257,7 +301,53 @@ static DeprecationIssue chainedMultiFieldsCheck(IndexMetadata indexMetadata) {
return null;
}

private static boolean containsChainedMultiFields(Map<?, ?> property) {
static DeprecationIssue chainedMultiFieldsDynamicTemplateCheck(IndexMetadata indexMetadata) {
List<String> issues = new ArrayList<>();
fieldLevelMappingIssue(
indexMetadata,
((mappingMetadata, sourceAsMap) -> issues.addAll(
findInDynamicTemplates(
mappingMetadata.type(),
sourceAsMap,
IndexDeprecationChecks::containsMappingWithChainedMultiFields,
IndexDeprecationChecks::formatField,
"",
""
)
))
);
if (issues.size() > 0) {
return new DeprecationIssue(
DeprecationIssue.Level.WARNING,
"Defining multi-fields within multi-fields inside dynamic templates is deprecated",
"https://ela.st/es-deprecation-7-chained-multi-fields",
String.format(
Locale.ROOT,
"Remove chained multi-fields from the \"%s\" dynamic template%s. Multi-fields within multi-fields "
+ "are not supported in 8.0.",
issues.stream().collect(Collectors.joining(",")),
issues.size() > 1 ? "s" : ""
),
false,
null
);
}
return null;
}

/**
* Predicate for dynamic templates that contain `fields` within `fields` entries
* @param property A dynamic mapping template body
*/
static boolean containsMappingWithChainedMultiFields(Map<?, ?> property) {
if (property.containsKey("mapping")) {
Map<?, ?> mappings = (Map<?, ?>) property.get("mapping");
return containsChainedMultiFields(mappings);
}
return false;
}

static boolean containsChainedMultiFields(Map<?, ?> property) {
if (property.containsKey("fields")) {
Map<?, ?> fields = (Map<?, ?>) property.get("fields");
for (Object rawSubField : fields.values()) {
Expand Down

0 comments on commit 48cbeda

Please sign in to comment.