Skip to content

Commit

Permalink
Index Template: parse and validate mappings in template when template…
Browse files Browse the repository at this point in the history 
… puts

Share applying template with MetaDataCreateIndexService and MetaDataIndexTemplateService
Add some unit test
Collapse addMappingsToMapperService and move it to MapperService
Extract validateTemplate method
use expectThrows in testcase
Add TODO comment

Closes #2415
  • Loading branch information
@johtani
johtani committed Jun 1, 2016
1 parent 93c5a9d commit fd76291
Show file tree
Hide file tree
Showing 8 changed files with 359 additions and 113 deletions.
39 changes: 8 additions & 31 deletions core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,7 @@
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexNotFoundException;
Expand Down Expand Up @@ -220,7 +218,7 @@ public ClusterState execute(ClusterState currentState) throws Exception {
List<String> templateNames = new ArrayList<>();

for (Map.Entry<String, String> entry : request.mappings().entrySet()) {
mappings.put(entry.getKey(), parseMapping(entry.getValue()));
mappings.put(entry.getKey(), MapperService.parseMapping(entry.getValue()));
}

for (Map.Entry<String, Custom> entry : request.customs().entrySet()) {
Expand All @@ -232,9 +230,9 @@ public ClusterState execute(ClusterState currentState) throws Exception {
templateNames.add(template.getName());
for (ObjectObjectCursor<String, CompressedXContent> cursor : template.mappings()) {
if (mappings.containsKey(cursor.key)) {
XContentHelper.mergeDefaults(mappings.get(cursor.key), parseMapping(cursor.value.string()));
XContentHelper.mergeDefaults(mappings.get(cursor.key), MapperService.parseMapping(cursor.value.string()));
} else {
mappings.put(cursor.key, parseMapping(cursor.value.string()));
mappings.put(cursor.key, MapperService.parseMapping(cursor.value.string()));
}
}
// handle custom
Expand Down Expand Up @@ -315,26 +313,11 @@ public ClusterState execute(ClusterState currentState) throws Exception {
createdIndex = indexService.index();
// now add the mappings
MapperService mapperService = indexService.mapperService();
// first, add the default mapping
if (mappings.containsKey(MapperService.DEFAULT_MAPPING)) {
try {
mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(MapperService.DEFAULT_MAPPING)).string()), MapperService.MergeReason.MAPPING_UPDATE, request.updateAllTypes());
} catch (Exception e) {
removalReason = "failed on parsing default mapping on index creation";
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, MapperService.DEFAULT_MAPPING, e.getMessage());
}
}
for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) {
if (entry.getKey().equals(MapperService.DEFAULT_MAPPING)) {
continue;
}
try {
// apply the default here, its the first time we parse it
mapperService.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), MapperService.MergeReason.MAPPING_UPDATE, request.updateAllTypes());
} catch (Exception e) {
removalReason = "failed on parsing mappings on index creation";
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
}
try {
mapperService.merge(mappings, request.updateAllTypes());
} catch (MapperParsingException mpe) {
removalReason = "failed on parsing default mapping/mappings on index creation";
throw mpe;
}

final QueryShardContext queryShardContext = indexService.newQueryShardContext();
Expand Down Expand Up @@ -426,12 +409,6 @@ public ClusterState execute(ClusterState currentState) throws Exception {
});
}

private Map<String, Object> parseMapping(String mappingSource) throws Exception {
try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
return parser.map();
}
}

private List<IndexTemplateMetaData> findTemplates(CreateIndexClusterStateUpdateRequest request, ClusterState state, IndexTemplateFilter indexTemplateFilter) throws IOException {
List<IndexTemplateMetaData> templates = new ArrayList<>();
for (ObjectCursor<IndexTemplateMetaData> cursor : state.metaData().templates().values()) {
Expand Down
97 changes: 72 additions & 25 deletions core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,24 +19,33 @@
package org.elasticsearch.cluster.metadata;

import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.support.master.MasterNodeRequest;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateUpdateTask;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.ValidationException;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.NodeServicesProvider;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.indices.IndexTemplateAlreadyExistsException;
import org.elasticsearch.indices.IndexTemplateMissingException;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.InvalidIndexTemplateException;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
Expand All @@ -51,14 +60,18 @@ public class MetaDataIndexTemplateService extends AbstractComponent {

private final ClusterService clusterService;
private final AliasValidator aliasValidator;
private final IndicesService indicesService;
private final MetaDataCreateIndexService metaDataCreateIndexService;
private final NodeServicesProvider nodeServicesProvider;

@Inject
public MetaDataIndexTemplateService(Settings settings, ClusterService clusterService, MetaDataCreateIndexService metaDataCreateIndexService, AliasValidator aliasValidator) {
public MetaDataIndexTemplateService(Settings settings, ClusterService clusterService, MetaDataCreateIndexService metaDataCreateIndexService, AliasValidator aliasValidator, IndicesService indicesService, NodeServicesProvider nodeServicesProvider) {
super(settings);
this.clusterService = clusterService;
this.aliasValidator = aliasValidator;
this.indicesService = indicesService;
this.metaDataCreateIndexService = metaDataCreateIndexService;
this.nodeServicesProvider = nodeServicesProvider;
}

public void removeTemplates(final RemoveRequest request, final RemoveListener listener) {
Expand Down Expand Up @@ -126,28 +139,7 @@ public void putTemplate(final PutRequest request, final PutListener listener) {
return;
}

IndexTemplateMetaData.Builder templateBuilder;
try {
templateBuilder = IndexTemplateMetaData.builder(request.name);
templateBuilder.order(request.order);
templateBuilder.template(request.template);
templateBuilder.settings(request.settings);
for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
templateBuilder.putMapping(entry.getKey(), entry.getValue());
}
for (Alias alias : request.aliases) {
AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
.indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build();
templateBuilder.putAlias(aliasMetaData);
}
for (Map.Entry<String, IndexMetaData.Custom> entry : request.customs.entrySet()) {
templateBuilder.putCustom(entry.getKey(), entry.getValue());
}
} catch (Throwable e) {
listener.onFailure(e);
return;
}
final IndexTemplateMetaData template = templateBuilder.build();
final IndexTemplateMetaData.Builder templateBuilder = IndexTemplateMetaData.builder(request.name);

clusterService.submitStateUpdateTask("create-index-template [" + request.name + "], cause [" + request.cause + "]",
new ClusterStateUpdateTask(Priority.URGENT) {
Expand All @@ -163,22 +155,77 @@ public void onFailure(String source, Throwable t) {
}

@Override
public ClusterState execute(ClusterState currentState) {
public ClusterState execute(ClusterState currentState) throws Exception {
if (request.create && currentState.metaData().templates().containsKey(request.name)) {
throw new IndexTemplateAlreadyExistsException(request.name);
}

validateAndAddTemplate(request, templateBuilder, indicesService, nodeServicesProvider, metaDataCreateIndexService);

for (Alias alias : request.aliases) {
AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
.indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build();
templateBuilder.putAlias(aliasMetaData);
}
for (Map.Entry<String, IndexMetaData.Custom> entry : request.customs.entrySet()) {
templateBuilder.putCustom(entry.getKey(), entry.getValue());
}
IndexTemplateMetaData template = templateBuilder.build();

MetaData.Builder builder = MetaData.builder(currentState.metaData()).put(template);

return ClusterState.builder(currentState).metaData(builder).build();
}

@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
listener.onResponse(new PutResponse(true, template));
listener.onResponse(new PutResponse(true, templateBuilder.build()));
}
});
}

private static void validateAndAddTemplate(final PutRequest request, IndexTemplateMetaData.Builder templateBuilder, IndicesService indicesService,
NodeServicesProvider nodeServicesProvider, MetaDataCreateIndexService metaDataCreateIndexService) throws Exception {
Index createdIndex = null;
final String temporaryIndexName = UUIDs.randomBase64UUID();
try {

//create index service for parsing and validating "mappings"
Settings dummySettings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(request.settings)
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
.build();

final IndexMetaData tmpIndexMetadata = IndexMetaData.builder(temporaryIndexName).settings(dummySettings).build();
IndexService dummyIndexService = indicesService.createIndex(nodeServicesProvider, tmpIndexMetadata, Collections.emptyList());
createdIndex = dummyIndexService.index();

templateBuilder.order(request.order);
templateBuilder.template(request.template);
templateBuilder.settings(request.settings);

Map<String, Map<String, Object>> mappingsForValidation = new HashMap<>();
for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
try {
templateBuilder.putMapping(entry.getKey(), entry.getValue());
} catch (Exception e) {
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
}
mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(entry.getValue()));
}

dummyIndexService.mapperService().merge(mappingsForValidation, false);

} finally {
if (createdIndex != null) {
indicesService.removeIndex(createdIndex, " created for parsing template mapping");
}
}
}

private void validate(PutRequest request) {
List<String> validationErrors = new ArrayList<>();
if (request.name.contains(" ")) {
Expand Down
31 changes: 31 additions & 0 deletions core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisService;
Expand Down Expand Up @@ -175,6 +177,35 @@ public DocumentMapperParser documentMapperParser() {
return this.documentParser;
}

public static Map<String, Object> parseMapping(String mappingSource) throws Exception {
try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
return parser.map();
}
}

//TODO: make this atomic
public void merge(Map<String, Map<String, Object>> mappings, boolean updateAllTypes) throws MapperParsingException {
// first, add the default mapping
if (mappings.containsKey(DEFAULT_MAPPING)) {
try {
this.merge(DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(DEFAULT_MAPPING)).string()), MergeReason.MAPPING_UPDATE, updateAllTypes);
} catch (Exception e) {
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, DEFAULT_MAPPING, e.getMessage());
}
}
for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) {
if (entry.getKey().equals(DEFAULT_MAPPING)) {
continue;
}
try {
// apply the default here, its the first time we parse it
this.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), MergeReason.MAPPING_UPDATE, updateAllTypes);
} catch (Exception e) {
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
}
}
}

public DocumentMapper merge(String type, CompressedXContent mappingSource, MergeReason reason, boolean updateAllTypes) {
if (DEFAULT_MAPPING.equals(type)) {
// verify we can parse it
Expand Down

0 comments on commit fd76291

Please sign in to comment.