Skip to content

Commit

Permalink
[7.x][ML] Refactor ML mappings and templates into JSON resources (#51… (
Browse files Browse the repository at this point in the history 
#52353)

ML mappings and index templates have so far been created
programmatically. While this had its merits due to static typing,
there is consensus it would be clear to maintain those in json files.
In addition, we are going to adding ILM policies to these indices
and the component for a plugin to register ILM policies is
`IndexTemplateRegistry`. It expects the templates to be in resource
json files.

For the above reasons this commit refactors ML mappings and index
templates into json resource files that are registered via
`MlIndexTemplateRegistry`.

Backport of #51765
  • Loading branch information
@dimitris-athanasiou
dimitris-athanasiou committed Feb 14, 2020
1 parent cabc176 commit ad56802
Show file tree
Hide file tree
Showing 51 changed files with 1,705 additions and 2,107 deletions.
28 changes: 28 additions & 0 deletions x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java
View file
@@ -0,0 +1,28 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.ml;

import org.elasticsearch.Version;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.xpack.core.template.TemplateUtils;

import java.util.Collections;

public class MlConfigIndex {

private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";

private MlConfigIndex() {}

public static String mapping() {
return mapping(MapperService.SINGLE_MAPPING_NAME);
}

public static String mapping(String mappingType) {
return TemplateUtils.loadTemplate("/org/elasticsearch/xpack/core/ml/config_index_mappings.json",
Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE, Collections.singletonMap("xpack.ml.mapping_type", mappingType));
}
}
39 changes: 1 addition & 38 deletions x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java
View file
Expand Up @@ -5,50 +5,13 @@
*/
package org.elasticsearch.xpack.core.ml;

import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.core.ml.calendars.Calendar;
import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;

import java.io.IOException;

import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;

public final class MlMetaIndex {
/**
* Where to store the ml info in Elasticsearch - must match what's
* expected by kibana/engineAPI/app/directives/mlLogUsage.js
*/
public static final String INDEX_NAME = ".ml-meta";

private MlMetaIndex() {}

public static XContentBuilder docMapping() throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(SINGLE_MAPPING_NAME);
ElasticsearchMappings.addMetaInformation(builder);
ElasticsearchMappings.addDefaultMapping(builder);
builder.startObject(ElasticsearchMappings.PROPERTIES)
.startObject(Calendar.ID.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
.endObject()
.startObject(Calendar.JOB_IDS.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
.endObject()
.startObject(Calendar.DESCRIPTION.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
.endObject()
.startObject(ScheduledEvent.START_TIME.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
.endObject()
.startObject(ScheduledEvent.END_TIME.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
.endObject()
.endObject()
.endObject()
.endObject();
return builder;
}
private MlMetaIndex() {}
}
100 changes: 25 additions & 75 deletions ...lugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java
View file
Expand Up @@ -6,6 +6,7 @@
package org.elasticsearch.xpack.core.ml.annotations;

import org.elasticsearch.ResourceAlreadyExistsException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
Expand All @@ -15,19 +16,13 @@
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.AliasOrIndex;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.UnassignedInfo;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.core.ml.MachineLearningField;
import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.template.TemplateUtils;

import java.io.IOException;
import java.util.SortedMap;

import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
Expand All @@ -40,6 +35,8 @@ public class AnnotationIndex {
public static final String INDEX_NAME = ".ml-annotations-6";
public static final String INDEX_PATTERN = ".ml-annotations*";

private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";

/**
* Create the .ml-annotations index with correct mappings if it does not already
* exist. This index is read and written by the UI results views, so needs to
Expand All @@ -64,39 +61,26 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c
// Create the annotations index if it doesn't exist already.
if (mlLookup.containsKey(INDEX_NAME) == false) {

final TimeValue delayedNodeTimeOutSetting;
// Whether we are using native process is a good way to detect whether we are in dev / test mode:
if (MachineLearningField.AUTODETECT_PROCESS.get(settings)) {
delayedNodeTimeOutSetting = UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(settings);
} else {
delayedNodeTimeOutSetting = TimeValue.ZERO;
}

CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME);
try (XContentBuilder annotationsMapping = AnnotationIndex.annotationsMapping()) {
createIndexRequest.mapping(SINGLE_MAPPING_NAME, annotationsMapping);
createIndexRequest.settings(Settings.builder()
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delayedNodeTimeOutSetting));
createIndexRequest.mapping(SINGLE_MAPPING_NAME, annotationsMapping(), XContentType.JSON);
createIndexRequest.settings(Settings.builder()
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1"));

executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest,
ActionListener.<CreateIndexResponse>wrap(
r -> createAliasListener.onResponse(r.isAcknowledged()),
e -> {
// Possible that the index was created while the request was executing,
// so we need to handle that possibility
if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) {
// Create the alias
createAliasListener.onResponse(true);
} else {
finalListener.onFailure(e);
}
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest,
ActionListener.<CreateIndexResponse>wrap(
r -> createAliasListener.onResponse(r.isAcknowledged()),
e -> {
// Possible that the index was created while the request was executing,
// so we need to handle that possibility
if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) {
// Create the alias
createAliasListener.onResponse(true);
} else {
finalListener.onFailure(e);
}
), client.admin().indices()::create);
} catch (IOException e) {
finalListener.onFailure(e);
}
}
), client.admin().indices()::create);
return;
}

Expand All @@ -111,42 +95,8 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c
finalListener.onResponse(false);
}

public static XContentBuilder annotationsMapping() throws IOException {
XContentBuilder builder = jsonBuilder()
.startObject()
.startObject(SINGLE_MAPPING_NAME);
ElasticsearchMappings.addMetaInformation(builder);
builder.startObject(ElasticsearchMappings.PROPERTIES)
.startObject(Annotation.ANNOTATION.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.TEXT)
.endObject()
.startObject(Annotation.CREATE_TIME.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
.endObject()
.startObject(Annotation.CREATE_USERNAME.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
.endObject()
.startObject(Annotation.TIMESTAMP.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
.endObject()
.startObject(Annotation.END_TIMESTAMP.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
.endObject()
.startObject(Job.ID.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
.endObject()
.startObject(Annotation.MODIFIED_TIME.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
.endObject()
.startObject(Annotation.MODIFIED_USERNAME.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
.endObject()
.startObject(Annotation.TYPE.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
.endObject()
.endObject()
.endObject()
.endObject();
return builder;
public static String annotationsMapping() {
return TemplateUtils.loadTemplate("/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json",
Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE);
}
}
14 changes: 14 additions & 0 deletions .../src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java
View file
Expand Up @@ -6,6 +6,7 @@
package org.elasticsearch.xpack.core.ml.job.persistence;

import org.elasticsearch.ResourceAlreadyExistsException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
Expand All @@ -16,7 +17,9 @@
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.template.TemplateUtils;

import java.util.Arrays;
import java.util.Collections;
Expand All @@ -31,6 +34,9 @@ public final class AnomalyDetectorsIndex {

public static final int CONFIG_INDEX_MAX_RESULTS_WINDOW = 10_000;

private static final String RESULTS_MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
private static final String RESOURCE_PATH = "/org/elasticsearch/xpack/core/ml/anomalydetection/";

private AnomalyDetectorsIndex() {
}

Expand Down Expand Up @@ -144,4 +150,12 @@ public static void createStateIndexAndAliasIfNecessary(Client client, ClusterSta
}
}

public static String resultsMapping() {
return resultsMapping(MapperService.SINGLE_MAPPING_NAME);
}

public static String resultsMapping(String mappingType) {
return TemplateUtils.loadTemplate(RESOURCE_PATH + "results_index_mappings.json",
Version.CURRENT.toString(), RESULTS_MAPPINGS_VERSION_VARIABLE, Collections.singletonMap("xpack.ml.mapping_type", mappingType));
}
}

0 comments on commit ad56802

Please sign in to comment.