From ad56802ac606b30a66f9e037eba0a1a2e6e32317 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Fri, 14 Feb 2020 17:16:06 +0200 Subject: [PATCH] =?UTF-8?q?[7.x][ML]=20Refactor=20ML=20mappings=20and=20te?= =?UTF-8?q?mplates=20into=20JSON=20resources=20(#51=E2=80=A6=20(#52353)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ML mappings and index templates have so far been created programmatically. While this had its merits due to static typing, there is consensus it would be clear to maintain those in json files. In addition, we are going to adding ILM policies to these indices and the component for a plugin to register ILM policies is `IndexTemplateRegistry`. It expects the templates to be in resource json files. For the above reasons this commit refactors ML mappings and index templates into json resource files that are registered via `MlIndexTemplateRegistry`. Backport of #51765 --- .../xpack/core/ml/MlConfigIndex.java | 28 + .../xpack/core/ml/MlMetaIndex.java | 39 +- .../core/ml/annotations/AnnotationIndex.java | 100 +- .../persistence/AnomalyDetectorsIndex.java | 14 + .../persistence/ElasticsearchMappings.java | 1129 +---------------- .../ml/job/results/ReservedFieldNames.java | 1 + ...itorField.java => NotificationsIndex.java} | 4 +- .../exporter/MonitoringTemplateUtils.java | 3 +- .../core/template/IndexTemplateConfig.java | 30 +- .../core/template/IndexTemplateRegistry.java | 15 + .../xpack/core/template/TemplateUtils.java | 51 +- .../core/ml/annotations_index_mappings.json | 36 + .../results_index_mappings.json | 478 +++++++ .../results_index_template.json | 19 + .../state_index_template.json | 21 + .../xpack/core/ml/config_index_mappings.json | 364 ++++++ .../xpack/core/ml/config_index_template.json | 15 + .../core/ml/inference_index_template.json | 72 ++ .../xpack/core/ml/meta_index_template.json | 47 + .../core/ml/notifications_index_template.json | 46 + .../ElasticsearchMappingsTests.java | 57 +- .../authz/store/ReservedRolesStoreTests.java | 6 +- .../core/template/TemplateUtilsTests.java | 75 +- .../template_with_variables-test.json | 14 + .../xpack/logstash/Logstash.java | 6 +- .../ml/integration/DatafeedJobsRestIT.java | 6 +- .../ml/integration/DeleteExpiredDataIT.java | 6 +- .../ml/integration/DetectionRulesIT.java | 4 +- ...NativeDataFrameAnalyticsIntegTestCase.java | 8 +- .../ml/integration/ScheduledEventsIT.java | 6 +- .../xpack/ml/MachineLearning.java | 114 +- .../xpack/ml/MlConfigMigrator.java | 5 +- .../xpack/ml/MlIndexTemplateRegistry.java | 114 ++ .../TransportPutDataFrameAnalyticsAction.java | 3 +- .../ml/action/TransportPutDatafeedAction.java | 3 +- .../persistence/InferenceInternalIndex.java | 134 -- .../xpack/ml/job/JobManager.java | 5 +- .../job/persistence/JobResultsProvider.java | 77 +- .../autodetect/AutodetectProcessManager.java | 2 +- .../AnomalyDetectionAuditor.java | 4 +- .../DataFrameAnalyticsAuditor.java | 4 +- .../ml/notifications/InferenceAuditor.java | 4 +- .../action/TransportOpenJobActionTests.java | 4 +- .../ml/integration/JobResultsProviderIT.java | 111 +- .../persistence/JobResultsProviderTests.java | 209 +-- .../ml/job/persistence/MockClientBuilder.java | 239 +--- .../support/SecurityIndexManager.java | 5 +- .../support/SecurityIndexManagerTests.java | 12 +- .../xpack/test/rest/XPackRestIT.java | 4 +- ...nfigIndexMappingsFullClusterRestartIT.java | 58 +- .../upgrades/MlMappingsUpgradeIT.java | 1 + 51 files changed, 1705 insertions(+), 2107 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/{AuditorField.java => NotificationsIndex.java} (83%) create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json create mode 100644 x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json create mode 100644 x-pack/plugin/core/src/test/resources/template_with_variables-test.json create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java delete mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/InferenceInternalIndex.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java new file mode 100644 index 0000000000000..1847c14c5e5da --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml; + +import org.elasticsearch.Version; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.xpack.core.template.TemplateUtils; + +import java.util.Collections; + +public class MlConfigIndex { + + private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; + + private MlConfigIndex() {} + + public static String mapping() { + return mapping(MapperService.SINGLE_MAPPING_NAME); + } + + public static String mapping(String mappingType) { + return TemplateUtils.loadTemplate("/org/elasticsearch/xpack/core/ml/config_index_mappings.json", + Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE, Collections.singletonMap("xpack.ml.mapping_type", mappingType)); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java index e4f82ad53fed1..54ab18bd7537d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java @@ -5,16 +5,6 @@ */ package org.elasticsearch.xpack.core.ml; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.ml.calendars.Calendar; -import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent; -import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; - -import java.io.IOException; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; - public final class MlMetaIndex { /** * Where to store the ml info in Elasticsearch - must match what's @@ -22,33 +12,6 @@ public final class MlMetaIndex { */ public static final String INDEX_NAME = ".ml-meta"; - private MlMetaIndex() {} - public static XContentBuilder docMapping() throws IOException { - XContentBuilder builder = jsonBuilder(); - builder.startObject(); - builder.startObject(SINGLE_MAPPING_NAME); - ElasticsearchMappings.addMetaInformation(builder); - ElasticsearchMappings.addDefaultMapping(builder); - builder.startObject(ElasticsearchMappings.PROPERTIES) - .startObject(Calendar.ID.getPreferredName()) - .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD) - .endObject() - .startObject(Calendar.JOB_IDS.getPreferredName()) - .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD) - .endObject() - .startObject(Calendar.DESCRIPTION.getPreferredName()) - .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD) - .endObject() - .startObject(ScheduledEvent.START_TIME.getPreferredName()) - .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE) - .endObject() - .startObject(ScheduledEvent.END_TIME.getPreferredName()) - .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE) - .endObject() - .endObject() - .endObject() - .endObject(); - return builder; - } + private MlMetaIndex() {} } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java index e4b00b5ce096f..c229b588fc06f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.ml.annotations; import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; @@ -15,19 +16,13 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasOrIndex; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.ml.MachineLearningField; -import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.template.TemplateUtils; -import java.io.IOException; import java.util.SortedMap; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; @@ -40,6 +35,8 @@ public class AnnotationIndex { public static final String INDEX_NAME = ".ml-annotations-6"; public static final String INDEX_PATTERN = ".ml-annotations*"; + private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; + /** * Create the .ml-annotations index with correct mappings if it does not already * exist. This index is read and written by the UI results views, so needs to @@ -64,39 +61,26 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c // Create the annotations index if it doesn't exist already. if (mlLookup.containsKey(INDEX_NAME) == false) { - final TimeValue delayedNodeTimeOutSetting; - // Whether we are using native process is a good way to detect whether we are in dev / test mode: - if (MachineLearningField.AUTODETECT_PROCESS.get(settings)) { - delayedNodeTimeOutSetting = UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(settings); - } else { - delayedNodeTimeOutSetting = TimeValue.ZERO; - } - CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME); - try (XContentBuilder annotationsMapping = AnnotationIndex.annotationsMapping()) { - createIndexRequest.mapping(SINGLE_MAPPING_NAME, annotationsMapping); - createIndexRequest.settings(Settings.builder() - .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1") - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1") - .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delayedNodeTimeOutSetting)); + createIndexRequest.mapping(SINGLE_MAPPING_NAME, annotationsMapping(), XContentType.JSON); + createIndexRequest.settings(Settings.builder() + .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")); - executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest, - ActionListener.wrap( - r -> createAliasListener.onResponse(r.isAcknowledged()), - e -> { - // Possible that the index was created while the request was executing, - // so we need to handle that possibility - if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) { - // Create the alias - createAliasListener.onResponse(true); - } else { - finalListener.onFailure(e); - } + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest, + ActionListener.wrap( + r -> createAliasListener.onResponse(r.isAcknowledged()), + e -> { + // Possible that the index was created while the request was executing, + // so we need to handle that possibility + if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) { + // Create the alias + createAliasListener.onResponse(true); + } else { + finalListener.onFailure(e); } - ), client.admin().indices()::create); - } catch (IOException e) { - finalListener.onFailure(e); - } + } + ), client.admin().indices()::create); return; } @@ -111,42 +95,8 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c finalListener.onResponse(false); } - public static XContentBuilder annotationsMapping() throws IOException { - XContentBuilder builder = jsonBuilder() - .startObject() - .startObject(SINGLE_MAPPING_NAME); - ElasticsearchMappings.addMetaInformation(builder); - builder.startObject(ElasticsearchMappings.PROPERTIES) - .startObject(Annotation.ANNOTATION.getPreferredName()) - .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.TEXT) - .endObject() - .startObject(Annotation.CREATE_TIME.getPreferredName()) - .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE) - .endObject() - .startObject(Annotation.CREATE_USERNAME.getPreferredName()) - .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD) - .endObject() - .startObject(Annotation.TIMESTAMP.getPreferredName()) - .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE) - .endObject() - .startObject(Annotation.END_TIMESTAMP.getPreferredName()) - .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE) - .endObject() - .startObject(Job.ID.getPreferredName()) - .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD) - .endObject() - .startObject(Annotation.MODIFIED_TIME.getPreferredName()) - .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE) - .endObject() - .startObject(Annotation.MODIFIED_USERNAME.getPreferredName()) - .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD) - .endObject() - .startObject(Annotation.TYPE.getPreferredName()) - .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD) - .endObject() - .endObject() - .endObject() - .endObject(); - return builder; + public static String annotationsMapping() { + return TemplateUtils.loadTemplate("/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json", + Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java index 7d4e2367cceff..b74a80563fb11 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.ml.job.persistence; import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; @@ -16,7 +17,9 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.template.TemplateUtils; import java.util.Arrays; import java.util.Collections; @@ -31,6 +34,9 @@ public final class AnomalyDetectorsIndex { public static final int CONFIG_INDEX_MAX_RESULTS_WINDOW = 10_000; + private static final String RESULTS_MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; + private static final String RESOURCE_PATH = "/org/elasticsearch/xpack/core/ml/anomalydetection/"; + private AnomalyDetectorsIndex() { } @@ -144,4 +150,12 @@ public static void createStateIndexAndAliasIfNecessary(Client client, ClusterSta } } + public static String resultsMapping() { + return resultsMapping(MapperService.SINGLE_MAPPING_NAME); + } + + public static String resultsMapping(String mappingType) { + return TemplateUtils.loadTemplate(RESOURCE_PATH + "results_index_mappings.json", + Version.CURRENT.toString(), RESULTS_MAPPINGS_VERSION_VARIABLE, Collections.singletonMap("xpack.ml.mapping_type", mappingType)); + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java index a90f0d919707f..37b13c9a725c5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java @@ -20,60 +20,16 @@ import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.plugins.MapperPlugin; -import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats; -import org.elasticsearch.xpack.core.ml.datafeed.DelayedDataCheckConfig; -import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; -import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest; -import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsSource; -import org.elasticsearch.xpack.core.ml.dataframe.analyses.BoostedTreeParams; -import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification; -import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetection; -import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression; -import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; -import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; -import org.elasticsearch.xpack.core.ml.job.config.DataDescription; -import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; -import org.elasticsearch.xpack.core.ml.job.config.Detector; -import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig; -import org.elasticsearch.xpack.core.ml.job.config.Operator; -import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; -import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; -import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; -import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; -import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshotField; -import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats; -import org.elasticsearch.xpack.core.ml.job.results.AnomalyCause; -import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord; -import org.elasticsearch.xpack.core.ml.job.results.Bucket; -import org.elasticsearch.xpack.core.ml.job.results.BucketInfluencer; -import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition; -import org.elasticsearch.xpack.core.ml.job.results.Forecast; -import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats; -import org.elasticsearch.xpack.core.ml.job.results.GeoResults; -import org.elasticsearch.xpack.core.ml.job.results.Influence; -import org.elasticsearch.xpack.core.ml.job.results.Influencer; -import org.elasticsearch.xpack.core.ml.job.results.ModelPlot; -import org.elasticsearch.xpack.core.ml.job.results.ReservedFieldNames; -import org.elasticsearch.xpack.core.ml.job.results.Result; -import org.elasticsearch.xpack.core.ml.notifications.AnomalyDetectionAuditMessage; -import org.elasticsearch.xpack.core.ml.utils.ExponentialAverageCalculationContext; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; @@ -109,7 +65,6 @@ public class ElasticsearchMappings { public static final String PROPERTIES = "properties"; public static final String TYPE = "type"; public static final String DYNAMIC = "dynamic"; - public static final String FIELDS = "fields"; /** * Name of the custom 'all' field for results @@ -132,1091 +87,16 @@ public class ElasticsearchMappings { public static final String BOOLEAN = "boolean"; public static final String DATE = "date"; public static final String DOUBLE = "double"; - public static final String GEO_POINT = "geo_point"; public static final String INTEGER = "integer"; public static final String KEYWORD = "keyword"; public static final String LONG = "long"; public static final String TEXT = "text"; - static final String RAW = "raw"; - private static final Logger logger = LogManager.getLogger(ElasticsearchMappings.class); private ElasticsearchMappings() { } - public static XContentBuilder configMapping() throws IOException { - return configMapping(SINGLE_MAPPING_NAME); - } - - public static XContentBuilder configMapping(String mappingType) throws IOException { - XContentBuilder builder = jsonBuilder(); - builder.startObject(); - builder.startObject(mappingType); - addMetaInformation(builder); - addDefaultMapping(builder); - builder.startObject(PROPERTIES); - - addJobConfigFields(builder); - addDatafeedConfigFields(builder); - addDataFrameAnalyticsFields(builder); - - builder.endObject() - .endObject() - .endObject(); - return builder; - } - - public static void addJobConfigFields(XContentBuilder builder) throws IOException { - - builder.startObject(CONFIG_TYPE) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Job.ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Job.JOB_TYPE.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Job.JOB_VERSION.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Job.GROUPS.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Job.ANALYSIS_CONFIG.getPreferredName()) - .startObject(PROPERTIES) - .startObject(AnalysisConfig.BUCKET_SPAN.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnalysisConfig.CATEGORIZATION_FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnalysisConfig.CATEGORIZATION_FILTERS.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnalysisConfig.CATEGORIZATION_ANALYZER.getPreferredName()) - .field(ENABLED, false) - .endObject() - .startObject(AnalysisConfig.LATENCY.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnalysisConfig.SUMMARY_COUNT_FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnalysisConfig.DETECTORS.getPreferredName()) - .startObject(PROPERTIES) - .startObject(Detector.DETECTOR_DESCRIPTION_FIELD.getPreferredName()) - .field(TYPE, TEXT) - .endObject() - .startObject(Detector.FUNCTION_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Detector.FIELD_NAME_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Detector.BY_FIELD_NAME_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Detector.OVER_FIELD_NAME_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Detector.PARTITION_FIELD_NAME_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Detector.USE_NULL_FIELD.getPreferredName()) - .field(TYPE, BOOLEAN) - .endObject() - .startObject(Detector.EXCLUDE_FREQUENT_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Detector.CUSTOM_RULES_FIELD.getPreferredName()) - .field(TYPE, NESTED) - .startObject(PROPERTIES) - .startObject(DetectionRule.ACTIONS_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - // RuleScope is a map - .startObject(DetectionRule.SCOPE_FIELD.getPreferredName()) - .field(ENABLED, false) - .endObject() - .startObject(DetectionRule.CONDITIONS_FIELD.getPreferredName()) - .field(TYPE, NESTED) - .startObject(PROPERTIES) - .startObject(RuleCondition.APPLIES_TO_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Operator.OPERATOR_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(RuleCondition.VALUE_FIELD.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - .startObject(Detector.DETECTOR_INDEX.getPreferredName()) - .field(TYPE, INTEGER) - .endObject() - .endObject() - .endObject() - - .startObject(AnalysisConfig.INFLUENCERS.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnalysisConfig.MULTIVARIATE_BY_FIELDS.getPreferredName()) - .field(TYPE, BOOLEAN) - .endObject() - .endObject() - .endObject() - - .startObject(Job.ANALYSIS_LIMITS.getPreferredName()) - .startObject(PROPERTIES) - .startObject(AnalysisLimits.MODEL_MEMORY_LIMIT.getPreferredName()) - .field(TYPE, KEYWORD) // TODO Should be a ByteSizeValue - .endObject() - .startObject(AnalysisLimits.CATEGORIZATION_EXAMPLES_LIMIT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .endObject() - .endObject() - - .startObject(Job.CREATE_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - - .startObject(Job.CUSTOM_SETTINGS.getPreferredName()) - // Custom settings are an untyped map - .field(ENABLED, false) - .endObject() - - .startObject(Job.DATA_DESCRIPTION.getPreferredName()) - .startObject(PROPERTIES) - .startObject(DataDescription.FORMAT_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(DataDescription.TIME_FIELD_NAME_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(DataDescription.TIME_FORMAT_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(DataDescription.FIELD_DELIMITER_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(DataDescription.QUOTE_CHARACTER_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .endObject() - .endObject() - - .startObject(Job.DESCRIPTION.getPreferredName()) - .field(TYPE, TEXT) - .endObject() - .startObject(Job.FINISHED_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - - .startObject(Job.MODEL_PLOT_CONFIG.getPreferredName()) - .startObject(PROPERTIES) - .startObject(ModelPlotConfig.ENABLED_FIELD.getPreferredName()) - .field(TYPE, BOOLEAN) - .endObject() - .startObject(ModelPlotConfig.TERMS_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .endObject() - .endObject() - - .startObject(Job.RENORMALIZATION_WINDOW_DAYS.getPreferredName()) - .field(TYPE, LONG) // TODO should be TimeValue - .endObject() - .startObject(Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Job.MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName()) - .field(TYPE, LONG) // TODO should be TimeValue - .endObject() - .startObject(Job.RESULTS_RETENTION_DAYS.getPreferredName()) - .field(TYPE, LONG) // TODO should be TimeValue - .endObject() - .startObject(Job.MODEL_SNAPSHOT_ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Job.MODEL_SNAPSHOT_MIN_VERSION.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Job.RESULTS_INDEX_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject(); - } - - public static void addDatafeedConfigFields(XContentBuilder builder) throws IOException { - builder.startObject(DatafeedConfig.ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(DatafeedConfig.QUERY_DELAY.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(DatafeedConfig.FREQUENCY.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(DatafeedConfig.INDICES.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(DatafeedConfig.QUERY.getPreferredName()) - .field(ENABLED, false) - .endObject() - .startObject(DatafeedConfig.SCROLL_SIZE.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DatafeedConfig.AGGREGATIONS.getPreferredName()) - .field(ENABLED, false) - .endObject() - .startObject(DatafeedConfig.SCRIPT_FIELDS.getPreferredName()) - .field(ENABLED, false) - .endObject() - .startObject(DatafeedConfig.CHUNKING_CONFIG.getPreferredName()) - .startObject(PROPERTIES) - .startObject(ChunkingConfig.MODE_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(ChunkingConfig.TIME_SPAN_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .endObject() - .endObject() - .startObject(DatafeedConfig.DELAYED_DATA_CHECK_CONFIG.getPreferredName()) - .startObject(PROPERTIES) - .startObject(DelayedDataCheckConfig.ENABLED.getPreferredName()) - .field(TYPE, BOOLEAN) - .endObject() - .startObject(DelayedDataCheckConfig.CHECK_WINDOW.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .endObject() - .endObject() - .startObject(DatafeedConfig.HEADERS.getPreferredName()) - .field(ENABLED, false) - .endObject(); - } - - /** - * {@link DataFrameAnalyticsConfig} mapping. - * Does not include mapping for CREATE_TIME as this mapping is added by {@link #addJobConfigFields} method. - */ - public static void addDataFrameAnalyticsFields(XContentBuilder builder) throws IOException { - builder.startObject(DataFrameAnalyticsConfig.ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(DataFrameAnalyticsConfig.SOURCE.getPreferredName()) - .startObject(PROPERTIES) - .startObject(DataFrameAnalyticsSource.INDEX.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(DataFrameAnalyticsSource.QUERY.getPreferredName()) - .field(ENABLED, false) - .endObject() - .startObject(DataFrameAnalyticsSource._SOURCE.getPreferredName()) - .field(ENABLED, false) - .endObject() - .endObject() - .endObject() - .startObject(DataFrameAnalyticsConfig.DEST.getPreferredName()) - .startObject(PROPERTIES) - .startObject(DataFrameAnalyticsDest.INDEX.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(DataFrameAnalyticsDest.RESULTS_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .endObject() - .endObject() - .startObject(DataFrameAnalyticsConfig.ANALYZED_FIELDS.getPreferredName()) - .field(ENABLED, false) - .endObject() - .startObject(DataFrameAnalyticsConfig.ANALYSIS.getPreferredName()) - .startObject(PROPERTIES) - .startObject(OutlierDetection.NAME.getPreferredName()) - .startObject(PROPERTIES) - .startObject(OutlierDetection.N_NEIGHBORS.getPreferredName()) - .field(TYPE, INTEGER) - .endObject() - .startObject(OutlierDetection.METHOD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(OutlierDetection.FEATURE_INFLUENCE_THRESHOLD.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .endObject() - .endObject() - .startObject(Regression.NAME.getPreferredName()) - .startObject(PROPERTIES) - .startObject(Regression.DEPENDENT_VARIABLE.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(BoostedTreeParams.LAMBDA.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(BoostedTreeParams.GAMMA.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(BoostedTreeParams.ETA.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(BoostedTreeParams.MAXIMUM_NUMBER_TREES.getPreferredName()) - .field(TYPE, INTEGER) - .endObject() - .startObject(BoostedTreeParams.FEATURE_BAG_FRACTION.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(BoostedTreeParams.NUM_TOP_FEATURE_IMPORTANCE_VALUES.getPreferredName()) - .field(TYPE, INTEGER) - .endObject() - .startObject(Regression.PREDICTION_FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Regression.TRAINING_PERCENT.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .endObject() - .endObject() - .startObject(Classification.NAME.getPreferredName()) - .startObject(PROPERTIES) - .startObject(Classification.DEPENDENT_VARIABLE.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(BoostedTreeParams.LAMBDA.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(BoostedTreeParams.GAMMA.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(BoostedTreeParams.ETA.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(BoostedTreeParams.MAXIMUM_NUMBER_TREES.getPreferredName()) - .field(TYPE, INTEGER) - .endObject() - .startObject(BoostedTreeParams.FEATURE_BAG_FRACTION.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(BoostedTreeParams.NUM_TOP_FEATURE_IMPORTANCE_VALUES.getPreferredName()) - .field(TYPE, INTEGER) - .endObject() - .startObject(Classification.PREDICTION_FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Classification.NUM_TOP_CLASSES.getPreferredName()) - .field(TYPE, INTEGER) - .endObject() - .startObject(Classification.TRAINING_PERCENT.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - // re-used: CREATE_TIME - .startObject(DataFrameAnalyticsConfig.VERSION.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject(); - } - - /** - * Creates a default mapping which has a dynamic template that - * treats all dynamically added fields as keywords. This is needed - * so that the per-job term fields will not be automatically added - * as fields of type 'text' to the index mappings of newly rolled indices. - * - * @throws IOException On write error - */ - public static void addDefaultMapping(XContentBuilder builder) throws IOException { - builder.startArray("dynamic_templates") - .startObject() - .startObject("strings_as_keywords") - .field("match", "*") - .startObject("mapping") - .field(TYPE, KEYWORD) - .endObject() - .endObject() - .endObject() - .endArray(); - } - - /** - * Inserts "_meta" containing useful information like the version into the mapping - * template. - * - * @param builder The builder for the mappings - * @throws IOException On write error - */ - public static void addMetaInformation(XContentBuilder builder) throws IOException { - builder.startObject("_meta") - .field("version", Version.CURRENT) - .endObject(); - } - - public static XContentBuilder resultsMapping(String mappingType) throws IOException { - return resultsMapping(mappingType, Collections.emptyList()); - } - - public static XContentBuilder resultsMapping(String mappingType, Collection extraTermFields) throws IOException { - XContentBuilder builder = jsonBuilder(); - builder.startObject(); - builder.startObject(mappingType); - addMetaInformation(builder); - addDefaultMapping(builder); - builder.startObject(PROPERTIES); - - // Add result all field for easy searches in kibana - builder.startObject(ALL_FIELD_VALUES) - .field(TYPE, TEXT) - .field(ANALYZER, WHITESPACE) - .endObject(); - - builder.startObject(Job.ID.getPreferredName()) - .field(TYPE, KEYWORD) - .field(COPY_TO, ALL_FIELD_VALUES) - .endObject(); - - builder.startObject(Result.TIMESTAMP.getPreferredName()) - .field(TYPE, DATE) - .endObject(); - - addResultsMapping(builder); - addCategoryDefinitionMapping(builder); - addDataCountsMapping(builder); - addTimingStatsExceptBucketCountMapping(builder); - addDatafeedTimingStats(builder); - addModelSnapshotMapping(builder); - - addTermFields(builder, extraTermFields); - - // end properties - builder.endObject(); - // end type - builder.endObject(); - // end mapping - builder.endObject(); - - - return builder; - } - - /** - * Create the Elasticsearch mapping for results objects - * {@link Bucket}s, {@link AnomalyRecord}s, {@link Influencer} and - * {@link BucketInfluencer} - * - * The mapping has a custom all field containing the *_FIELD_VALUE fields - * e.g. BY_FIELD_VALUE, OVER_FIELD_VALUE, etc. The custom all field {@link #ALL_FIELD_VALUES} - * must be set in the index settings. A custom all field is preferred over the usual - * '_all' field as most fields do not belong in '_all', disabling '_all' and - * using a custom all field simplifies the mapping. - * - * These fields are copied to the custom all field - *
    - *
  • by_field_value
    • - *
  • partition_field_value
    • - *
  • over_field_value
    • - *
  • AnomalyCause.correlated_by_field_value
    • - *
  • AnomalyCause.by_field_value
    • - *
  • AnomalyCause.partition_field_value
    • - *
  • AnomalyCause.over_field_value
    • - *
  • AnomalyRecord.Influencers.influencer_field_values
    • - *
  • Influencer.influencer_field_value
    • - *
- * - * @throws IOException On write error - */ - private static void addResultsMapping(XContentBuilder builder) throws IOException { - builder.startObject(Result.RESULT_TYPE.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Bucket.ANOMALY_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(Result.IS_INTERIM.getPreferredName()) - .field(TYPE, BOOLEAN) - .endObject() - .startObject(Bucket.EVENT_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(Bucket.BUCKET_SPAN.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(Bucket.PROCESSING_TIME_MS.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(Bucket.SCHEDULED_EVENTS.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - - .startObject(Bucket.BUCKET_INFLUENCERS.getPreferredName()) - .field(TYPE, NESTED) - .startObject(PROPERTIES) - .startObject(Job.ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Result.RESULT_TYPE.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(BucketInfluencer.INFLUENCER_FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(BucketInfluencer.INITIAL_ANOMALY_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(BucketInfluencer.ANOMALY_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(BucketInfluencer.PROBABILITY.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(Result.TIMESTAMP.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(BucketInfluencer.BUCKET_SPAN.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(Result.IS_INTERIM.getPreferredName()) - .field(TYPE, BOOLEAN) - .endObject() - .endObject() - .endObject() - - // Model Plot Output - .startObject(ModelPlot.MODEL_FEATURE.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(ModelPlot.MODEL_LOWER.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(ModelPlot.MODEL_UPPER.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(ModelPlot.MODEL_MEDIAN.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject(); - - addForecastFieldsToMapping(builder); - addAnomalyRecordFieldsToMapping(builder); - addInfluencerFieldsToMapping(builder); - addModelSizeStatsFieldsToMapping(builder); - } - - /** - * Generate a keyword mapping for {@code termFields} for the default type - * {@link org.elasticsearch.index.mapper.MapperService#SINGLE_MAPPING_NAME} - * - * If the returned mapping is used in index creation and the new index has a matching template - * then the mapping type ({@link org.elasticsearch.index.mapper.MapperService#SINGLE_MAPPING_NAME}) - * must match the mapping type of the template otherwise the mappings will not be merged correctly. - * - * @param termFields Fields to generate mapping for - * @return The mapping - */ - public static XContentBuilder termFieldsMapping(Collection termFields) { - try { - XContentBuilder builder = jsonBuilder().startObject(); - builder.startObject(SINGLE_MAPPING_NAME); - builder.startObject(PROPERTIES); - addTermFields(builder, termFields); - builder.endObject(); - builder.endObject(); - return builder.endObject(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static void addTermFields(XContentBuilder builder, Collection termFields) throws IOException { - for (String fieldName : termFields) { - if (ReservedFieldNames.isValidFieldName(fieldName)) { - builder.startObject(fieldName).field(TYPE, KEYWORD).endObject(); - } - } - } - - private static void addForecastFieldsToMapping(XContentBuilder builder) throws IOException { - - // Forecast Output - builder.startObject(Forecast.FORECAST_LOWER.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(Forecast.FORECAST_UPPER.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(Forecast.FORECAST_PREDICTION.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(Forecast.FORECAST_ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject(); - - // Forecast Stats Output - // re-used: TIMESTAMP, PROCESSING_TIME_MS, PROCESSED_RECORD_COUNT, LATEST_RECORD_TIME - builder.startObject(ForecastRequestStats.START_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(ForecastRequestStats.END_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(ForecastRequestStats.CREATE_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(ForecastRequestStats.EXPIRY_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(ForecastRequestStats.MESSAGES.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(ForecastRequestStats.PROGRESS.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(ForecastRequestStats.STATUS.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(ForecastRequestStats.MEMORY_USAGE.getPreferredName()) - .field(TYPE, LONG) - .endObject(); - } - - /** - * AnomalyRecord fields to be added under the 'properties' section of the mapping - * @param builder Add properties to this builder - * @throws IOException On write error - */ - private static void addAnomalyRecordFieldsToMapping(XContentBuilder builder) throws IOException { - builder.startObject(Detector.DETECTOR_INDEX.getPreferredName()) - .field(TYPE, INTEGER) - .endObject() - .startObject(AnomalyRecord.ACTUAL.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(AnomalyRecord.TYPICAL.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(AnomalyRecord.PROBABILITY.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(AnomalyRecord.MULTI_BUCKET_IMPACT.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(AnomalyRecord.FUNCTION.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyRecord.FUNCTION_DESCRIPTION.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyRecord.BY_FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyRecord.BY_FIELD_VALUE.getPreferredName()) - .field(TYPE, KEYWORD) - .field(COPY_TO, ALL_FIELD_VALUES) - .endObject() - .startObject(AnomalyRecord.FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyRecord.PARTITION_FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName()) - .field(TYPE, KEYWORD) - .field(COPY_TO, ALL_FIELD_VALUES) - .endObject() - .startObject(AnomalyRecord.OVER_FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyRecord.OVER_FIELD_VALUE.getPreferredName()) - .field(TYPE, KEYWORD) - .field(COPY_TO, ALL_FIELD_VALUES) - .endObject() - .startObject(AnomalyRecord.RECORD_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(AnomalyRecord.INITIAL_RECORD_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(AnomalyRecord.CAUSES.getPreferredName()) - .field(TYPE, NESTED) - .startObject(PROPERTIES) - .startObject(AnomalyCause.ACTUAL.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(AnomalyCause.TYPICAL.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(AnomalyCause.PROBABILITY.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(AnomalyCause.FUNCTION.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyCause.FUNCTION_DESCRIPTION.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyCause.BY_FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyCause.BY_FIELD_VALUE.getPreferredName()) - .field(TYPE, KEYWORD) - .field(COPY_TO, ALL_FIELD_VALUES) - .endObject() - .startObject(AnomalyCause.CORRELATED_BY_FIELD_VALUE.getPreferredName()) - .field(TYPE, KEYWORD) - .field(COPY_TO, ALL_FIELD_VALUES) - .endObject() - .startObject(AnomalyCause.FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyCause.PARTITION_FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyCause.PARTITION_FIELD_VALUE.getPreferredName()) - .field(TYPE, KEYWORD) - .field(COPY_TO, ALL_FIELD_VALUES) - .endObject() - .startObject(AnomalyCause.OVER_FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyCause.OVER_FIELD_VALUE.getPreferredName()) - .field(TYPE, KEYWORD) - .field(COPY_TO, ALL_FIELD_VALUES) - .endObject() - .startObject(AnomalyCause.GEO_RESULTS.getPreferredName()) - .startObject(PROPERTIES) - .startObject(GeoResults.ACTUAL_POINT.getPreferredName()) - .field(TYPE, GEO_POINT) - .endObject() - .startObject(GeoResults.TYPICAL_POINT.getPreferredName()) - .field(TYPE, GEO_POINT) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - .startObject(AnomalyRecord.INFLUENCERS.getPreferredName()) - /* Array of influences */ - .field(TYPE, NESTED) - .startObject(PROPERTIES) - .startObject(Influence.INFLUENCER_FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Influence.INFLUENCER_FIELD_VALUES.getPreferredName()) - .field(TYPE, KEYWORD) - .field(COPY_TO, ALL_FIELD_VALUES) - .endObject() - .endObject() - .endObject() - .startObject(AnomalyRecord.GEO_RESULTS.getPreferredName()) - .startObject(PROPERTIES) - .startObject(GeoResults.ACTUAL_POINT.getPreferredName()) - .field(TYPE, GEO_POINT) - .endObject() - .startObject(GeoResults.TYPICAL_POINT.getPreferredName()) - .field(TYPE, GEO_POINT) - .endObject() - .endObject() - .endObject(); - } - - private static void addInfluencerFieldsToMapping(XContentBuilder builder) throws IOException { - builder.startObject(Influencer.INFLUENCER_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(Influencer.INITIAL_INFLUENCER_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(Influencer.INFLUENCER_FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Influencer.INFLUENCER_FIELD_VALUE.getPreferredName()) - .field(TYPE, KEYWORD) - .field(COPY_TO, ALL_FIELD_VALUES) - .endObject(); - } - - /** - * {@link DataCounts} mapping. - * - * @throws IOException On builder write error - */ - private static void addDataCountsMapping(XContentBuilder builder) throws IOException { - builder.startObject(DataCounts.PROCESSED_RECORD_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.PROCESSED_FIELD_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.INPUT_BYTES.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.INPUT_RECORD_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.INPUT_FIELD_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.INVALID_DATE_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.MISSING_FIELD_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.OUT_OF_ORDER_TIME_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.EMPTY_BUCKET_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.SPARSE_BUCKET_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.BUCKET_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.EARLIEST_RECORD_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(DataCounts.LATEST_RECORD_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(DataCounts.LATEST_EMPTY_BUCKET_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(DataCounts.LATEST_SPARSE_BUCKET_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(DataCounts.LAST_DATA_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject(); - } - - /** - * {@link TimingStats} mapping. - * Does not include mapping for BUCKET_COUNT as this mapping is added by {@link #addDataCountsMapping} method. - * - * @throws IOException On builder write error - */ - private static void addTimingStatsExceptBucketCountMapping(XContentBuilder builder) throws IOException { - builder - // re-used: BUCKET_COUNT - .startObject(TimingStats.MIN_BUCKET_PROCESSING_TIME_MS.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(TimingStats.MAX_BUCKET_PROCESSING_TIME_MS.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(TimingStats.AVG_BUCKET_PROCESSING_TIME_MS.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(TimingStats.EXPONENTIAL_AVG_BUCKET_PROCESSING_TIME_MS.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(TimingStats.EXPONENTIAL_AVG_CALCULATION_CONTEXT.getPreferredName()) - .startObject(PROPERTIES) - .startObject(ExponentialAverageCalculationContext.INCREMENTAL_METRIC_VALUE_MS.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(ExponentialAverageCalculationContext.LATEST_TIMESTAMP.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(ExponentialAverageCalculationContext.PREVIOUS_EXPONENTIAL_AVERAGE_MS.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .endObject() - .endObject(); - } - - /** - * {@link DatafeedTimingStats} mapping. - * Does not include mapping for BUCKET_COUNT as this mapping is added by {@link #addDataCountsMapping} method. - * Does not include mapping for EXPONENTIAL_AVG_CALCULATION_CONTEXT as this mapping is added by - * {@link #addTimingStatsExceptBucketCountMapping} method. - * - * @throws IOException On builder write error - */ - private static void addDatafeedTimingStats(XContentBuilder builder) throws IOException { - builder - .startObject(DatafeedTimingStats.SEARCH_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - // re-used: BUCKET_COUNT - .startObject(DatafeedTimingStats.TOTAL_SEARCH_TIME_MS.getPreferredName()) - .field(TYPE, DOUBLE) - // re-used: EXPONENTIAL_AVG_CALCULATION_CONTEXT - .endObject(); - } - - /** - * Create the Elasticsearch mapping for {@linkplain CategoryDefinition}. - * The '_all' field is disabled as the document isn't meant to be searched. - * - * @throws IOException On builder error - */ - private static void addCategoryDefinitionMapping(XContentBuilder builder) throws IOException { - builder.startObject(CategoryDefinition.CATEGORY_ID.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(CategoryDefinition.TERMS.getPreferredName()) - .field(TYPE, TEXT) - .endObject() - .startObject(CategoryDefinition.REGEX.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(CategoryDefinition.MAX_MATCHING_LENGTH.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(CategoryDefinition.EXAMPLES.getPreferredName()) - .field(TYPE, TEXT) - .endObject(); - } - - /** - * Create the Elasticsearch mapping for state. State could potentially be - * huge (target document size is 16MB and there can be many documents) so all - * analysis by Elasticsearch is disabled. The only way to retrieve state is - * by knowing the ID of a particular document. - */ - public static XContentBuilder stateMapping() throws IOException { - XContentBuilder builder = jsonBuilder(); - builder.startObject(); - builder.startObject(SINGLE_MAPPING_NAME); - addMetaInformation(builder); - builder.field(ENABLED, false); - builder.endObject(); - builder.endObject(); - - return builder; - } - - /** - * Create the Elasticsearch mapping for {@linkplain ModelSnapshot}. - * The '_all' field is disabled but the type is searchable - */ - private static void addModelSnapshotMapping(XContentBuilder builder) throws IOException { - builder.startObject(ModelSnapshot.DESCRIPTION.getPreferredName()) - .field(TYPE, TEXT) - .endObject() - .startObject(ModelSnapshotField.SNAPSHOT_ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(ModelSnapshot.SNAPSHOT_DOC_COUNT.getPreferredName()) - .field(TYPE, INTEGER) - .endObject() - .startObject(ModelSnapshot.RETAIN.getPreferredName()) - .field(TYPE, BOOLEAN) - .endObject() - .startObject(ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName()) - .startObject(PROPERTIES) - .startObject(Job.ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Result.RESULT_TYPE.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(ModelSizeStats.TIMESTAMP_FIELD.getPreferredName()) - .field(TYPE, DATE) - .endObject(); - - addModelSizeStatsFieldsToMapping(builder); - - // end model size stats properties - builder.endObject(); - // end model size stats mapping - builder.endObject(); - - builder.startObject(ModelSnapshot.QUANTILES.getPreferredName()) - .field(ENABLED, false) - .endObject().startObject(ModelSnapshot.MIN_VERSION.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(ModelSnapshot.LATEST_RECORD_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(ModelSnapshot.LATEST_RESULT_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject(); - } - - /** - * {@link ModelSizeStats} fields to be added under the 'properties' section of the mapping - * @param builder Add properties to this builder - * @throws IOException On write error - */ - private static void addModelSizeStatsFieldsToMapping(XContentBuilder builder) throws IOException { - builder.startObject(ModelSizeStats.MODEL_BYTES_FIELD.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(ModelSizeStats.TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(ModelSizeStats.TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(ModelSizeStats.TOTAL_PARTITION_FIELD_COUNT_FIELD.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(ModelSizeStats.BUCKET_ALLOCATION_FAILURES_COUNT_FIELD.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(ModelSizeStats.MEMORY_STATUS_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(ModelSizeStats.LOG_TIME_FIELD.getPreferredName()) - .field(TYPE, DATE) - .endObject(); - } - - public static XContentBuilder auditMessageMapping() throws IOException { - XContentBuilder builder = jsonBuilder().startObject(); - builder.startObject(SINGLE_MAPPING_NAME); - addMetaInformation(builder); - builder.field(DYNAMIC, "false"); - builder.startObject(PROPERTIES) - .startObject(Job.ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyDetectionAuditMessage.LEVEL.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyDetectionAuditMessage.MESSAGE.getPreferredName()) - .field(TYPE, TEXT) - .startObject(FIELDS) - .startObject(RAW) - .field(TYPE, KEYWORD) - .endObject() - .endObject() - .endObject() - .startObject(AnomalyDetectionAuditMessage.TIMESTAMP.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(AnomalyDetectionAuditMessage.NODE_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyDetectionAuditMessage.JOB_TYPE.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .endObject() - .endObject() - .endObject(); - - return builder; - } - static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndices, Version minVersion) throws IOException { List indicesToUpdate = new ArrayList<>(); @@ -1266,7 +146,7 @@ static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndic } public static void addDocMappingIfMissing(String alias, - CheckedFunction mappingSupplier, + CheckedFunction mappingSupplier, Client client, ClusterState state, ActionListener listener) { AliasOrIndex aliasOrIndex = state.metaData().getAliasAndIndexLookup().get(alias); if (aliasOrIndex == null) { @@ -1290,10 +170,11 @@ public static void addDocMappingIfMissing(String alias, IndexMetaData indexMetaData = state.metaData().index(indicesThatRequireAnUpdate[0]); String mappingType = indexMetaData.mapping().type(); - try (XContentBuilder mapping = mappingSupplier.apply(mappingType)) { + try { + String mapping = mappingSupplier.apply(mappingType); PutMappingRequest putMappingRequest = new PutMappingRequest(indicesThatRequireAnUpdate); putMappingRequest.type(mappingType); - putMappingRequest.source(mapping); + putMappingRequest.source(mapping, XContentType.JSON); executeAsyncWithOrigin(client, ML_ORIGIN, PutMappingAction.INSTANCE, putMappingRequest, ActionListener.wrap(response -> { if (response.isAcknowledged()) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java index 23075b2b9df23..890ecbb87e539 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java @@ -234,6 +234,7 @@ public final class ReservedFieldNames { Job.MODEL_SNAPSHOT_ID.getPreferredName(), Job.MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), Job.RESULTS_INDEX_NAME.getPreferredName(), + Job.ALLOW_LAZY_OPEN.getPreferredName(), AnalysisConfig.BUCKET_SPAN.getPreferredName(), AnalysisConfig.CATEGORIZATION_FIELD_NAME.getPreferredName(), diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/AuditorField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java similarity index 83% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/AuditorField.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java index 307ff01fa45a1..3535d33b8c69f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/AuditorField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java @@ -5,9 +5,9 @@ */ package org.elasticsearch.xpack.core.ml.notifications; -public final class AuditorField { +public final class NotificationsIndex { public static final String NOTIFICATIONS_INDEX = ".ml-notifications-000001"; - private AuditorField() {} + private NotificationsIndex() {} } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/exporter/MonitoringTemplateUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/exporter/MonitoringTemplateUtils.java index 9d4a941a24c2c..181d3fea2f460 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/exporter/MonitoringTemplateUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/exporter/MonitoringTemplateUtils.java @@ -17,12 +17,11 @@ import java.io.IOException; import java.time.Instant; import java.util.Locale; -import java.util.regex.Pattern; public final class MonitoringTemplateUtils { private static final String TEMPLATE_FILE = "/monitoring-%s.json"; - private static final String TEMPLATE_VERSION_PROPERTY = Pattern.quote("${monitoring.template.version}"); + private static final String TEMPLATE_VERSION_PROPERTY = "monitoring.template.version"; /** * The last version of X-Pack that updated the templates and pipelines. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateConfig.java index 4565f7a93b149..315ea79af400c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateConfig.java @@ -7,6 +7,9 @@ package org.elasticsearch.xpack.core.template; import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; import java.util.regex.Pattern; /** @@ -18,6 +21,7 @@ public class IndexTemplateConfig { private final String fileName; private final int version; private final String versionProperty; + private final Map variables; /** * Describes a template to be loaded from a resource file. Includes handling for substituting a version property into the template. @@ -38,10 +42,33 @@ public class IndexTemplateConfig { * @param versionProperty The property that will be replaced with the {@code version} string as described above. */ public IndexTemplateConfig(String templateName, String fileName, int version, String versionProperty) { + this(templateName, fileName, version, versionProperty, Collections.emptyMap()); + } + + /** + * Describes a template to be loaded from a resource file. Includes handling for substituting a version property into the template. + * + * The {@code versionProperty} parameter will be used to substitute the value of {@code version} into the template. For example, + * this template: + * {@code {"myTemplateVersion": "${my.version.property}"}} + * With {@code version = "42"; versionProperty = "my.version.property"} will result in {@code {"myTemplateVersion": "42"}}. + * + * @param templateName The name that will be used for the index template. Literal, include the version in this string if + * it should be used. + * @param fileName The filename the template should be loaded from. Literal, should include leading {@literal /} and + * extension if necessary. + * @param version The version of the template. Substituted for {@code versionProperty} as described above. + * @param versionProperty The property that will be replaced with the {@code version} string as described above. + * @param variables A map of additional variable substitutions. The map's keys are the variable names. + * The corresponding values will replace the variable names. + */ + public IndexTemplateConfig(String templateName, String fileName, int version, String versionProperty, Map variables) + { this.templateName = templateName; this.fileName = fileName; this.version = version; this.versionProperty = versionProperty; + this.variables = Objects.requireNonNull(variables); } public String getFileName() { @@ -61,8 +88,7 @@ public int getVersion() { * @return The template as a UTF-8 byte array. */ public byte[] loadBytes() { - final String versionPattern = Pattern.quote("${" + versionProperty + "}"); - String template = TemplateUtils.loadTemplate(fileName, Integer.toString(version), versionPattern); + String template = TemplateUtils.loadTemplate(fileName, Integer.toString(version), versionProperty, variables); assert template != null && template.length() > 0; assert Pattern.compile("\"version\"\\s*:\\s*" + version).matcher(template).find() : "index template must have a version property set to the given version property"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java index f5d95204a7995..79802d08ee4da 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java @@ -119,6 +119,12 @@ public void clusterChanged(ClusterChangedEvent event) { return; } + // This registry requires to run on a master node. + // If not a master node, exit. + if (requiresMasterNode() && state.nodes().isLocalNodeElectedMaster() == false) { + return; + } + // if this node is newer than the master node, we probably need to add the template, which might be newer than the // template the master node has, so we need potentially add new templates despite being not the master node DiscoveryNode localNode = event.state().getNodes().getLocalNode(); @@ -130,6 +136,15 @@ public void clusterChanged(ClusterChangedEvent event) { } } + /** + * Whether the registry should only apply changes when running on the master node. + * This is useful for plugins where certain actions are performed on master nodes + * and the templates should match the respective version. + */ + protected boolean requiresMasterNode() { + return false; + } + private void addTemplatesIfMissing(ClusterState state) { final List indexTemplates = getTemplateConfigs(); for (IndexTemplateConfig newTemplate : indexTemplates) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java index 3c7e9cdfb9095..57e0cce1d1ea7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java @@ -11,11 +11,11 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.compress.NotXContentException; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -23,8 +23,10 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import java.io.IOException; +import java.util.Collections; import java.util.Map; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -57,12 +59,18 @@ public static void loadTemplateIntoMap(String resource, Map variables) { + try { + String source = load(resource); + source = replaceVariables(source, version, versionProperty, variables); + validate(source); + return source; } catch (Exception e) { throw new IllegalArgumentException("Unable to load template [" + resource + "]", e); } @@ -71,34 +79,43 @@ public static String loadTemplate(String resource, String version, String versio /** * Loads a resource from the classpath and returns it as a {@link BytesReference} */ - public static BytesReference load(String name) throws IOException { - return Streams.readFully(TemplateUtils.class.getResourceAsStream(name)); + public static String load(String name) throws IOException { + return Streams.readFully(TemplateUtils.class.getResourceAsStream(name)).utf8ToString(); } /** * Parses and validates that the source is not empty. */ - public static void validate(BytesReference source) { + public static void validate(String source) { if (source == null) { throw new ElasticsearchParseException("Template must not be null"); } + if (Strings.isEmpty(source)) { + throw new ElasticsearchParseException("Template must not be empty"); + } try { - XContentHelper.convertToMap(source, false, XContentType.JSON).v2(); - } catch (NotXContentException e) { - throw new ElasticsearchParseException("Template must not be empty"); + XContentHelper.convertToMap(JsonXContent.jsonXContent, source, false); } catch (Exception e) { throw new ElasticsearchParseException("Invalid template", e); } } + private static String replaceVariables(String input, String version, String versionProperty, Map variables) { + String template = replaceVariable(input, versionProperty, version); + for (Map.Entry variable : variables.entrySet()) { + template = replaceVariable(template, variable.getKey(), variable.getValue()); + } + return template; + } + /** - * Filters the source: replaces any template version property with the version number + * Replaces all occurences of given variable with the value */ - public static String filter(BytesReference source, String version, String versionProperty) { - return Pattern.compile(versionProperty) - .matcher(source.utf8ToString()) - .replaceAll(version); + public static String replaceVariable(String input, String variable, String value) { + return Pattern.compile("${" + variable + "}", Pattern.LITERAL) + .matcher(input) + .replaceAll(value); } /** diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json new file mode 100644 index 0000000000000..41c26b0f83d4f --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json @@ -0,0 +1,36 @@ +{ + "_doc": { + "_meta" : { + "version" : "${xpack.ml.version}" + }, + "properties" : { + "annotation" : { + "type" : "text" + }, + "create_time" : { + "type" : "date" + }, + "create_username" : { + "type" : "keyword" + }, + "end_timestamp" : { + "type" : "date" + }, + "job_id" : { + "type" : "keyword" + }, + "modified_time" : { + "type" : "date" + }, + "modified_username" : { + "type" : "keyword" + }, + "timestamp" : { + "type" : "date" + }, + "type" : { + "type" : "keyword" + } + } + } +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json new file mode 100644 index 0000000000000..8ff990d2e5020 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json @@ -0,0 +1,478 @@ +{ + "${xpack.ml.mapping_type}" : { + "_meta" : { + "version" : "${xpack.ml.version}" + }, + "dynamic_templates" : [ + { + "strings_as_keywords" : { + "match" : "*", + "mapping" : { + "type" : "keyword" + } + } + } + ], + "properties" : { + "actual" : { + "type" : "double" + }, + "all_field_values" : { + "type" : "text", + "analyzer" : "whitespace" + }, + "anomaly_score" : { + "type" : "double" + }, + "average_bucket_processing_time_ms" : { + "type" : "double" + }, + "bucket_allocation_failures_count" : { + "type" : "long" + }, + "bucket_count" : { + "type" : "long" + }, + "bucket_influencers" : { + "type" : "nested", + "properties" : { + "anomaly_score" : { + "type" : "double" + }, + "bucket_span" : { + "type" : "long" + }, + "influencer_field_name" : { + "type" : "keyword" + }, + "initial_anomaly_score" : { + "type" : "double" + }, + "is_interim" : { + "type" : "boolean" + }, + "job_id" : { + "type" : "keyword" + }, + "probability" : { + "type" : "double" + }, + "raw_anomaly_score" : { + "type" : "double" + }, + "result_type" : { + "type" : "keyword" + }, + "timestamp" : { + "type" : "date" + } + } + }, + "bucket_span" : { + "type" : "long" + }, + "by_field_name" : { + "type" : "keyword" + }, + "by_field_value" : { + "type" : "keyword", + "copy_to" : [ + "all_field_values" + ] + }, + "category_id" : { + "type" : "long" + }, + "causes" : { + "type" : "nested", + "properties" : { + "actual" : { + "type" : "double" + }, + "by_field_name" : { + "type" : "keyword" + }, + "by_field_value" : { + "type" : "keyword", + "copy_to" : [ + "all_field_values" + ] + }, + "correlated_by_field_value" : { + "type" : "keyword", + "copy_to" : [ + "all_field_values" + ] + }, + "field_name" : { + "type" : "keyword" + }, + "function" : { + "type" : "keyword" + }, + "function_description" : { + "type" : "keyword" + }, + "geo_results" : { + "properties" : { + "actual_point" : { + "type" : "geo_point" + }, + "typical_point" : { + "type" : "geo_point" + } + } + }, + "over_field_name" : { + "type" : "keyword" + }, + "over_field_value" : { + "type" : "keyword", + "copy_to" : [ + "all_field_values" + ] + }, + "partition_field_name" : { + "type" : "keyword" + }, + "partition_field_value" : { + "type" : "keyword", + "copy_to" : [ + "all_field_values" + ] + }, + "probability" : { + "type" : "double" + }, + "typical" : { + "type" : "double" + } + } + }, + "description" : { + "type" : "text" + }, + "detector_index" : { + "type" : "integer" + }, + "earliest_record_timestamp" : { + "type" : "date" + }, + "empty_bucket_count" : { + "type" : "long" + }, + "event_count" : { + "type" : "long" + }, + "examples" : { + "type" : "text" + }, + "exponential_average_bucket_processing_time_ms" : { + "type" : "double" + }, + "exponential_average_calculation_context" : { + "properties" : { + "incremental_metric_value_ms" : { + "type" : "double" + }, + "latest_timestamp" : { + "type" : "date" + }, + "previous_exponential_average_ms" : { + "type" : "double" + } + } + }, + "field_name" : { + "type" : "keyword" + }, + "forecast_create_timestamp" : { + "type" : "date" + }, + "forecast_end_timestamp" : { + "type" : "date" + }, + "forecast_expiry_timestamp" : { + "type" : "date" + }, + "forecast_id" : { + "type" : "keyword" + }, + "forecast_lower" : { + "type" : "double" + }, + "forecast_memory_bytes" : { + "type" : "long" + }, + "forecast_messages" : { + "type" : "keyword" + }, + "forecast_prediction" : { + "type" : "double" + }, + "forecast_progress" : { + "type" : "double" + }, + "forecast_start_timestamp" : { + "type" : "date" + }, + "forecast_status" : { + "type" : "keyword" + }, + "forecast_upper" : { + "type" : "double" + }, + "function" : { + "type" : "keyword" + }, + "function_description" : { + "type" : "keyword" + }, + "geo_results" : { + "properties" : { + "actual_point" : { + "type" : "geo_point" + }, + "typical_point" : { + "type" : "geo_point" + } + } + }, + "influencer_field_name" : { + "type" : "keyword" + }, + "influencer_field_value" : { + "type" : "keyword", + "copy_to" : [ + "all_field_values" + ] + }, + "influencer_score" : { + "type" : "double" + }, + "influencers" : { + "type" : "nested", + "properties" : { + "influencer_field_name" : { + "type" : "keyword" + }, + "influencer_field_values" : { + "type" : "keyword", + "copy_to" : [ + "all_field_values" + ] + } + } + }, + "initial_anomaly_score" : { + "type" : "double" + }, + "initial_influencer_score" : { + "type" : "double" + }, + "initial_record_score" : { + "type" : "double" + }, + "input_bytes" : { + "type" : "long" + }, + "input_field_count" : { + "type" : "long" + }, + "input_record_count" : { + "type" : "long" + }, + "invalid_date_count" : { + "type" : "long" + }, + "is_interim" : { + "type" : "boolean" + }, + "job_id" : { + "type" : "keyword", + "copy_to" : [ + "all_field_values" + ] + }, + "last_data_time" : { + "type" : "date" + }, + "latest_empty_bucket_timestamp" : { + "type" : "date" + }, + "latest_record_time_stamp" : { + "type" : "date" + }, + "latest_record_timestamp" : { + "type" : "date" + }, + "latest_result_time_stamp" : { + "type" : "date" + }, + "latest_sparse_bucket_timestamp" : { + "type" : "date" + }, + "log_time" : { + "type" : "date" + }, + "max_matching_length" : { + "type" : "long" + }, + "maximum_bucket_processing_time_ms" : { + "type" : "double" + }, + "memory_status" : { + "type" : "keyword" + }, + "min_version" : { + "type" : "keyword" + }, + "minimum_bucket_processing_time_ms" : { + "type" : "double" + }, + "missing_field_count" : { + "type" : "long" + }, + "model_bytes" : { + "type" : "long" + }, + "model_feature" : { + "type" : "keyword" + }, + "model_lower" : { + "type" : "double" + }, + "model_median" : { + "type" : "double" + }, + "model_size_stats" : { + "properties" : { + "bucket_allocation_failures_count" : { + "type" : "long" + }, + "job_id" : { + "type" : "keyword" + }, + "log_time" : { + "type" : "date" + }, + "memory_status" : { + "type" : "keyword" + }, + "model_bytes" : { + "type" : "long" + }, + "result_type" : { + "type" : "keyword" + }, + "timestamp" : { + "type" : "date" + }, + "total_by_field_count" : { + "type" : "long" + }, + "total_over_field_count" : { + "type" : "long" + }, + "total_partition_field_count" : { + "type" : "long" + } + } + }, + "model_upper" : { + "type" : "double" + }, + "multi_bucket_impact" : { + "type" : "double" + }, + "out_of_order_timestamp_count" : { + "type" : "long" + }, + "over_field_name" : { + "type" : "keyword" + }, + "over_field_value" : { + "type" : "keyword", + "copy_to" : [ + "all_field_values" + ] + }, + "partition_field_name" : { + "type" : "keyword" + }, + "partition_field_value" : { + "type" : "keyword", + "copy_to" : [ + "all_field_values" + ] + }, + "probability" : { + "type" : "double" + }, + "processed_field_count" : { + "type" : "long" + }, + "processed_record_count" : { + "type" : "long" + }, + "processing_time_ms" : { + "type" : "long" + }, + "quantiles" : { + "type" : "object", + "enabled" : false + }, + "raw_anomaly_score" : { + "type" : "double" + }, + "record_score" : { + "type" : "double" + }, + "regex" : { + "type" : "keyword" + }, + "result_type" : { + "type" : "keyword" + }, + "retain" : { + "type" : "boolean" + }, + "scheduled_events" : { + "type" : "keyword" + }, + "search_count" : { + "type" : "long" + }, + "snapshot_doc_count" : { + "type" : "integer" + }, + "snapshot_id" : { + "type" : "keyword" + }, + "sparse_bucket_count" : { + "type" : "long" + }, + "terms" : { + "type" : "text" + }, + "timestamp" : { + "type" : "date" + }, + "total_by_field_count" : { + "type" : "long" + }, + "total_over_field_count" : { + "type" : "long" + }, + "total_partition_field_count" : { + "type" : "long" + }, + "total_search_time_ms" : { + "type" : "double" + }, + "typical" : { + "type" : "double" + } + } + } +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json new file mode 100644 index 0000000000000..37a759cba7325 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json @@ -0,0 +1,19 @@ +{ + "order" : 0, + "version" : ${xpack.ml.version.id}, + "index_patterns" : [ + ".ml-anomalies-*" + ], + "settings" : { + "index" : { + "translog" : { + "durability" : "async" + }, + "auto_expand_replicas" : "0-1", + "query" : { + "default_field" : "all_field_values" + } + } + }, + "mappings": ${xpack.ml.anomalydetection.results.mappings} +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json new file mode 100644 index 0000000000000..39211ae5b20ff --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json @@ -0,0 +1,21 @@ +{ + "order" : 0, + "version" : ${xpack.ml.version.id}, + "index_patterns" : [ + ".ml-state*" + ], + "settings" : { + "index" : { + "auto_expand_replicas" : "0-1" + } + }, + "mappings" : { + "_doc": { + "_meta": { + "version": "${xpack.ml.version}" + }, + "enabled": false + } + }, + "aliases" : { } +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json new file mode 100644 index 0000000000000..aa7211cf44836 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json @@ -0,0 +1,364 @@ +{ + "${xpack.ml.mapping_type}" : { + "_meta" : { + "version" : "${xpack.ml.version}" + }, + "dynamic_templates" : [ + { + "strings_as_keywords" : { + "match" : "*", + "mapping" : { + "type" : "keyword" + } + } + } + ], + "properties" : { + "aggregations" : { + "type" : "object", + "enabled" : false + }, + "allow_lazy_open" : { + "type" : "keyword" + }, + "analysis" : { + "properties" : { + "classification" : { + "properties" : { + "dependent_variable" : { + "type" : "keyword" + }, + "eta" : { + "type" : "double" + }, + "feature_bag_fraction" : { + "type" : "double" + }, + "gamma" : { + "type" : "double" + }, + "lambda" : { + "type" : "double" + }, + "maximum_number_trees" : { + "type" : "integer" + }, + "num_top_classes" : { + "type" : "integer" + }, + "num_top_feature_importance_values" : { + "type" : "integer" + }, + "prediction_field_name" : { + "type" : "keyword" + }, + "training_percent" : { + "type" : "double" + } + } + }, + "outlier_detection" : { + "properties" : { + "feature_influence_threshold" : { + "type" : "double" + }, + "method" : { + "type" : "keyword" + }, + "n_neighbors" : { + "type" : "integer" + } + } + }, + "regression" : { + "properties" : { + "dependent_variable" : { + "type" : "keyword" + }, + "eta" : { + "type" : "double" + }, + "feature_bag_fraction" : { + "type" : "double" + }, + "gamma" : { + "type" : "double" + }, + "lambda" : { + "type" : "double" + }, + "maximum_number_trees" : { + "type" : "integer" + }, + "num_top_feature_importance_values" : { + "type" : "integer" + }, + "prediction_field_name" : { + "type" : "keyword" + }, + "training_percent" : { + "type" : "double" + } + } + } + } + }, + "analysis_config" : { + "properties" : { + "bucket_span" : { + "type" : "keyword" + }, + "categorization_analyzer" : { + "type" : "object", + "enabled" : false + }, + "categorization_field_name" : { + "type" : "keyword" + }, + "categorization_filters" : { + "type" : "keyword" + }, + "detectors" : { + "properties" : { + "by_field_name" : { + "type" : "keyword" + }, + "custom_rules" : { + "type" : "nested", + "properties" : { + "actions" : { + "type" : "keyword" + }, + "conditions" : { + "type" : "nested", + "properties" : { + "applies_to" : { + "type" : "keyword" + }, + "operator" : { + "type" : "keyword" + }, + "value" : { + "type" : "double" + } + } + }, + "scope" : { + "type" : "object", + "enabled" : false + } + } + }, + "detector_description" : { + "type" : "text" + }, + "detector_index" : { + "type" : "integer" + }, + "exclude_frequent" : { + "type" : "keyword" + }, + "field_name" : { + "type" : "keyword" + }, + "function" : { + "type" : "keyword" + }, + "over_field_name" : { + "type" : "keyword" + }, + "partition_field_name" : { + "type" : "keyword" + }, + "use_null" : { + "type" : "boolean" + } + } + }, + "influencers" : { + "type" : "keyword" + }, + "latency" : { + "type" : "keyword" + }, + "multivariate_by_fields" : { + "type" : "boolean" + }, + "summary_count_field_name" : { + "type" : "keyword" + } + } + }, + "analysis_limits" : { + "properties" : { + "categorization_examples_limit" : { + "type" : "long" + }, + "model_memory_limit" : { + "type" : "keyword" + } + } + }, + "analyzed_fields" : { + "type" : "object", + "enabled" : false + }, + "background_persist_interval" : { + "type" : "keyword" + }, + "chunking_config" : { + "properties" : { + "mode" : { + "type" : "keyword" + }, + "time_span" : { + "type" : "keyword" + } + } + }, + "config_type" : { + "type" : "keyword" + }, + "create_time" : { + "type" : "date" + }, + "custom_settings" : { + "type" : "object", + "enabled" : false + }, + "data_description" : { + "properties" : { + "field_delimiter" : { + "type" : "keyword" + }, + "format" : { + "type" : "keyword" + }, + "quote_character" : { + "type" : "keyword" + }, + "time_field" : { + "type" : "keyword" + }, + "time_format" : { + "type" : "keyword" + } + } + }, + "datafeed_id" : { + "type" : "keyword" + }, + "delayed_data_check_config" : { + "properties" : { + "check_window" : { + "type" : "keyword" + }, + "enabled" : { + "type" : "boolean" + } + } + }, + "description" : { + "type" : "text" + }, + "dest" : { + "properties" : { + "index" : { + "type" : "keyword" + }, + "results_field" : { + "type" : "keyword" + } + } + }, + "finished_time" : { + "type" : "date" + }, + "frequency" : { + "type" : "keyword" + }, + "groups" : { + "type" : "keyword" + }, + "headers" : { + "type" : "object", + "enabled" : false + }, + "id" : { + "type" : "keyword" + }, + "indices" : { + "type" : "keyword" + }, + "job_id" : { + "type" : "keyword" + }, + "job_type" : { + "type" : "keyword" + }, + "job_version" : { + "type" : "keyword" + }, + "model_plot_config" : { + "properties" : { + "enabled" : { + "type" : "boolean" + }, + "terms" : { + "type" : "keyword" + } + } + }, + "model_snapshot_id" : { + "type" : "keyword" + }, + "model_snapshot_min_version" : { + "type" : "keyword" + }, + "model_snapshot_retention_days" : { + "type" : "long" + }, + "query" : { + "type" : "object", + "enabled" : false + }, + "query_delay" : { + "type" : "keyword" + }, + "renormalization_window_days" : { + "type" : "long" + }, + "results_index_name" : { + "type" : "keyword" + }, + "results_retention_days" : { + "type" : "long" + }, + "script_fields" : { + "type" : "object", + "enabled" : false + }, + "scroll_size" : { + "type" : "long" + }, + "source" : { + "properties" : { + "_source" : { + "type" : "object", + "enabled" : false + }, + "index" : { + "type" : "keyword" + }, + "query" : { + "type" : "object", + "enabled" : false + } + } + }, + "version" : { + "type" : "keyword" + } + } + } +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json new file mode 100644 index 0000000000000..8c6c352ab245a --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json @@ -0,0 +1,15 @@ +{ + "order" : 0, + "version" : ${xpack.ml.version.id}, + "index_patterns" : [ + ".ml-config" + ], + "settings" : { + "index" : { + "max_result_window" : "${xpack.ml.config.max_result_window}", + "number_of_shards" : "1", + "auto_expand_replicas" : "0-1" + } + }, + "mappings": ${xpack.ml.config.mappings} +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json new file mode 100644 index 0000000000000..5cbee23ca8094 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json @@ -0,0 +1,72 @@ +{ + "order" : 0, + "version" : ${xpack.ml.version.id}, + "index_patterns" : [ + ".ml-inference-000001" + ], + "settings" : { + "index" : { + "number_of_shards" : "1", + "auto_expand_replicas" : "0-1" + } + }, + "mappings" : { + "_doc": { + "_meta": { + "version": "${xpack.ml.version}" + }, + "dynamic": "false", + "properties": { + "doc_type": { + "type": "keyword" + }, + "model_id": { + "type": "keyword" + }, + "created_by": { + "type": "keyword" + }, + "input": { + "enabled": false + }, + "version": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "create_time": { + "type": "date" + }, + "tags": { + "type": "keyword" + }, + "metadata": { + "enabled": false + }, + "estimated_operations": { + "type": "long" + }, + "estimated_heap_memory_usage_bytes": { + "type": "long" + }, + "doc_num": { + "type": "long" + }, + "definition": { + "enabled": false + }, + "compression_version": { + "type": "long" + }, + "definition_length": { + "type": "long" + }, + "total_definition_length": { + "type": "long" + } + } + } + }, + "aliases" : { } +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json new file mode 100644 index 0000000000000..19df45c52509c --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json @@ -0,0 +1,47 @@ +{ + "order" : 0, + "version" : ${xpack.ml.version.id}, + "index_patterns" : [ + ".ml-meta" + ], + "settings" : { + "index" : { + "number_of_shards" : "1", + "auto_expand_replicas" : "0-1" + } + }, + "mappings" : { + "_doc": { + "_meta": { + "version": "${xpack.ml.version}" + }, + "dynamic_templates": [ + { + "strings_as_keywords": { + "match": "*", + "mapping": { + "type": "keyword" + } + } + } + ], + "properties": { + "calendar_id": { + "type": "keyword" + }, + "job_ids": { + "type": "keyword" + }, + "description": { + "type": "keyword" + }, + "start_time": { + "type": "date" + }, + "end_time": { + "type": "date" + } + } + } + } +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json new file mode 100644 index 0000000000000..804707dfb189f --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json @@ -0,0 +1,46 @@ +{ + "order" : 0, + "version" : ${xpack.ml.version.id}, + "index_patterns" : [ + ".ml-notifications-000001" + ], + "settings" : { + "index" : { + "number_of_shards" : "1", + "auto_expand_replicas" : "0-1" + } + }, + "mappings" : { + "_doc": { + "_meta" : { + "version" : "${xpack.ml.version}" + }, + "dynamic" : "false", + "properties" : { + "job_id": { + "type": "keyword" + }, + "level": { + "type": "keyword" + }, + "message": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + } + } + }, + "timestamp": { + "type": "date" + }, + "node_name": { + "type": "keyword" + }, + "job_type": { + "type": "keyword" + } + } + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java index 168dd24109d90..7469105097475 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java @@ -20,15 +20,13 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ml.MlConfigIndex; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats; import org.elasticsearch.xpack.core.ml.job.config.Job; @@ -38,7 +36,6 @@ import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats; -import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord; import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition; import org.elasticsearch.xpack.core.ml.job.results.ReservedFieldNames; import org.elasticsearch.xpack.core.ml.job.results.Result; @@ -56,8 +53,6 @@ import java.util.Map; import java.util.Set; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; @@ -125,7 +120,6 @@ public void testConfigMappingReservedFields() throws Exception { compareFields(expected, ReservedFieldNames.RESERVED_CONFIG_FIELD_NAMES); } - private void compareFields(Set expected, Set reserved) { if (reserved.size() != expected.size()) { Set diff = new HashSet<>(reserved); @@ -146,32 +140,6 @@ private void compareFields(Set expected, Set reserved) { } } - @SuppressWarnings("unchecked") - public void testTermFieldMapping() throws IOException { - - XContentBuilder builder = ElasticsearchMappings.termFieldsMapping(Arrays.asList("apple", "strawberry", - AnomalyRecord.BUCKET_SPAN.getPreferredName())); - - XContentParser parser = createParser(builder); - Map mapping = (Map) parser.map().get(SINGLE_MAPPING_NAME); - Map properties = (Map) mapping.get(ElasticsearchMappings.PROPERTIES); - - Map instanceMapping = (Map) properties.get("apple"); - assertNotNull(instanceMapping); - String dataType = (String)instanceMapping.get(ElasticsearchMappings.TYPE); - assertEquals(ElasticsearchMappings.KEYWORD, dataType); - - instanceMapping = (Map) properties.get("strawberry"); - assertNotNull(instanceMapping); - dataType = (String)instanceMapping.get(ElasticsearchMappings.TYPE); - assertEquals(ElasticsearchMappings.KEYWORD, dataType); - - // check no mapping for the reserved field - instanceMapping = (Map) properties.get(AnomalyRecord.BUCKET_SPAN.getPreferredName()); - assertNull(instanceMapping); - } - - public void testMappingRequiresUpdateNoMapping() throws IOException { ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); ClusterState cs = csBuilder.build(); @@ -240,7 +208,7 @@ public void testAddDocMappingIfMissing() throws IOException { ClusterState clusterState = getClusterStateWithMappingsWithMetaData(Collections.singletonMap("index-name", "0.0")); ElasticsearchMappings.addDocMappingIfMissing( "index-name", - ElasticsearchMappingsTests::fakeMapping, + mappingType -> "{\"_doc\":{\"properties\":{\"some-field\":{\"type\":\"long\"}}}}", client, clusterState, ActionListener.wrap( @@ -260,19 +228,6 @@ public void testAddDocMappingIfMissing() throws IOException { assertThat(request.source(), equalTo("{\"_doc\":{\"properties\":{\"some-field\":{\"type\":\"long\"}}}}")); } - private static XContentBuilder fakeMapping(String mappingType) throws IOException { - return jsonBuilder() - .startObject() - .startObject(mappingType) - .startObject(ElasticsearchMappings.PROPERTIES) - .startObject("some-field") - .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.LONG) - .endObject() - .endObject() - .endObject() - .endObject(); - } - private ClusterState getClusterStateWithMappingsWithMetaData(Map namesAndVersions) throws IOException { MetaData.Builder metaDataBuilder = MetaData.builder(); @@ -311,17 +266,17 @@ private ClusterState getClusterStateWithMappingsWithMetaData(Map private Set collectResultsDocFieldNames() throws IOException { // Only the mappings for the results index should be added below. Do NOT add mappings for other indexes here. - return collectFieldNames(ElasticsearchMappings.resultsMapping("_doc")); + return collectFieldNames(AnomalyDetectorsIndex.resultsMapping()); } private Set collectConfigDocFieldNames() throws IOException { // Only the mappings for the config index should be added below. Do NOT add mappings for other indexes here. - return collectFieldNames(ElasticsearchMappings.configMapping()); + return collectFieldNames(MlConfigIndex.mapping()); } - private Set collectFieldNames(XContentBuilder mapping) throws IOException { + private Set collectFieldNames(String mapping) throws IOException { BufferedInputStream inputStream = - new BufferedInputStream(new ByteArrayInputStream(Strings.toString(mapping).getBytes(StandardCharsets.UTF_8))); + new BufferedInputStream(new ByteArrayInputStream(mapping.getBytes(StandardCharsets.UTF_8))); JsonParser parser = new JsonFactory().createParser(inputStream); Set fieldNames = new HashSet<>(); boolean isAfterPropertiesStart = false; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 100b8feec898f..edc8dec11e3f0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -108,7 +108,7 @@ import org.elasticsearch.xpack.core.ml.action.ValidateJobConfigAction; import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; -import org.elasticsearch.xpack.core.ml.notifications.AuditorField; +import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkAction; import org.elasticsearch.xpack.core.security.action.DelegatePkiAuthenticationAction; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction; @@ -1135,7 +1135,7 @@ public void testMachineLearningAdminRole() { assertOnlyReadAllowed(role, MlMetaIndex.INDEX_NAME); assertOnlyReadAllowed(role, AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX); assertOnlyReadAllowed(role, AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT); - assertOnlyReadAllowed(role, AuditorField.NOTIFICATIONS_INDEX); + assertOnlyReadAllowed(role, NotificationsIndex.NOTIFICATIONS_INDEX); assertReadWriteDocsButNotDeleteIndexAllowed(role, AnnotationIndex.INDEX_NAME); assertNoAccessAllowed(role, RestrictedIndicesNames.RESTRICTED_NAMES); @@ -1222,7 +1222,7 @@ public void testMachineLearningUserRole() { assertNoAccessAllowed(role, MlMetaIndex.INDEX_NAME); assertNoAccessAllowed(role, AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX); assertOnlyReadAllowed(role, AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT); - assertOnlyReadAllowed(role, AuditorField.NOTIFICATIONS_INDEX); + assertOnlyReadAllowed(role, NotificationsIndex.NOTIFICATIONS_INDEX); assertReadWriteDocsButNotDeleteIndexAllowed(role, AnnotationIndex.INDEX_NAME); assertNoAccessAllowed(role, RestrictedIndicesNames.RESTRICTED_NAMES); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TemplateUtilsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TemplateUtilsTests.java index 57ba25af63feb..09ce23e741fc1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TemplateUtilsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TemplateUtilsTests.java @@ -8,15 +8,13 @@ import org.apache.lucene.util.Constants; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.template.TemplateUtils; import org.hamcrest.Matcher; import java.io.IOException; +import java.util.HashMap; import java.util.Locale; -import java.util.regex.Pattern; +import java.util.Map; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -25,12 +23,13 @@ public class TemplateUtilsTests extends ESTestCase { - private static final String TEST_TEMPLATE = "/monitoring-%s.json"; + private static final String SIMPLE_TEST_TEMPLATE = "/monitoring-%s.json"; + private static final String TEST_TEMPLATE_WITH_VARIABLES = "/template_with_variables-test.json"; - public void testLoadTemplate() throws IOException { + public void testLoadTemplate() { final int version = randomIntBetween(0, 10_000); - String resource = String.format(Locale.ROOT, TEST_TEMPLATE, "test"); - String source = TemplateUtils.loadTemplate(resource, String.valueOf(version), Pattern.quote("${monitoring.template.version}")); + String resource = String.format(Locale.ROOT, SIMPLE_TEST_TEMPLATE, "test"); + String source = TemplateUtils.loadTemplate(resource, String.valueOf(version), "monitoring.template.version"); assertThat(source, notNullValue()); assertThat(source.length(), greaterThan(0)); @@ -46,9 +45,36 @@ public void testLoadTemplate() throws IOException { "}\n")); } + public void testLoadTemplate_GivenTemplateWithVariables() { + final int version = randomIntBetween(0, 10_000); + Map variables = new HashMap<>(); + variables.put("test.template.field_1", "test_field_1"); + variables.put("test.template.field_2", "\"test_field_2\": {\"type\": \"long\"}"); + + String source = TemplateUtils.loadTemplate(TEST_TEMPLATE_WITH_VARIABLES, String.valueOf(version), + "test.template.version", variables); + + assertThat(source, notNullValue()); + assertThat(source.length(), greaterThan(0)); + assertTemplate(source, equalTo("{\n" + + " \"index_patterns\": \".test-" + version + "\",\n" + + " \"mappings\": {\n" + + " \"doc\": {\n" + + " \"_meta\": {\n" + + " \"template.version\": \"" + version + "\"\n" + + " },\n" + + " \"properties\": {\n" + + " \"test_field_1\": {\"type\": \"keyword\"},\n" + + " \"test_field_2\": {\"type\": \"long\"}\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n")); + } + public void testLoad() throws IOException { - String resource = String.format(Locale.ROOT, TEST_TEMPLATE, "test"); - BytesReference source = TemplateUtils.load(resource); + String resource = String.format(Locale.ROOT, SIMPLE_TEST_TEMPLATE, "test"); + String source = TemplateUtils.load(resource); assertThat(source, notNullValue()); assertThat(source.length(), greaterThan(0)); } @@ -60,35 +86,34 @@ public void testValidateNullSource() { public void testValidateEmptySource() { ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class, - () -> TemplateUtils.validate(new BytesArray(""))); + () -> TemplateUtils.validate("")); assertThat(exception.getMessage(), is("Template must not be empty")); } public void testValidateInvalidSource() { ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class, - () -> TemplateUtils.validate(new BytesArray("{\"foo\": \"bar"))); + () -> TemplateUtils.validate("{\"foo\": \"bar")); assertThat(exception.getMessage(), is("Invalid template")); } public void testValidate() throws IOException { - String resource = String.format(Locale.ROOT, TEST_TEMPLATE, "test"); + String resource = String.format(Locale.ROOT, SIMPLE_TEST_TEMPLATE, "test"); TemplateUtils.validate(TemplateUtils.load(resource)); } - public void testFilter() { - assertTemplate(TemplateUtils.filter(new BytesArray("${monitoring.template.version}"), "0", - Pattern.quote("${monitoring.template.version}")), equalTo("0")); - assertTemplate(TemplateUtils.filter(new BytesArray("{\"template\": \"test-${monitoring.template.version}\"}"), "1", - Pattern.quote("${monitoring.template.version}")), equalTo("{\"template\": \"test-1\"}")); - assertTemplate(TemplateUtils.filter(new BytesArray("{\"template\": \"${monitoring.template.version}-test\"}"), "2", - Pattern.quote("${monitoring.template.version}")), equalTo("{\"template\": \"2-test\"}")); - assertTemplate(TemplateUtils.filter(new BytesArray("{\"template\": \"test-${monitoring.template.version}-test\"}"), "3", - Pattern.quote("${monitoring.template.version}")), equalTo("{\"template\": \"test-3-test\"}")); + public void testReplaceVariable() { + assertTemplate(TemplateUtils.replaceVariable("${monitoring.template.version}", + "monitoring.template.version", "0"), equalTo("0")); + assertTemplate(TemplateUtils.replaceVariable("{\"template\": \"test-${monitoring.template.version}\"}", + "monitoring.template.version", "1"), equalTo("{\"template\": \"test-1\"}")); + assertTemplate(TemplateUtils.replaceVariable("{\"template\": \"${monitoring.template.version}-test\"}", + "monitoring.template.version", "2"), equalTo("{\"template\": \"2-test\"}")); + assertTemplate(TemplateUtils.replaceVariable("{\"template\": \"test-${monitoring.template.version}-test\"}", + "monitoring.template.version", "3"), equalTo("{\"template\": \"test-3-test\"}")); final int version = randomIntBetween(0, 100); - assertTemplate(TemplateUtils.filter(new BytesArray("{\"foo-${monitoring.template.version}\": " + - "\"bar-${monitoring.template.version}\"}"), String.valueOf(version), - Pattern.quote("${monitoring.template.version}")), + assertTemplate(TemplateUtils.replaceVariable("{\"foo-${monitoring.template.version}\": " + + "\"bar-${monitoring.template.version}\"}", "monitoring.template.version", String.valueOf(version)), equalTo("{\"foo-" + version + "\": \"bar-" + version + "\"}")); } diff --git a/x-pack/plugin/core/src/test/resources/template_with_variables-test.json b/x-pack/plugin/core/src/test/resources/template_with_variables-test.json new file mode 100644 index 0000000000000..d41bd84a919bb --- /dev/null +++ b/x-pack/plugin/core/src/test/resources/template_with_variables-test.json @@ -0,0 +1,14 @@ +{ + "index_patterns": ".test-${test.template.version}", + "mappings": { + "doc": { + "_meta": { + "template.version": "${test.template.version}" + }, + "properties": { + "${test.template.field_1}": {"type": "keyword"}, + ${test.template.field_2} + } + } + } +} diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java index fcf22834a8e5c..2c8d209782889 100644 --- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java +++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java @@ -24,7 +24,6 @@ import java.util.List; import java.util.Map; import java.util.function.UnaryOperator; -import java.util.regex.Pattern; /** * This class activates/deactivates the logstash modules depending if we're running a node client or transport client @@ -35,8 +34,7 @@ public class Logstash extends Plugin implements SystemIndexPlugin { private static final String LOGSTASH_TEMPLATE_FILE_NAME = "logstash-management"; private static final String LOGSTASH_INDEX_TEMPLATE_NAME = ".logstash-management"; private static final String OLD_LOGSTASH_INDEX_NAME = "logstash-index-template"; - private static final String TEMPLATE_VERSION_PATTERN = - Pattern.quote("${logstash.template.version}"); + private static final String TEMPLATE_VERSION_VARIABLE = "logstash.template.version"; private final boolean enabled; private final boolean transportClientMode; @@ -66,7 +64,7 @@ public UnaryOperator> getIndexTemplateMetaDat return templates -> { templates.keySet().removeIf(OLD_LOGSTASH_INDEX_NAME::equals); TemplateUtils.loadTemplateIntoMap("/" + LOGSTASH_TEMPLATE_FILE_NAME + ".json", templates, LOGSTASH_INDEX_TEMPLATE_NAME, - Version.CURRENT.toString(), TEMPLATE_VERSION_PATTERN, LogManager.getLogger(Logstash.class)); + Version.CURRENT.toString(), TEMPLATE_VERSION_VARIABLE, LogManager.getLogger(Logstash.class)); //internal representation of typeless templates requires the default "_doc" type, which is also required for internal templates assert templates.get(LOGSTASH_INDEX_TEMPLATE_NAME).mappings().get(MapperService.SINGLE_MAPPING_NAME) != null; return templates; diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java index 8fc6bdeef55d1..3e6837affb046 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java @@ -16,7 +16,7 @@ import org.elasticsearch.test.SecuritySettingsSourceField; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.core.ml.integration.MlRestTestStateCleaner; -import org.elasticsearch.xpack.core.ml.notifications.AuditorField; +import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.elasticsearch.xpack.core.rollup.job.RollupJob; import org.elasticsearch.xpack.ml.MachineLearning; import org.junit.After; @@ -749,7 +749,7 @@ public void testLookbackWithoutPermissions() throws Exception { // There should be a notification saying that there was a problem extracting data client().performRequest(new Request("POST", "/_refresh")); Response notificationsResponse = client().performRequest( - new Request("GET", AuditorField.NOTIFICATIONS_INDEX + "/_search?size=1000&q=job_id:" + jobId)); + new Request("GET", NotificationsIndex.NOTIFICATIONS_INDEX + "/_search?size=1000&q=job_id:" + jobId)); String notificationsResponseAsString = EntityUtils.toString(notificationsResponse.getEntity()); assertThat(notificationsResponseAsString, containsString("\"message\":\"Datafeed is encountering errors extracting data: " + "action [indices:data/read/search] is unauthorized for user [ml_admin_plus_data]\"")); @@ -956,7 +956,7 @@ public void testLookbackWithoutPermissionsAndRollup() throws Exception { // There should be a notification saying that there was a problem extracting data client().performRequest(new Request("POST", "/_refresh")); Response notificationsResponse = client().performRequest( - new Request("GET", AuditorField.NOTIFICATIONS_INDEX + "/_search?size=1000&q=job_id:" + jobId)); + new Request("GET", NotificationsIndex.NOTIFICATIONS_INDEX + "/_search?size=1000&q=job_id:" + jobId)); String notificationsResponseAsString = EntityUtils.toString(notificationsResponse.getEntity()); assertThat(notificationsResponseAsString, containsString("\"message\":\"Datafeed is encountering errors extracting data: " + "action [indices:admin/xpack/rollup/search] is unauthorized for user [ml_admin_plus_data]\"")); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java index 8bc6cdb69eccd..6480eccc852cb 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java @@ -29,7 +29,7 @@ import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.job.results.Bucket; import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats; -import org.elasticsearch.xpack.core.ml.notifications.AuditorField; +import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.junit.After; import org.junit.Before; @@ -185,7 +185,7 @@ public void testDeleteExpiredData() throws Exception { .setQuery(QueryBuilders.termQuery("result_type", "model_size_stats")) .get().getHits().getTotalHits().value; long totalNotificationsCountBeforeDelete = - client().prepareSearch(AuditorField.NOTIFICATIONS_INDEX).get().getHits().getTotalHits().value; + client().prepareSearch(NotificationsIndex.NOTIFICATIONS_INDEX).get().getHits().getTotalHits().value; assertThat(totalModelSizeStatsBeforeDelete, greaterThan(0L)); assertThat(totalNotificationsCountBeforeDelete, greaterThan(0L)); @@ -233,7 +233,7 @@ public void testDeleteExpiredData() throws Exception { .setQuery(QueryBuilders.termQuery("result_type", "model_size_stats")) .get().getHits().getTotalHits().value; long totalNotificationsCountAfterDelete = - client().prepareSearch(AuditorField.NOTIFICATIONS_INDEX).get().getHits().getTotalHits().value; + client().prepareSearch(NotificationsIndex.NOTIFICATIONS_INDEX).get().getHits().getTotalHits().value; assertThat(totalModelSizeStatsAfterDelete, equalTo(totalModelSizeStatsBeforeDelete)); assertThat(totalNotificationsCountAfterDelete, greaterThanOrEqualTo(totalNotificationsCountBeforeDelete)); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java index fb1a4a6f004c9..2e6730674eff5 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java @@ -23,7 +23,7 @@ import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; import org.elasticsearch.xpack.core.ml.job.config.RuleScope; import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord; -import org.elasticsearch.xpack.core.ml.notifications.AuditorField; +import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.junit.After; import java.io.IOException; @@ -188,7 +188,7 @@ public void testScope() throws Exception { // Wait until the notification that the filter was updated is indexed assertBusy(() -> { SearchResponse searchResponse = - client().prepareSearch(AuditorField.NOTIFICATIONS_INDEX) + client().prepareSearch(NotificationsIndex.NOTIFICATIONS_INDEX) .setSize(1) .addSort("timestamp", SortOrder.DESC) .setQuery(QueryBuilders.boolQuery() diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java index 2c586b34e28e9..665be4551df8a 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java @@ -39,7 +39,7 @@ import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.core.ml.notifications.AuditorField; +import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.elasticsearch.xpack.core.ml.utils.PhaseProgress; import org.elasticsearch.xpack.core.ml.utils.QueryProvider; import org.elasticsearch.xpack.ml.dataframe.StoredProgress; @@ -232,7 +232,7 @@ protected static void assertThatAuditMessagesMatch(String configId, String... ex // Make sure we wrote to the audit // Since calls to write the AbstractAuditor are sent and forgot (async) we could have returned from the start, // finished the job (as this is a very short analytics job), all without the audit being fully written. - assertBusy(() -> assertTrue(indexExists(AuditorField.NOTIFICATIONS_INDEX))); + assertBusy(() -> assertTrue(indexExists(NotificationsIndex.NOTIFICATIONS_INDEX))); @SuppressWarnings("unchecked") Matcher[] itemMatchers = Arrays.stream(expectedAuditMessagePrefixes).map(Matchers::startsWith).toArray(Matcher[]::new); assertBusy(() -> { @@ -244,12 +244,12 @@ protected static void assertThatAuditMessagesMatch(String configId, String... ex } private static List fetchAllAuditMessages(String dataFrameAnalyticsId) { - RefreshRequest refreshRequest = new RefreshRequest(AuditorField.NOTIFICATIONS_INDEX); + RefreshRequest refreshRequest = new RefreshRequest(NotificationsIndex.NOTIFICATIONS_INDEX); RefreshResponse refreshResponse = client().execute(RefreshAction.INSTANCE, refreshRequest).actionGet(); assertThat(refreshResponse.getStatus().getStatus(), anyOf(equalTo(200), equalTo(201))); SearchRequest searchRequest = new SearchRequestBuilder(client(), SearchAction.INSTANCE) - .setIndices(AuditorField.NOTIFICATIONS_INDEX) + .setIndices(NotificationsIndex.NOTIFICATIONS_INDEX) .addSort("timestamp", SortOrder.ASC) .setQuery(QueryBuilders.termQuery("job_id", dataFrameAnalyticsId)) .request(); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java index fd728f39545c4..cfc7b9c14084a 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java @@ -21,7 +21,7 @@ import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord; import org.elasticsearch.xpack.core.ml.job.results.Bucket; -import org.elasticsearch.xpack.core.ml.notifications.AuditorField; +import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.junit.After; import java.io.IOException; @@ -225,7 +225,7 @@ public void testAddEventsToOpenJob() throws Exception { // Wait until the notification that the process was updated is indexed assertBusy(() -> { SearchResponse searchResponse = - client().prepareSearch(AuditorField.NOTIFICATIONS_INDEX) + client().prepareSearch(NotificationsIndex.NOTIFICATIONS_INDEX) .setSize(1) .addSort("timestamp", SortOrder.DESC) .setQuery(QueryBuilders.boolQuery() @@ -301,7 +301,7 @@ public void testAddOpenedJobToGroupWithCalendar() throws Exception { // Wait until the notification that the job was updated is indexed assertBusy(() -> { SearchResponse searchResponse = - client().prepareSearch(AuditorField.NOTIFICATIONS_INDEX) + client().prepareSearch(NotificationsIndex.NOTIFICATIONS_INDEX) .setSize(1) .addSort("timestamp", SortOrder.DESC) .setQuery(QueryBuilders.boolQuery() diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 956d6b824104d..5dd7d8f833151 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -9,21 +9,18 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SetOnce; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.OriginSettingClient; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; @@ -37,10 +34,8 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.TokenizerFactory; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; @@ -135,8 +130,7 @@ import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; -import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; -import org.elasticsearch.xpack.core.ml.notifications.AuditorField; +import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.elasticsearch.xpack.core.template.TemplateUtils; import org.elasticsearch.xpack.ml.action.TransportCloseJobAction; import org.elasticsearch.xpack.ml.action.TransportDeleteCalendarAction; @@ -215,7 +209,6 @@ import org.elasticsearch.xpack.ml.dataframe.process.results.MemoryUsageEstimationResult; import org.elasticsearch.xpack.ml.inference.ingest.InferenceProcessor; import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; -import org.elasticsearch.xpack.ml.inference.persistence.InferenceInternalIndex; import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider; import org.elasticsearch.xpack.ml.job.JobManager; import org.elasticsearch.xpack.ml.job.JobManagerHolder; @@ -321,7 +314,6 @@ import java.util.function.UnaryOperator; import static java.util.Collections.emptyList; -import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; public class MachineLearning extends Plugin implements SystemIndexPlugin, AnalysisPlugin, IngestPlugin, PersistentTaskPlugin { public static final String NAME = "ml"; @@ -525,6 +517,8 @@ public Collection createComponents(Client client, ClusterService cluster return Collections.singletonList(new JobManagerHolder()); } + new MlIndexTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry); + AnomalyDetectionAuditor anomalyDetectionAuditor = new AnomalyDetectionAuditor(client, clusterService.getNodeName()); DataFrameAnalyticsAuditor dataFrameAnalyticsAuditor = new DataFrameAnalyticsAuditor(client, clusterService.getNodeName()); InferenceAuditor inferenceAuditor = new InferenceAuditor(client, clusterService.getNodeName()); @@ -898,112 +892,14 @@ public Map> getTokenizers() { @Override public UnaryOperator> getIndexTemplateMetaDataUpgrader() { - return templates -> { - - try (XContentBuilder auditMapping = ElasticsearchMappings.auditMessageMapping()) { - IndexTemplateMetaData notificationMessageTemplate = - IndexTemplateMetaData.builder(AuditorField.NOTIFICATIONS_INDEX) - .putMapping(SINGLE_MAPPING_NAME, Strings.toString(auditMapping)) - .patterns(Collections.singletonList(AuditorField.NOTIFICATIONS_INDEX)) - .version(Version.CURRENT.id) - .settings(Settings.builder() - // Our indexes are small and one shard puts the - // least possible burden on Elasticsearch - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")) - .build(); - templates.put(AuditorField.NOTIFICATIONS_INDEX, notificationMessageTemplate); - } catch (IOException e) { - logger.warn("Error loading the template for the notification message index", e); - } - - try (XContentBuilder docMapping = MlMetaIndex.docMapping()) { - IndexTemplateMetaData metaTemplate = - IndexTemplateMetaData.builder(MlMetaIndex.INDEX_NAME) - .patterns(Collections.singletonList(MlMetaIndex.INDEX_NAME)) - .settings(Settings.builder() - // Our indexes are small and one shard puts the - // least possible burden on Elasticsearch - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")) - .version(Version.CURRENT.id) - .putMapping(SINGLE_MAPPING_NAME, Strings.toString(docMapping)) - .build(); - templates.put(MlMetaIndex.INDEX_NAME, metaTemplate); - } catch (IOException e) { - logger.warn("Error loading the template for the " + MlMetaIndex.INDEX_NAME + " index", e); - } - - try (XContentBuilder configMapping = ElasticsearchMappings.configMapping()) { - IndexTemplateMetaData configTemplate = - IndexTemplateMetaData.builder(AnomalyDetectorsIndex.configIndexName()) - .patterns(Collections.singletonList(AnomalyDetectorsIndex.configIndexName())) - .settings(Settings.builder() - // Our indexes are small and one shard puts the - // least possible burden on Elasticsearch - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1") - .put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), - AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)) - .version(Version.CURRENT.id) - .putMapping(SINGLE_MAPPING_NAME, Strings.toString(configMapping)) - .build(); - templates.put(AnomalyDetectorsIndex.configIndexName(), configTemplate); - } catch (IOException e) { - logger.warn("Error loading the template for the " + AnomalyDetectorsIndex.configIndexName() + " index", e); - } - - try (XContentBuilder stateMapping = ElasticsearchMappings.stateMapping()) { - IndexTemplateMetaData stateTemplate = - IndexTemplateMetaData.builder(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX) - .patterns(Collections.singletonList(AnomalyDetectorsIndex.jobStateIndexPattern())) - // TODO review these settings - .settings(Settings.builder() - .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")) - .putMapping(SINGLE_MAPPING_NAME, Strings.toString(stateMapping)) - .version(Version.CURRENT.id) - .build(); - - templates.put(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, stateTemplate); - } catch (IOException e) { - logger.error("Error loading the template for the " + AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX + " index", e); - } - - try (XContentBuilder docMapping = ElasticsearchMappings.resultsMapping(SINGLE_MAPPING_NAME)) { - IndexTemplateMetaData jobResultsTemplate = - IndexTemplateMetaData.builder(AnomalyDetectorsIndex.jobResultsIndexPrefix()) - .patterns(Collections.singletonList(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*")) - .settings(Settings.builder() - .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1") - // Sacrifice durability for performance: in the event of power - // failure we can lose the last 5 seconds of changes, but it's - // much faster - .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), "async") - // set the default all search field - .put(IndexSettings.DEFAULT_FIELD_SETTING.getKey(), ElasticsearchMappings.ALL_FIELD_VALUES)) - .putMapping(SINGLE_MAPPING_NAME, Strings.toString(docMapping)) - .version(Version.CURRENT.id) - .build(); - templates.put(AnomalyDetectorsIndex.jobResultsIndexPrefix(), jobResultsTemplate); - } catch (IOException e) { - logger.error("Error loading the template for the " + AnomalyDetectorsIndex.jobResultsIndexPrefix() + " indices", e); - } - - try { - templates.put(InferenceIndexConstants.LATEST_INDEX_NAME, InferenceInternalIndex.getIndexTemplateMetaData()); - } catch (IOException e) { - logger.error("Error loading the template for the " + InferenceIndexConstants.LATEST_INDEX_NAME + " index", e); - } - - return templates; - }; + return UnaryOperator.identity(); } public static boolean allTemplatesInstalled(ClusterState clusterState) { boolean allPresent = true; List templateNames = Arrays.asList( - AuditorField.NOTIFICATIONS_INDEX, + NotificationsIndex.NOTIFICATIONS_INDEX, MlMetaIndex.INDEX_NAME, AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, AnomalyDetectorsIndex.jobResultsIndexPrefix(), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java index d8407d67b9ac4..b2bcb43689be9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java @@ -32,9 +32,11 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.core.ml.MlConfigIndex; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.OpenJobAction; @@ -43,7 +45,6 @@ import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; @@ -494,7 +495,7 @@ private void createConfigIndex(ActionListener listener) { .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1") .put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW) ); - createIndexRequest.mapping(SINGLE_MAPPING_NAME, ElasticsearchMappings.configMapping()); + createIndexRequest.mapping(SINGLE_MAPPING_NAME, MlConfigIndex.mapping(), XContentType.JSON); } catch (Exception e) { logger.error("error writing the .ml-config mappings", e); listener.onFailure(e); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java new file mode 100644 index 0000000000000..be52cc9202c78 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml; + +import org.elasticsearch.Version; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.ml.MlConfigIndex; +import org.elasticsearch.xpack.core.ml.MlMetaIndex; +import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; +import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; +import org.elasticsearch.xpack.core.template.IndexTemplateConfig; +import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; +import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class MlIndexTemplateRegistry extends IndexTemplateRegistry { + + private static final String ROOT_RESOURCE_PATH = "/org/elasticsearch/xpack/core/ml/"; + private static final String ANOMALY_DETECTION_PATH = ROOT_RESOURCE_PATH + "anomalydetection/"; + private static final String VERSION_PATTERN = "xpack.ml.version"; + private static final String VERSION_ID_PATTERN = "xpack.ml.version.id"; + + private static final IndexTemplateConfig ANOMALY_DETECTION_RESULTS_TEMPLATE = anomalyDetectionResultsTemplate(); + + private static final IndexTemplateConfig ANOMALY_DETECTION_STATE_TEMPLATE = new IndexTemplateConfig( + AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,ANOMALY_DETECTION_PATH + "state_index_template.json", + Version.CURRENT.id, VERSION_PATTERN, + Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id))); + + private static final IndexTemplateConfig META_TEMPLATE = new IndexTemplateConfig(MlMetaIndex.INDEX_NAME, + ROOT_RESOURCE_PATH + "meta_index_template.json", Version.CURRENT.id, VERSION_PATTERN, + Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id))); + + private static final IndexTemplateConfig NOTIFICATIONS_TEMPLATE = new IndexTemplateConfig(NotificationsIndex.NOTIFICATIONS_INDEX, + ROOT_RESOURCE_PATH + "notifications_index_template.json", Version.CURRENT.id, VERSION_PATTERN, + Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id))); + + private static final IndexTemplateConfig CONFIG_TEMPLATE = configTemplate(); + + private static final IndexTemplateConfig INFERENCE_TEMPLATE = new IndexTemplateConfig(InferenceIndexConstants.LATEST_INDEX_NAME, + ROOT_RESOURCE_PATH + "inference_index_template.json", Version.CURRENT.id, VERSION_PATTERN, + Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id))); + + private static IndexTemplateConfig configTemplate() { + Map variables = new HashMap<>(); + variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)); + variables.put("xpack.ml.config.max_result_window", + String.valueOf(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)); + variables.put("xpack.ml.config.mappings", MlConfigIndex.mapping()); + + return new IndexTemplateConfig(AnomalyDetectorsIndex.configIndexName(), + ROOT_RESOURCE_PATH + "config_index_template.json", + Version.CURRENT.id, VERSION_PATTERN, + variables); + } + + private static IndexTemplateConfig anomalyDetectionResultsTemplate() { + Map variables = new HashMap<>(); + variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)); + variables.put("xpack.ml.anomalydetection.results.mappings", AnomalyDetectorsIndex.resultsMapping()); + + return new IndexTemplateConfig(AnomalyDetectorsIndex.jobResultsIndexPrefix(), + ANOMALY_DETECTION_PATH + "results_index_template.json", + Version.CURRENT.id, VERSION_PATTERN, + variables); + } + + public MlIndexTemplateRegistry(Settings nodeSettings, ClusterService clusterService, ThreadPool threadPool, Client client, + NamedXContentRegistry xContentRegistry) { + super(nodeSettings, clusterService, threadPool, client, xContentRegistry); + } + + @Override + protected boolean requiresMasterNode() { + return true; + } + + @Override + protected List getTemplateConfigs() { + return Arrays.asList( + ANOMALY_DETECTION_RESULTS_TEMPLATE, + ANOMALY_DETECTION_STATE_TEMPLATE, + CONFIG_TEMPLATE, + INFERENCE_TEMPLATE, + META_TEMPLATE, + NOTIFICATIONS_TEMPLATE + ); + } + + @Override + protected List getPolicyConfigs() { + return Collections.emptyList(); + } + + @Override + protected String getOrigin() { + return ClientHelper.ML_ORIGIN; + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java index 89306c0a60c08..2af917157ffd5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java @@ -35,6 +35,7 @@ import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.common.validation.SourceDestValidator; import org.elasticsearch.xpack.core.ml.MachineLearningField; +import org.elasticsearch.xpack.core.ml.MlConfigIndex; import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; import org.elasticsearch.xpack.core.ml.job.messages.Messages; @@ -208,7 +209,7 @@ private void updateDocMappingAndPutConfig(DataFrameAnalyticsConfig config, } ElasticsearchMappings.addDocMappingIfMissing( AnomalyDetectorsIndex.configIndexName(), - ElasticsearchMappings::configMapping, + MlConfigIndex::mapping, client, clusterState, ActionListener.wrap( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java index 1179268b1527d..8f00956949df3 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java @@ -36,6 +36,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.ml.MlConfigIndex; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; @@ -210,7 +211,7 @@ private void putDatafeed(PutDatafeedAction.Request request, } ElasticsearchMappings.addDocMappingIfMissing( AnomalyDetectorsIndex.configIndexName(), - ElasticsearchMappings::configMapping, + MlConfigIndex::mapping, client, clusterState, ActionListener.wrap(mappingsUpdated, listener::onFailure)); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/InferenceInternalIndex.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/InferenceInternalIndex.java deleted file mode 100644 index 8e72d025b56d0..0000000000000 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/InferenceInternalIndex.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.inference.persistence; - -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; -import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants; - -import java.io.IOException; -import java.util.Collections; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; -import static org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants.LATEST_INDEX_NAME; -import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.DATE; -import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.DYNAMIC; -import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.ENABLED; -import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.KEYWORD; -import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.LONG; -import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.PROPERTIES; -import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.TEXT; -import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.TYPE; -import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.addMetaInformation; - -/** - * Changelog of internal index versions - * - * Please list changes, increase the version in {@link InferenceInternalIndex} if you are 1st in this release cycle - * - * version 1 (7.5): initial - */ -public final class InferenceInternalIndex { - - private InferenceInternalIndex() {} - - public static XContentBuilder mappings() throws IOException { - return configMapping(SINGLE_MAPPING_NAME); - } - - public static IndexTemplateMetaData getIndexTemplateMetaData() throws IOException { - IndexTemplateMetaData inferenceTemplate = IndexTemplateMetaData.builder(LATEST_INDEX_NAME) - .patterns(Collections.singletonList(LATEST_INDEX_NAME)) - .version(Version.CURRENT.id) - .settings(Settings.builder() - // the configurations are expected to be small - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")) - .putMapping(SINGLE_MAPPING_NAME, Strings.toString(mappings())) - .build(); - return inferenceTemplate; - } - - public static XContentBuilder configMapping(String mappingType) throws IOException { - XContentBuilder builder = jsonBuilder(); - builder.startObject(); - builder.startObject(mappingType); - addMetaInformation(builder); - - // do not allow anything outside of the defined schema - builder.field(DYNAMIC, "false"); - - builder.startObject(PROPERTIES); - - // Add the doc_type field - builder.startObject(InferenceIndexConstants.DOC_TYPE.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject(); - - addInferenceDocFields(builder); - addDefinitionDocFields(builder); - return builder.endObject() - .endObject() - .endObject(); - } - - private static void addInferenceDocFields(XContentBuilder builder) throws IOException { - builder.startObject(TrainedModelConfig.MODEL_ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(TrainedModelConfig.CREATED_BY.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(TrainedModelConfig.INPUT.getPreferredName()) - .field(ENABLED, false) - .endObject() - .startObject(TrainedModelConfig.VERSION.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(TrainedModelConfig.DESCRIPTION.getPreferredName()) - .field(TYPE, TEXT) - .endObject() - .startObject(TrainedModelConfig.CREATE_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(TrainedModelConfig.TAGS.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(TrainedModelConfig.METADATA.getPreferredName()) - .field(ENABLED, false) - .endObject() - .startObject(TrainedModelConfig.ESTIMATED_OPERATIONS.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(TrainedModelConfig.ESTIMATED_HEAP_MEMORY_USAGE_BYTES.getPreferredName()) - .field(TYPE, LONG) - .endObject(); - } - - private static void addDefinitionDocFields(XContentBuilder builder) throws IOException { - builder.startObject(TrainedModelDefinitionDoc.DOC_NUM.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(TrainedModelDefinitionDoc.DEFINITION.getPreferredName()) - .field(ENABLED, false) - .endObject() - .startObject(TrainedModelDefinitionDoc.COMPRESSION_VERSION.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(TrainedModelDefinitionDoc.DEFINITION_LENGTH.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(TrainedModelDefinitionDoc.TOTAL_DEFINITION_LENGTH.getPreferredName()) - .field(TYPE, LONG) - .endObject(); - } -} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index b134cf59c193c..c35351e586d75 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -30,13 +30,14 @@ import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.action.util.QueryPage; import org.elasticsearch.xpack.core.ml.MachineLearningField; +import org.elasticsearch.xpack.core.ml.MlConfigIndex; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction; import org.elasticsearch.xpack.core.ml.action.UpdateJobAction; -import org.elasticsearch.xpack.core.action.util.QueryPage; import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; @@ -294,7 +295,7 @@ public void onFailure(Exception e) { return; } ElasticsearchMappings.addDocMappingIfMissing( - AnomalyDetectorsIndex.configIndexName(), ElasticsearchMappings::configMapping, client, state, putJobListener); + AnomalyDetectorsIndex.configIndexName(), MlConfigIndex::mapping, client, state, putJobListener); }, putJobListener::onFailure ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java index 16b666c8a9b9a..979b145c4eb93 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java @@ -56,6 +56,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -101,6 +102,7 @@ import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats; import org.elasticsearch.xpack.core.ml.job.results.Influencer; import org.elasticsearch.xpack.core.ml.job.results.ModelPlot; +import org.elasticsearch.xpack.core.ml.job.results.ReservedFieldNames; import org.elasticsearch.xpack.core.ml.job.results.Result; import org.elasticsearch.xpack.core.ml.stats.CountAccumulator; import org.elasticsearch.xpack.core.ml.stats.ForecastStats; @@ -131,7 +133,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; @@ -279,7 +280,7 @@ public void createJobResultIndex(Job job, ClusterState state, final ActionListen } final String indexName = tempIndexName; - final ActionListener createAliasListener = ActionListener.wrap(success -> { + ActionListener indexAndMappingsListener = ActionListener.wrap(success -> { final IndicesAliasesRequest request = client.admin().indices().prepareAliases() .addAlias(indexName, readAliasName, QueryBuilders.termQuery(Job.ID.getPreferredName(), job.getId())) .addAlias(indexName, writeAliasName).request(); @@ -293,54 +294,50 @@ public void createJobResultIndex(Job job, ClusterState state, final ActionListen if (!state.getMetaData().hasIndex(indexName)) { LOGGER.trace("ES API CALL: create index {}", indexName); CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName); - // This assumes the requested mapping will be merged with mappings from the template, - // and may need to be revisited if template merging is ever refactored - try (XContentBuilder termFieldsMapping = ElasticsearchMappings.termFieldsMapping(termFields)) { - createIndexRequest.mapping(SINGLE_MAPPING_NAME, termFieldsMapping); - } executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest, ActionListener.wrap( - r -> createAliasListener.onResponse(r.isAcknowledged()), + // Add the term field mappings and alias. The complication is that the state at the + // beginning of the operation doesn't have any knowledge of the index, as it's only + // just been created. So we need yet another operation to get the mappings for it. + r -> getLatestIndexMappingsAndAddTerms(indexName, termFields, indexAndMappingsListener), e -> { // Possible that the index was created while the request was executing, // so we need to handle that possibility if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) { - LOGGER.info("Index already exists"); - // Add the term field mappings and alias. The complication is that the state at the - // beginning of the operation doesn't have any knowledge of the index, as it's only - // just been created. So we need yet another operation to get the mappings for it. - getLatestIndexMappings(indexName, ActionListener.wrap( - response -> { - // Expect one index and one type. If this is not the case then it means the - // index has been deleted almost immediately after being created, and this is - // so unlikely that it's reasonable to fail the whole operation. - ImmutableOpenMap indexMappings = - response.getMappings().iterator().next().value; - MappingMetaData typeMappings = indexMappings.iterator().next().value; - addTermsAndAliases(typeMappings, indexName, termFields, createAliasListener); - }, - finalListener::onFailure - )); + LOGGER.info("Index [{}] already exists", indexName); + getLatestIndexMappingsAndAddTerms(indexName, termFields, indexAndMappingsListener); } else { finalListener.onFailure(e); } } ), client.admin().indices()::create); } else { - MappingMetaData mapping = state.metaData().index(indexName).mapping(); - addTermsAndAliases(mapping, indexName, termFields, createAliasListener); + MappingMetaData indexMappings = state.metaData().index(indexName).mapping(); + addTermsMapping(indexMappings, indexName, termFields, indexAndMappingsListener); } } - private void getLatestIndexMappings(final String indexName, final ActionListener listener) { + private void getLatestIndexMappingsAndAddTerms(String indexName, Collection termFields, ActionListener listener) { + + ActionListener getMappingsListener = ActionListener.wrap( + getMappingsResponse -> { + // Expect one index and one type. If this is not the case then it means the + // index has been deleted almost immediately after being created, and this is + // so unlikely that it's reasonable to fail the whole operation. + ImmutableOpenMap indexMappings = getMappingsResponse.getMappings().iterator().next().value; + MappingMetaData typeMappings = indexMappings.iterator().next().value; + addTermsMapping(typeMappings, indexName, termFields, listener); + }, + listener::onFailure + ); GetMappingsRequest getMappingsRequest = client.admin().indices().prepareGetMappings(indexName).request(); - executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, getMappingsRequest, listener, + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, getMappingsRequest, getMappingsListener, client.admin().indices()::getMappings); } - private void addTermsAndAliases(final MappingMetaData mapping, final String indexName, final Collection termFields, - final ActionListener listener) { + private void addTermsMapping(MappingMetaData mapping, String indexName, Collection termFields, + ActionListener listener) { long fieldCountLimit = MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.get(settings); if (violatedFieldCountLimit(termFields.size(), fieldCountLimit, mapping)) { @@ -380,8 +377,9 @@ public static int countFields(Map mapping) { private void updateIndexMappingWithTermFields(String indexName, String mappingType, Collection termFields, ActionListener listener) { - // Put the whole mapping, not just the term fields, otherwise we'll wipe the _meta section of the mapping - try (XContentBuilder termFieldsMapping = ElasticsearchMappings.resultsMapping(mappingType, termFields)) { + + try (XContentBuilder termFieldsMapping = JsonXContent.contentBuilder()) { + createTermFieldsMapping(termFieldsMapping, mappingType, termFields); final PutMappingRequest request = client.admin().indices().preparePutMapping(indexName) .setType(mappingType) .setSource(termFieldsMapping).request(); @@ -401,6 +399,21 @@ public void onFailure(Exception e) { } } + // Visible for testing + static void createTermFieldsMapping(XContentBuilder builder, String mappingType, Collection termFields) throws IOException { + builder.startObject(); + builder.startObject(mappingType); + builder.startObject("properties"); + for (String fieldName : termFields) { + if (ReservedFieldNames.isValidFieldName(fieldName)) { + builder.startObject(fieldName).field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD).endObject(); + } + } + builder.endObject(); + builder.endObject(); + builder.endObject(); + } + /** * Get the job's data counts * diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java index 9ef733198114e..9eef65bcbbe35 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -441,7 +441,7 @@ protected void doRun() { // Try adding the results doc mapping - this updates to the latest version if an old mapping is present ElasticsearchMappings.addDocMappingIfMissing(AnomalyDetectorsIndex.jobResultsAliasedName(jobId), - ElasticsearchMappings::resultsMapping, client, clusterState, resultsMappingUpdateHandler); + AnomalyDetectorsIndex::resultsMapping, client, clusterState, resultsMappingUpdateHandler); } private boolean createProcessAndSetRunning(ProcessContext processContext, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/AnomalyDetectionAuditor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/AnomalyDetectionAuditor.java index 48c5872b057ed..6e3af2b8928d9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/AnomalyDetectionAuditor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/AnomalyDetectionAuditor.java @@ -7,7 +7,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.xpack.core.common.notifications.AbstractAuditor; -import org.elasticsearch.xpack.core.ml.notifications.AuditorField; +import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.elasticsearch.xpack.core.ml.notifications.AnomalyDetectionAuditMessage; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; @@ -15,6 +15,6 @@ public class AnomalyDetectionAuditor extends AbstractAuditor { public AnomalyDetectionAuditor(Client client, String nodeName) { - super(client, nodeName, AuditorField.NOTIFICATIONS_INDEX, ML_ORIGIN, AnomalyDetectionAuditMessage::new); + super(client, nodeName, NotificationsIndex.NOTIFICATIONS_INDEX, ML_ORIGIN, AnomalyDetectionAuditMessage::new); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/DataFrameAnalyticsAuditor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/DataFrameAnalyticsAuditor.java index 1c9be78d241c4..1acccaafcdaf0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/DataFrameAnalyticsAuditor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/DataFrameAnalyticsAuditor.java @@ -7,7 +7,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.xpack.core.common.notifications.AbstractAuditor; -import org.elasticsearch.xpack.core.ml.notifications.AuditorField; +import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.elasticsearch.xpack.core.ml.notifications.DataFrameAnalyticsAuditMessage; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; @@ -15,6 +15,6 @@ public class DataFrameAnalyticsAuditor extends AbstractAuditor { public DataFrameAnalyticsAuditor(Client client, String nodeName) { - super(client, nodeName, AuditorField.NOTIFICATIONS_INDEX, ML_ORIGIN, DataFrameAnalyticsAuditMessage::new); + super(client, nodeName, NotificationsIndex.NOTIFICATIONS_INDEX, ML_ORIGIN, DataFrameAnalyticsAuditMessage::new); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/InferenceAuditor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/InferenceAuditor.java index dfce44af7c9a4..2be3e76e85b58 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/InferenceAuditor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/InferenceAuditor.java @@ -7,7 +7,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.xpack.core.common.notifications.AbstractAuditor; -import org.elasticsearch.xpack.core.ml.notifications.AuditorField; +import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.elasticsearch.xpack.core.ml.notifications.InferenceAuditMessage; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; @@ -15,6 +15,6 @@ public class InferenceAuditor extends AbstractAuditor { public InferenceAuditor(Client client, String nodeName) { - super(client, nodeName, AuditorField.NOTIFICATIONS_INDEX, ML_ORIGIN, InferenceAuditMessage::new); + super(client, nodeName, NotificationsIndex.NOTIFICATIONS_INDEX, ML_ORIGIN, InferenceAuditMessage::new); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java index 2809d7e57efc1..949684227082c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java @@ -47,7 +47,7 @@ import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; -import org.elasticsearch.xpack.core.ml.notifications.AuditorField; +import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.job.JobNodeSelector; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; @@ -235,7 +235,7 @@ private void addIndices(MetaData.Builder metaData, RoutingTable.Builder routingT indices.add(AnomalyDetectorsIndex.configIndexName()); indices.add(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX); indices.add(MlMetaIndex.INDEX_NAME); - indices.add(AuditorField.NOTIFICATIONS_INDEX); + indices.add(NotificationsIndex.NOTIFICATIONS_INDEX); indices.add(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT); for (String indexName : indices) { IndexMetaData.Builder indexMetaData = IndexMetaData.builder(indexName); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java index c7b7faa817556..3fa6a8721bd5d 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java @@ -5,8 +5,11 @@ */ package org.elasticsearch.xpack.ml.integration; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; @@ -15,6 +18,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.OriginSettingClient; +import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; @@ -32,10 +36,10 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.action.util.QueryPage; import org.elasticsearch.xpack.core.ml.MlMetaIndex; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.PutJobAction; -import org.elasticsearch.xpack.core.action.util.QueryPage; import org.elasticsearch.xpack.core.ml.calendars.Calendar; import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; @@ -78,8 +82,13 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.isIn; import static org.hamcrest.Matchers.not; @@ -116,6 +125,73 @@ public void createComponents() throws Exception { waitForMlTemplates(); } + public void testPutJob_CreatesResultsIndex() { + + Job.Builder job1 = new Job.Builder("first_job"); + job1.setAnalysisConfig(createAnalysisConfig("by_field_1", Collections.emptyList())); + job1.setDataDescription(new DataDescription.Builder()); + + // Put fist job. This should create the results index as it's the first job. + client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(job1)).actionGet(); + + String sharedResultsIndex = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; + Map mappingProperties = getIndexMappingProperties(sharedResultsIndex); + + // Assert mappings have a few fields from the template + assertThat(mappingProperties.keySet(), hasItems("anomaly_score", "bucket_count")); + // Assert mappings have the by field + assertThat(mappingProperties.keySet(), hasItem("by_field_1")); + + // Check aliases have been created + assertThat(getAliases(sharedResultsIndex), containsInAnyOrder(AnomalyDetectorsIndex.jobResultsAliasedName(job1.getId()), + AnomalyDetectorsIndex.resultsWriteAlias(job1.getId()))); + + // Now let's create a second job to test things work when the index exists already + assertThat(mappingProperties.keySet(), not(hasItem("by_field_2"))); + + Job.Builder job2 = new Job.Builder("second_job"); + job2.setAnalysisConfig(createAnalysisConfig("by_field_2", Collections.emptyList())); + job2.setDataDescription(new DataDescription.Builder()); + + client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(job2)).actionGet(); + + mappingProperties = getIndexMappingProperties(sharedResultsIndex); + + // Assert mappings have a few fields from the template + assertThat(mappingProperties.keySet(), hasItems("anomaly_score", "bucket_count")); + // Assert mappings have the by field + assertThat(mappingProperties.keySet(), hasItems("by_field_1", "by_field_2")); + + // Check aliases have been created + assertThat(getAliases(sharedResultsIndex), containsInAnyOrder( + AnomalyDetectorsIndex.jobResultsAliasedName(job1.getId()), + AnomalyDetectorsIndex.resultsWriteAlias(job1.getId()), + AnomalyDetectorsIndex.jobResultsAliasedName(job2.getId()), + AnomalyDetectorsIndex.resultsWriteAlias(job2.getId()) + )); + } + + public void testPutJob_WithCustomResultsIndex() { + Job.Builder job = new Job.Builder("foo"); + job.setResultsIndexName("bar"); + job.setAnalysisConfig(createAnalysisConfig("by_field", Collections.emptyList())); + job.setDataDescription(new DataDescription.Builder()); + + client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(job)).actionGet(); + + String customIndex = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-bar"; + Map mappingProperties = getIndexMappingProperties(customIndex); + + // Assert mappings have a few fields from the template + assertThat(mappingProperties.keySet(), hasItems("anomaly_score", "bucket_count")); + // Assert mappings have the by field + assertThat(mappingProperties.keySet(), hasItem("by_field")); + + // Check aliases have been created + assertThat(getAliases(customIndex), containsInAnyOrder(AnomalyDetectorsIndex.jobResultsAliasedName(job.getId()), + AnomalyDetectorsIndex.resultsWriteAlias(job.getId()))); + } + @AwaitsFix(bugUrl ="https://github.com/elastic/elasticsearch/issues/40134") public void testMultipleSimultaneousJobCreations() { @@ -268,6 +344,39 @@ public void testRemoveJobFromCalendar() throws Exception { } } + private Map getIndexMappingProperties(String index) { + GetMappingsRequest request = new GetMappingsRequest().indices(index); + GetMappingsResponse response = client().execute(GetMappingsAction.INSTANCE, request).actionGet(); + ImmutableOpenMap> indexMappings = response.getMappings(); + assertNotNull(indexMappings); + ImmutableOpenMap typeMappings = indexMappings.get(index); + assertNotNull("expected " + index + " in " + indexMappings, typeMappings); + assertEquals("expected 1 type in " + typeMappings, 1, typeMappings.size()); + Map mappings = typeMappings.iterator().next().value.getSourceAsMap(); + assertNotNull(mappings); + + // Assert _meta info is present + assertThat(mappings.keySet(), hasItem("_meta")); + @SuppressWarnings("unchecked") + Map meta = (Map) mappings.get("_meta"); + assertThat(meta.keySet(), hasItem("version")); + assertThat(meta.get("version"), equalTo(Version.CURRENT.toString())); + + @SuppressWarnings("unchecked") + Map properties = (Map) mappings.get("properties"); + assertNotNull("expected 'properties' field in " + mappings, properties); + return properties; + } + + private Set getAliases(String index) { + GetAliasesResponse getAliasesResponse = client().admin().indices().getAliases( + new GetAliasesRequest().indices(index)).actionGet(); + ImmutableOpenMap> aliases = getAliasesResponse.getAliases(); + assertThat(aliases.containsKey(index), is(true)); + List aliasMetaData = aliases.get(index); + return aliasMetaData.stream().map(AliasMetaData::alias).collect(Collectors.toSet()); + } + private List getCalendars(String jobId) throws Exception { CountDownLatch latch = new CountDownLatch(1); AtomicReference exceptionHolder = new AtomicReference<>(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java index bc2358e0f861f..c2619f56e5187 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java @@ -9,8 +9,6 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.search.MultiSearchAction; import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.MultiSearchRequestBuilder; @@ -19,26 +17,19 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.AckedClusterStateUpdateTask; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.Index; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.test.ESTestCase; @@ -47,17 +38,14 @@ import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats; import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord; import org.elasticsearch.xpack.core.ml.job.results.Bucket; import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition; import org.elasticsearch.xpack.core.ml.job.results.Influencer; -import org.elasticsearch.xpack.core.ml.job.results.Result; import org.elasticsearch.xpack.core.ml.utils.ExponentialAverageCalculationContext; import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder.InfluencersQuery; -import org.mockito.ArgumentCaptor; import java.io.IOException; import java.time.Instant; @@ -68,183 +56,20 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; -import static org.elasticsearch.xpack.core.ml.job.config.JobTests.buildJobBuilder; import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; public class JobResultsProviderTests extends ESTestCase { - private static final String CLUSTER_NAME = "myCluster"; - - @SuppressWarnings("unchecked") - public void testCreateJobResultsIndex() { - String resultsIndexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; - QueryBuilder jobFilter = QueryBuilders.termQuery("job_id", "foo"); - - MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME); - ArgumentCaptor captor = ArgumentCaptor.forClass(CreateIndexRequest.class); - clientBuilder.createIndexRequest(captor, resultsIndexName); - clientBuilder.prepareAlias(resultsIndexName, AnomalyDetectorsIndex.jobResultsAliasedName("foo"), jobFilter); - clientBuilder.prepareAlias(resultsIndexName, AnomalyDetectorsIndex.resultsWriteAlias("foo")); - - Job.Builder job = buildJobBuilder("foo"); - JobResultsProvider provider = createProvider(clientBuilder.build()); - AtomicReference resultHolder = new AtomicReference<>(); - - ClusterState cs = ClusterState.builder(new ClusterName("_name")) - .metaData(MetaData.builder().indices(ImmutableOpenMap.of())) - .build(); - - ClusterService clusterService = mock(ClusterService.class); - - doAnswer(invocationOnMock -> { - AckedClusterStateUpdateTask task = (AckedClusterStateUpdateTask) invocationOnMock.getArguments()[1]; - task.execute(cs); - return null; - }).when(clusterService).submitStateUpdateTask(eq("put-job-foo"), any(AckedClusterStateUpdateTask.class)); - - provider.createJobResultIndex(job.build(), cs, new ActionListener() { - @Override - public void onResponse(Boolean aBoolean) { - CreateIndexRequest request = captor.getValue(); - assertNotNull(request); - assertEquals(resultsIndexName, request.index()); - clientBuilder.verifyIndexCreated(resultsIndexName); - resultHolder.set(aBoolean); - } - - @Override - public void onFailure(Exception e) { - fail(e.toString()); - } - }); - - assertNotNull(resultHolder.get()); - assertTrue(resultHolder.get()); - } - - @SuppressWarnings("unchecked") - public void testCreateJobWithExistingIndex() { - QueryBuilder jobFilter = QueryBuilders.termQuery("job_id", "foo"); - MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME); - clientBuilder.prepareAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"), - AnomalyDetectorsIndex.jobResultsAliasedName("foo123"), jobFilter); - clientBuilder.preparePutMapping(mock(AcknowledgedResponse.class), Result.TYPE.getPreferredName()); - - GetMappingsResponse getMappingsResponse = mock(GetMappingsResponse.class); - ImmutableOpenMap typeMappings = ImmutableOpenMap.of(); - - ImmutableOpenMap> mappings = - ImmutableOpenMap.>builder() - .fPut(AnomalyDetectorsIndex.jobResultsAliasedName("foo"), typeMappings).build(); - when(getMappingsResponse.mappings()).thenReturn(mappings); - clientBuilder.prepareGetMapping(getMappingsResponse); - - Job.Builder job = buildJobBuilder("foo123"); - job.setResultsIndexName("foo"); - JobResultsProvider provider = createProvider(clientBuilder.build()); - - Index index = mock(Index.class); - when(index.getName()).thenReturn(AnomalyDetectorsIndex.jobResultsAliasedName("foo")); - IndexMetaData indexMetaData = mock(IndexMetaData.class); - when(indexMetaData.getIndex()).thenReturn(index); - - ImmutableOpenMap aliases = ImmutableOpenMap.of(); - when(indexMetaData.getAliases()).thenReturn(aliases); - when(indexMetaData.getSettings()).thenReturn(Settings.EMPTY); - - ImmutableOpenMap indexMap = ImmutableOpenMap.builder() - .fPut(AnomalyDetectorsIndex.jobResultsAliasedName("foo"), indexMetaData).build(); - - ClusterState cs2 = ClusterState.builder(new ClusterName("_name")) - .metaData(MetaData.builder().indices(indexMap)).build(); - - ClusterService clusterService = mock(ClusterService.class); - - doAnswer(invocationOnMock -> { - AckedClusterStateUpdateTask task = (AckedClusterStateUpdateTask) invocationOnMock.getArguments()[1]; - task.execute(cs2); - return null; - }).when(clusterService).submitStateUpdateTask(eq("put-job-foo123"), any(AckedClusterStateUpdateTask.class)); - - doAnswer(invocationOnMock -> { - AckedClusterStateUpdateTask task = (AckedClusterStateUpdateTask) invocationOnMock.getArguments()[1]; - task.execute(cs2); - return null; - }).when(clusterService).submitStateUpdateTask(eq("index-aliases"), any(AckedClusterStateUpdateTask.class)); - - provider.createJobResultIndex(job.build(), cs2, new ActionListener() { - @Override - public void onResponse(Boolean aBoolean) { - assertTrue(aBoolean); - verify(clientBuilder.build().admin().indices(), times(1)).preparePutMapping(any()); - } - - @Override - public void onFailure(Exception e) { - fail(e.toString()); - } - }); - } - - @SuppressWarnings("unchecked") - public void testCreateJobRelatedIndicies_createsAliasBecauseIndexNameIsSet() { - String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-bar"; - String readAliasName = AnomalyDetectorsIndex.jobResultsAliasedName("foo"); - String writeAliasName = AnomalyDetectorsIndex.resultsWriteAlias("foo"); - QueryBuilder jobFilter = QueryBuilders.termQuery("job_id", "foo"); - - MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME); - ArgumentCaptor captor = ArgumentCaptor.forClass(CreateIndexRequest.class); - clientBuilder.createIndexRequest(captor, indexName); - clientBuilder.prepareAlias(indexName, readAliasName, jobFilter); - clientBuilder.prepareAlias(indexName, writeAliasName); - clientBuilder.preparePutMapping(mock(AcknowledgedResponse.class), Result.TYPE.getPreferredName()); - - Job.Builder job = buildJobBuilder("foo"); - job.setResultsIndexName("bar"); - Client client = clientBuilder.build(); - JobResultsProvider provider = createProvider(client); - - ImmutableOpenMap indexMap = ImmutableOpenMap.builder().build(); - - ClusterState cs = ClusterState.builder(new ClusterName("_name")) - .metaData(MetaData.builder().indices(indexMap)).build(); - - ClusterService clusterService = mock(ClusterService.class); - - doAnswer(invocationOnMock -> { - AckedClusterStateUpdateTask task = (AckedClusterStateUpdateTask) invocationOnMock.getArguments()[1]; - task.execute(cs); - return null; - }).when(clusterService).submitStateUpdateTask(eq("put-job-foo"), any(AckedClusterStateUpdateTask.class)); - - provider.createJobResultIndex(job.build(), cs, new ActionListener() { - @Override - public void onResponse(Boolean aBoolean) { - verify(client.admin().indices(), times(1)).prepareAliases(); - verify(client.admin().indices().prepareAliases(), times(1)).addAlias(indexName, readAliasName, jobFilter); - verify(client.admin().indices().prepareAliases(), times(1)).addAlias(indexName, writeAliasName); - } - - @Override - public void onFailure(Exception e) { - fail(e.toString()); - } - }); - } public void testBuckets_OneBucketNoInterim() throws IOException { String jobId = "TestJobIdentification"; @@ -853,7 +678,7 @@ public void testTimingStats_Ok() throws IOException { contextMap.put(ExponentialAverageCalculationContext.LATEST_TIMESTAMP.getPreferredName(), Instant.ofEpochMilli(1000_000_000)); contextMap.put(ExponentialAverageCalculationContext.PREVIOUS_EXPONENTIAL_AVERAGE_MS.getPreferredName(), 200.0); timingStatsMap.put(DatafeedTimingStats.EXPONENTIAL_AVG_CALCULATION_CONTEXT.getPreferredName(), contextMap); - + List> source = Arrays.asList(timingStatsMap); SearchResponse response = createSearchResponse(source); Client client = getMockedClient( @@ -1041,6 +866,32 @@ public void testDatafeedTimingStats_NotFound() throws IOException { verifyNoMoreInteractions(client); } + @SuppressWarnings("unchecked") + public void testCreateTermFieldsMapping() throws IOException { + + XContentBuilder termFieldsMapping = JsonXContent.contentBuilder(); + JobResultsProvider.createTermFieldsMapping(termFieldsMapping, "_doc", Arrays.asList("apple", "strawberry", + AnomalyRecord.BUCKET_SPAN.getPreferredName())); + + XContentParser parser = createParser(termFieldsMapping); + Map typeMappings = (Map) parser.map().get("_doc"); + Map properties = (Map) typeMappings.get("properties"); + + Map instanceMapping = (Map) properties.get("apple"); + assertNotNull(instanceMapping); + String dataType = (String)instanceMapping.get("type"); + assertEquals("keyword", dataType); + + instanceMapping = (Map) properties.get("strawberry"); + assertNotNull(instanceMapping); + dataType = (String)instanceMapping.get("type"); + assertEquals("keyword", dataType); + + // check no mapping for the reserved field + instanceMapping = (Map) properties.get(AnomalyRecord.BUCKET_SPAN.getPreferredName()); + assertNull(instanceMapping); + } + private JobResultsProvider createProvider(Client client) { return new JobResultsProvider(client, Settings.EMPTY); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java index 75a99e2899da7..a2b628df85885 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java @@ -10,37 +10,22 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.get.GetRequestBuilder; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchScrollRequestBuilder; import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.Client; import org.elasticsearch.client.ClusterAdminClient; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.document.DocumentField; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -54,22 +39,14 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutionException; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyInt; -import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class MockClientBuilder { @@ -79,14 +56,11 @@ public class MockClientBuilder { private ClusterAdminClient clusterAdminClient; private IndicesAdminClient indicesAdminClient; - private IndicesAliasesRequestBuilder aliasesRequestBuilder; - public MockClientBuilder(String clusterName) { client = mock(Client.class); adminClient = mock(AdminClient.class); clusterAdminClient = mock(ClusterAdminClient.class); indicesAdminClient = mock(IndicesAdminClient.class); - aliasesRequestBuilder = mock(IndicesAliasesRequestBuilder.class); when(client.admin()).thenReturn(adminClient); when(adminClient.cluster()).thenReturn(clusterAdminClient); @@ -99,7 +73,7 @@ public MockClientBuilder(String clusterName) { } @SuppressWarnings({ "unchecked" }) - public MockClientBuilder addClusterStatusYellowResponse() throws InterruptedException, ExecutionException { + public MockClientBuilder addClusterStatusYellowResponse() { PlainActionFuture actionFuture = mock(PlainActionFuture.class); ClusterHealthRequestBuilder clusterHealthRequestBuilder = mock(ClusterHealthRequestBuilder.class); @@ -110,64 +84,6 @@ public MockClientBuilder addClusterStatusYellowResponse() throws InterruptedExce return this; } - @SuppressWarnings({ "unchecked" }) - public MockClientBuilder addClusterStatusYellowResponse(String index) throws InterruptedException, ExecutionException { - PlainActionFuture actionFuture = mock(PlainActionFuture.class); - ClusterHealthRequestBuilder clusterHealthRequestBuilder = mock(ClusterHealthRequestBuilder.class); - - when(clusterAdminClient.prepareHealth(index)).thenReturn(clusterHealthRequestBuilder); - when(clusterHealthRequestBuilder.setWaitForYellowStatus()).thenReturn(clusterHealthRequestBuilder); - when(clusterHealthRequestBuilder.execute()).thenReturn(actionFuture); - when(actionFuture.actionGet()).thenReturn(mock(ClusterHealthResponse.class)); - return this; - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - public MockClientBuilder addIndicesExistsResponse(String index, boolean exists) throws InterruptedException, ExecutionException { - ActionFuture actionFuture = mock(ActionFuture.class); - ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(IndicesExistsRequest.class); - - when(indicesAdminClient.exists(requestCaptor.capture())).thenReturn(actionFuture); - doAnswer(invocation -> { - IndicesExistsRequest request = (IndicesExistsRequest) invocation.getArguments()[0]; - return request.indices()[0].equals(index) ? actionFuture : null; - }).when(indicesAdminClient).exists(any(IndicesExistsRequest.class)); - when(actionFuture.get()).thenReturn(new IndicesExistsResponse(exists)); - when(actionFuture.actionGet()).thenReturn(new IndicesExistsResponse(exists)); - return this; - } - - @SuppressWarnings({ "unchecked" }) - public MockClientBuilder addIndicesDeleteResponse(String index, boolean exists, boolean exception, - ActionListener actionListener) throws InterruptedException, ExecutionException, IOException { - StreamInput si = mock(StreamInput.class); - // this looks complicated but Mockito can't mock the final method - // DeleteIndexResponse.isAcknowledged() and the only way to create - // one with a true response is reading from a stream. - when(si.readByte()).thenReturn((byte) 0x01); - AcknowledgedResponse response = DeleteIndexAction.INSTANCE.getResponseReader().read(si); - - doAnswer(invocation -> { - DeleteIndexRequest deleteIndexRequest = (DeleteIndexRequest) invocation.getArguments()[0]; - assertArrayEquals(new String[] { index }, deleteIndexRequest.indices()); - if (exception) { - actionListener.onFailure(new InterruptedException()); - } else { - actionListener.onResponse(new AcknowledgedResponse(true)); - } - return null; - }).when(indicesAdminClient).delete(any(DeleteIndexRequest.class), any(ActionListener.class)); - return this; - } - - public MockClientBuilder prepareGet(String index, String type, String id, GetResponse response) { - GetRequestBuilder getRequestBuilder = mock(GetRequestBuilder.class); - when(getRequestBuilder.get()).thenReturn(response); - when(getRequestBuilder.setFetchSource(false)).thenReturn(getRequestBuilder); - when(client.prepareGet(index, type, id)).thenReturn(getRequestBuilder); - return this; - } - @SuppressWarnings("unchecked") public MockClientBuilder get(GetResponse response) { doAnswer(new Answer() { @@ -192,64 +108,6 @@ public MockClientBuilder prepareCreate(String index) { return this; } - @SuppressWarnings({ "rawtypes", "unchecked" }) - public MockClientBuilder createIndexRequest(ArgumentCaptor requestCapture, final String index) { - - doAnswer(invocation -> { - CreateIndexResponse response = new CreateIndexResponse(true, true, index) {}; - ((ActionListener) invocation.getArguments()[1]).onResponse(response); - return null; - }).when(indicesAdminClient).create(requestCapture.capture(), any(ActionListener.class)); - return this; - } - - @SuppressWarnings("unchecked") - public MockClientBuilder prepareSearchExecuteListener(String index, SearchResponse response) { - SearchRequestBuilder builder = mock(SearchRequestBuilder.class); - when(builder.setTypes(anyString())).thenReturn(builder); - when(builder.addSort(any(SortBuilder.class))).thenReturn(builder); - when(builder.setFetchSource(anyBoolean())).thenReturn(builder); - when(builder.setScroll(anyString())).thenReturn(builder); - when(builder.addDocValueField(any(String.class))).thenReturn(builder); - when(builder.addDocValueField(any(String.class), any(String.class))).thenReturn(builder); - when(builder.addSort(any(String.class), any(SortOrder.class))).thenReturn(builder); - when(builder.setQuery(any())).thenReturn(builder); - when(builder.setSize(anyInt())).thenReturn(builder); - - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocationOnMock) throws Throwable { - ActionListener listener = (ActionListener) invocationOnMock.getArguments()[0]; - listener.onResponse(response); - return null; - } - }).when(builder).execute(any()); - - when(client.prepareSearch(eq(index))).thenReturn(builder); - - return this; - } - - @SuppressWarnings("unchecked") - public MockClientBuilder prepareSearchScrollExecuteListener(SearchResponse response) { - SearchScrollRequestBuilder builder = mock(SearchScrollRequestBuilder.class); - when(builder.setScroll(anyString())).thenReturn(builder); - when(builder.setScrollId(anyString())).thenReturn(builder); - - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocationOnMock) throws Throwable { - ActionListener listener = (ActionListener) invocationOnMock.getArguments()[0]; - listener.onResponse(response); - return null; - } - }).when(builder).execute(any()); - - when(client.prepareSearchScroll(anyString())).thenReturn(builder); - - return this; - } - public MockClientBuilder prepareSearch(String index, String type, int from, int size, SearchResponse response, ArgumentCaptor filter) { SearchRequestBuilder builder = mock(SearchRequestBuilder.class); @@ -352,38 +210,6 @@ public Void answer(InvocationOnMock invocationOnMock) { return this; } - @SuppressWarnings("unchecked") - public MockClientBuilder prepareAlias(String indexName, String alias, QueryBuilder filter) { - when(aliasesRequestBuilder.addAlias(eq(indexName), eq(alias), eq(filter))).thenReturn(aliasesRequestBuilder); - when(indicesAdminClient.prepareAliases()).thenReturn(aliasesRequestBuilder); - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocationOnMock) throws Throwable { - ActionListener listener = - (ActionListener) invocationOnMock.getArguments()[0]; - listener.onResponse(mock(AcknowledgedResponse.class)); - return null; - } - }).when(aliasesRequestBuilder).execute(any()); - return this; - } - - @SuppressWarnings("unchecked") - public MockClientBuilder prepareAlias(String indexName, String alias) { - when(aliasesRequestBuilder.addAlias(eq(indexName), eq(alias))).thenReturn(aliasesRequestBuilder); - when(indicesAdminClient.prepareAliases()).thenReturn(aliasesRequestBuilder); - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocationOnMock) throws Throwable { - ActionListener listener = - (ActionListener) invocationOnMock.getArguments()[1]; - listener.onResponse(mock(AcknowledgedResponse.class)); - return null; - } - }).when(indicesAdminClient).aliases(any(IndicesAliasesRequest.class), any(ActionListener.class)); - return this; - } - @SuppressWarnings("unchecked") public MockClientBuilder prepareBulk(BulkResponse response) { PlainActionFuture actionFuture = mock(PlainActionFuture.class); @@ -402,70 +228,7 @@ public MockClientBuilder bulk(BulkResponse response) { return this; } - @SuppressWarnings("unchecked") - public MockClientBuilder preparePutMapping(AcknowledgedResponse response, String type) { - PutMappingRequestBuilder requestBuilder = mock(PutMappingRequestBuilder.class); - when(requestBuilder.setType(eq(type))).thenReturn(requestBuilder); - when(requestBuilder.setSource(any(XContentBuilder.class))).thenReturn(requestBuilder); - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocationOnMock) throws Throwable { - ActionListener listener = - (ActionListener) invocationOnMock.getArguments()[0]; - listener.onResponse(response); - return null; - } - }).when(requestBuilder).execute(any()); - - when(indicesAdminClient.preparePutMapping(any())).thenReturn(requestBuilder); - return this; - } - - @SuppressWarnings("unchecked") - public MockClientBuilder prepareGetMapping(GetMappingsResponse response) { - GetMappingsRequestBuilder builder = mock(GetMappingsRequestBuilder.class); - - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocationOnMock) throws Throwable { - ActionListener listener = - (ActionListener) invocationOnMock.getArguments()[0]; - listener.onResponse(response); - return null; - } - }).when(builder).execute(any()); - - when(indicesAdminClient.prepareGetMappings(any())).thenReturn(builder); - return this; - } - - @SuppressWarnings("unchecked") - public MockClientBuilder putTemplate(ArgumentCaptor requestCaptor) { - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocationOnMock) throws Throwable { - ActionListener listener = - (ActionListener) invocationOnMock.getArguments()[1]; - listener.onResponse(mock(AcknowledgedResponse.class)); - return null; - } - }).when(indicesAdminClient).putTemplate(requestCaptor.capture(), any(ActionListener.class)); - return this; - } - - public Client build() { return client; } - - public void verifyIndexCreated(String index) { - ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(CreateIndexRequest.class); - verify(indicesAdminClient).create(requestCaptor.capture(), any()); - assertEquals(index, requestCaptor.getValue().index()); - } - - public void resetIndices() { - reset(indicesAdminClient); - } - } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 2c6f3902b3877..f53fa59fde987 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -66,7 +66,6 @@ import java.util.function.Consumer; import java.util.function.Predicate; import java.util.function.Supplier; -import java.util.regex.Pattern; import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING; @@ -85,7 +84,7 @@ public class SecurityIndexManager implements ClusterStateListener { public static final String SECURITY_MAIN_TEMPLATE_7 = "security-index-template-7"; public static final String SECURITY_TOKENS_TEMPLATE_7 = "security-tokens-index-template-7"; public static final String SECURITY_VERSION_STRING = "security-version"; - public static final String TEMPLATE_VERSION_PATTERN = Pattern.quote("${security.template.version}"); + public static final String TEMPLATE_VERSION_VARIABLE = "security.template.version"; private static final Logger logger = LogManager.getLogger(SecurityIndexManager.class); @@ -434,7 +433,7 @@ public static boolean isIndexDeleted(State previousState, State currentState) { private static byte[] readTemplateAsBytes(String templateName) { return TemplateUtils.loadTemplate("/" + templateName + ".json", Version.CURRENT.toString(), - SecurityIndexManager.TEMPLATE_VERSION_PATTERN).getBytes(StandardCharsets.UTF_8); + SecurityIndexManager.TEMPLATE_VERSION_VARIABLE).getBytes(StandardCharsets.UTF_8); } private static Tuple parseMappingAndSettingsFromTemplateBytes(byte[] template) throws IOException { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index 6396757eb482a..daa33a9e7a6ed 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -19,10 +19,10 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; -import org.elasticsearch.action.ActionType; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.client.Client; import org.elasticsearch.client.FilterClient; @@ -61,16 +61,16 @@ import org.junit.Before; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7; -import static org.elasticsearch.xpack.security.support.SecurityIndexManager.TEMPLATE_VERSION_PATTERN; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.TEMPLATE_VERSION_VARIABLE; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class SecurityIndexManagerTests extends ESTestCase { @@ -444,7 +444,7 @@ private static Map getTemplateMappings(String templateName) { private static String loadTemplate(String templateName) { final String resource = "/" + templateName + ".json"; - return TemplateUtils.loadTemplate(resource, Version.CURRENT.toString(), TEMPLATE_VERSION_PATTERN); + return TemplateUtils.loadTemplate(resource, Version.CURRENT.toString(), TEMPLATE_VERSION_VARIABLE); } public void testMappingVersionMatching() throws IOException { @@ -535,7 +535,7 @@ private ClusterState.Builder createClusterStateWithMappingAndTemplate(String sec private static IndexMetaData.Builder createIndexMetadata(String indexName, String templateString) throws IOException { String template = TemplateUtils.loadTemplate(templateString, Version.CURRENT.toString(), - SecurityIndexManager.TEMPLATE_VERSION_PATTERN); + SecurityIndexManager.TEMPLATE_VERSION_VARIABLE); PutIndexTemplateRequest request = new PutIndexTemplateRequest(); request.source(template, XContentType.JSON); IndexMetaData.Builder indexMetaData = IndexMetaData.builder(indexName); @@ -574,7 +574,7 @@ private ClusterState.Builder createClusterStateWithIndex(String securityTemplate private static IndexTemplateMetaData.Builder getIndexTemplateMetaData(String templateName, String templateString) throws IOException { String template = TemplateUtils.loadTemplate(templateString, Version.CURRENT.toString(), - SecurityIndexManager.TEMPLATE_VERSION_PATTERN); + SecurityIndexManager.TEMPLATE_VERSION_VARIABLE); PutIndexTemplateRequest request = new PutIndexTemplateRequest(); request.source(template, XContentType.JSON); IndexTemplateMetaData.Builder templateBuilder = IndexTemplateMetaData.builder(templateName) diff --git a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java index 3f133ee539e95..5ed2b8c570a58 100644 --- a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java +++ b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java @@ -26,7 +26,7 @@ import org.elasticsearch.xpack.core.ml.integration.MlRestTestStateCleaner; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; -import org.elasticsearch.xpack.core.ml.notifications.AuditorField; +import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.elasticsearch.xpack.core.rollup.job.RollupJob; import org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField; import org.junit.After; @@ -89,7 +89,7 @@ private void waitForTemplates() throws Exception { List templates = new ArrayList<>(); templates.addAll( Arrays.asList( - AuditorField.NOTIFICATIONS_INDEX, + NotificationsIndex.NOTIFICATIONS_INDEX, MlMetaIndex.INDEX_NAME, AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, AnomalyDetectorsIndex.jobResultsIndexPrefix(), diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java index 452f8d48fc2e4..461378e27da2f 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java @@ -11,22 +11,21 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.WarningFailureException; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; -import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.MlConfigIndex; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; import org.elasticsearch.xpack.test.rest.XPackRestTestHelper; import org.junit.Before; @@ -39,8 +38,8 @@ import java.util.Map; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractFullClusterRestartTestCase { @@ -48,24 +47,6 @@ public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractFullClust private static final String OLD_CLUSTER_JOB_ID = "ml-config-mappings-old-cluster-job"; private static final String NEW_CLUSTER_JOB_ID = "ml-config-mappings-new-cluster-job"; - private static final Map EXPECTED_DATA_FRAME_ANALYSIS_MAPPINGS = getDataFrameAnalysisMappings(); - - @SuppressWarnings("unchecked") - private static Map getDataFrameAnalysisMappings() { - try (XContentBuilder builder = JsonXContent.contentBuilder()) { - builder.startObject(); - ElasticsearchMappings.addDataFrameAnalyticsFields(builder); - builder.endObject(); - - Map asMap = builder.generator().contentType().xContent().createParser( - NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, BytesReference.bytes(builder).streamInput()).map(); - return (Map) asMap.get(DataFrameAnalyticsConfig.ANALYSIS.getPreferredName()); - } catch (IOException e) { - fail("Failed to initialize expected data frame analysis mappings"); - } - return null; - } - @Override protected Settings restClientSettings() { String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); @@ -90,16 +71,16 @@ public void testMlConfigIndexMappingsAfterMigration() throws Exception { createAnomalyDetectorJob(OLD_CLUSTER_JOB_ID); if (getOldClusterVersion().onOrAfter(Version.V_7_3_0)) { // .ml-config has mappings for analytics as the feature was introduced in 7.3.0 - assertThat(mappingsForDataFrameAnalysis(), is(notNullValue())); + assertThat(getDataFrameAnalysisMappings().keySet(), hasItem("outlier_detection")); } else { // .ml-config does not yet have correct mappings, it will need an update after cluster is upgraded - assertThat(mappingsForDataFrameAnalysis(), is(nullValue())); + assertThat(getDataFrameAnalysisMappings(), is(nullValue())); } } else { // trigger .ml-config index mappings update createAnomalyDetectorJob(NEW_CLUSTER_JOB_ID); // assert that the mappings are updated - assertThat(mappingsForDataFrameAnalysis(), is(equalTo(EXPECTED_DATA_FRAME_ANALYSIS_MAPPINGS))); + assertThat(getDataFrameAnalysisMappings(), equalTo(loadDataFrameAnalysisMappings())); } } @@ -110,8 +91,7 @@ private void assertThatMlConfigIndexDoesNotExist() { } private void createAnomalyDetectorJob(String jobId) throws IOException { - Detector.Builder detector = new Detector.Builder("metric", "responsetime") - .setByFieldName("airline"); + Detector.Builder detector = new Detector.Builder("metric", "responsetime"); AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build())) .setBucketSpan(TimeValue.timeValueMinutes(10)); Job.Builder job = new Job.Builder(jobId) @@ -125,7 +105,7 @@ private void createAnomalyDetectorJob(String jobId) throws IOException { } @SuppressWarnings("unchecked") - private Map mappingsForDataFrameAnalysis() throws Exception { + private Map getConfigIndexMappings() throws Exception { Request getIndexMappingsRequest = new Request("GET", ".ml-config/_mappings"); Response getIndexMappingsResponse; try { @@ -140,7 +120,25 @@ private Map mappingsForDataFrameAnalysis() throws Exception { if (mappings.containsKey("doc")) { mappings = (Map) XContentMapValues.extractValue(mappings, "doc"); } - mappings = (Map) XContentMapValues.extractValue(mappings, "properties", "analysis"); + mappings = (Map) XContentMapValues.extractValue(mappings, "properties"); + return mappings; + } + + @SuppressWarnings("unchecked") + private Map getDataFrameAnalysisMappings() throws Exception { + Map mappings = getConfigIndexMappings(); + mappings = (Map) XContentMapValues.extractValue(mappings, "analysis", "properties"); return mappings; } + + @SuppressWarnings("unchecked") + private Map loadDataFrameAnalysisMappings() throws IOException { + String mapping = MlConfigIndex.mapping(); + try (XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, new BytesArray(mapping).streamInput())) { + Map mappings = parser.map(); + mappings = (Map) XContentMapValues.extractValue(mappings, "_doc", "properties", "analysis", "properties"); + return mappings; + } + } } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java index 13ed2dafc5f31..6d984317288d6 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java @@ -99,6 +99,7 @@ private void assertUpgradedMappings() throws Exception { assertNotNull(indexLevel); Map mappingsLevel = (Map) indexLevel.get("mappings"); assertNotNull(mappingsLevel); + Map metaLevel = (Map) mappingsLevel.get("_meta"); assertEquals(Collections.singletonMap("version", Version.CURRENT.toString()), metaLevel); Map propertiesLevel = (Map) mappingsLevel.get("properties");