diff --git a/docs/changelog/127824.yaml b/docs/changelog/127824.yaml new file mode 100644 index 0000000000000..6868f3c39297a --- /dev/null +++ b/docs/changelog/127824.yaml @@ -0,0 +1,6 @@ +pr: 127824 +summary: Skip the validation when retrieving the index mode during reindexing a time + series data stream +area: TSDB +type: bug +issues: [] diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index efba9c06b4b02..a9475730c122f 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -14,7 +14,7 @@ esplugin { restResources { restApi { include 'bulk', 'count', 'search', '_common', 'indices', 'index', 'cluster', 'rank_eval', 'reindex', 'update_by_query', 'delete_by_query', - 'eql', 'data_stream', 'ingest', 'cat', 'capabilities' + 'eql', 'data_stream', 'ingest', 'cat', 'capabilities', 'reindex' } } @@ -22,6 +22,7 @@ dependencies { testImplementation project(path: ':test:test-clusters') testImplementation project(":modules:mapper-extras") internalClusterTestImplementation project(":modules:mapper-extras") + internalClusterTestImplementation project(':modules:reindex') } tasks.withType(StandaloneRestIntegTestTask).configureEach { diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index 434a8bced8895..eaef99d86a86e 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.admin.indices.diskusage.TransportAnalyzeIndexDiskUsageAction; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest; @@ -20,6 +21,8 @@ import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; @@ -27,6 +30,7 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; @@ -34,10 +38,15 @@ import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.FormatNames; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.ReindexAction; +import org.elasticsearch.index.reindex.ReindexRequest; import org.elasticsearch.indices.InvalidIndexTemplateException; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -53,6 +62,7 @@ import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -98,7 +108,7 @@ public class TSDBIndexingIT extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return List.of(DataStreamsPlugin.class, InternalSettingsPlugin.class); + return List.of(DataStreamsPlugin.class, InternalSettingsPlugin.class, ReindexPlugin.class); } @Override @@ -557,6 +567,60 @@ public void testTrimId() throws Exception { }); } + public void testReindexing() throws Exception { + String dataStreamName = "my-ds"; + String reindexedDataStreamName = "my-reindexed-ds"; + var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName, reindexedDataStreamName)) + .template( + new Template( + Settings.builder().put("index.mode", "time_series").build(), + new CompressedXContent(MAPPING_TEMPLATE), + null + ) + ) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest)); + + // index doc + long docCount = randomLongBetween(10, 50); + Instant startTime = Instant.now(); + BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); + bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (int i = 0; i < docCount; i++) { + IndexRequest indexRequest = new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE); + indexRequest.source(DOC.replace("$time", formatInstant(startTime.plusSeconds(i))), XContentType.JSON); + bulkRequestBuilder.add(indexRequest); + } + BulkResponse bulkResponse = bulkRequestBuilder.get(); + assertThat(bulkResponse.hasFailures(), is(false)); + + BulkByScrollResponse reindexResponse = safeGet( + client().execute( + ReindexAction.INSTANCE, + new ReindexRequest().setSourceIndices(dataStreamName).setDestIndex(reindexedDataStreamName).setDestOpType("create") + ) + ); + assertThat(reindexResponse.getCreated(), equalTo(docCount)); + + GetIndexResponse getIndexResponse = safeGet( + indicesAdmin().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(dataStreamName, reindexedDataStreamName)) + ); + assertThat(getIndexResponse.getIndices().length, equalTo(2)); + var index1 = getIndexResponse.getIndices()[0]; + var index2 = getIndexResponse.getIndices()[1]; + assertThat(getIndexResponse.getSetting(index1, IndexSettings.MODE.getKey()), equalTo(IndexMode.TIME_SERIES.getName())); + assertThat(getIndexResponse.getSetting(index2, IndexSettings.MODE.getKey()), equalTo(IndexMode.TIME_SERIES.getName())); + assertThat( + getIndexResponse.getSetting(index2, IndexMetadata.INDEX_ROUTING_PATH.getKey()), + equalTo(getIndexResponse.getSetting(index1, IndexMetadata.INDEX_ROUTING_PATH.getKey())) + ); + } + static String formatInstant(Instant instant) { return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index 91ce987ff78c5..9a6c6a8839bf3 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -254,7 +254,9 @@ private IndexMode destinationIndexMode(ClusterState state) { return IndexMode.STANDARD; } Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), template); - return IndexSettings.MODE.get(settings); + // We retrieve the setting without performing any validation because that the template has already been validated + String indexMode = settings.get(IndexSettings.MODE.getKey()); + return indexMode == null ? IndexMode.STANDARD : IndexMode.fromString(indexMode); } @Override