diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index 1b875c28f7f43..d1dd008e27977 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -42,6 +42,7 @@ import org.elasticsearch.cluster.SimpleBatchedExecutor; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -796,7 +797,7 @@ private void maybeExecuteRollover(ClusterState state, DataStream dataStream) { RolloverRequest rolloverRequest = getDefaultRolloverRequest( rolloverConfiguration, dataStream.getName(), - dataStream.getLifecycle().getEffectiveDataRetention() + dataStream.getLifecycle().getEffectiveDataRetention(DataStreamGlobalRetention.getFromClusterState(state)) ); transportActionsDeduplicator.executeOnce( rolloverRequest, @@ -823,14 +824,15 @@ private void maybeExecuteRollover(ClusterState state, DataStream dataStream) { */ private Set maybeExecuteRetention(ClusterState state, DataStream dataStream, Set indicesToExcludeForRemainingRun) { Metadata metadata = state.metadata(); - List backingIndicesOlderThanRetention = dataStream.getIndicesPastRetention(metadata::index, nowSupplier); + DataStreamGlobalRetention globalRetention = DataStreamGlobalRetention.getFromClusterState(state); + List backingIndicesOlderThanRetention = dataStream.getIndicesPastRetention(metadata::index, nowSupplier, globalRetention); if (backingIndicesOlderThanRetention.isEmpty()) { return Set.of(); } Set indicesToBeRemoved = new HashSet<>(); // We know that there is lifecycle and retention because there are indices to be deleted assert dataStream.getLifecycle() != null; - TimeValue effectiveDataRetention = dataStream.getLifecycle().getEffectiveDataRetention(); + TimeValue effectiveDataRetention = dataStream.getLifecycle().getEffectiveDataRetention(globalRetention); for (Index index : backingIndicesOlderThanRetention) { if (indicesToExcludeForRemainingRun.contains(index) == false) { IndexMetadata backingIndex = metadata.index(index); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index a5f424f875eb7..776fb9fd87740 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -759,13 +759,17 @@ public DataStream snapshot(Collection indicesInSnapshot) { * NOTE that this specifically does not return the write index of the data stream as usually retention * is treated differently for the write index (i.e. they first need to be rolled over) */ - public List getIndicesPastRetention(Function indexMetadataSupplier, LongSupplier nowSupplier) { - if (lifecycle == null || lifecycle.isEnabled() == false || lifecycle.getEffectiveDataRetention() == null) { + public List getIndicesPastRetention( + Function indexMetadataSupplier, + LongSupplier nowSupplier, + DataStreamGlobalRetention globalRetention + ) { + if (lifecycle == null || lifecycle.isEnabled() == false || lifecycle.getEffectiveDataRetention(globalRetention) == null) { return List.of(); } List indicesPastRetention = getNonWriteIndicesOlderThan( - lifecycle.getEffectiveDataRetention(), + lifecycle.getEffectiveDataRetention(globalRetention), indexMetadataSupplier, this::isIndexManagedByDataStreamLifecycle, nowSupplier @@ -1098,14 +1102,18 @@ public static DataStream fromXContent(XContentParser parser) throws IOException @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return toXContent(builder, params, null); + return toXContent(builder, params, null, null); } /** * Converts the data stream to XContent and passes the RolloverConditions, when provided, to the lifecycle. */ - public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nullable RolloverConfiguration rolloverConfiguration) - throws IOException { + public XContentBuilder toXContent( + XContentBuilder builder, + Params params, + @Nullable RolloverConfiguration rolloverConfiguration, + @Nullable DataStreamGlobalRetention globalRetention + ) throws IOException { builder.startObject(); builder.field(NAME_FIELD.getPreferredName(), name); builder.field(TIMESTAMP_FIELD_FIELD.getPreferredName()) @@ -1132,7 +1140,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla } if (lifecycle != null) { builder.field(LIFECYCLE.getPreferredName()); - lifecycle.toXContent(builder, params, rolloverConfiguration); + lifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention); } builder.field(ROLLOVER_ON_WRITE_FIELD.getPreferredName(), rolloverOnWrite); if (autoShardingEvent != null) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java index b4a3a1eb3502a..a8b094bafde2e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.xcontent.AbstractObjectParser; import org.elasticsearch.xcontent.ConstructingObjectParser; @@ -34,6 +35,7 @@ import java.io.IOException; import java.util.List; +import java.util.Locale; import java.util.Objects; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; @@ -131,15 +133,52 @@ public boolean isEnabled() { /** * The least amount of time data should be kept by elasticsearch. * @return the time period or null, null represents that data should never be deleted. + * @deprecated use {@link #getEffectiveDataRetention(DataStreamGlobalRetention)} */ + @Deprecated @Nullable public TimeValue getEffectiveDataRetention() { - return getDataStreamRetention(); + return getEffectiveDataRetention(null); + } + + /** + * The least amount of time data should be kept by elasticsearch. + * @return the time period or null, null represents that data should never be deleted. + */ + @Nullable + public TimeValue getEffectiveDataRetention(@Nullable DataStreamGlobalRetention globalRetention) { + return getEffectiveDataRetentionWithSource(globalRetention).v1(); + } + + /** + * The least amount of time data should be kept by elasticsearch. + * @return the time period or null, null represents that data should never be deleted. + */ + @Nullable + public Tuple getEffectiveDataRetentionWithSource(@Nullable DataStreamGlobalRetention globalRetention) { + // If lifecycle is disabled there is no effective retention + if (enabled == false) { + return Tuple.tuple(null, RetentionSource.DATA_STREAM_CONFIGURATION); + } + var dataStreamRetention = getDataStreamRetention(); + if (globalRetention == null) { + return Tuple.tuple(dataStreamRetention, RetentionSource.DATA_STREAM_CONFIGURATION); + } + if (dataStreamRetention == null) { + return globalRetention.getDefaultRetention() != null + ? Tuple.tuple(globalRetention.getDefaultRetention(), RetentionSource.DEFAULT_GLOBAL_RETENTION) + : Tuple.tuple(globalRetention.getMaxRetention(), RetentionSource.MAX_GLOBAL_RETENTION); + } + if (globalRetention.getMaxRetention() != null && globalRetention.getMaxRetention().getMillis() < dataStreamRetention.getMillis()) { + return Tuple.tuple(globalRetention.getMaxRetention(), RetentionSource.MAX_GLOBAL_RETENTION); + } else { + return Tuple.tuple(dataStreamRetention, RetentionSource.DATA_STREAM_CONFIGURATION); + } } /** * The least amount of time data the data stream is requesting es to keep the data. - * NOTE: this can be overriden by the {@link DataStreamLifecycle#getEffectiveDataRetention()}. + * NOTE: this can be overridden by the {@link DataStreamLifecycle#getEffectiveDataRetention(DataStreamGlobalRetention)}. * @return the time period or null, null represents that data should never be deleted. */ @Nullable @@ -232,14 +271,28 @@ public String toString() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return toXContent(builder, params, null); + return toXContent(builder, params, null, null); } /** * Converts the data stream lifecycle to XContent and injects the RolloverConditions if they exist. + * @deprecated use {@link #toXContent(XContentBuilder, Params, RolloverConfiguration, DataStreamGlobalRetention)} */ + @Deprecated public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nullable RolloverConfiguration rolloverConfiguration) throws IOException { + return toXContent(builder, params, rolloverConfiguration, null); + } + + /** + * Converts the data stream lifecycle to XContent and injects the RolloverConditions and the global retention if they exist. + */ + public XContentBuilder toXContent( + XContentBuilder builder, + Params params, + @Nullable RolloverConfiguration rolloverConfiguration, + @Nullable DataStreamGlobalRetention globalRetention + ) throws IOException { builder.startObject(); builder.field(ENABLED_FIELD.getPreferredName(), enabled); if (dataRetention != null) { @@ -255,7 +308,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla } if (rolloverConfiguration != null) { builder.field(ROLLOVER_FIELD.getPreferredName()); - rolloverConfiguration.evaluateAndConvertToXContent(builder, params, getEffectiveDataRetention()); + rolloverConfiguration.evaluateAndConvertToXContent(builder, params, getEffectiveDataRetention(globalRetention)); } builder.endObject(); return builder; @@ -466,4 +519,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } } + + /** + * This enum represents all configuration sources that can influence the retention of a data stream. + */ + public enum RetentionSource { + DATA_STREAM_CONFIGURATION, + DEFAULT_GLOBAL_RETENTION, + MAX_GLOBAL_RETENTION; + + public String displayName() { + return this.toString().toLowerCase(Locale.ROOT); + } + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java index 441e8491b4b92..e3bf5260a7445 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.test.AbstractXContentSerializingTestCase; import org.elasticsearch.test.ESTestCase; @@ -33,6 +34,9 @@ import java.util.Set; import java.util.stream.Stream; +import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.RetentionSource.DATA_STREAM_CONFIGURATION; +import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.RetentionSource.DEFAULT_GLOBAL_RETENTION; +import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.RetentionSource.MAX_GLOBAL_RETENTION; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -107,10 +111,11 @@ public void testXContentSerializationWithRollover() throws IOException { try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { builder.humanReadable(true); RolloverConfiguration rolloverConfiguration = RolloverConfigurationTests.randomRolloverConditions(); - lifecycle.toXContent(builder, ToXContent.EMPTY_PARAMS, rolloverConfiguration); + DataStreamGlobalRetention globalRetention = DataStreamGlobalRetentionSerializationTests.randomGlobalRetention(); + lifecycle.toXContent(builder, ToXContent.EMPTY_PARAMS, rolloverConfiguration, globalRetention); String serialized = Strings.toString(builder); assertThat(serialized, containsString("rollover")); - for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention()) + for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention)) .getConditions() .keySet()) { assertThat(serialized, containsString(label)); @@ -253,6 +258,72 @@ public void testInvalidDownsamplingConfiguration() { } } + public void testEffectiveRetention() { + // No retention in the data stream lifecycle + { + DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.newBuilder().downsampling(randomDownsampling()).build(); + TimeValue maxRetention = TimeValue.timeValueDays(randomIntBetween(50, 100)); + TimeValue defaultRetention = TimeValue.timeValueDays(randomIntBetween(1, 50)); + Tuple effectiveDataRetentionWithSource = noRetentionLifecycle + .getEffectiveDataRetentionWithSource(null); + assertThat(effectiveDataRetentionWithSource.v1(), nullValue()); + assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DATA_STREAM_CONFIGURATION)); + + effectiveDataRetentionWithSource = noRetentionLifecycle.getEffectiveDataRetentionWithSource( + new DataStreamGlobalRetention(null, maxRetention) + ); + assertThat(effectiveDataRetentionWithSource.v1(), equalTo(maxRetention)); + assertThat(effectiveDataRetentionWithSource.v2(), equalTo(MAX_GLOBAL_RETENTION)); + + effectiveDataRetentionWithSource = noRetentionLifecycle.getEffectiveDataRetentionWithSource( + new DataStreamGlobalRetention(defaultRetention, null) + ); + assertThat(effectiveDataRetentionWithSource.v1(), equalTo(defaultRetention)); + assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DEFAULT_GLOBAL_RETENTION)); + + effectiveDataRetentionWithSource = noRetentionLifecycle.getEffectiveDataRetentionWithSource( + new DataStreamGlobalRetention(defaultRetention, maxRetention) + ); + assertThat(effectiveDataRetentionWithSource.v1(), equalTo(defaultRetention)); + assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DEFAULT_GLOBAL_RETENTION)); + } + + // With retention in the data stream lifecycle + { + TimeValue dataStreamRetention = TimeValue.timeValueDays(randomIntBetween(5, 100)); + DataStreamLifecycle lifecycleRetention = DataStreamLifecycle.newBuilder() + .dataRetention(dataStreamRetention) + .downsampling(randomDownsampling()) + .build(); + TimeValue defaultRetention = TimeValue.timeValueDays(randomIntBetween(1, (int) dataStreamRetention.getDays() - 1)); + + Tuple effectiveDataRetentionWithSource = lifecycleRetention + .getEffectiveDataRetentionWithSource(null); + assertThat(effectiveDataRetentionWithSource.v1(), equalTo(dataStreamRetention)); + assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DATA_STREAM_CONFIGURATION)); + + effectiveDataRetentionWithSource = lifecycleRetention.getEffectiveDataRetentionWithSource( + new DataStreamGlobalRetention(defaultRetention, null) + ); + assertThat(effectiveDataRetentionWithSource.v1(), equalTo(dataStreamRetention)); + assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DATA_STREAM_CONFIGURATION)); + + TimeValue maxGlobalRetention = randomBoolean() ? dataStreamRetention : TimeValue.timeValueDays(dataStreamRetention.days() + 1); + effectiveDataRetentionWithSource = lifecycleRetention.getEffectiveDataRetentionWithSource( + new DataStreamGlobalRetention(defaultRetention, maxGlobalRetention) + ); + assertThat(effectiveDataRetentionWithSource.v1(), equalTo(dataStreamRetention)); + assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DATA_STREAM_CONFIGURATION)); + + TimeValue maxRetentionLessThanDataStream = TimeValue.timeValueDays(dataStreamRetention.days() - 1); + effectiveDataRetentionWithSource = lifecycleRetention.getEffectiveDataRetentionWithSource( + new DataStreamGlobalRetention(randomBoolean() ? null : TimeValue.timeValueDays(10), maxRetentionLessThanDataStream) + ); + assertThat(effectiveDataRetentionWithSource.v1(), equalTo(maxRetentionLessThanDataStream)); + assertThat(effectiveDataRetentionWithSource.v2(), equalTo(MAX_GLOBAL_RETENTION)); + } + } + @Nullable public static DataStreamLifecycle randomLifecycle() { return DataStreamLifecycle.newBuilder() diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 8e1ce495fdf5c..3e758df17c432 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -50,6 +50,7 @@ import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; +import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.randomGlobalRetention; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.randomIndexInstances; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.randomNonEmptyIndexInstances; import static org.elasticsearch.index.IndexSettings.LIFECYCLE_ORIGINATION_DATE; @@ -1146,11 +1147,15 @@ public void testGetIndicesPastRetention() { ); Metadata metadata = builder.build(); - assertThat(dataStream.getIndicesPastRetention(metadata::index, () -> now).isEmpty(), is(true)); + assertThat(dataStream.getIndicesPastRetention(metadata::index, () -> now, randomGlobalRetention()).isEmpty(), is(true)); } { - // no retention configured so we expect an empty list + // no retention configured but we have default retention + DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention( + TimeValue.timeValueMillis(2500), + randomBoolean() ? TimeValue.timeValueMillis(randomIntBetween(2500, 5000)) : null + ); Metadata.Builder builder = Metadata.builder(); DataStream dataStream = createDataStream( builder, @@ -1161,7 +1166,29 @@ public void testGetIndicesPastRetention() { ); Metadata metadata = builder.build(); - assertThat(dataStream.getIndicesPastRetention(metadata::index, () -> now).isEmpty(), is(true)); + List backingIndices = dataStream.getIndicesPastRetention(metadata::index, () -> now, globalRetention); + assertThat(backingIndices.size(), is(2)); + assertThat(backingIndices.get(0).getName(), is(dataStream.getIndices().get(0).getName())); + assertThat(backingIndices.get(1).getName(), is(dataStream.getIndices().get(1).getName())); + } + + { + // no retention configured but we have max retention + DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(null, TimeValue.timeValueMillis(2500)); + Metadata.Builder builder = Metadata.builder(); + DataStream dataStream = createDataStream( + builder, + dataStreamName, + creationAndRolloverTimes, + settings(IndexVersion.current()), + new DataStreamLifecycle() + ); + Metadata metadata = builder.build(); + + List backingIndices = dataStream.getIndicesPastRetention(metadata::index, () -> now, globalRetention); + assertThat(backingIndices.size(), is(2)); + assertThat(backingIndices.get(0).getName(), is(dataStream.getIndices().get(0).getName())); + assertThat(backingIndices.get(1).getName(), is(dataStream.getIndices().get(1).getName())); } { @@ -1175,10 +1202,10 @@ public void testGetIndicesPastRetention() { ); Metadata metadata = builder.build(); - List backingIndices = dataStream.getIndicesPastRetention(metadata::index, () -> now); + List backingIndices = dataStream.getIndicesPastRetention(metadata::index, () -> now, randomGlobalRetention()); assertThat(backingIndices.size(), is(2)); - assertThat(backingIndices.get(0).getName(), is(DataStream.getDefaultBackingIndexName(dataStreamName, 1))); - assertThat(backingIndices.get(1).getName(), is(DataStream.getDefaultBackingIndexName(dataStreamName, 2))); + assertThat(backingIndices.get(0).getName(), is(dataStream.getIndices().get(0).getName())); + assertThat(backingIndices.get(1).getName(), is(dataStream.getIndices().get(1).getName())); } { @@ -1193,13 +1220,13 @@ public void testGetIndicesPastRetention() { ); Metadata metadata = builder.build(); - List backingIndices = dataStream.getIndicesPastRetention(metadata::index, () -> now); + List backingIndices = dataStream.getIndicesPastRetention(metadata::index, () -> now, randomGlobalRetention()); assertThat(backingIndices.size(), is(4)); - assertThat(backingIndices.get(0).getName(), is(DataStream.getDefaultBackingIndexName(dataStreamName, 1))); - assertThat(backingIndices.get(1).getName(), is(DataStream.getDefaultBackingIndexName(dataStreamName, 2))); - assertThat(backingIndices.get(2).getName(), is(DataStream.getDefaultBackingIndexName(dataStreamName, 3))); - assertThat(backingIndices.get(3).getName(), is(DataStream.getDefaultBackingIndexName(dataStreamName, 4))); + assertThat(backingIndices.get(0).getName(), is(dataStream.getIndices().get(0).getName())); + assertThat(backingIndices.get(1).getName(), is(dataStream.getIndices().get(1).getName())); + assertThat(backingIndices.get(2).getName(), is(dataStream.getIndices().get(2).getName())); + assertThat(backingIndices.get(3).getName(), is(dataStream.getIndices().get(3).getName())); } { @@ -1214,7 +1241,7 @@ public void testGetIndicesPastRetention() { ); Metadata metadata = builder.build(); - List backingIndices = dataStream.getIndicesPastRetention(metadata::index, () -> now); + List backingIndices = dataStream.getIndicesPastRetention(metadata::index, () -> now, randomGlobalRetention()); assertThat(backingIndices.isEmpty(), is(true)); } @@ -1232,13 +1259,13 @@ public void testGetIndicesPastRetention() { ); Metadata metadata = builder.build(); - List backingIndices = dataStream.getIndicesPastRetention(metadata::index, () -> now); + List backingIndices = dataStream.getIndicesPastRetention(metadata::index, () -> now, randomGlobalRetention()); assertThat(backingIndices.isEmpty(), is(true)); } } public void testGetIndicesPastRetentionWithOriginationDate() { - // First, build an ordinary datastream: + // First, build an ordinary data stream: String dataStreamName = "metrics-foo"; long now = System.currentTimeMillis(); List creationAndRolloverTimes = List.of( @@ -1267,37 +1294,37 @@ public TimeValue getDataStreamRetention() { { // no retention configured so we expect an empty list testRetentionReference.set(null); - assertThat(dataStream.getIndicesPastRetention(metadata::index, () -> now).isEmpty(), is(true)); + assertThat(dataStream.getIndicesPastRetention(metadata::index, () -> now, null).isEmpty(), is(true)); } { // retention period where oldIndex is too old, but newIndex should be retained testRetentionReference.set(TimeValue.timeValueMillis(2500)); - List backingIndices = dataStream.getIndicesPastRetention(metadata::index, () -> now); + List backingIndices = dataStream.getIndicesPastRetention(metadata::index, () -> now, null); assertThat(backingIndices.size(), is(3)); - assertThat(backingIndices.get(0).getName(), is(DataStream.getDefaultBackingIndexName(dataStreamName, 1))); - assertThat(backingIndices.get(1).getName(), is(DataStream.getDefaultBackingIndexName(dataStreamName, 2))); - assertThat(backingIndices.get(2).getName(), is(DataStream.getDefaultBackingIndexName(dataStreamName, 6))); + assertThat(backingIndices.get(0).getName(), is(dataStream.getIndices().get(0).getName())); + assertThat(backingIndices.get(1).getName(), is(dataStream.getIndices().get(1).getName())); + assertThat(backingIndices.get(2).getName(), is(dataStream.getIndices().get(5).getName())); } { // even though all indices match the write index should not be returned testRetentionReference.set(TimeValue.timeValueMillis(0)); - List backingIndices = dataStream.getIndicesPastRetention(metadata::index, () -> now); + List backingIndices = dataStream.getIndicesPastRetention(metadata::index, () -> now, null); assertThat(backingIndices.size(), is(6)); - assertThat(backingIndices.get(0).getName(), is(DataStream.getDefaultBackingIndexName(dataStreamName, 1))); - assertThat(backingIndices.get(1).getName(), is(DataStream.getDefaultBackingIndexName(dataStreamName, 2))); - assertThat(backingIndices.get(2).getName(), is(DataStream.getDefaultBackingIndexName(dataStreamName, 3))); - assertThat(backingIndices.get(3).getName(), is(DataStream.getDefaultBackingIndexName(dataStreamName, 4))); - assertThat(backingIndices.get(4).getName(), is(DataStream.getDefaultBackingIndexName(dataStreamName, 5))); - assertThat(backingIndices.get(5).getName(), is(DataStream.getDefaultBackingIndexName(dataStreamName, 6))); + assertThat(backingIndices.get(0).getName(), is(dataStream.getIndices().get(0).getName())); + assertThat(backingIndices.get(1).getName(), is(dataStream.getIndices().get(1).getName())); + assertThat(backingIndices.get(2).getName(), is(dataStream.getIndices().get(2).getName())); + assertThat(backingIndices.get(3).getName(), is(dataStream.getIndices().get(3).getName())); + assertThat(backingIndices.get(4).getName(), is(dataStream.getIndices().get(4).getName())); + assertThat(backingIndices.get(5).getName(), is(dataStream.getIndices().get(5).getName())); } { // no index matches the retention age testRetentionReference.set(TimeValue.timeValueMillis(9000)); - List backingIndices = dataStream.getIndicesPastRetention(metadata::index, () -> now); + List backingIndices = dataStream.getIndicesPastRetention(metadata::index, () -> now, null); assertThat(backingIndices.isEmpty(), is(true)); } } @@ -1670,10 +1697,11 @@ public void testXContentSerializationWithRollover() throws IOException { try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { builder.humanReadable(true); RolloverConfiguration rolloverConfiguration = RolloverConfigurationTests.randomRolloverConditions(); - dataStream.toXContent(builder, ToXContent.EMPTY_PARAMS, rolloverConfiguration); + DataStreamGlobalRetention globalRetention = DataStreamGlobalRetentionSerializationTests.randomGlobalRetention(); + dataStream.toXContent(builder, ToXContent.EMPTY_PARAMS, rolloverConfiguration, globalRetention); String serialized = Strings.toString(builder); assertThat(serialized, containsString("rollover")); - for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention()) + for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention)) .getConditions() .keySet()) { assertThat(serialized, containsString(label)); diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index 5d6ba6c3a6d1d..4cc019a300e8b 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; @@ -372,6 +373,18 @@ public static DataStreamAlias randomAliasInstance() { ); } + @Nullable + public static DataStreamGlobalRetention randomGlobalRetention() { + if (randomBoolean()) { + return null; + } + boolean withDefault = randomBoolean(); + return new DataStreamGlobalRetention( + withDefault ? TimeValue.timeValueDays(randomIntBetween(1, 30)) : null, + withDefault == false || randomBoolean() ? TimeValue.timeValueDays(randomIntBetween(31, 100)) : null + ); + } + /** * Constructs {@code ClusterState} with the specified data streams and indices. * diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportAction.java index fb49ba6c7e7a7..947adf9f8462f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportAction.java @@ -85,8 +85,8 @@ public static Tuple calculateStats(Collection