Skip to content

[8.x] Refactor data stream lifecycle to use the template paradigm (#124593) #125199

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Mar 21, 2025
Merged

[8.x] Refactor data stream lifecycle to use the template paradigm (#124593) #125199

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6d03314
Refactor data stream lifecycle to use the template paradigm (#124593)
gmarouli Mar 18, 2025
24875c5
Fix missed update
gmarouli Mar 19, 2025
2f658d4
Merge branch '8.x' into backport/8.x/pr-124593
gmarouli Mar 20, 2025
447d916
Merge branch '8.x' into backport/8.x/pr-124593
gmarouli Mar 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ...data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1372,7 +1372,7 @@ public void testSearchAllResolvesDataStreams() throws Exception {

public void testGetDataStream() throws Exception {
Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, maximumNumberOfReplicas() + 2).build();
DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build();
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build();
putComposableIndexTemplate("template_for_foo", null, List.of("metrics-foo*"), settings, null, null, lifecycle, false);
int numDocsFoo = randomIntBetween(2, 16);
indexDocs("metrics-foo", numDocsFoo);
Expand All @@ -1388,7 +1388,7 @@ public void testGetDataStream() throws Exception {
assertThat(metricsFooDataStream.getDataStreamStatus(), is(ClusterHealthStatus.YELLOW));
assertThat(metricsFooDataStream.getIndexTemplate(), is("template_for_foo"));
assertThat(metricsFooDataStream.getIlmPolicy(), is(nullValue()));
assertThat(dataStream.getLifecycle(), is(lifecycle));
assertThat(dataStream.getLifecycle(), is(lifecycle.toDataStreamLifecycle()));
assertThat(metricsFooDataStream.templatePreferIlmValue(), is(true));
GetDataStreamAction.Response.IndexProperties indexProperties = metricsFooDataStream.getIndexSettingsValues()
.get(dataStream.getWriteIndex());
Expand Down Expand Up @@ -2435,7 +2435,7 @@ static void putComposableIndexTemplate(
@Nullable Settings settings,
@Nullable Map<String, Object> metadata,
@Nullable Map<String, AliasMetadata> aliases,
@Nullable DataStreamLifecycle lifecycle,
@Nullable DataStreamLifecycle.Template lifecycle,
boolean withFailureStore
) throws IOException {
TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request(id);
Expand Down
28 changes: 14 additions & 14 deletions ...alClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import java.util.List;

import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.putComposableIndexTemplate;
import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomLifecycle;
import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomLifecycleTemplate;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
Expand All @@ -39,7 +39,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
}

public void testGetLifecycle() throws Exception {
DataStreamLifecycle lifecycle = randomLifecycle();
DataStreamLifecycle.Template lifecycle = randomLifecycleTemplate();
putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle);
putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null);
{
Expand Down Expand Up @@ -82,9 +82,9 @@ public void testGetLifecycle() throws Exception {
).get();
assertThat(response.getDataStreamLifecycles().size(), equalTo(3));
assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
assertThat(response.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-2"));
assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle));
assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
assertThat(response.getDataStreamLifecycles().get(2).dataStreamName(), equalTo("without-lifecycle"));
assertThat(response.getDataStreamLifecycles().get(2).lifecycle(), is(nullValue()));
assertThat(response.getRolloverConfiguration(), nullValue());
Expand All @@ -102,9 +102,9 @@ public void testGetLifecycle() throws Exception {
).get();
assertThat(response.getDataStreamLifecycles().size(), equalTo(2));
assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
assertThat(response.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-2"));
assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), is(lifecycle));
assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), is(lifecycle.toDataStreamLifecycle()));
assertThat(response.getRolloverConfiguration(), nullValue());
}

Expand All @@ -120,7 +120,7 @@ public void testGetLifecycle() throws Exception {
).get();
assertThat(response.getDataStreamLifecycles().size(), equalTo(2));
assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
assertThat(response.getRolloverConfiguration(), nullValue());
}

Expand All @@ -135,9 +135,9 @@ public void testGetLifecycle() throws Exception {
).get();
assertThat(responseWithRollover.getDataStreamLifecycles().size(), equalTo(3));
assertThat(responseWithRollover.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
assertThat(responseWithRollover.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
assertThat(responseWithRollover.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
assertThat(responseWithRollover.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-2"));
assertThat(responseWithRollover.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle));
assertThat(responseWithRollover.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
assertThat(responseWithRollover.getDataStreamLifecycles().get(2).dataStreamName(), equalTo("without-lifecycle"));
assertThat(responseWithRollover.getDataStreamLifecycles().get(2).lifecycle(), is(nullValue()));
assertThat(responseWithRollover.getRolloverConfiguration(), notNullValue());
Expand Down Expand Up @@ -192,8 +192,8 @@ public void testPutLifecycle() throws Exception {
).get();
assertThat(response.getDataStreamLifecycles().size(), equalTo(1));
assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("my-data-stream"));
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().getDataStreamRetention(), equalTo(dataRetention));
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().isEnabled(), equalTo(true));
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().dataRetention(), equalTo(dataRetention));
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().enabled(), equalTo(true));
}

// Disable the lifecycle
Expand All @@ -220,13 +220,13 @@ public void testPutLifecycle() throws Exception {
).get();
assertThat(response.getDataStreamLifecycles().size(), equalTo(1));
assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("my-data-stream"));
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().getDataStreamRetention(), equalTo(dataRetention));
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().isEnabled(), equalTo(false));
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().dataRetention(), equalTo(dataRetention));
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().enabled(), equalTo(false));
}
}

public void testDeleteLifecycle() throws Exception {
DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build();
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build();
putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle);
putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null);
{
Expand Down
2 changes: 1 addition & 1 deletion ...terTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
Template.builder()
.settings(Settings.EMPTY)
.mappings(mappings)
.lifecycle(DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build())
.lifecycle(DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build())
)
.dataStreamTemplate(new DataStreamTemplate())
.build(),
Expand Down
36 changes: 17 additions & 19 deletions ...lusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ public void cleanup() {

public void testRolloverLifecycle() throws Exception {
// empty lifecycle contains the default rollover
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;

putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
String dataStreamName = "metrics-foo";
Expand Down Expand Up @@ -178,7 +178,7 @@ public void testRolloverLifecycle() throws Exception {
}

public void testRolloverAndRetention() throws Exception {
DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(0).build();
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(TimeValue.ZERO).build();

putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);

Expand Down Expand Up @@ -321,7 +321,7 @@ public void testOriginationDate() throws Exception {
* days ago, and one with an origination date 1 day ago. After data stream lifecycle runs, we expect the one with the old
* origination date to have been deleted, and the one with the newer origination date to remain.
*/
DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(TimeValue.timeValueDays(7)).build();
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(TimeValue.timeValueDays(7)).build();

putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);

Expand Down Expand Up @@ -393,7 +393,7 @@ public void testOriginationDate() throws Exception {
}

public void testUpdatingLifecycleAppliesToAllBackingIndices() throws Exception {
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;

putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);

Expand Down Expand Up @@ -437,7 +437,7 @@ public void testAutomaticForceMerge() throws Exception {
* because all necessary merging has already happened automatically. So in order to detect whether forcemerge has been called, we
* use a SendRequestBehavior in the MockTransportService to detect it.
*/
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
disableDataStreamLifecycle();
String dataStreamName = "metrics-foo";
putComposableIndexTemplate(
Expand Down Expand Up @@ -539,7 +539,7 @@ private static void disableDataStreamLifecycle() {

public void testErrorRecordingOnRollover() throws Exception {
// empty lifecycle contains the default rollover
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
/*
* We set index.auto_expand_replicas to 0-1 so that if we get a single-node cluster it is not yellow. The cluster being yellow
* could result in data stream lifecycle's automatic forcemerge failing, which would result in an unexpected error in the error
Expand Down Expand Up @@ -697,7 +697,7 @@ public void testErrorRecordingOnRollover() throws Exception {
public void testErrorRecordingOnRetention() throws Exception {
// starting with a lifecycle without retention so we can rollover the data stream and manipulate the second generation index such
// that its retention execution fails
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;

/*
* We set index.auto_expand_replicas to 0-1 so that if we get a single-node cluster it is not yellow. The cluster being yellow
Expand Down Expand Up @@ -871,7 +871,7 @@ public void testErrorRecordingOnRetention() throws Exception {
}

public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception {
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;

putComposableIndexTemplate(
"id1",
Expand Down Expand Up @@ -970,7 +970,7 @@ public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception

public void testReenableDataStreamLifecycle() throws Exception {
// start with a lifecycle that's not enabled
DataStreamLifecycle lifecycle = new DataStreamLifecycle(null, null, false);
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().enabled(false).build();

putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
String dataStreamName = "metrics-foo";
Expand Down Expand Up @@ -1029,15 +1029,13 @@ public void testReenableDataStreamLifecycle() throws Exception {

public void testLifecycleAppliedToFailureStore() throws Exception {
// We configure a lifecycle with downsampling to ensure it doesn't fail
DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder()
.dataRetention(20_000)
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder()
.dataRetention(TimeValue.timeValueSeconds(20))
.downsampling(
new DataStreamLifecycle.Downsampling(
List.of(
new DataStreamLifecycle.Downsampling.Round(
TimeValue.timeValueMillis(10),
new DownsampleConfig(new DateHistogramInterval("10m"))
)
List.of(
new DataStreamLifecycle.DownsamplingRound(
TimeValue.timeValueMillis(10),
new DownsampleConfig(new DateHistogramInterval("10m"))
)
)
)
Expand Down Expand Up @@ -1202,7 +1200,7 @@ static void putComposableIndexTemplate(
List<String> patterns,
@Nullable Settings settings,
@Nullable Map<String, Object> metadata,
@Nullable DataStreamLifecycle lifecycle,
@Nullable DataStreamLifecycle.Template lifecycle,
boolean withFailureStore
) throws IOException {
TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request(id);
Expand Down Expand Up @@ -1265,7 +1263,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
Template.builder()
.settings(Settings.EMPTY)
.lifecycle(
DataStreamLifecycle.newBuilder()
DataStreamLifecycle.Template.builder()
.dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS))
)
)
Expand Down
Loading