Skip to content

Commit

Permalink
Remove timestampField from DataStreamTestHelper#newInstance(...) help…
Browse files Browse the repository at this point in the history 
…er methods (#85260)

This makes tests a bit cleaner.

The timestamp field is always '@timestamp' and in the future will likely
remove the 'timeStampField' field from `DataStream` class. Since in practice
it is always '@timestamp'.

When data streams was developed the idea was the timestamp field would be configurable.
We left it hardcode to '@timestamp', since that is the timestamp field for all Elastic
solutions. In the last years we have not found a reason to use any other timestamp field
than '@timestamp', so in the future will likely remove this 'timeStampField' field from
`DataStream` class, which currently is hardcoded to '@timestamp'.
  • Loading branch information
@martijnvg
martijnvg committed Mar 23, 2022
1 parent 142292b commit d0c2b09
Show file tree
Hide file tree
Showing 30 changed files with 87 additions and 239 deletions.
28 changes: 9 additions & 19 deletions server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@
import java.util.Map;
import java.util.Set;

import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createTimestampField;
import static org.hamcrest.Matchers.arrayContaining;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
Expand All @@ -59,9 +58,9 @@ public class ResolveIndexTests extends ESTestCase {
{ ".test-system-index", false, false, true, false, null, new String[] {} } };

private final Object[][] dataStreams = new Object[][] {
// name, timestampField, numBackingIndices
{ "logs-mysql-prod", "@timestamp", 4 },
{ "logs-mysql-test", "@timestamp", 2 } };
// name, numBackingIndices
{ "logs-mysql-prod", 4 },
{ "logs-mysql-test", 2 } };

private Metadata metadata;
private final IndexAbstractionResolver resolver = new IndexAbstractionResolver(TestIndexNameExpressionResolver.newInstance());
Expand Down Expand Up @@ -179,11 +178,7 @@ public void testResolvePreservesBackingIndexOrdering() {
builder.put(index, false);
}

DataStream ds = DataStreamTestHelper.newInstance(
dataStreamName,
createTimestampField("@timestamp"),
backingIndices.stream().map(IndexMetadata::getIndex).toList()
);
DataStream ds = DataStreamTestHelper.newInstance(dataStreamName, backingIndices.stream().map(IndexMetadata::getIndex).toList());
builder.put(ds);

IndicesOptions indicesOptions = IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN;
Expand Down Expand Up @@ -279,8 +274,8 @@ private void validateDataStreams(List<ResolvedDataStream> resolvedDataStreams, S
Object[] dataStreamInfo = findInfo(dataStreams, expectedDataStreams[k]);
assertThat(dataStreamInfo, notNullValue());
assertThat(resolvedDataStream.getName(), equalTo((String) dataStreamInfo[0]));
assertThat(resolvedDataStream.getTimestampField(), equalTo((String) dataStreamInfo[1]));
int numBackingIndices = (int) dataStreamInfo[2];
assertThat(resolvedDataStream.getTimestampField(), equalTo("@timestamp"));
int numBackingIndices = (int) dataStreamInfo[1];
List<String> expectedBackingIndices = new ArrayList<>();
for (int m = 1; m <= numBackingIndices; m++) {
expectedBackingIndices.add(DataStream.getDefaultBackingIndexName(resolvedDataStream.getName(), m, epochMillis));
Expand All @@ -295,8 +290,7 @@ Metadata buildMetadata(Object[][] dataStreams, Object[][] indices) {
List<IndexMetadata> allIndices = new ArrayList<>();
for (Object[] dsInfo : dataStreams) {
String dataStreamName = (String) dsInfo[0];
String timestampField = (String) dsInfo[1];
int numBackingIndices = (int) dsInfo[2];
int numBackingIndices = (int) dsInfo[1];
List<IndexMetadata> backingIndices = new ArrayList<>();
for (int backingIndexNumber = 1; backingIndexNumber <= numBackingIndices; backingIndexNumber++) {
backingIndices.add(
Expand All @@ -305,11 +299,7 @@ Metadata buildMetadata(Object[][] dataStreams, Object[][] indices) {
}
allIndices.addAll(backingIndices);

DataStream ds = DataStreamTestHelper.newInstance(
dataStreamName,
createTimestampField(timestampField),
backingIndices.stream().map(IndexMetadata::getIndex).toList()
);
DataStream ds = DataStreamTestHelper.newInstance(dataStreamName, backingIndices.stream().map(IndexMetadata::getIndex).toList());
builder.put(ds);
}

Expand Down Expand Up @@ -373,7 +363,7 @@ private static Object[] findInfo(Object[][] indexSource, String indexName) {
private Object[] findBackingIndexInfo(Object[][] dataStreamSource, String indexName) {
for (Object[] info : dataStreamSource) {
String dataStreamName = (String) info[0];
int generations = (int) info[2];
int generations = (int) info[1];
for (int k = 1; k <= generations; k++) {
if (DataStream.getDefaultBackingIndexName(dataStreamName, k, epochMillis).equals(indexName)) {
return new Object[] {
Expand Down
2 changes: 0 additions & 2 deletions server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.support.ActiveShardCount;
import org.elasticsearch.action.support.replication.ReplicationResponse;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.common.bytes.BytesArray;
Expand Down Expand Up @@ -326,7 +325,6 @@ public void testGetConcreteWriteIndex() {
.put(
DataStreamTestHelper.newInstance(
regularDataStream,
new DataStream.TimestampField("@timestamp"),
List.of(backingIndex1.getIndex(), backingIndex2.getIndex()),
2,
null
Expand Down
18 changes: 3 additions & 15 deletions server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -478,11 +478,7 @@ public void run() {
public void testCanMatchFilteringOnCoordinatorThatCanBeSkipped() throws Exception {
Index dataStreamIndex1 = new Index(".ds-mydata0001", UUIDs.base64UUID());
Index dataStreamIndex2 = new Index(".ds-mydata0002", UUIDs.base64UUID());
DataStream dataStream = DataStreamTestHelper.newInstance(
"mydata",
new DataStream.TimestampField("@timestamp"),
List.of(dataStreamIndex1, dataStreamIndex2)
);
DataStream dataStream = DataStreamTestHelper.newInstance("mydata", List.of(dataStreamIndex1, dataStreamIndex2));

List<Index> regularIndices = randomList(0, 2, () -> new Index(randomAlphaOfLength(10), UUIDs.base64UUID()));

Expand Down Expand Up @@ -549,11 +545,7 @@ public void testCanMatchFilteringOnCoordinatorThatCanBeSkipped() throws Exceptio
public void testCanMatchFilteringOnCoordinatorParsingFails() throws Exception {
Index dataStreamIndex1 = new Index(".ds-mydata0001", UUIDs.base64UUID());
Index dataStreamIndex2 = new Index(".ds-mydata0002", UUIDs.base64UUID());
DataStream dataStream = DataStreamTestHelper.newInstance(
"mydata",
new DataStream.TimestampField("@timestamp"),
List.of(dataStreamIndex1, dataStreamIndex2)
);
DataStream dataStream = DataStreamTestHelper.newInstance("mydata", List.of(dataStreamIndex1, dataStreamIndex2));

List<Index> regularIndices = randomList(0, 2, () -> new Index(randomAlphaOfLength(10), UUIDs.base64UUID()));

Expand Down Expand Up @@ -590,11 +582,7 @@ public void testCanMatchFilteringOnCoordinatorThatCanNotBeSkipped() throws Excep
// Generate indices
Index dataStreamIndex1 = new Index(".ds-mydata0001", UUIDs.base64UUID());
Index dataStreamIndex2 = new Index(".ds-mydata0002", UUIDs.base64UUID());
DataStream dataStream = DataStreamTestHelper.newInstance(
"mydata",
new DataStream.TimestampField("@timestamp"),
List.of(dataStreamIndex1, dataStreamIndex2)
);
DataStream dataStream = DataStreamTestHelper.newInstance("mydata", List.of(dataStreamIndex1, dataStreamIndex2));

List<Index> regularIndices = randomList(0, 2, () -> new Index(randomAlphaOfLength(10), UUIDs.base64UUID()));

Expand Down
3 changes: 1 addition & 2 deletions ...r/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import java.util.stream.Stream;

import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createFirstBackingIndex;
import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createTimestampField;
import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
Expand Down Expand Up @@ -102,7 +101,7 @@ private Metadata randomMeta() {
for (int i = 0; i < numDataStreams; i++) {
String dataStreamName = "name" + 1;
IndexMetadata backingIndex = createFirstBackingIndex(dataStreamName).build();
mdBuilder.put(newInstance(dataStreamName, createTimestampField("@timestamp"), List.of(backingIndex.getIndex())));
mdBuilder.put(newInstance(dataStreamName, List.of(backingIndex.getIndex())));
}
}
mdBuilder.indexGraveyard(graveyard.build());
Expand Down
23 changes: 11 additions & 12 deletions server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import java.util.List;
import java.util.Locale;

import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createTimestampField;
import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.everyItem;
Expand Down Expand Up @@ -160,7 +159,7 @@ public void testRemoveBackingIndex() {
for (int k = 1; k <= numBackingIndices; k++) {
indices.add(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, k), UUIDs.randomBase64UUID(random())));
}
DataStream original = DataStreamTestHelper.newInstance(dataStreamName, createTimestampField("@timestamp"), indices);
DataStream original = DataStreamTestHelper.newInstance(dataStreamName, indices);
DataStream updated = original.removeBackingIndex(indices.get(indexToRemove - 1));
assertThat(updated.getName(), equalTo(original.getName()));
assertThat(updated.getGeneration(), equalTo(original.getGeneration() + 1));
Expand All @@ -179,7 +178,7 @@ public void testRemoveBackingIndexThatDoesNotExist() {
for (int k = 1; k <= numBackingIndices; k++) {
indices.add(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, k), UUIDs.randomBase64UUID(random())));
}
DataStream original = DataStreamTestHelper.newInstance(dataStreamName, createTimestampField("@timestamp"), indices);
DataStream original = DataStreamTestHelper.newInstance(dataStreamName, indices);

final Index indexToRemove = new Index(randomAlphaOfLength(4), UUIDs.randomBase64UUID(random()));

Expand All @@ -198,7 +197,7 @@ public void testRemoveBackingWriteIndex() {
for (int k = 1; k <= numBackingIndices; k++) {
indices.add(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, k), UUIDs.randomBase64UUID(random())));
}
DataStream original = DataStreamTestHelper.newInstance(dataStreamName, createTimestampField("@timestamp"), indices);
DataStream original = DataStreamTestHelper.newInstance(dataStreamName, indices);

IllegalArgumentException e = expectThrows(
IllegalArgumentException.class,
Expand Down Expand Up @@ -236,7 +235,7 @@ public void testAddBackingIndex() {
.build();
builder.put(im, false);
}
DataStream original = DataStreamTestHelper.newInstance(dataStreamName, createTimestampField("@timestamp"), indices);
DataStream original = DataStreamTestHelper.newInstance(dataStreamName, indices);
builder.put(original);
Index indexToAdd = new Index(randomAlphaOfLength(4), UUIDs.randomBase64UUID(random()));
builder.put(
Expand Down Expand Up @@ -284,8 +283,8 @@ public void testAddBackingIndexThatIsPartOfAnotherDataStream() {
.build();
builder.put(im, false);
}
DataStream ds1 = DataStreamTestHelper.newInstance(dsName1, createTimestampField("@timestamp"), indices1);
DataStream ds2 = DataStreamTestHelper.newInstance(dsName2, createTimestampField("@timestamp"), indices2);
DataStream ds1 = DataStreamTestHelper.newInstance(dsName1, indices1);
DataStream ds2 = DataStreamTestHelper.newInstance(dsName2, indices2);
builder.put(ds1);
builder.put(ds2);

Expand Down Expand Up @@ -325,7 +324,7 @@ public void testAddExistingBackingIndex() {
.build();
builder.put(im, false);
}
DataStream original = DataStreamTestHelper.newInstance(dataStreamName, createTimestampField("@timestamp"), indices);
DataStream original = DataStreamTestHelper.newInstance(dataStreamName, indices);
builder.put(original);
Index indexToAdd = randomFrom(indices.toArray(Index.EMPTY_ARRAY));

Expand Down Expand Up @@ -358,7 +357,7 @@ public void testAddBackingIndexWithAliases() {
.build();
builder.put(im, false);
}
DataStream original = DataStreamTestHelper.newInstance(dataStreamName, createTimestampField("@timestamp"), indices);
DataStream original = DataStreamTestHelper.newInstance(dataStreamName, indices);
builder.put(original);

Index indexToAdd = new Index(randomAlphaOfLength(4), UUIDs.randomBase64UUID(random()));
Expand Down Expand Up @@ -414,7 +413,7 @@ public void testReplaceBackingIndex() {
for (int i = 1; i <= numBackingIndices; i++) {
indices.add(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, i), UUIDs.randomBase64UUID(random())));
}
DataStream original = DataStreamTestHelper.newInstance(dataStreamName, createTimestampField("@timestamp"), indices);
DataStream original = DataStreamTestHelper.newInstance(dataStreamName, indices);

Index newBackingIndex = new Index("replacement-index", UUIDs.randomBase64UUID(random()));
DataStream updated = original.replaceBackingIndex(indices.get(indexToReplace), newBackingIndex);
Expand All @@ -439,7 +438,7 @@ public void testReplaceBackingIndexThrowsExceptionIfIndexNotPartOfDataStream() {
for (int i = 1; i <= numBackingIndices; i++) {
indices.add(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, i), UUIDs.randomBase64UUID(random())));
}
DataStream original = DataStreamTestHelper.newInstance(dataStreamName, createTimestampField("@timestamp"), indices);
DataStream original = DataStreamTestHelper.newInstance(dataStreamName, indices);

Index standaloneIndex = new Index("index-foo", UUIDs.randomBase64UUID(random()));
Index newBackingIndex = new Index("replacement-index", UUIDs.randomBase64UUID(random()));
Expand All @@ -456,7 +455,7 @@ public void testReplaceBackingIndexThrowsExceptionIfReplacingWriteIndex() {
indices.add(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, i), UUIDs.randomBase64UUID(random())));
}
int generation = randomBoolean() ? numBackingIndices : numBackingIndices + randomIntBetween(1, 5);
DataStream original = newInstance(dataStreamName, createTimestampField("@timestamp"), indices, generation, null);
DataStream original = newInstance(dataStreamName, indices, generation, null);

Index newBackingIndex = new Index("replacement-index", UUIDs.randomBase64UUID(random()));
IllegalArgumentException e = expectThrows(
Expand Down

0 comments on commit d0c2b09

Please sign in to comment.