Skip to content

Commit

Permalink
Fixing elastic#69799 and cleaning up test
Browse files Browse the repository at this point in the history
  • Loading branch information
@csoulios
csoulios committed Jun 2, 2022
1 parent e2ca6d3 commit 02741fb
Showing 1 changed file with 27 additions and 19 deletions.
46 changes: 27 additions & 19 deletions ...n/rollup/src/test/java/org/elasticsearch/xpack/rollup/v2/RollupActionSingleNodeTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ public class RollupActionSingleNodeTests extends ESSingleNodeTestCase {
private static final int MAX_DIM_VALUES = 5;
private static final long MAX_NUM_BUCKETS = 10;

private String sourceIndex, sourceIndexClone, rollupIndex;
private String sourceIndex, rollupIndex;
private long startTime;
private int docCount, numOfShards, numOfReplicas;
private List<String> dimensionValues;
Expand All @@ -112,13 +112,12 @@ protected Collection<Class<? extends Plugin>> getPlugins() {

@Before
public void setup() {
sourceIndex = randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
sourceIndexClone = sourceIndex + "-clone";
rollupIndex = randomAlphaOfLength(6).toLowerCase(Locale.ROOT);
sourceIndex = randomAlphaOfLength(12).toLowerCase(Locale.ROOT);
rollupIndex = "rollup-" + sourceIndex + "-" + randomAlphaOfLength(4).toLowerCase(Locale.ROOT);
startTime = randomLongBetween(946769284000L, 1607470084000L); // random date between 2000-2020
docCount = randomIntBetween(10, 9000);
numOfShards = randomIntBetween(1, 4);
numOfReplicas = randomIntBetween(0, 3);
numOfReplicas = 0; // Since this is a single node, we cannot have replicas

// Values for keyword dimensions
dimensionValues = new ArrayList<>(MAX_DIM_VALUES);
Expand Down Expand Up @@ -169,6 +168,8 @@ public void testRollupIndex() throws IOException {
};
bulkIndex(sourceSupplier);
prepareSourceIndex(sourceIndex);
// Clone the source index before rollup deletes it
String sourceIndexClone = cloneSourceIndex(sourceIndex);
rollup(sourceIndex, rollupIndex, config);
assertRollupIndex(config, sourceIndex, sourceIndexClone, rollupIndex);
}
Expand Down Expand Up @@ -217,6 +218,8 @@ public void testRollupSparseMetrics() throws IOException {
};
bulkIndex(sourceSupplier);
prepareSourceIndex(sourceIndex);
// Clone the source index before rollup deletes it
String sourceIndexClone = cloneSourceIndex(sourceIndex);
rollup(sourceIndex, rollupIndex, config);
assertRollupIndex(config, sourceIndex, sourceIndexClone, rollupIndex);
}
Expand All @@ -238,14 +241,15 @@ public void testRollupEmptyIndex() {
RollupActionConfig config = new RollupActionConfig(randomInterval());
// Source index has been created in the setup() method
prepareSourceIndex(sourceIndex);
// Clone the source index before rollup deletes it
String sourceIndexClone = cloneSourceIndex(sourceIndex);
rollup(sourceIndex, rollupIndex, config);
assertRollupIndex(config, sourceIndex, sourceIndexClone, rollupIndex);
}

public void testCannotRollupIndexWithNoMetrics() {
// Create a source index that contains no metric fields in its mapping
sourceIndex = randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
sourceIndexClone = sourceIndex + "-clone";
String sourceIndex = "no-metrics-idx-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
client().admin()
.indices()
.prepareCreate(sourceIndex)
Expand All @@ -270,7 +274,6 @@ public void testCannotRollupIndexWithNoMetrics() {
.get();

RollupActionConfig config = new RollupActionConfig(randomInterval());
// Source index has been created in the setup() method
prepareSourceIndex(sourceIndex);
Exception exception = expectThrows(RollupActionRequestValidationException.class, () -> rollup(sourceIndex, rollupIndex, config));
assertThat(exception.getMessage(), containsString("does not contain any metric fields"));
Expand Down Expand Up @@ -325,10 +328,11 @@ public void testRollupDatastream() throws Exception {
};
bulkIndex(dataStreamName, sourceSupplier);

this.sourceIndex = rollover(dataStreamName).getOldIndex();
this.sourceIndexClone = sourceIndex + "-clone";
this.rollupIndex = ".rollup-" + sourceIndex;
String sourceIndex = rollover(dataStreamName).getOldIndex();
prepareSourceIndex(sourceIndex);
// Clone the source index before rollup deletes it
String sourceIndexClone = cloneSourceIndex(sourceIndex);
String rollupIndex = "rollup-" + sourceIndex + "-" + randomAlphaOfLength(4).toLowerCase(Locale.ROOT);
rollup(sourceIndex, rollupIndex, config);
assertRollupIndex(config, sourceIndex, sourceIndexClone, rollupIndex);

Expand All @@ -354,7 +358,15 @@ private String randomDateForRange(long start, long end) {
return DATE_FORMATTER.formatMillis(randomLongBetween(start, end));
}

private void cloneSourceIndex(String sourceIndex, String sourceIndexClone) {
/**
* The source index is deleted at the end of the rollup process.
* We clone the source index, so that we validate rollup results against the
* source index clone.
*
* @return the name of the source index clone
*/
private String cloneSourceIndex(String sourceIndex) {
String sourceIndexClone = "clone-" + sourceIndex;
ResizeResponse r = client().admin()
.indices()
.prepareResizeIndex(sourceIndex, sourceIndexClone)
Expand All @@ -364,6 +376,7 @@ private void cloneSourceIndex(String sourceIndex, String sourceIndexClone) {
)
.get();
assertTrue(r.isAcknowledged());
return sourceIndexClone;
}

private void bulkIndex(SourceSupplier sourceSupplier) throws IOException {
Expand All @@ -386,15 +399,15 @@ private void bulkIndex(String indexName, SourceSupplier sourceSupplier) throws I
if (response.getFailure().getCause() instanceof VersionConflictEngineException) {
// A duplicate event was created by random generator. We should not fail for this
// reason.
logger.info("We tried to insert a duplicate: " + response.getFailureMessage());
logger.debug("We tried to insert a duplicate: [{}]", response.getFailureMessage());
duplicates++;
} else {
fail("Failed to index data: " + bulkResponse.buildFailureMessage());
}
}
}
int docsIndexed = docCount - duplicates;
logger.info("Indexed [" + docsIndexed + "] documents");
logger.info("Indexed [{}] documents. Dropped [{}] duplicates.", docsIndexed, duplicates);
assertHitCount(client().prepareSearch(indexName).setSize(0).get(), docsIndexed);
}

Expand All @@ -406,11 +419,6 @@ private void prepareSourceIndex(String sourceIndex) {
.setSettings(Settings.builder().put(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey(), true).build())
.get();
assertTrue(r.isAcknowledged());

// The source index is deleted at the end of the rollup process.
// We clone the source index, so that we validate rollup results against the
// source index clone.
cloneSourceIndex(sourceIndex, sourceIndexClone);
}

private void rollup(String sourceIndex, String rollupIndex, RollupActionConfig config) {
Expand Down

0 comments on commit 02741fb

Please sign in to comment.