Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Give more time to AutoFollowIT tests #51938

Merged
merged 1 commit into from
Feb 5, 2020
Merged

Give more time to AutoFollowIT tests #51938

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
102 changes: 56 additions & 46 deletions x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@
*/
package org.elasticsearch.xpack.ccr;

import org.apache.logging.log4j.message.ParameterizedMessage;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.CheckedRunnable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
Expand Down Expand Up @@ -80,7 +82,7 @@ public void testAutoFollow() throws Exception {
assertTrue(ESIntegTestCase.indexExists("copy-logs-201901", followerClient()));
});
createLeaderIndex("transactions-201901", leaderIndexSettings);
assertBusy(() -> {
assertLongBusy(() -> {
AutoFollowStats autoFollowStats = getAutoFollowStats();
assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(2L));
assertTrue(ESIntegTestCase.indexExists("copy-transactions-201901", followerClient()));
Expand All @@ -98,7 +100,7 @@ public void testCleanFollowedLeaderIndexUUIDs() throws Exception {

putAutoFollowPatterns("my-pattern", new String[] {"logs-*"});
createLeaderIndex("logs-201901", leaderIndexSettings);
assertBusy(() -> {
assertLongBusy(() -> {
AutoFollowStats autoFollowStats = getAutoFollowStats();
assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(1L));

Expand All @@ -118,7 +120,7 @@ public void testCleanFollowedLeaderIndexUUIDs() throws Exception {
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest("logs-201901");
assertAcked(leaderClient().admin().indices().delete(deleteIndexRequest).actionGet());

assertBusy(() -> {
assertLongBusy(() -> {
AutoFollowMetadata autoFollowMetadata = getFollowerCluster().clusterService().state()
.metaData()
.custom(AutoFollowMetadata.TYPE);
Expand All @@ -143,15 +145,15 @@ public void testAutoFollowManyIndices() throws Exception {
MetaData[] metaData = new MetaData[1];
AutoFollowStats[] autoFollowStats = new AutoFollowStats[1];
try {
assertBusy(() -> {
assertLongBusy(() -> {
metaData[0] = getFollowerCluster().clusterService().state().metaData();
autoFollowStats[0] = getAutoFollowStats();

assertThat(metaData[0].indices().size(), equalTo((int) expectedVal1));
AutoFollowMetadata autoFollowMetadata = metaData[0].custom(AutoFollowMetadata.TYPE);
assertThat(autoFollowMetadata.getFollowedLeaderIndexUUIDs().get("my-pattern"), hasSize((int) expectedVal1));
assertThat(autoFollowStats[0].getNumberOfSuccessfulFollowIndices(), equalTo(expectedVal1));
}, 30, TimeUnit.SECONDS);
});
} catch (AssertionError ae) {
logger.warn("indices={}", Arrays.toString(metaData[0].indices().keys().toArray(String.class)));
logger.warn("auto follow stats={}", Strings.toString(autoFollowStats[0]));
Expand All @@ -162,15 +164,15 @@ public void testAutoFollowManyIndices() throws Exception {
// then the leader index created after that should never be auto followed:
deleteAutoFollowPattern("my-pattern");
try {
assertBusy(() -> {
assertLongBusy(() -> {
metaData[0] = getFollowerCluster().clusterService().state().metaData();
autoFollowStats[0] = getAutoFollowStats();

assertThat(metaData[0].indices().size(), equalTo((int )expectedVal1));
AutoFollowMetadata autoFollowMetadata = metaData[0].custom(AutoFollowMetadata.TYPE);
assertThat(autoFollowMetadata.getFollowedLeaderIndexUUIDs().get("my-pattern"), nullValue());
assertThat(autoFollowStats[0].getAutoFollowedClusters().size(), equalTo(0));
}, 30, TimeUnit.SECONDS);
});
} catch (AssertionError ae) {
logger.warn("indices={}", Arrays.toString(metaData[0].indices().keys().toArray(String.class)));
logger.warn("auto follow stats={}", Strings.toString(autoFollowStats[0]));
Expand All @@ -186,27 +188,21 @@ public void testAutoFollowManyIndices() throws Exception {
}
long expectedVal2 = numIndices;

try {
assertBusy(() -> {
metaData[0] = getFollowerCluster().clusterService().state().metaData();
autoFollowStats[0] = getAutoFollowStats();

assertThat(metaData[0].indices().size(), equalTo((int) expectedVal2));
AutoFollowMetadata autoFollowMetadata = metaData[0].custom(AutoFollowMetadata.TYPE);
// expectedVal2 + 1, because logs-does-not-count is also marked as auto followed.
// (This is because indices created before a pattern exists are not auto followed and are just marked as such.)
assertThat(autoFollowMetadata.getFollowedLeaderIndexUUIDs().get("my-pattern"), hasSize((int) expectedVal2 + 1));
long count = Arrays.stream(metaData[0].getConcreteAllIndices()).filter(s -> s.startsWith("copy-")).count();
assertThat(count, equalTo(expectedVal2));
// Ensure that there are no auto follow errors:
// (added specifically to see that there are no leader indices auto followed multiple times)
assertThat(autoFollowStats[0].getRecentAutoFollowErrors().size(), equalTo(0));
}, 30, TimeUnit.SECONDS);
} catch (AssertionError ae) {
logger.warn("indices={}", Arrays.toString(metaData[0].indices().keys().toArray(String.class)));
logger.warn("auto follow stats={}", Strings.toString(autoFollowStats[0]));
throw ae;
}
assertLongBusy(() -> {
metaData[0] = getFollowerCluster().clusterService().state().metaData();
autoFollowStats[0] = getAutoFollowStats();

assertThat(metaData[0].indices().size(), equalTo((int) expectedVal2));
AutoFollowMetadata autoFollowMetadata = metaData[0].custom(AutoFollowMetadata.TYPE);
// expectedVal2 + 1, because logs-does-not-count is also marked as auto followed.
// (This is because indices created before a pattern exists are not auto followed and are just marked as such.)
assertThat(autoFollowMetadata.getFollowedLeaderIndexUUIDs().get("my-pattern"), hasSize((int) expectedVal2 + 1));
long count = Arrays.stream(metaData[0].getConcreteAllIndices()).filter(s -> s.startsWith("copy-")).count();
assertThat(count, equalTo(expectedVal2));
// Ensure that there are no auto follow errors:
// (added specifically to see that there are no leader indices auto followed multiple times)
assertThat(autoFollowStats[0].getRecentAutoFollowErrors().size(), equalTo(0));
});
}

public void testAutoFollowParameterAreDelegated() throws Exception {
Expand Down Expand Up @@ -322,7 +318,7 @@ public void testConflictingPatterns() throws Exception {
putAutoFollowPatterns("my-pattern2", new String[] {"logs-2018*"});

createLeaderIndex("logs-201701", leaderIndexSettings);
assertBusy(() -> {
assertLongBusy(() -> {
AutoFollowStats autoFollowStats = getAutoFollowStats();
assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(1L));
assertThat(autoFollowStats.getNumberOfFailedFollowIndices(), equalTo(0L));
Expand All @@ -331,7 +327,7 @@ public void testConflictingPatterns() throws Exception {
assertTrue(ESIntegTestCase.indexExists("copy-logs-201701", followerClient()));

createLeaderIndex("logs-201801", leaderIndexSettings);
assertBusy(() -> {
assertLongBusy(() -> {
AutoFollowStats autoFollowStats = getAutoFollowStats();
assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(1L));
assertThat(autoFollowStats.getNumberOfFailedFollowIndices(), greaterThanOrEqualTo(1L));
Expand Down Expand Up @@ -363,15 +359,15 @@ public void testPauseAndResumeAutoFollowPattern() throws Exception {

// create the auto follow pattern
putAutoFollowPatterns("test-pattern", new String[]{"test-*", "tests-*"});
assertBusy(() -> {
assertLongBusy(() -> {
final AutoFollowStats autoFollowStats = getAutoFollowStats();
assertThat(autoFollowStats.getAutoFollowedClusters().size(), equalTo(1));
assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(0L));
});

// index created in the remote cluster are auto followed
createLeaderIndex("test-new-index-is-auto-followed", leaderIndexSettings);
assertBusy(() -> {
assertLongBusy(() -> {
final AutoFollowStats autoFollowStats = getAutoFollowStats();
assertThat(autoFollowStats.getAutoFollowedClusters().size(), equalTo(1));
assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(1L));
Expand Down Expand Up @@ -411,10 +407,11 @@ public void testPauseAndResumeAutoFollowPattern() throws Exception {

// resume the auto follow pattern, indices created while the pattern was paused are picked up for auto-following
resumeAutoFollowPattern("test-pattern");
assertBusy(() -> {
assertLongBusy(() -> {
final Client client = followerClient();
assertThat(getAutoFollowStats().getAutoFollowedClusters().size(), equalTo(1));
assertThat(client.admin().indices().prepareStats("copy-*").get().getIndices().size(), equalTo(1 + nbIndicesCreatedWhilePaused));
assertThat(client.admin().cluster().prepareState().clear().setIndices("copy-*").setMetaData(true).get()
.getState().getMetaData().getIndices().size(), equalTo(1 + nbIndicesCreatedWhilePaused));
for (int i = 0; i < nbIndicesCreatedWhilePaused; i++) {
assertTrue(ESIntegTestCase.indexExists("copy-test-index-created-while-pattern-is-paused-" + i, client));
}
Expand Down Expand Up @@ -474,31 +471,27 @@ public void testPauseAndResumeWithMultipleAutoFollowPatterns() throws Exception
createNewLeaderIndicesThread.start();

// wait for 3 leader indices to be created on the remote cluster
latchThree.await(30L, TimeUnit.SECONDS);
latchThree.await(60L, TimeUnit.SECONDS);
assertThat(leaderIndices.get(), greaterThanOrEqualTo(3));
assertBusy(() -> assertThat(getAutoFollowStats().getNumberOfSuccessfulFollowIndices(), greaterThanOrEqualTo(3L)),
30L, TimeUnit.SECONDS);
assertLongBusy(() -> assertThat(getAutoFollowStats().getNumberOfSuccessfulFollowIndices(), greaterThanOrEqualTo(3L)));

// now pause some random patterns
pausedAutoFollowerPatterns.forEach(this::pauseAutoFollowPattern);
assertBusy(() -> autoFollowPatterns.forEach(pattern ->
assertThat(getAutoFollowPattern(pattern).isActive(), equalTo(pausedAutoFollowerPatterns.contains(pattern) == false))),
30L, TimeUnit.SECONDS);
assertLongBusy(() -> autoFollowPatterns.forEach(pattern ->
assertThat(getAutoFollowPattern(pattern).isActive(), equalTo(pausedAutoFollowerPatterns.contains(pattern) == false))));

// wait for more leader indices to be created on the remote cluster
latchSix.await(30L, TimeUnit.SECONDS);
latchSix.await(60L, TimeUnit.SECONDS);
assertThat(leaderIndices.get(), greaterThanOrEqualTo(6));

// resume auto follow patterns
pausedAutoFollowerPatterns.forEach(this::resumeAutoFollowPattern);
assertBusy(() -> autoFollowPatterns.forEach(pattern -> assertTrue(getAutoFollowPattern(pattern).isActive())),
30L, TimeUnit.SECONDS);
assertLongBusy(() -> autoFollowPatterns.forEach(pattern -> assertTrue(getAutoFollowPattern(pattern).isActive())));

// wait for more leader indices to be created on the remote cluster
latchNine.await(30L, TimeUnit.SECONDS);
latchNine.await(60L, TimeUnit.SECONDS);
assertThat(leaderIndices.get(), greaterThanOrEqualTo(9));
assertBusy(() -> assertThat(getAutoFollowStats().getNumberOfSuccessfulFollowIndices(), greaterThanOrEqualTo(9L)),
30L, TimeUnit.SECONDS);
assertLongBusy(() -> assertThat(getAutoFollowStats().getNumberOfSuccessfulFollowIndices(), greaterThanOrEqualTo(9L)));

running.set(false);
createNewLeaderIndicesThread.join();
Expand Down Expand Up @@ -563,4 +556,21 @@ private AutoFollowMetadata.AutoFollowPattern getAutoFollowPattern(final String n
assertTrue(response.getAutoFollowPatterns().containsKey(name));
return response.getAutoFollowPatterns().get(name);
}

private void assertLongBusy(CheckedRunnable<Exception> codeBlock) throws Exception {
try {
assertBusy(codeBlock, 60L, TimeUnit.SECONDS);
} catch (AssertionError ae) {
AutoFollowStats autoFollowStats = null;
try {
autoFollowStats = getAutoFollowStats();
} catch (Exception e) {
ae.addSuppressed(e);
}
final AutoFollowStats finalAutoFollowStats = autoFollowStats;
logger.warn(() -> new ParameterizedMessage("AssertionError when waiting for auto-follower, auto-follow stats are: {}",
finalAutoFollowStats != null ? Strings.toString(finalAutoFollowStats) : "null"), ae);
throw ae;
}
}
}