Skip to content

Enforce min master nodes in test cluster #22065

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Dec 14, 2016
Merged

Enforce min master nodes in test cluster #22065

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1bd6a68
enfore min master nodes to be set
bleskes Dec 8, 2016
966e6e8
fix testBeforeTest and testDataFolderAssignmentAndCleaning
bleskes Dec 8, 2016
5dcb05e
don't creates clients (without settings) unless we need to and set mo…
bleskes Dec 8, 2016
7e0dc9d
Merge remote-tracking branch 'upstream/master' into enforce_min_maste…
bleskes Dec 14, 2016
60281c6
feedback
bleskes Dec 14, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion core/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.discovery.MasterNotDiscoveredException;
import org.elasticsearch.discovery.zen.ElectMasterService;
import org.elasticsearch.gateway.GatewayService;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
Expand All @@ -36,7 +37,9 @@
public class IndicesExistsIT extends ESIntegTestCase {

public void testIndexExistsWithBlocksInPlace() throws IOException {
Settings settings = Settings.builder().put(GatewayService.RECOVER_AFTER_NODES_SETTING.getKey(), 99).build();
Settings settings = Settings.builder()
.put(GatewayService.RECOVER_AFTER_NODES_SETTING.getKey(), 99)
.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 1).build();
String node = internalCluster().startNode(settings);

assertThrows(client(node).admin().indices().prepareExists("test").setMasterNodeTimeout(TimeValue.timeValueSeconds(0)),
Expand Down
2 changes: 1 addition & 1 deletion core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ public void testMultipleNodesShutdownNonMasterNodes() throws Exception {
public void testDynamicUpdateMinimumMasterNodes() throws Exception {
Settings settings = Settings.builder()
.put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "400ms")
.put("discovery.initial_state_timeout", "500ms")
.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), "1")
.build();

logger.info("--> start first node and wait for it to be a master");
Expand Down
50 changes: 37 additions & 13 deletions core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.discovery.zen.ElectMasterService;
import org.elasticsearch.node.Node;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
Expand All @@ -49,20 +50,22 @@ public Set<ClusterBlock> waitForNoBlocksOnNode(TimeValue timeout, Client nodeCli
return blocks;
}

public Client startNode(Settings.Builder settings) {
String name = internalCluster().startNode(settings);
public Client startNode(Settings.Builder settings, int minMasterNodes) {
String name = internalCluster().startNode(
Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes)
.put(settings.build()));
return internalCluster().client(name);
}

public void testRecoverAfterNodes() throws Exception {
logger.info("--> start node (1)");
Client clientNode1 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3));
Client clientNode1 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1);
assertThat(clientNode1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
.getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));

logger.info("--> start node (2)");
Client clientNode2 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3));
Client clientNode2 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1);
Thread.sleep(BLOCK_WAIT_TIMEOUT.millis());
assertThat(clientNode1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
.getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
Expand All @@ -72,7 +75,7 @@ public void testRecoverAfterNodes() throws Exception {
hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));

logger.info("--> start node (3)");
Client clientNode3 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3));
Client clientNode3 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1);

assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, clientNode1).isEmpty(), equalTo(true));
assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, clientNode2).isEmpty(), equalTo(true));
Expand All @@ -81,13 +84,17 @@ public void testRecoverAfterNodes() throws Exception {

public void testRecoverAfterMasterNodes() throws Exception {
logger.info("--> start master_node (1)");
Client master1 = startNode(Settings.builder().put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
Client master1 = startNode(Settings.builder()
.put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false)
.put(Node.NODE_MASTER_SETTING.getKey(), true), 1);
assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
.getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));

logger.info("--> start data_node (1)");
Client data1 = startNode(Settings.builder().put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
Client data1 = startNode(Settings.builder()
.put("gateway.recover_after_master_nodes", 2)
.put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false), 1);
assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
.getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
Expand All @@ -96,7 +103,9 @@ public void testRecoverAfterMasterNodes() throws Exception {
hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));

logger.info("--> start data_node (2)");
Client data2 = startNode(Settings.builder().put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
Client data2 = startNode(Settings.builder()
.put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), true)
.put(Node.NODE_MASTER_SETTING.getKey(), false), 1);
assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
.getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
Expand All @@ -108,7 +117,10 @@ public void testRecoverAfterMasterNodes() throws Exception {
hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));

logger.info("--> start master_node (2)");
Client master2 = startNode(Settings.builder().put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
Client master2 = startNode(Settings.builder()
.put("gateway.recover_after_master_nodes", 2)
.put(Node.NODE_DATA_SETTING.getKey(), false)
.put(Node.NODE_MASTER_SETTING.getKey(), true), 1);
assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master1).isEmpty(), equalTo(true));
assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master2).isEmpty(), equalTo(true));
assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, data1).isEmpty(), equalTo(true));
Expand All @@ -117,13 +129,19 @@ public void testRecoverAfterMasterNodes() throws Exception {

public void testRecoverAfterDataNodes() throws Exception {
logger.info("--> start master_node (1)");
Client master1 = startNode(Settings.builder().put("gateway.recover_after_data_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
Client master1 = startNode(Settings.builder()
.put("gateway.recover_after_data_nodes", 2)
.put(Node.NODE_DATA_SETTING.getKey(), false)
.put(Node.NODE_MASTER_SETTING.getKey(), true), 1);
assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
.getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));

logger.info("--> start data_node (1)");
Client data1 = startNode(Settings.builder().put("gateway.recover_after_data_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
Client data1 = startNode(Settings.builder()
.put("gateway.recover_after_data_nodes", 2)
.put(Node.NODE_DATA_SETTING.getKey(), true)
.put(Node.NODE_MASTER_SETTING.getKey(), false), 1);
assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet()
.getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
Expand All @@ -132,7 +150,10 @@ public void testRecoverAfterDataNodes() throws Exception {
hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));

logger.info("--> start master_node (2)");
Client master2 = startNode(Settings.builder().put("gateway.recover_after_data_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_MASTER_SETTING.getKey(), true));
Client master2 = startNode(Settings.builder()
.put("gateway.recover_after_data_nodes", 2)
.put(Node.NODE_DATA_SETTING.getKey(), false)
.put(Node.NODE_MASTER_SETTING.getKey(), true), 1);
assertThat(master2.admin().cluster().prepareState().setLocal(true).execute().actionGet()
.getState().blocks().global(ClusterBlockLevel.METADATA_WRITE),
hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));
Expand All @@ -144,7 +165,10 @@ public void testRecoverAfterDataNodes() throws Exception {
hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK));

logger.info("--> start data_node (2)");
Client data2 = startNode(Settings.builder().put("gateway.recover_after_data_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
Client data2 = startNode(Settings.builder()
.put("gateway.recover_after_data_nodes", 2)
.put(Node.NODE_DATA_SETTING.getKey(), true)
.put(Node.NODE_MASTER_SETTING.getKey(), false), 1);
assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master1).isEmpty(), equalTo(true));
assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master2).isEmpty(), equalTo(true));
assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, data1).isEmpty(), equalTo(true));
Expand Down
4 changes: 2 additions & 2 deletions test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1065,7 +1065,7 @@ public void logMemoryStats() {
}

protected void ensureClusterSizeConsistency() {
if (cluster() != null) { // if static init fails the cluster can be null
if (cluster() != null && cluster().size() > 0) { // if static init fails the cluster can be null
logger.trace("Check consistency for [{}] nodes", cluster().size());
assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(cluster().size())).get());
}
Expand All @@ -1075,7 +1075,7 @@ protected void ensureClusterSizeConsistency() {
* Verifies that all nodes that have the same version of the cluster state as master have same cluster state
*/
protected void ensureClusterStateConsistency() throws IOException {
if (cluster() != null) {
if (cluster() != null && cluster().size() > 0) {
ClusterState masterClusterState = client().admin().cluster().prepareState().all().get().getState();
byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterClusterState);
// remove local node reference
Expand Down
17 changes: 12 additions & 5 deletions test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@

import static org.apache.lucene.util.LuceneTestCase.TEST_NIGHTLY;
import static org.apache.lucene.util.LuceneTestCase.rarely;
import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING;
import static org.elasticsearch.test.ESTestCase.assertBusy;
import static org.elasticsearch.test.ESTestCase.randomFrom;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
Expand Down Expand Up @@ -589,12 +590,14 @@ private NodeAndClient buildNode(int nodeId, long seed, Settings settings,
.put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed);

if (autoManageMinMasterNodes) {
assert finalSettings.get(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null :
assert finalSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null :
"min master nodes may not be set when auto managed";
finalSettings
// don't wait too long not to slow down tests
.put(ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.getKey(), "5s")
.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), defaultMinMasterNodes);
.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), defaultMinMasterNodes);
} else if (finalSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null) {
throw new IllegalArgumentException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " must be configured");
}
MockNode node = new MockNode(finalSettings.build(), plugins);
return new NodeAndClient(name, node, nodeId);
Expand Down Expand Up @@ -883,8 +886,8 @@ void recreateNodeOnRestart(RestartCallback callback, boolean clearDataIfNeeded,
newSettings.put(callbackSettings);
}
if (minMasterNodes >= 0) {
assert ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(newSettings.build()) == false : "min master nodes is auto managed";
newSettings.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes).build();
assert DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(newSettings.build()) == false : "min master nodes is auto managed";
newSettings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes).build();
}
if (clearDataIfNeeded) {
clearDataIfNeeded(callback);
Expand All @@ -908,6 +911,10 @@ private void clearDataIfNeeded(RestartCallback callback) throws IOException {
private void createNewNode(final Settings newSettings) {
final long newIdSeed = NodeEnvironment.NODE_ID_SEED_SETTING.get(node.settings()) + 1; // use a new seed to make sure we have new node id
Settings finalSettings = Settings.builder().put(node.settings()).put(newSettings).put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), newIdSeed).build();
if (DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(finalSettings) == false) {
throw new IllegalStateException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() +
" is not configured after restart of [" + name + "]");
}
Collection<Class<? extends Plugin>> plugins = node.getClasspathPlugins();
node = new MockNode(finalSettings, plugins);
markNodeDataDirsAsNotEligableForWipe(node);
Expand Down Expand Up @@ -1694,7 +1701,7 @@ private int updateMinMasterNodes(int eligibleMasterNodeCount) {
logger.debug("updating min_master_nodes to [{}]", minMasterNodes);
try {
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(
Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes)
Settings.builder().put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes)
));
} catch (Exception e) {
throw new ElasticsearchException("failed to update minimum master node to [{}] (current masters [{}])", e,
Expand Down
Loading