From ee1837b85c3884238853894a047aa9232ffd114b Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 30 May 2014 00:49:46 +0200 Subject: [PATCH] Have a dedicated join timeout that is higher than ping.timeout for node join Using ping.timeout, which defaults to 3s, to use as a timeout value on the join request a node makes to the master once its discovered can be too small, specifically when there is a large cluster state involved (and by definition, all the buffers and such on the nio layer will be "cold"). Introduce a dedicated join.timeout setting, that by default is 10x the ping.timeout (so 30s by default). closes #6342 --- .../org/elasticsearch/discovery/zen/ZenDiscovery.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java index b72436cf21264..1c8f8667d4183 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java @@ -93,6 +93,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen private final TimeValue pingTimeout; + private final TimeValue joinTimeout; // a flag that should be used only for testing private final boolean sendLeaveRequest; @@ -134,12 +135,13 @@ public ZenDiscovery(Settings settings, ClusterName clusterName, ThreadPool threa // also support direct discovery.zen settings, for cases when it gets extended this.pingTimeout = settings.getAsTime("discovery.zen.ping.timeout", settings.getAsTime("discovery.zen.ping_timeout", componentSettings.getAsTime("ping_timeout", componentSettings.getAsTime("initial_ping_timeout", timeValueSeconds(3))))); + this.joinTimeout = settings.getAsTime("discovery.zen.join_timeout", TimeValue.timeValueMillis(pingTimeout.millis() * 10)); this.sendLeaveRequest = componentSettings.getAsBoolean("send_leave_request", true); this.masterElectionFilterClientNodes = settings.getAsBoolean("discovery.zen.master_election.filter_client", true); this.masterElectionFilterDataNodes = settings.getAsBoolean("discovery.zen.master_election.filter_data", false); - logger.debug("using ping.timeout [{}], master_election.filter_client [{}], master_election.filter_data [{}]", pingTimeout, masterElectionFilterClientNodes, masterElectionFilterDataNodes); + logger.debug("using ping.timeout [{}], join.timeout [{}], master_election.filter_client [{}], master_election.filter_data [{}]", pingTimeout, joinTimeout, masterElectionFilterClientNodes, masterElectionFilterDataNodes); this.electMaster = new ElectMasterService(settings); nodeSettingsService.addListener(new ApplySettings()); @@ -343,7 +345,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS } // send join request try { - membership.sendJoinRequestBlocking(masterNode, localNode, pingTimeout); + membership.sendJoinRequestBlocking(masterNode, localNode, joinTimeout); } catch (Exception e) { if (e instanceof ElasticsearchException) { logger.info("failed to send join request to master [{}], reason [{}]", masterNode, ((ElasticsearchException) e).getDetailedMessage()); @@ -713,7 +715,7 @@ private ClusterState handleJoinRequest(final DiscoveryNode node) { // validate the join request, will throw a failure if it fails, which will get back to the // node calling the join request - membership.sendValidateJoinRequestBlocking(node, state, pingTimeout); + membership.sendValidateJoinRequestBlocking(node, state, joinTimeout); clusterService.submitStateUpdateTask("zen-disco-receive(join from node[" + node + "])", Priority.IMMEDIATE, new ClusterStateUpdateTask() { @Override