diff --git a/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md b/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md index 7a4773fd921..5f19fb92c93 100644 --- a/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md +++ b/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md @@ -1088,6 +1088,19 @@ As an example, this will enable all four letter word commands: properly, check your operating system's options regarding TCP keepalive for more information. Defaults to **false**. + +* *electionPortBindRetry* : + (Java system property only: **zookeeper.electionPortBindRetry**) + Property set max retry count when Zookeeper server fails to bind + leader election port. Such errors can be temporary and recoverable, + such as DNS issue described in [ZOOKEEPER-3320](https://issues.apache.org/jira/projects/ZOOKEEPER/issues/ZOOKEEPER-3320), + or non-retryable, such as port already in use. + In case of transient errors, this property can improve availability + of Zookeeper server and help it to self recover. + Default value 3. In container environment, especially in Kubernetes, + this value should be increased or set to 0(infinite retry) to overcome issues + related to DNS name resolving. + * *observer.reconnectDelayMs* : (Java system property: **zookeeper.observer.reconnectDelayMs**) diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumCnxManager.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumCnxManager.java index d97da2a074a..6ac430ead1b 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumCnxManager.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumCnxManager.java @@ -18,6 +18,8 @@ package org.apache.zookeeper.server.quorum; +import static org.apache.zookeeper.common.NetUtils.formatInetAddr; + import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.DataInputStream; @@ -36,6 +38,7 @@ import java.util.Enumeration; import java.util.HashSet; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -43,24 +46,20 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.NoSuchElementException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; - +import javax.net.ssl.SSLSocket; import org.apache.zookeeper.common.X509Exception; import org.apache.zookeeper.server.ExitCode; -import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException; -import org.apache.zookeeper.server.util.ConfigUtils; import org.apache.zookeeper.server.ZooKeeperThread; +import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException; import org.apache.zookeeper.server.quorum.auth.QuorumAuthLearner; import org.apache.zookeeper.server.quorum.auth.QuorumAuthServer; import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; +import org.apache.zookeeper.server.util.ConfigUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.net.ssl.SSLSocket; -import static org.apache.zookeeper.common.NetUtils.formatInetAddr; - /** * This class implements a connection manager for leader election using TCP. It * maintains one connection for every pair of servers. The tricky part is to @@ -848,12 +847,39 @@ private void resetConnectionThreadCount() { */ public class Listener extends ZooKeeperThread { + private static final String ELECTION_PORT_BIND_RETRY = "zookeeper.electionPortBindRetry"; + private static final int DEFAULT_PORT_BIND_MAX_RETRY = 3; + + private final int portBindMaxRetry; + private Runnable socketBindErrorHandler = () -> System.exit(ExitCode.UNABLE_TO_BIND_QUORUM_PORT.getValue()); volatile ServerSocket ss = null; public Listener() { // During startup of thread, thread name will be overridden to // specific election address super("ListenerThread"); + + // maximum retry count while trying to bind to election port + // see ZOOKEEPER-3320 for more details + final Integer maxRetry = Integer.getInteger(ELECTION_PORT_BIND_RETRY, + DEFAULT_PORT_BIND_MAX_RETRY); + if (maxRetry >= 0) { + LOG.info("Election port bind maximum retries is {}", + maxRetry == 0 ? "infinite" : maxRetry); + portBindMaxRetry = maxRetry; + } else { + LOG.info("'{}' contains invalid value: {}(must be >= 0). " + + "Use default value of {} instead.", + ELECTION_PORT_BIND_RETRY, maxRetry, DEFAULT_PORT_BIND_MAX_RETRY); + portBindMaxRetry = DEFAULT_PORT_BIND_MAX_RETRY; + } + } + + /** + * Change socket bind error handler. Used for testing. + */ + void setSocketBindErrorHandler(Runnable errorHandler) { + this.socketBindErrorHandler = errorHandler; } /** @@ -865,7 +891,7 @@ public void run() { InetSocketAddress addr; Socket client = null; Exception exitException = null; - while((!shutdown) && (numRetries < 3)){ + while ((!shutdown) && (portBindMaxRetry == 0 || numRetries < portBindMaxRetry)) { try { if (self.shouldUsePortUnification()) { LOG.info("Creating TLS-enabled quorum server socket"); @@ -935,15 +961,18 @@ public void run() { } LOG.info("Leaving listener"); if (!shutdown) { - LOG.error("As I'm leaving the listener thread, " - + "I won't be able to participate in leader " - + "election any longer: " - + formatInetAddr(self.getElectionAddress())); - if (exitException instanceof BindException) { + LOG.error("As I'm leaving the listener thread after " + + numRetries + " errors. " + + "I won't be able to participate in leader " + + "election any longer: " + + formatInetAddr(self.getElectionAddress()) + + ". Use " + ELECTION_PORT_BIND_RETRY + " property to " + + "increase retry count."); + if (exitException instanceof SocketException) { // After leaving listener thread, the host cannot join the // quorum anymore, this is a severe error that we cannot // recover from, so we need to exit - System.exit(ExitCode.UNABLE_TO_BIND_QUORUM_PORT.getValue()); + socketBindErrorHandler.run(); } } else if (ss != null) { // Clean up for shutdown. diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/CnxManagerTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/CnxManagerTest.java index 878e41b4c00..276f35f4767 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/CnxManagerTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/CnxManagerTest.java @@ -36,6 +36,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.net.Socket; +import java.util.concurrent.atomic.AtomicBoolean; import javax.net.ssl.SSLSession; import javax.net.ssl.SSLSocket; import javax.net.ssl.HandshakeCompletedListener; @@ -290,6 +291,36 @@ public void testCnxManagerSpinLock() throws Exception { Assert.assertFalse(cnxManager.listener.isAlive()); } + /** + * Test for bug described in {@link https://issues.apache.org/jira/browse/ZOOKEEPER-3320}. + * Test create peer with address which contains unresolvable DNS name, + * leader election listener thread should stop after N errors. + * + * @throws Exception + */ + @Test + public void testCnxManagerListenerThreadConfigurableRetry() throws Exception { + final Map unresolvablePeers = new HashMap<>(); + final long myid = 1L; + unresolvablePeers.put(myid, new QuorumServer(myid, "unresolvable-domain.org:2182:2183;2181")); + final QuorumPeer peer = new QuorumPeer(unresolvablePeers, + ClientBase.createTmpDir(), + ClientBase.createTmpDir(), + 2181, 3, myid, 1000, 2, 2, 2); + final QuorumCnxManager cnxManager = peer.createCnxnManager(); + final QuorumCnxManager.Listener listener = cnxManager.listener; + final AtomicBoolean errorHappend = new AtomicBoolean(); + listener.setSocketBindErrorHandler(() -> errorHappend.set(true)); + listener.start(); + // listener thread should stop and throws error which notify QuorumPeer about error. + // QuorumPeer should start shutdown process + listener.join(15000); // set wait time, if listener contains bug and thread not stops. + Assert.assertFalse(listener.isAlive()); + Assert.assertTrue(errorHappend.get()); + Assert.assertFalse(QuorumPeer.class.getSimpleName() + " not stopped after " + + "listener thread death", listener.isAlive()); + } + /** * Tests a bug in QuorumCnxManager that causes a NPE when a 3.4.6 * observer connects to a 3.5.0 server.