ZOOKEEPER-3320(backport to branch-3.5): Leader election port stop listen when hostname unresolvable for some time

zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md

@@ -862,6 +862,19 @@ As an example, this will enable all four letter word commands:
     properly, check your operating system's options regarding TCP
     keepalive for more information.  Defaults to
     **false**.
+
+* *electionPortBindRetry* :
+    (Java system property only: **zookeeper.electionPortBindRetry**)
+    Property set max retry count when Zookeeper server fails to bind 
+    leader election port. Such errors can be temporary and recoverable, 
+    such as DNS issue described in [ZOOKEEPER-3320](https://issues.apache.org/jira/projects/ZOOKEEPER/issues/ZOOKEEPER-3320),
+    or non-retryable, such as port already in use.  
+    In case of transient errors, this property can improve availability 
+    of Zookeeper server and help it to self recover. 
+    Default value 3. In container environment, especially in Kubernetes, 
+    this value should be increased or set to 0(infinite retry) to overcome issues 
+    related to DNS name resolving.
+
 
 <a name="sc_authOptions"></a>
 

zookeeper-server/src/main/java/org/apache/zookeeper/common/NetUtils.java

@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.common;
+
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+
+/**
+ * This class contains common utilities for netstuff. Like printing IPv6 literals correctly
+ */
+public class NetUtils {
+
+    public static String formatInetAddr(InetSocketAddress addr) {
+        InetAddress ia = addr.getAddress();
+
+        if (ia == null) {
+            return String.format("%s:%s", addr.getHostString(), addr.getPort());
+        }
+
+        if (ia instanceof Inet6Address) {
+            return String.format("[%s]:%s", ia.getHostAddress(), addr.getPort());
+        } else {
+            return String.format("%s:%s", ia.getHostAddress(), addr.getPort());
+        }
+    }
+}

zookeeper-server/src/main/java/org/apache/zookeeper/server/ExitCode.java

@@ -20,8 +20,43 @@
 /**
  * Exit code used to exit server
  */
-public class ExitCode {
+public enum ExitCode {
 
-    /* Represents unexpected error */
-    public final static int UNEXPECTED_ERROR = 1;
-}
+    /** Execution finished normally */
+    EXECUTION_FINISHED(0),
+
+    /** Unexpected errors like IO Exceptions */
+    UNEXPECTED_ERROR(1),
+
+    /** Invalid arguments during invocations */
+    INVALID_INVOCATION(2),
+
+    /** Cannot access datadir when trying to replicate server */
+    UNABLE_TO_ACCESS_DATADIR(3),
+
+    /** Unable to start admin server at ZooKeeper startup */
+    ERROR_STARTING_ADMIN_SERVER(4),
+
+    /** Severe error during snapshot IO */
+    TXNLOG_ERROR_TAKING_SNAPSHOT(10),
+
+    /** zxid from COMMIT does not match the one from pendingTxns queue */
+    UNMATCHED_TXN_COMMIT(12),
+
+    /** Unexpected packet from leader, or unable to truncate log on Leader.TRUNC */
+    QUORUM_PACKET_ERROR(13),
+
+    /** Unable to bind to the quorum (election) port after multiple retry */
+    UNABLE_TO_BIND_QUORUM_PORT(14);
+
+    private final int value;
+
+    ExitCode(final int newValue) {
+        value = newValue;
+    }
+
+    public int getValue() {
+        return value;
+    }
+
+}

zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperCriticalThread.java

@@ -46,6 +46,6 @@ public ZooKeeperCriticalThread(String threadName, ZooKeeperServerListener listen
     @Override
     protected void handleException(String threadName, Throwable e) {
         LOG.error("Severe unrecoverable error, from thread : {}", threadName, e);
-        listener.notifyStopping(threadName, ExitCode.UNEXPECTED_ERROR);
+        listener.notifyStopping(threadName, ExitCode.UNEXPECTED_ERROR.getValue());
     }
 }