ZOOKEEPER-3188: Improve resilience to network

zookeeper-server/src/main/java/org/apache/zookeeper/server/ObserverBean.java

@@ -18,6 +18,7 @@
 
 package org.apache.zookeeper.server;
 
+import java.net.InetSocketAddress;
 import org.apache.zookeeper.server.quorum.Observer;
 import org.apache.zookeeper.server.quorum.ObserverMXBean;
 import org.apache.zookeeper.server.quorum.QuorumPeer;
@@ -49,10 +50,11 @@ public String getQuorumAddress() {
 
     public String getLearnerMaster() {
         QuorumPeer.QuorumServer learnerMaster = observer.getCurrentLearnerMaster();
-        if (learnerMaster == null || learnerMaster.addr == null) {
+        if (learnerMaster == null || learnerMaster.addr.isEmpty()) {
             return "Unknown";
         }
-        return learnerMaster.addr.getAddress().getHostAddress() + ":" + learnerMaster.addr.getPort();
+        InetSocketAddress address = learnerMaster.addr.getReachableOrOne();
+        return address.getAddress().getHostAddress() + ":" + address.getPort();
     }
 
     public void setLearnerMaster(String learnerMaster) {

zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java

@@ -18,7 +18,9 @@
 
 package org.apache.zookeeper.server.admin;
 
-import com.fasterxml.jackson.annotation.JsonAnyGetter;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import java.net.InetSocketAddress;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -43,7 +45,9 @@
 import org.apache.zookeeper.server.quorum.FollowerZooKeeperServer;
 import org.apache.zookeeper.server.quorum.Leader;
 import org.apache.zookeeper.server.quorum.LeaderZooKeeperServer;
+import org.apache.zookeeper.server.quorum.MultipleAddresses;
 import org.apache.zookeeper.server.quorum.QuorumPeer;
+import org.apache.zookeeper.server.quorum.QuorumPeer.LearnerType;
 import org.apache.zookeeper.server.quorum.QuorumZooKeeperServer;
 import org.apache.zookeeper.server.quorum.ReadOnlyZooKeeperServer;
 import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier;
@@ -672,53 +676,62 @@ public CommandResponse run(ZooKeeperServer zkServer, Map<String, String> kwargs)
             CommandResponse response = initializeResponse();
             if (zkServer instanceof QuorumZooKeeperServer) {
                 QuorumPeer peer = ((QuorumZooKeeperServer) zkServer).self;
-                VotingView votingView = new VotingView(peer.getVotingView());
+                Map<Long, QuorumServerView> votingView = peer.getVotingView().entrySet().stream()
+                        .collect(Collectors.toMap(Map.Entry::getKey, e -> new QuorumServerView(e.getValue())));
                 response.put("current_config", votingView);
             } else {
                 response.put("current_config", Collections.emptyMap());
             }
             return response;
         }
 
-        private static class VotingView {
-
-            private final Map<Long, String> view;
-
-            VotingView(Map<Long, QuorumPeer.QuorumServer> view) {
-                this.view = view.entrySet()
-                                .stream()
-                                .filter(e -> e.getValue().addr != null)
-                                .collect(Collectors.toMap(
-                                    Map.Entry::getKey,
-                                    e -> String.format(
-                                        "%s:%d%s:%s%s",
-                                        QuorumPeer.QuorumServer.delimitedHostString(e.getValue().addr),
-                                        e.getValue().addr.getPort(),
-                                        e.getValue().electionAddr == null ? "" : ":" + e.getValue().electionAddr.getPort(),
-                                        e.getValue().type.equals(QuorumPeer.LearnerType.PARTICIPANT) ? "participant" : "observer",
-                                        e.getValue().clientAddr == null || e.getValue().isClientAddrFromStatic
-                                            ? ""
-                                            : String.format(
-                                                ";%s:%d",
-                                                QuorumPeer.QuorumServer.delimitedHostString(e.getValue().clientAddr),
-                                                e.getValue().clientAddr.getPort())),
-                                    (v1, v2) -> v1, // cannot get duplicates as this straight draws from the other map
-                                    TreeMap::new));
+        @SuppressFBWarnings(value = "URF_UNREAD_FIELD", justification = "class is used only for JSON serialization")
+        private static class QuorumServerView {
+
+            @JsonProperty
+            private List<String> serverAddresses;
+
+            @JsonProperty
+            private List<String> electionAddresses;
+
+            @JsonProperty
+            private String clientAddress;
+
+            @JsonProperty
+            private String learnerType;
+
+            public QuorumServerView(QuorumPeer.QuorumServer quorumServer) {
+                this.serverAddresses = getMultiAddressString(quorumServer.addr);
+                this.electionAddresses = getMultiAddressString(quorumServer.electionAddr);
+                this.learnerType = quorumServer.type.equals(LearnerType.PARTICIPANT) ? "participant" : "observer";
+                this.clientAddress = getAddressString(quorumServer.clientAddr);
             }
 
-            @JsonAnyGetter
-            public Map<Long, String> getView() {
-                return view;
+            private static List<String> getMultiAddressString(MultipleAddresses multipleAddresses) {
+                if (multipleAddresses == null) {
+                    return Collections.emptyList();
+                }
+
+                return multipleAddresses.getAllAddresses().stream()
+                        .map(QuorumServerView::getAddressString)
+                        .collect(Collectors.toList());
             }
 
-        }
+            private static String getAddressString(InetSocketAddress address) {
+                if (address == null) {
+                    return "";
+                }
+                return String.format("%s:%d", QuorumPeer.QuorumServer.delimitedHostString(address), address.getPort());
+            }
+       }
 
     }
 
     /**
      * Watch information aggregated by session. Returned Map contains:
      *   - "session_id_to_watched_paths": Map&lt;Long, Set&lt;String&gt;&gt; session ID -&gt; watched paths
      * @see DataTree#getWatches()
+     * @see DataTree#getWatches()
      */
     public static class WatchCommand extends CommandBase {
 

zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/AuthFastLeaderElection.java

@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.net.DatagramPacket;
 import java.net.DatagramSocket;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.SocketException;
 import java.nio.ByteBuffer;
@@ -713,7 +714,8 @@ private void process(ToSend m) {
             }
 
             for (QuorumServer server : self.getVotingView().values()) {
-                InetSocketAddress saddr = new InetSocketAddress(server.addr.getAddress(), port);
+                InetAddress address = server.addr.getReachableOrOne().getAddress();
+                InetSocketAddress saddr = new InetSocketAddress(address, port);
                 addrChallengeMap.put(saddr, new ConcurrentHashMap<Long, Long>());
             }
 
@@ -741,7 +743,7 @@ public AuthFastLeaderElection(QuorumPeer self) {
 
     private void starter(QuorumPeer self) {
         this.self = self;
-        port = self.getVotingView().get(self.getId()).electionAddr.getPort();
+        port = self.getVotingView().get(self.getId()).electionAddr.getAllPorts().get(0);
         proposedLeader = -1;
         proposedZxid = -1;
 
@@ -764,14 +766,15 @@ private void leaveInstance() {
     private void sendNotifications() {
         for (QuorumServer server : self.getView().values()) {
 
+            InetSocketAddress address = self.getView().get(server.id).electionAddr.getReachableOrOne();
             ToSend notmsg = new ToSend(
                 ToSend.mType.notification,
                 AuthFastLeaderElection.sequencer++,
                 proposedLeader,
                 proposedZxid,
                 logicalclock.get(),
                 QuorumPeer.ServerState.LOOKING,
-                self.getView().get(server.id).electionAddr);
+                address);
 
             sendqueue.offer(notmsg);
         }