diff --git a/hazelcast/src/main/java/com/hazelcast/client/impl/ClientEngineImpl.java b/hazelcast/src/main/java/com/hazelcast/client/impl/ClientEngineImpl.java index b1251c9ad198..89e1fcdc8561 100644 --- a/hazelcast/src/main/java/com/hazelcast/client/impl/ClientEngineImpl.java +++ b/hazelcast/src/main/java/com/hazelcast/client/impl/ClientEngineImpl.java @@ -316,9 +316,9 @@ public boolean bind(final ClientEndpoint endpoint) { // On such a case, `ClientEngine#connectionRemoved` will not be called for this connection since // we did not register the connection. // Endpoint removal logic(inside `ClientEngine#connectionRemoved`) will not be able to run, instead endpoint - // will be cleaned up by ClientHearbeatMonitor#cleanupEndpointsWithDeadConnections later. - if (conn.getRemoteAddress() != null) { - node.getServer().getConnectionManager(CLIENT).register(conn.getRemoteAddress(), conn); + // will be cleaned up by ClientHeartbeatMonitor#cleanupEndpointsWithDeadConnections later. + if (conn.getRemoteAddress() != null && endpoint.getUuid() != null) { + node.getServer().getConnectionManager(CLIENT).register(conn.getRemoteAddress(), endpoint.getUuid(), conn); } } @@ -438,7 +438,11 @@ public void connectionRemoved(Connection c) { logger.finest("connectionRemoved: No endpoint for connection:" + connection); return; } - + UUID clientUuid = endpoint.getUuid(); + if (clientUuid != null) { + node.getLocalAddressRegistry().tryRemoveRegistration(clientUuid, + endpoint.getConnection().getRemoteAddress()); + } endpointManager.removeEndpoint(endpoint); } } @@ -480,10 +484,8 @@ Map getClientsInCluster() { if (endpoints == null) { continue; } - //Merge connected clients according to their UUID - for (Map.Entry entry : endpoints.entrySet()) { - clientsMap.put(entry.getKey(), entry.getValue()); - } + // Merge connected clients according to their UUID + clientsMap.putAll(endpoints); } catch (Exception e) { logger.warning("Cannot get client information from: " + target.toString(), e); } diff --git a/hazelcast/src/main/java/com/hazelcast/client/impl/connection/ClientConnection.java b/hazelcast/src/main/java/com/hazelcast/client/impl/connection/ClientConnection.java index ffdd90065046..3a2b1d65d6fc 100644 --- a/hazelcast/src/main/java/com/hazelcast/client/impl/connection/ClientConnection.java +++ b/hazelcast/src/main/java/com/hazelcast/client/impl/connection/ClientConnection.java @@ -21,7 +21,6 @@ import com.hazelcast.internal.nio.Connection; import java.util.Map; -import java.util.UUID; /** * The ClientConnection is connection that lives on the client side on behalf of a Java client. @@ -48,5 +47,4 @@ public interface ClientConnection extends Connection { // used in tests Map getEventHandlers(); - UUID getRemoteUuid(); } diff --git a/hazelcast/src/main/java/com/hazelcast/client/impl/proxy/ClientClusterProxy.java b/hazelcast/src/main/java/com/hazelcast/client/impl/proxy/ClientClusterProxy.java index 1b56f491938c..7d7171e477c0 100644 --- a/hazelcast/src/main/java/com/hazelcast/client/impl/proxy/ClientClusterProxy.java +++ b/hazelcast/src/main/java/com/hazelcast/client/impl/proxy/ClientClusterProxy.java @@ -45,6 +45,7 @@ public ClientClusterProxy(ClientClusterServiceImpl clusterService) { } @Override + @Nonnull public UUID addMembershipListener(@Nonnull MembershipListener listener) { return clusterService.addMembershipListener(listener); } @@ -55,12 +56,14 @@ public boolean removeMembershipListener(@Nonnull UUID registrationId) { } @Override + @Nonnull public Set getMembers() { final Collection members = clusterService.getMemberList(); return new LinkedHashSet<>(members); } @Override + @Nonnull public Member getLocalMember() { throw new UnsupportedOperationException("Client has no local member!"); } @@ -82,6 +85,7 @@ public void changeClusterState(@Nonnull ClusterState newState) { } @Override + @Nonnull public Version getClusterVersion() { throw new UnsupportedOperationException(); } @@ -92,6 +96,7 @@ public HotRestartService getHotRestartService() { } @Override + @Nonnull public PersistenceService getPersistenceService() { throw new UnsupportedOperationException(); } diff --git a/hazelcast/src/main/java/com/hazelcast/cluster/Cluster.java b/hazelcast/src/main/java/com/hazelcast/cluster/Cluster.java index 696bd1f02acb..4ab8ba5313db 100644 --- a/hazelcast/src/main/java/com/hazelcast/cluster/Cluster.java +++ b/hazelcast/src/main/java/com/hazelcast/cluster/Cluster.java @@ -54,6 +54,7 @@ public interface Cluster { * @throws java.lang.NullPointerException if listener is null * @see #removeMembershipListener(UUID) */ + @Nonnull UUID addMembershipListener(@Nonnull MembershipListener listener); /** @@ -79,6 +80,7 @@ public interface Cluster { * * @return current members in the cluster */ + @Nonnull Set getMembers(); /** @@ -93,6 +95,7 @@ public interface Cluster { * * @return this Hazelcast instance member */ + @Nonnull Member getLocalMember(); /** @@ -225,6 +228,7 @@ public interface Cluster { * @return the version at which this cluster operates. * @since 3.8 */ + @Nonnull Version getClusterVersion(); /** @@ -246,6 +250,7 @@ public interface Cluster { * supported on this instance (e.g. on client) * @since 5.0 */ + @Nonnull PersistenceService getPersistenceService(); /** diff --git a/hazelcast/src/main/java/com/hazelcast/instance/AddressPicker.java b/hazelcast/src/main/java/com/hazelcast/instance/AddressPicker.java index b068dd910b9c..f58cadcf266e 100644 --- a/hazelcast/src/main/java/com/hazelcast/instance/AddressPicker.java +++ b/hazelcast/src/main/java/com/hazelcast/instance/AddressPicker.java @@ -53,8 +53,28 @@ public interface AddressPicker { */ Address getPublicAddress(EndpointQualifier qualifier); + /** + * Returns all public {@link Address}es of this member which are advertised to other + * members, mapped by corresponding {@link EndpointQualifier}. Also, see + * {@link com.hazelcast.internal.cluster.impl.MemberHandshake} and + * {@link com.hazelcast.internal.server.tcp.SendMemberHandshakeTask}. + * + * @return a {@code Map} of this member's public addresses + * or an empty map if called before {@link #pickAddress()} + * @since 3.12 + */ Map getPublicAddressMap(); + /** + * Returns all bound server socket {@link Address}es of this member, mapped by + * corresponding {@link EndpointQualifier} + * + * @return a {@code Map} of the bound addresses of + * this member's server sockets or an empty map if called before {@link #pickAddress()} + * @since 5.1 + */ + Map getBindAddressMap(); + /** * Returns a server channel. * diff --git a/hazelcast/src/main/java/com/hazelcast/instance/impl/AdvancedNetworkAddressPicker.java b/hazelcast/src/main/java/com/hazelcast/instance/impl/AdvancedNetworkAddressPicker.java index 1c719edc65c1..a2fd806157a1 100644 --- a/hazelcast/src/main/java/com/hazelcast/instance/impl/AdvancedNetworkAddressPicker.java +++ b/hazelcast/src/main/java/com/hazelcast/instance/impl/AdvancedNetworkAddressPicker.java @@ -92,6 +92,16 @@ public Map getPublicAddressMap() { return pubAddressMap; } + @Override + public Map getBindAddressMap() { + Map bindAddressMap = new HashMap<>(pickers.size()); + for (Map.Entry entry : pickers.entrySet()) { + bindAddressMap.put(entry.getKey(), entry.getValue().getBindAddress(entry.getKey())); + } + + return bindAddressMap; + } + @Override public ServerSocketChannel getServerSocketChannel(EndpointQualifier qualifier) { return pickers.get(qualifier).getServerSocketChannel(qualifier); diff --git a/hazelcast/src/main/java/com/hazelcast/instance/impl/DefaultAddressPicker.java b/hazelcast/src/main/java/com/hazelcast/instance/impl/DefaultAddressPicker.java index 6b82ec2399d0..62a66632cc45 100644 --- a/hazelcast/src/main/java/com/hazelcast/instance/impl/DefaultAddressPicker.java +++ b/hazelcast/src/main/java/com/hazelcast/instance/impl/DefaultAddressPicker.java @@ -439,6 +439,13 @@ public Map getPublicAddressMap() { return publicAddressMap; } + @Override + public Map getBindAddressMap() { + HashMap bindAddressMap = new HashMap<>(); + bindAddressMap.put(MEMBER, bindAddress); + return bindAddressMap; + } + void setHostnameResolver(HostnameResolver hostnameResolver) { this.hostnameResolver = checkNotNull(hostnameResolver); } diff --git a/hazelcast/src/main/java/com/hazelcast/instance/impl/DefaultNodeContext.java b/hazelcast/src/main/java/com/hazelcast/instance/impl/DefaultNodeContext.java index 659d9f0c62e0..3ce1cb405cb2 100644 --- a/hazelcast/src/main/java/com/hazelcast/instance/impl/DefaultNodeContext.java +++ b/hazelcast/src/main/java/com/hazelcast/instance/impl/DefaultNodeContext.java @@ -24,13 +24,14 @@ import com.hazelcast.internal.metrics.MetricsRegistry; import com.hazelcast.internal.networking.ChannelErrorHandler; import com.hazelcast.internal.networking.Networking; -import com.hazelcast.internal.server.tcp.ServerSocketRegistry; import com.hazelcast.internal.networking.nio.NioNetworking; import com.hazelcast.internal.nio.ClassLoaderUtil; import com.hazelcast.internal.server.Server; -import com.hazelcast.internal.server.tcp.TcpServerContext; -import com.hazelcast.internal.server.tcp.TcpServerConnectionChannelErrorHandler; +import com.hazelcast.internal.server.tcp.LocalAddressRegistry; +import com.hazelcast.internal.server.tcp.ServerSocketRegistry; import com.hazelcast.internal.server.tcp.TcpServer; +import com.hazelcast.internal.server.tcp.TcpServerConnectionChannelErrorHandler; +import com.hazelcast.internal.server.tcp.TcpServerContext; import com.hazelcast.internal.util.InstantiationUtils; import com.hazelcast.logging.ILogger; import com.hazelcast.logging.impl.LoggingServiceImpl; @@ -143,7 +144,7 @@ public Joiner createJoiner(Node node) { } @Override - public Server createServer(Node node, ServerSocketRegistry registry) { + public Server createServer(Node node, ServerSocketRegistry registry, LocalAddressRegistry addressRegistry) { TcpServerContext context = new TcpServerContext(node, node.nodeEngine); Networking networking = createNetworking(node); Config config = node.getConfig(); @@ -152,6 +153,7 @@ public Server createServer(Node node, ServerSocketRegistry registry) { return new TcpServer(config, context, registry, + addressRegistry, metricsRegistry, networking, node.getNodeExtension().createChannelInitializerFn(context)); diff --git a/hazelcast/src/main/java/com/hazelcast/instance/impl/DelegatingAddressPicker.java b/hazelcast/src/main/java/com/hazelcast/instance/impl/DelegatingAddressPicker.java index 7c7456d41700..2004db979bd3 100644 --- a/hazelcast/src/main/java/com/hazelcast/instance/impl/DelegatingAddressPicker.java +++ b/hazelcast/src/main/java/com/hazelcast/instance/impl/DelegatingAddressPicker.java @@ -195,7 +195,15 @@ public Map getPublicAddressMap() { for (Map.Entry entry : publicAddresses.entrySet()) { mappings.put(entry.getKey(), new Address(entry.getValue())); } + return mappings; + } + @Override + public Map getBindAddressMap() { + Map mappings = new HashMap<>(bindAddresses.size()); + for (Map.Entry entry : bindAddresses.entrySet()) { + mappings.put(entry.getKey(), new Address(entry.getValue())); + } return mappings; } } diff --git a/hazelcast/src/main/java/com/hazelcast/instance/impl/Node.java b/hazelcast/src/main/java/com/hazelcast/instance/impl/Node.java index 80fa9b3ae391..6bbd48be8d00 100644 --- a/hazelcast/src/main/java/com/hazelcast/instance/impl/Node.java +++ b/hazelcast/src/main/java/com/hazelcast/instance/impl/Node.java @@ -71,6 +71,7 @@ import com.hazelcast.internal.serialization.InternalSerializationService; import com.hazelcast.internal.serialization.impl.compact.schema.MemberSchemaService; import com.hazelcast.internal.server.Server; +import com.hazelcast.internal.server.tcp.LocalAddressRegistry; import com.hazelcast.internal.server.tcp.ServerSocketRegistry; import com.hazelcast.internal.services.GracefulShutdownAwareService; import com.hazelcast.internal.usercodedeployment.UserCodeDeploymentClassLoader; @@ -162,6 +163,7 @@ public class Node { */ public final Address address; public final SecurityContext securityContext; + private final ILogger logger; private final AtomicBoolean shuttingDown = new AtomicBoolean(false); private final NodeShutdownHookThread shutdownHookThread; @@ -173,8 +175,11 @@ public class Node { private final BuildInfo buildInfo; private final HealthMonitor healthMonitor; private final Joiner joiner; + private final LocalAddressRegistry localAddressRegistry; private ManagementCenterService managementCenterService; + // it can be changed on cluster service reset see: ClusterServiceImpl#resetLocalMemberUuid + private volatile UUID thisUuid; private volatile NodeState state = NodeState.STARTING; /** @@ -218,14 +223,15 @@ public Node(HazelcastInstanceImpl hazelcastInstance, Config staticConfig, NodeCo try { boolean liteMember = config.isLiteMember(); - address = addressPicker.getPublicAddress(MEMBER); nodeExtension = nodeContext.createNodeExtension(this); + address = addressPicker.getPublicAddress(MEMBER); + thisUuid = nodeExtension.createMemberUuid(); final Map memberAttributes = findMemberAttributes( new MemberAttributeConfigReadOnly(config.getMemberAttributeConfig())); MemberImpl localMember = new MemberImpl.Builder(addressPicker.getPublicAddressMap()) .version(version) .localMember(true) - .uuid(nodeExtension.createMemberUuid()) + .uuid(thisUuid) .attributes(memberAttributes) .liteMember(liteMember) .instance(hazelcastInstance) @@ -248,8 +254,8 @@ public Node(HazelcastInstanceImpl hazelcastInstance, Config staticConfig, NodeCo config.onSecurityServiceUpdated(getSecurityService()); MetricsRegistry metricsRegistry = nodeEngine.getMetricsRegistry(); metricsRegistry.provideMetrics(nodeExtension); - - server = nodeContext.createServer(this, serverSocketRegistry); + localAddressRegistry = new LocalAddressRegistry(this, addressPicker); + server = nodeContext.createServer(this, serverSocketRegistry, localAddressRegistry); healthMonitor = new HealthMonitor(this); clientEngine = hasClientServerSocket() ? new ClientEngineImpl(this) : new NoOpClientEngine(); JoinConfig joinConfig = getActiveMemberNetworkConfig(this.config).getJoin(); @@ -428,6 +434,14 @@ public Address getThisAddress() { return address; } + public UUID getThisUuid() { + return thisUuid; + } + + public void setThisUuid(UUID uuid) { + thisUuid = uuid; + } + public MemberImpl getLocalMember() { return clusterService.getLocalMember(); } @@ -725,6 +739,10 @@ public DiscoveryService getDiscoveryService() { return discoveryService; } + public LocalAddressRegistry getLocalAddressRegistry() { + return localAddressRegistry; + } + private enum ShutdownHookPolicy { TERMINATE, GRACEFUL @@ -857,10 +875,6 @@ private boolean usePublicAddress(JoinConfig join) { || allUsePublicAddress(AliasedDiscoveryConfigUtils.aliasedDiscoveryConfigsFrom(join)); } - public UUID getThisUuid() { - return clusterService.getThisUuid(); - } - public Config getConfig() { return config; } diff --git a/hazelcast/src/main/java/com/hazelcast/instance/impl/NodeContext.java b/hazelcast/src/main/java/com/hazelcast/instance/impl/NodeContext.java index 7837af294b63..a2a667fc857e 100644 --- a/hazelcast/src/main/java/com/hazelcast/instance/impl/NodeContext.java +++ b/hazelcast/src/main/java/com/hazelcast/instance/impl/NodeContext.java @@ -18,6 +18,7 @@ import com.hazelcast.internal.cluster.Joiner; import com.hazelcast.instance.AddressPicker; +import com.hazelcast.internal.server.tcp.LocalAddressRegistry; import com.hazelcast.internal.server.tcp.ServerSocketRegistry; import com.hazelcast.internal.server.Server; @@ -36,6 +37,5 @@ public interface NodeContext { Joiner createJoiner(Node node); - // TODO Consider the changes here (JET?) - Server createServer(Node node, ServerSocketRegistry registry); + Server createServer(Node node, ServerSocketRegistry registry, LocalAddressRegistry addressRegistry); } diff --git a/hazelcast/src/main/java/com/hazelcast/internal/cluster/ClusterService.java b/hazelcast/src/main/java/com/hazelcast/internal/cluster/ClusterService.java index 1cea9faf586a..a8d5dcf28cc7 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/cluster/ClusterService.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/cluster/ClusterService.java @@ -23,6 +23,8 @@ import com.hazelcast.cluster.Address; import com.hazelcast.internal.services.CoreService; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.util.Collection; import java.util.UUID; @@ -39,7 +41,8 @@ public interface ClusterService extends CoreService, Cluster { * @param address the address of the member to lookup * @return the found member, or {@code null} if not found (if the address is {@code null}, {@code null} is returned) */ - MemberImpl getMember(Address address); + @Nullable + MemberImpl getMember(@Nullable Address address); /** * Gets the member with the given UUID. @@ -47,7 +50,8 @@ public interface ClusterService extends CoreService, Cluster { * @param uuid the UUID of the member * @return the found member, or {@code null} if not found (if the UUID is {@code null}, {@code null} is returned) */ - MemberImpl getMember(UUID uuid); + @Nullable + MemberImpl getMember(@Nullable UUID uuid); /** * Gets the member with the given UUID and address. @@ -57,7 +61,8 @@ public interface ClusterService extends CoreService, Cluster { * @return the found member, or {@code null} if not found * (if the UUID and/or address is {@code null}, {@code null} is returned) */ - MemberImpl getMember(Address address, UUID uuid); + @Nullable + MemberImpl getMember(@Nullable Address address, @Nullable UUID uuid); /** * Gets the collection of members. @@ -66,6 +71,7 @@ public interface ClusterService extends CoreService, Cluster { * * @return the collection of member (the returned value will never be {@code null}) */ + @Nonnull Collection getMemberImpls(); /** @@ -81,6 +87,7 @@ public interface ClusterService extends CoreService, Cluster { * * @return the address of the master member (can be {@code null} if the master is not yet known) */ + @Nullable Address getMasterAddress(); /** @@ -102,8 +109,17 @@ public interface ClusterService extends CoreService, Cluster { * * @return the address of this member (the returned value will never be {@code null}) */ + @Nonnull Address getThisAddress(); + /** + * Gets the uuid of this member. + * + * @return the uuid of this member (the returned value will never be {@code null}) + */ + @Nonnull + UUID getThisUuid(); + /** * Gets the local member instance. *

@@ -114,6 +130,7 @@ public interface ClusterService extends CoreService, Cluster { * * @return the local member instance (the returned value will never be {@code null}) */ + @Nonnull Member getLocalMember(); /** @@ -138,6 +155,7 @@ public interface ClusterService extends CoreService, Cluster { * * @return the ClusterClock */ + @Nonnull ClusterClock getClusterClock(); /** @@ -145,6 +163,7 @@ public interface ClusterService extends CoreService, Cluster { * * @return unique UUID for cluster */ + @Nullable UUID getClusterId(); /** diff --git a/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/AbstractJoiner.java b/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/AbstractJoiner.java index addde349447d..91af961ca45d 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/AbstractJoiner.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/AbstractJoiner.java @@ -111,8 +111,10 @@ public void setTargetAddress(Address targetAddress) { @Override public void blacklist(Address address, boolean permanent) { - logger.info(address + " is added to the blacklist."); - blacklistedAddresses.putIfAbsent(address, permanent); + Boolean prev = blacklistedAddresses.putIfAbsent(address, permanent); + if (prev == null) { + logger.info(address + " is " + (permanent ? "permanently " : "") + "added to the blacklist."); + } } @Override diff --git a/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/ClusterJoinManager.java b/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/ClusterJoinManager.java index bb467909e619..7e423d3cf501 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/ClusterJoinManager.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/ClusterJoinManager.java @@ -651,14 +651,14 @@ private void sendMasterAnswer(Address target) { @SuppressWarnings("checkstyle:cyclomaticcomplexity") private boolean checkIfJoinRequestFromAnExistingMember(JoinMessage joinMessage, ServerConnection connection) { - Address target = joinMessage.getAddress(); - MemberImpl member = clusterService.getMember(target); + Address targetAddress = joinMessage.getAddress(); + MemberImpl member = clusterService.getMember(targetAddress); if (member == null) { return checkIfUsingAnExistingMemberUuid(joinMessage); } if (joinMessage.getUuid().equals(member.getUuid())) { - sendMasterAnswer(target); + sendMasterAnswer(targetAddress); if (clusterService.isMaster() && !isMastershipClaimInProgress()) { if (logger.isFineEnabled()) { @@ -677,7 +677,7 @@ private boolean checkIfJoinRequestFromAnExistingMember(JoinMessage joinMessage, clusterClock.getClusterStartTime(), clusterStateManager.getState(), clusterService.getClusterVersion(), partitionRuntimeState, deferPartitionProcessing); op.setCallerUuid(clusterService.getThisUuid()); - invokeClusterOp(op, target); + invokeClusterOp(op, targetAddress); } return true; } @@ -685,18 +685,18 @@ private boolean checkIfJoinRequestFromAnExistingMember(JoinMessage joinMessage, // If I am the master, I will just suspect from the target. If it sends a new join request, it will be processed. // If I am not the current master, I can turn into the new master and start the claim process // after I suspect from the target. - if (clusterService.isMaster() || target.equals(clusterService.getMasterAddress())) { + if (clusterService.isMaster() || targetAddress.equals(clusterService.getMasterAddress())) { String msg = format("New join request has been received from an existing endpoint %s." + " Removing old member and processing join request...", member); logger.warning(msg); clusterService.suspectMember(member, msg, false); - ServerConnection existing = node.getServer().getConnectionManager(MEMBER).get(target); + ServerConnection existing = node.getServer().getConnectionManager(MEMBER).get(targetAddress); if (existing != connection) { if (existing != null) { existing.close(msg, null); } - node.getServer().getConnectionManager(MEMBER).register(target, connection); + node.getServer().getConnectionManager(MEMBER).register(targetAddress, joinMessage.getUuid(), connection); } } return true; diff --git a/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/ClusterServiceImpl.java b/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/ClusterServiceImpl.java index c5cb80314e9d..d636ee10c306 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/ClusterServiceImpl.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/ClusterServiceImpl.java @@ -340,7 +340,7 @@ private void resetLocalMemberUuid() { UUID newUuid = UuidUtil.newUnsecureUUID(); logger.warning("Resetting local member UUID. Previous: " + localMember.getUuid() + ", new: " + newUuid); - + node.setThisUuid(newUuid); localMember = new MemberImpl.Builder(addressMap) .version(localMember.getVersion()) .localMember(true) @@ -350,8 +350,8 @@ private void resetLocalMemberUuid() { .memberListJoinVersion(localMember.getMemberListJoinVersion()) .instance(node.hazelcastInstance) .build(); - node.loggingService.setThisMember(localMember); + node.getLocalAddressRegistry().setLocalUuid(newUuid); } public void resetJoinState() { @@ -585,6 +585,7 @@ public MemberImpl getMember(Address address, UUID uuid) { } @Override + @Nonnull public Collection getMemberImpls() { return membershipManager.getMembers(); } @@ -594,6 +595,7 @@ public Collection

getMemberAddresses() { } @Override + @Nonnull public Set getMembers() { return membershipManager.getMemberSet(); } @@ -663,17 +665,21 @@ public boolean isMaster() { } @Override + @Nonnull public Address getThisAddress() { return node.getThisAddress(); } @Override - public MemberImpl getLocalMember() { - return localMember; + @Nonnull + public UUID getThisUuid() { + return node.getThisUuid(); } - public UUID getThisUuid() { - return localMember.getUuid(); + @Override + @Nonnull + public MemberImpl getLocalMember() { + return localMember; } // should be called under lock @@ -706,6 +712,7 @@ public int getSize(MemberSelector selector) { } @Override + @Nonnull public ClusterClockImpl getClusterClock() { return clusterClock; } @@ -733,6 +740,7 @@ private void resetClusterId() { clusterId = null; } + @Nonnull public UUID addMembershipListener(@Nonnull MembershipListener listener) { checkNotNull(listener, "listener cannot be null"); @@ -827,6 +835,7 @@ private void changeClusterState(@Nonnull ClusterState newState, } @Override + @Nonnull public Version getClusterVersion() { return clusterStateManager.getClusterVersion(); } @@ -837,6 +846,7 @@ public HotRestartService getHotRestartService() { } @Override + @Nonnull public PersistenceService getPersistenceService() { return node.getNodeExtension().getHotRestartService(); } @@ -1080,12 +1090,11 @@ public String toString() { } /** - * - * @param millis + * @param timeoutMillis the maximum time in millis to block on join * @return true is cluster has been joined, false if timed out * @throws InterruptedException */ - public boolean blockOnJoin(long millis) throws InterruptedException { - return joined.get().latch.await(millis, TimeUnit.MILLISECONDS); + public boolean blockOnJoin(long timeoutMillis) throws InterruptedException { + return joined.get().latch.await(timeoutMillis, TimeUnit.MILLISECONDS); } } diff --git a/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/DiscoveryJoiner.java b/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/DiscoveryJoiner.java index 3f0eb6c6a94b..a2e65f0acdb2 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/DiscoveryJoiner.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/DiscoveryJoiner.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Set; import static com.hazelcast.spi.properties.ClusterProperty.WAIT_SECONDS_BEFORE_JOIN; import static com.hazelcast.internal.util.Preconditions.checkNotNull; @@ -72,12 +73,12 @@ protected Collection
getPossibleAddresses() { "Discovered nodes cannot be null!"); MemberImpl localMember = node.nodeEngine.getLocalMember(); - Address localAddress = localMember.getAddress(); + Set
localAddresses = node.getLocalAddressRegistry().getLocalAddresses(); Collection
possibleMembers = new ArrayList<>(); for (DiscoveryNode discoveryNode : discoveredNodes) { Address discoveredAddress = usePublicAddress ? discoveryNode.getPublicAddress() : discoveryNode.getPrivateAddress(); - if (localAddress.equals(discoveredAddress)) { + if (localAddresses.contains(discoveredAddress)) { if (!usePublicAddress && discoveryNode.getPublicAddress() != null) { // enrich member with client public address localMember.getAddressMap().put(EndpointQualifier.resolve(ProtocolType.CLIENT, "public"), diff --git a/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/MembershipManager.java b/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/MembershipManager.java index e903a18b31e5..0b530a9842ba 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/MembershipManager.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/MembershipManager.java @@ -27,8 +27,8 @@ import com.hazelcast.internal.cluster.impl.operations.FetchMembersViewOp; import com.hazelcast.internal.cluster.impl.operations.MembersUpdateOp; import com.hazelcast.internal.hotrestart.InternalHotRestartService; -import com.hazelcast.internal.nio.Connection; import com.hazelcast.internal.partition.impl.InternalPartitionServiceImpl; +import com.hazelcast.internal.server.ServerConnection; import com.hazelcast.internal.services.MembershipAwareService; import com.hazelcast.internal.services.MembershipServiceEvent; import com.hazelcast.internal.util.EmptyStatement; @@ -353,7 +353,7 @@ void updateMembers(MembersView membersView) { } for (MemberImpl member : removedMembers) { - closeConnection(member.getAddress(), "Member left event received from master"); + closeConnections(member.getAddress(), "Member left event received from master"); handleMemberRemove(memberMapRef.get(), member); } @@ -695,7 +695,7 @@ private boolean addSuspectedMember(MemberImpl suspectedMember, String reason, } if (shouldCloseConn) { - closeConnection(address, reason); + closeConnections(address, reason); } return true; } @@ -712,7 +712,7 @@ private void removeMember(MemberImpl member, String reason, boolean shouldCloseC Address address = member.getAddress(); if (shouldCloseConn) { - closeConnection(address, reason); + closeConnections(address, reason); } MemberMap currentMembers = memberMapRef.get(); @@ -751,11 +751,9 @@ private void removeMember(MemberImpl member, String reason, boolean shouldCloseC } } - private void closeConnection(Address address, String reason) { - Connection conn = node.getServer().getConnectionManager(MEMBER).get(address); - if (conn != null) { - conn.close(reason, null); - } + private void closeConnections(Address address, String reason) { + List connections = node.getServer().getConnectionManager(MEMBER).getAllConnections(address); + connections.forEach(conn -> conn.close(reason, null)); } private void handleMemberRemove(MemberMap newMembers, MemberImpl removedMember) { diff --git a/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/TcpIpJoiner.java b/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/TcpIpJoiner.java index 696b0046107f..a0cf032ce2f7 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/TcpIpJoiner.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/TcpIpJoiner.java @@ -27,6 +27,8 @@ import com.hazelcast.internal.cluster.impl.operations.JoinMastershipClaimOp; import com.hazelcast.internal.nio.Connection; import com.hazelcast.internal.server.ServerConnectionManager; +import com.hazelcast.internal.server.tcp.LinkedAddresses; +import com.hazelcast.internal.server.tcp.LocalAddressRegistry; import com.hazelcast.internal.util.AddressUtil; import com.hazelcast.internal.util.AddressUtil.AddressMatcher; import com.hazelcast.internal.util.AddressUtil.InvalidAddressException; @@ -41,6 +43,7 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.Set; +import java.util.UUID; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -116,7 +119,6 @@ private void joinViaTargetMember(Address targetAddress, long maxJoinMillis) { ServerConnectionManager connectionManager = node.getServer().getConnectionManager(MEMBER); connection = connectionManager.getOrConnect(targetAddress); if (connection == null) { - //noinspection BusyWait connectionManager.blockOnConnect(targetAddress, JOIN_RETRY_WAIT_TIME, 0); continue; } @@ -124,7 +126,7 @@ private void joinViaTargetMember(Address targetAddress, long maxJoinMillis) { logger.fine("Sending joinRequest " + targetAddress); } clusterJoinManager.sendJoinRequest(targetAddress); - //noinspection BusyWait + if (!clusterService.isJoined()) { clusterService.blockOnJoin(JOIN_RETRY_WAIT_TIME); } @@ -147,7 +149,6 @@ private void joinViaPossibleMembers() { if (clusterService.isJoined()) { return; } - if (isAllBlacklisted(possibleAddresses)) { logger.fine( "This node will assume master role since none of the possible members accepted join request."); @@ -203,8 +204,13 @@ private boolean claimMastership(Collection
possibleAddresses) { OperationServiceImpl operationService = node.getNodeEngine().getOperationService(); Collection> futures = new LinkedList<>(); for (Address address : possibleAddresses) { - if (isBlacklisted(address)) { - continue; + try { + if (isBlacklisted(address) || isLocalAddress(address)) { + continue; + } + } catch (UnknownHostException e) { + logger.warning(e); + ignore(e); } Future future = operationService @@ -228,6 +234,7 @@ private boolean claimMastership(Collection
possibleAddresses) { } } + @SuppressWarnings("checkstyle:NestedIfDepth") private boolean isThisNodeMasterCandidate(Collection
addresses) { int thisHashCode = node.getThisAddress().hashCode(); for (Address address : addresses) { @@ -235,8 +242,15 @@ private boolean isThisNodeMasterCandidate(Collection
addresses) { continue; } if (node.getServer().getConnectionManager(MEMBER).get(address) != null) { - if (thisHashCode > address.hashCode()) { - return false; + LocalAddressRegistry addressRegistry = node.getLocalAddressRegistry(); + UUID memberUuid = addressRegistry.uuidOf(address); + if (memberUuid != null) { + Address primaryAddress = addressRegistry.getPrimaryAddress(memberUuid); + if (primaryAddress != null) { + if (thisHashCode > primaryAddress.hashCode()) { + return false; + } + } } } } @@ -265,6 +279,18 @@ private void tryJoinAddresses(Collection
addresses) throws InterruptedE if (!clusterService.isJoined()) { clusterService.blockOnJoin(JOIN_RETRY_WAIT_TIME); } + + addresses.removeIf(address -> { + try { + return isLocalAddress(address); + } catch (UnknownHostException e) { + if (logger.isFineEnabled()) { + logger.fine("Error during resolving possible target address!", e); + } + ignore(e); + return false; + } + }); } } @@ -393,8 +419,14 @@ private void addPossibleAddresses(final Set
possibleAddresses, } private boolean isLocalAddress(final Address address) throws UnknownHostException { - final Address thisAddress = node.getThisAddress(); - final boolean local = thisAddress.getInetSocketAddress().equals(address.getInetSocketAddress()); + UUID memberUuid = node.getLocalAddressRegistry().uuidOf(address); + if (memberUuid == null) { + // also try to resolve this address + Address resolvedAddress = new Address(address.getInetSocketAddress()); + memberUuid = node.getLocalAddressRegistry().uuidOf(resolvedAddress); + } + boolean local = memberUuid != null && memberUuid.equals(node.getThisUuid()); + if (logger.isFineEnabled()) { logger.fine(address + " is local? " + local); } @@ -429,8 +461,21 @@ public void searchForOtherClusters() { logger.severe(e); return; } - possibleAddresses.remove(node.getThisAddress()); - possibleAddresses.removeAll(node.getClusterService().getMemberAddresses()); + LocalAddressRegistry addressRegistry = node.getLocalAddressRegistry(); + possibleAddresses.removeAll(addressRegistry.getLocalAddresses()); + node.getClusterService().getMembers().forEach( + member -> { + LinkedAddresses addresses = addressRegistry.linkedAddressesOf(member.getUuid()); + if (addresses != null) { + Set
knownMemberAddresses = addresses.getAllAddresses(); + possibleAddresses.removeAll(knownMemberAddresses); + } else { + // do not expect this case in the normal conditions, except for disconnections happens + // at the same time + possibleAddresses.remove(member.getAddress()); + } + } + ); if (possibleAddresses.isEmpty()) { return; diff --git a/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/operations/ClusterMismatchOp.java b/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/operations/ClusterMismatchOp.java index d0223698747e..47c93978ab79 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/operations/ClusterMismatchOp.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/cluster/impl/operations/ClusterMismatchOp.java @@ -16,8 +16,10 @@ package com.hazelcast.internal.cluster.impl.operations; +import com.hazelcast.cluster.Address; import com.hazelcast.instance.impl.Node; import com.hazelcast.internal.cluster.impl.ClusterDataSerializerHook; +import com.hazelcast.internal.server.tcp.LinkedAddresses; import com.hazelcast.logging.ILogger; import com.hazelcast.internal.nio.Connection; import com.hazelcast.spi.impl.NodeEngineImpl; @@ -35,13 +37,20 @@ public void run() { String message = "Node could not join cluster at node: " + connection.getRemoteAddress() + " Cause: the target cluster has a different cluster-name"; + Node node = nodeEngine.getNode(); + LinkedAddresses linkedAddresses = node.getLocalAddressRegistry().linkedAddressesOf(getCallerUuid()); + connection.close(message, null); ILogger logger = nodeEngine.getLogger("com.hazelcast.cluster"); logger.warning(message); - - Node node = nodeEngine.getNode(); - node.getJoiner().blacklist(getCallerAddress(), true); + if (linkedAddresses != null) { + for (Address address : linkedAddresses.getAllAddresses()) { + node.getJoiner().blacklist(address, true); + } + } else { + node.getJoiner().blacklist(getCallerAddress(), true); + } } @Override diff --git a/hazelcast/src/main/java/com/hazelcast/internal/nio/Connection.java b/hazelcast/src/main/java/com/hazelcast/internal/nio/Connection.java index a231fbe69f82..831eff904cab 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/nio/Connection.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/nio/Connection.java @@ -19,8 +19,10 @@ import com.hazelcast.cluster.Address; import com.hazelcast.internal.networking.OutboundFrame; +import javax.annotation.Nullable; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.util.UUID; import java.util.concurrent.ConcurrentMap; /** @@ -76,6 +78,7 @@ public interface Connection { *

* todo: do we really need this method because we have getInetAddress, InetSocketAddress and getEndPoint. */ + @Nullable InetSocketAddress getRemoteSocketAddress(); /** @@ -92,11 +95,28 @@ public interface Connection { */ void setRemoteAddress(Address remoteAddress); + /** + * Gets the {@link UUID} of the other side of this connection. + * It can be null if the other side of connection is not hz + * member or client (e.g. REST client) + * @return the uuid of the remote endpoint of the connection. + */ + @Nullable + UUID getRemoteUuid(); + + /** + * Sets the {@link UUID} of the other side of this connection. + * + * @param remoteUuid the uuid of the remote endpoint of the connection. + */ + void setRemoteUuid(UUID remoteUuid); + /** * Returns remote address of this Connection. * * @return the remote address. The returned value could be null if the connection is not alive. */ + @Nullable InetAddress getInetAddress(); /** diff --git a/hazelcast/src/main/java/com/hazelcast/internal/server/ServerConnectionManager.java b/hazelcast/src/main/java/com/hazelcast/internal/server/ServerConnectionManager.java index f909c77e62ab..2e04cd25c2bb 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/server/ServerConnectionManager.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/server/ServerConnectionManager.java @@ -24,6 +24,8 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.Collection; +import java.util.List; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.LockSupport; import java.util.function.Consumer; @@ -41,7 +43,8 @@ public interface ServerConnectionManager * In case of a member connection, it will also return connections that have not yet completed the * {@link com.hazelcast.internal.cluster.impl.MemberHandshake}. */ - @Nonnull Collection getConnections(); + @Nonnull + Collection getConnections(); /** * Returns the number of connections that satisfy some predicate. @@ -58,29 +61,46 @@ default int connectionCount(@Nullable Predicate predicate) { } /** - * Registers (i.e. stores) the connection for the given remote remoteAddress. - * Once this call finishes every subsequent call to {@link #get(Address)} will return - * the relevant {@link Connection} resource. + * Registers (i.e. stores) the connection for the given primaryAddress. + * Once this call finishes every subsequent call to {@link #get(Address)} + * will return the relevant {@link Connection} resource. * - * @param remoteAddress - The remote address to register the connection under + * @param primaryAddress - The primary address to register the connection under * @param connection - The connection to be registered * @return True if the call was successful */ - default boolean register(Address remoteAddress, ServerConnection connection) { - return register(remoteAddress, connection, 0); + default boolean register(Address primaryAddress, UUID remoteUuid, ServerConnection connection) { + return register(primaryAddress, primaryAddress, null, remoteUuid, connection, 0); } /** - * Registers (i.e. stores) the connection for the given remote remoteAddress. + * Registers (i.e. stores) the connection for the given primary address. * Once this call finishes every subsequent call to {@link #get(Address)} will return * the relevant {@link Connection} resource. * - * @param remoteAddress - The remote address to register the connection under + * @param primaryAddress - The primary address to register the connection under. + * If the remote side of connection is a Hazelcast member, we prefer one + * of the available public addresses as a primary address. If the underlying + * connection manager supports multiple endpoints, the public address corresponding + * to {@link com.hazelcast.instance.EndpointQualifier#MEMBER} has the priority. + * @param targetAddress - Requested target address on the connector side, null if acceptor side. + * We register this targetAddress as an alias to primaryAddress and also remove + * the connection from in progress connections set by using this target address. + * address. + * @param remoteAddressAliases the other address aliases to be registered which are incoming as part of remote + * member's MemberHandshake * @param connection - The connection to be registered * @param planeIndex - The index of the plane * @return True if the call was successful */ - boolean register(Address remoteAddress, ServerConnection connection, int planeIndex); + boolean register( + Address primaryAddress, + Address targetAddress, + Collection

remoteAddressAliases, + UUID remoteUuid, + ServerConnection connection, + int planeIndex + ); /** * Returns the number of connections. @@ -94,21 +114,36 @@ default int connectionCount() { /** * Gets the connection for a given address. If the connection does not exist, it returns null. * - * @param address the remote side of the connection + * @param address the address of remote side of the connection * @return the found Connection, or none if one doesn't exist */ - default ServerConnection get(Address address) { + @Nullable + default ServerConnection get(@Nonnull Address address) { return get(address, 0); } /** - * Gets the connection for a given address and streamId. If the connection does not exist, it returns null. + * Gets the connection for a given address and streamId. If the connection + * does not exist, it returns null. * - * @param address the remote side of the connection + * @param address the address of remote side of the connection * @param streamId the stream id * @return the found Connection, or none if one doesn't exist */ - ServerConnection get(Address address, int streamId); + @Nullable + ServerConnection get(@Nonnull Address address, int streamId); + + /** + * Gets all the connections for a given address on all planes. If + * there is no connection exist on any plane, it returns an + * empty list. + * + * @param address the address of connections + * @return the list of connections to the given address or an + * empty list if one doesn't exist + */ + @Nonnull + List getAllConnections(@Nonnull Address address); /** * Gets the existing connection for a given address or connects. @@ -122,7 +157,8 @@ default ServerConnection get(Address address) { * @return the found connection, or {@code null} if no connection exists * @see #getOrConnect(Address, boolean) */ - default ServerConnection getOrConnect(Address address) { + @Nullable + default ServerConnection getOrConnect(@Nonnull Address address) { return getOrConnect(address, false, 0); } @@ -139,7 +175,8 @@ default ServerConnection getOrConnect(Address address) { * @return the found connection, or {@code null} if no connection exists * @see #getOrConnect(Address, boolean) */ - ServerConnection getOrConnect(Address address, int streamId); + @Nullable + ServerConnection getOrConnect(@Nonnull Address address, int streamId); /** * Gets the existing connection for a given address. If it does not exist, the system will try to connect @@ -153,7 +190,8 @@ default ServerConnection getOrConnect(Address address) { * otherwise errors are not reported and logged on debug level * @return the existing connection */ - default ServerConnection getOrConnect(Address address, boolean silent) { + @Nullable + default ServerConnection getOrConnect(@Nonnull Address address, boolean silent) { return getOrConnect(address, silent, 0); } @@ -170,7 +208,8 @@ default ServerConnection getOrConnect(Address address, boolean silent) { * @param streamId the stream id * @return the existing connection */ - ServerConnection getOrConnect(Address address, boolean silent, int streamId); + @Nullable + ServerConnection getOrConnect(@Nonnull Address address, boolean silent, int streamId); /** * Transmits a packet to a certain address. @@ -179,12 +218,12 @@ default ServerConnection getOrConnect(Address address, boolean silent) { * true can be returned, even though the connection eventually can't be established. * * @param packet The Packet to transmit. - * @param target The address of the target machine where the Packet should be transmitted. + * @param targetAddress The address of the target machine where the Packet should be transmitted. * @return true if the transmit was a success, false if a failure. * @throws NullPointerException if packet or target is null. */ - default boolean transmit(Packet packet, Address target) { - return transmit(packet, target, 0); + default boolean transmit(Packet packet, Address targetAddress) { + return transmit(packet, targetAddress, 0); } /** @@ -194,12 +233,12 @@ default boolean transmit(Packet packet, Address target) { * true can be returned, even though the connection eventually can't be established. * * @param packet The Packet to transmit. - * @param target The address of the target machine where the Packet should be transmitted. + * @param targetAddress The address of the target machine where the packet should be transmitted. * @param streamId the stream id * @return true if the transmit was a success, false if a failure. * @throws NullPointerException if packet or target is null. */ - boolean transmit(Packet packet, Address target, int streamId); + boolean transmit(Packet packet, Address targetAddress, int streamId); /** * Returns network stats for inbound and outbound traffic. @@ -219,14 +258,14 @@ default boolean transmit(Packet packet, Address target) { /** * blocks the caller thread until a connection is established (or failed) * or the time runs out. Callers must ensure a connection is established after this method returns {@code true}. - * @param address - * @param millis - * @param streamId + * @param address the address of the remote side of connection that we're waiting for its establishment + * @param timeoutMillis the maximum time to block on + * @param streamId the stream id for the connection * @return true if connected successfully, false if timed out - * @throws java.lang.InterruptedException + * @throws java.lang.InterruptedException if the current thread was interrupted while blocking */ - default boolean blockOnConnect(Address address, long millis, int streamId) throws InterruptedException { - LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(millis)); + default boolean blockOnConnect(Address address, long timeoutMillis, int streamId) throws InterruptedException { + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(timeoutMillis)); return false; } } diff --git a/hazelcast/src/main/java/com/hazelcast/internal/server/ServerContext.java b/hazelcast/src/main/java/com/hazelcast/internal/server/ServerContext.java index 32ec07aaa282..a24c3d234363 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/server/ServerContext.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/server/ServerContext.java @@ -58,6 +58,13 @@ public interface ServerContext { // returns the MEMBER server socket address Address getThisAddress(); + /** + * Returns UUID of the local member. + * + * @return member UUID + */ + UUID getThisUuid(); + /** * @return all server socket addresses of this Hazelcast member, as picked by the * configured {@link com.hazelcast.instance.AddressPicker} @@ -82,7 +89,7 @@ public interface ServerContext { TextCommandService getTextCommandService(); - void removeEndpoint(Address endpoint); + void removeEndpoint(Address endpointAddress); void onSuccessfulConnection(Address address); @@ -117,11 +124,4 @@ public interface ServerContext { OutboundHandler[] createOutboundHandlers(EndpointQualifier qualifier, ServerConnection connection); AuditlogService getAuditLogService(); - - /** - * Returns UUID of the local member. - * - * @return member UUID - */ - UUID getUuid(); } diff --git a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/LinkedAddresses.java b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/LinkedAddresses.java new file mode 100644 index 000000000000..776300c55032 --- /dev/null +++ b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/LinkedAddresses.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2008-2021, Hazelcast, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hazelcast.internal.server.tcp; + +import com.hazelcast.cluster.Address; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import static com.hazelcast.internal.util.EmptyStatement.ignore; +import static java.util.Collections.newSetFromMap; +import static java.util.Collections.unmodifiableSet; + +/** + * LinkedAddresses keeps all network addresses pointing to the same Hazelcast + * instance together. Also, it specifically stores which of these addresses is + * the primary address. + */ +public final class LinkedAddresses { + + private final Address primaryAddress; + + // all linked addresses also includes primary address + private final Set
allLinkedAddresses; + + LinkedAddresses(Address primaryAddress) { + this.primaryAddress = primaryAddress; + allLinkedAddresses = newSetFromMap(new ConcurrentHashMap<>()); + allLinkedAddresses.add(primaryAddress); + } + + public Address getPrimaryAddress() { + return primaryAddress; + } + + public Set
getAllAddresses() { + return unmodifiableSet(allLinkedAddresses); + } + + public void addAllResolvedAddresses(Address address) { + this.addLinkedAddresses(getResolvedAddresses(address)); + } + + public void addLinkedAddresses(LinkedAddresses other) { + allLinkedAddresses.addAll(other.getAllAddresses()); + } + + public static LinkedAddresses getResolvedAddresses(Address primaryAddress) { + LinkedAddresses linkedAddresses = new LinkedAddresses(primaryAddress); + try { + InetAddress inetAddress = primaryAddress.getInetAddress(); + // ip address for the given primary address + String ip = inetAddress.getHostAddress(); + + Address addressIp = new Address(ip, primaryAddress.getPort()); + linkedAddresses.addAddress(addressIp); + } catch (UnknownHostException e) { + // we have a hostname here in `address`, but we can't resolve it + // how on earth we could come here? + ignore(e); + } + return linkedAddresses; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + LinkedAddresses that = (LinkedAddresses) o; + + return Objects.equals(primaryAddress, that.primaryAddress) + && allLinkedAddresses.size() == that.allLinkedAddresses.size() + && allLinkedAddresses.containsAll(that.allLinkedAddresses); + } + + @Override + public int hashCode() { + return primaryAddress != null ? primaryAddress.hashCode() : 0; + } + + public boolean contains(Address address) { + return allLinkedAddresses.contains(address); + } + + public boolean intersects(LinkedAddresses other) { + Set
tmp = new HashSet<>(allLinkedAddresses); + tmp.retainAll(other.getAllAddresses()); + return tmp.size() > 0; + } + + @Override + public String toString() { + return "LinkedAddresses{" + + "primaryAddress=" + primaryAddress + + ", allLinkedAddresses=" + allLinkedAddresses + + '}'; + } + + private void addAddress(Address address) { + allLinkedAddresses.add(address); + } +} diff --git a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/LocalAddressRegistry.java b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/LocalAddressRegistry.java new file mode 100644 index 000000000000..c5b72928ec08 --- /dev/null +++ b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/LocalAddressRegistry.java @@ -0,0 +1,280 @@ +/* + * Copyright (c) 2008-2021, Hazelcast, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hazelcast.internal.server.tcp; + +import com.hazelcast.cluster.Address; +import com.hazelcast.instance.AddressPicker; +import com.hazelcast.instance.EndpointQualifier; +import com.hazelcast.instance.impl.Node; +import com.hazelcast.logging.ILogger; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.nio.channels.ServerSocketChannel; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.hazelcast.internal.util.EmptyStatement.ignore; + +/** + * A LocalAddressRegistry contains maps to store `UUID -> Addresses` + * and its reverse `Address->UUID` mappings which is used to manage + * addresses of a Hazelcast instance. + */ +public class LocalAddressRegistry { + private final ConcurrentMap addressToUuid; + private final ConcurrentMap uuidToAddresses; + private final ILogger logger; + + // Since the requested lifecycle of local member's uuid and addresses are slightly different + // from the remote ones, I manage these separately. + private volatile UUID localUuid; + private volatile LinkedAddresses localAddresses; + + // protected for testing purposes + protected LocalAddressRegistry(ILogger logger) { + this.addressToUuid = new ConcurrentHashMap<>(); + this.uuidToAddresses = new ConcurrentHashMap<>(); + this.logger = logger; + } + + public LocalAddressRegistry(Node node, AddressPicker addressPicker) { + this(node.getLogger(LocalAddressRegistry.class)); + registerLocalAddresses(node.getThisUuid(), addressPicker); + } + + /** + * Binds a set of address to given instance uuid. While registering these + * addresses, we use the LinkedAddresses, it stores one of these addresses + * as the primary address. + * We count the registrations made to the same instance uuid, and we require + * to removal as much as this registration count in order to delete a + * registry entry. If multiple registration is performed on the same uuid + * (it can happen when there are multiple connections between the members), + * we require all the connections are closed to remove the registration entry. + *

+ * When multiple registration attempts are made to the same uuid, it goes like this: + * If the registration to the same uuid includes an address set that intersects + * with the current registered addresses, we combine these two address set and + * update the entry with this combined value. Also, then increment the registration + * count of the registry entry. + * + * If the registration to the same uuid includes a completely different + * address set than the already registered addresses, we call the old + * registration stale and remove it completely. In this case, we reset the + * registration count to 1. + * + * @param instanceUuid the uuid of instance (member or client) + * @param linkedAddresses a set of addresses + */ + public void register(@Nonnull UUID instanceUuid, @Nonnull LinkedAddresses linkedAddresses) { + if (instanceUuid.equals(localUuid)) { + localAddresses.addLinkedAddresses(linkedAddresses); + if (logger.isFineEnabled()) { + logger.fine("This member connected to itself since some its addresses are unknown to itself." + + linkedAddresses + "registered for the local member uuid=" + instanceUuid + + " currently all registered addresses for this local member: " + localAddresses); + } + return; + } + + // If the old linked addresses set and the new one intersect, suppose + // that the new ones are additional addresses and add them into old + // address set. Otherwise, If there is no intersection between these + // two sets, I'll consider the old addresses as stale and remove them. + uuidToAddresses.compute(instanceUuid, (uuid, linkedAddressesRegistrationCountPair) -> { + if (linkedAddressesRegistrationCountPair == null) { + linkedAddressesRegistrationCountPair = new Pair(linkedAddresses, new AtomicInteger(1)); + } else { + LinkedAddresses previousAddresses = linkedAddressesRegistrationCountPair.getAddresses(); + AtomicInteger registrationCount = linkedAddressesRegistrationCountPair.registrationCount; + + if (previousAddresses.intersects(linkedAddresses)) { + previousAddresses.addLinkedAddresses(linkedAddresses); + registrationCount.incrementAndGet(); + } else { + // override the value pair with the new one (removes previous addresses) + linkedAddressesRegistrationCountPair = new Pair(linkedAddresses, new AtomicInteger(1)); + // remove previous addresses from the addressToUuid map + previousAddresses.getAllAddresses().forEach(address -> addressToUuid.remove(address, uuid)); + } + } + linkedAddresses.getAllAddresses().forEach(address -> addressToUuid.put(address, instanceUuid)); + if (logger.isFinestEnabled()) { + logger.finest(linkedAddresses + " registered for the instance uuid=" + instanceUuid + + " currently all registered addresses for this instance uuid: " + + linkedAddressesRegistrationCountPair.getAddresses()); + } + return linkedAddressesRegistrationCountPair; + }); + } + + /** + * Try to remove the registry entry for given instance uuid and primary address. + * To make sure not delete a new entry of a rejoined member with the same uuid + * by a stale connection close event, both member uuid and primary address of + * the connection that was closed is checked if it matches with the entry inside + * the address registry. + *

+ * If the uuid and addresses match some entry in the address registry, we try to + * remove this entry if the all connections belong to this member is closed. We + * keep track the number of active connections belongs to this member uuid entry + * and remove if there is no active connection to this member uuid. + * + * @param instanceUuid instance uuid of which we try to remove its registration entry + * @param primaryAddress primary address which is set as a Connection#remoteAddress + * to remove the registration entry + */ + public void tryRemoveRegistration(@Nonnull UUID instanceUuid, @Nonnull Address primaryAddress) { + uuidToAddresses.computeIfPresent(instanceUuid, (uuid, linkedAddressesRegistrationCountPair) -> { + LinkedAddresses addresses = linkedAddressesRegistrationCountPair.getAddresses(); + if (addresses.contains(primaryAddress)) { + AtomicInteger registrationCount = linkedAddressesRegistrationCountPair.getRegistrationCount(); + // there is no active connection after this remove the entry + if (registrationCount.decrementAndGet() == 0) { + // not using removeIf due to https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8078645 + Iterator iterator = addressToUuid.values().iterator(); + while (iterator.hasNext()) { + UUID currUuid = iterator.next(); + if (currUuid.equals(instanceUuid)) { + iterator.remove(); + } + } + if (logger.isFineEnabled()) { + logger.fine(addresses + " previously registered for the instance uuid=" + instanceUuid + + " are removed from the registry"); + } + // remove the entry + return null; + } + } + return linkedAddressesRegistrationCountPair; + }); + } + + /** + * If this address or its resolved IP address has been registered before, + * it returns the instance uuid corresponding to this address. + * @param address address of hz instance + * @return the registered instance uuid corresponds to given address, + * null if the address is not registered + */ + @Nullable + public UUID uuidOf(@Nonnull Address address) { + if (localAddresses != null && localAddresses.contains(address)) { + return localUuid; + } + return addressToUuid.get(address); + } + + /** + * If this instance uuid and its addresses has been registered before, it returns + * the addresses corresponding to this instance uuid. + * @param uuid instance uuid + * @return the registered addresses corresponds to given instance uuid + */ + @Nullable + public LinkedAddresses linkedAddressesOf(@Nonnull UUID uuid) { + if (uuid.equals(localUuid)) { + return localAddresses; + } + Pair pair = uuidToAddresses.get(uuid); + return pair != null ? pair.getAddresses() : null; + } + + /** + * If this instance uuid and its addresses has been registered before, it returns + * the primary address corresponding to this instance uuid. + * @param uuid instance uuid + * @return the primary address for the instance corresponds to given uuid + */ + @Nullable + public Address getPrimaryAddress(@Nonnull UUID uuid) { + if (uuid.equals(localUuid)) { + return localAddresses.getPrimaryAddress(); + } + LinkedAddresses linkedAddresses = linkedAddressesOf(uuid); + return linkedAddresses != null ? linkedAddresses.getPrimaryAddress() : null; + } + + public void reset() { + addressToUuid.clear(); + uuidToAddresses.clear(); + } + + public void setLocalUuid(@Nonnull UUID newUuid) { + localUuid = newUuid; + } + + @Nonnull + public Set

getLocalAddresses() { + return localAddresses.getAllAddresses(); + } + + private void registerLocalAddresses(UUID thisUuid, AddressPicker addressPicker) { + LinkedAddresses addresses = + LinkedAddresses.getResolvedAddresses(addressPicker.getPublicAddress(EndpointQualifier.MEMBER)); + for (Map.Entry addressEntry : addressPicker.getBindAddressMap().entrySet()) { + addresses.addAllResolvedAddresses(addressPicker.getPublicAddress(addressEntry.getKey())); + addresses.addAllResolvedAddresses(addressEntry.getValue()); + ServerSocketChannel serverSocketChannel = addressPicker.getServerSocketChannel(addressEntry.getKey()); + if (serverSocketChannel != null && serverSocketChannel.socket().getInetAddress().isAnyLocalAddress()) { + int port = addressEntry.getValue().getPort(); + try { + Collections.list(NetworkInterface.getNetworkInterfaces()) + .forEach(networkInterface -> + Collections.list(networkInterface.getInetAddresses()) + .forEach(inetAddress -> + addresses.addAllResolvedAddresses(new Address(inetAddress, port)))); + } catch (SocketException e) { + ignore(e); + } + } + localUuid = thisUuid; + localAddresses = addresses; + } + if (logger.isFineEnabled()) { + logger.fine(localAddresses + " are registered for the local member with local uuid=" + localUuid); + } + } + + private static final class Pair { + private final LinkedAddresses addresses; + private final AtomicInteger registrationCount; + + private Pair(LinkedAddresses addresses, AtomicInteger connectionCount) { + this.addresses = addresses; + this.registrationCount = connectionCount; + } + + public LinkedAddresses getAddresses() { + return addresses; + } + + public AtomicInteger getRegistrationCount() { + return registrationCount; + } + } +} diff --git a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/SendMemberHandshakeTask.java b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/SendMemberHandshakeTask.java index 4cb60194b5db..f44041fe861e 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/SendMemberHandshakeTask.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/SendMemberHandshakeTask.java @@ -72,7 +72,7 @@ public void run() { getConfiguredLocalAddresses(), remoteAddress, reply, - serverContext.getUuid()) + serverContext.getThisUuid()) .addOption(OPTION_PLANE_COUNT, planeCount) .addOption(OPTION_PLANE_INDEX, planeIndex); byte[] bytes = serverContext.getSerializationService().toBytes(memberHandshake); diff --git a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServer.java b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServer.java index f99a97a58785..0ee6009376c5 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServer.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServer.java @@ -87,6 +87,7 @@ public final class TcpServer implements Server { public TcpServer(Config config, ServerContext context, ServerSocketRegistry registry, + LocalAddressRegistry addressRegistry, MetricsRegistry metricsRegistry, Networking networking, Function channelInitializerFn) { @@ -102,13 +103,25 @@ public TcpServer(Config config, if (registry.holdsUnifiedSocket()) { unifiedConnectionManager = new TcpServerConnectionManager( - this, null, channelInitializerFn, context, ProtocolType.valuesAsSet()); + this, + null, + addressRegistry, + channelInitializerFn, + context, + ProtocolType.valuesAsSet() + ); } else { unifiedConnectionManager = null; for (EndpointConfig endpointConfig : config.getAdvancedNetworkConfig().getEndpointConfigs().values()) { EndpointQualifier qualifier = endpointConfig.getQualifier(); TcpServerConnectionManager cm = new TcpServerConnectionManager( - this, endpointConfig, channelInitializerFn, context, singleton(endpointConfig.getProtocolType())); + this, + endpointConfig, + addressRegistry, + channelInitializerFn, + context, + singleton(endpointConfig.getProtocolType()) + ); connectionManagers.put(qualifier, cm); } refreshStatsTask.registerMetrics(metricsRegistry); diff --git a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerAcceptor.java b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerAcceptor.java index 721525d49611..6d8ba41e22a2 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerAcceptor.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerAcceptor.java @@ -58,11 +58,11 @@ import static java.util.concurrent.TimeUnit.SECONDS; /** - * Contains the logic for accepting TcpIpConnections. + * Contains the logic for accepting {@link TcpServerConnection}s. * * The {@link TcpServerAcceptor} and {@link TcpServerConnector} are 2 sides of the same coin. The * {@link TcpServerConnector} take care of the 'client' side of a connection and the {@link TcpServerAcceptor} - * is the 'server' side of a connection (each connection has a client and server-side + * is the 'server' side of a connection (each connection has a client and server-side) */ public class TcpServerAcceptor implements DynamicMetricsProvider { private static final long SHUTDOWN_TIMEOUT_MILLIS = SECONDS.toMillis(10); @@ -305,7 +305,7 @@ private void newConnection(final EndpointQualifier qualifier, SocketChannel sock private void newConnection0(TcpServerConnectionManager connectionManager, Channel channel) { try { serverContext.interceptSocket(connectionManager.getEndpointQualifier(), channel.socket(), true); - connectionManager.newConnection(channel, null); + connectionManager.newConnection(channel, null, true); } catch (Exception e) { exceptionCount.inc(); logger.warning(e.getClass().getName() + ": " + e.getMessage(), e); diff --git a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnection.java b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnection.java index 9f29f9b2d77e..08a50adb378f 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnection.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnection.java @@ -34,6 +34,7 @@ import java.net.InetSocketAddress; import java.nio.channels.CancelledKeyException; import java.util.Objects; +import java.util.UUID; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; @@ -69,12 +70,18 @@ public class TcpServerConnection implements ServerConnection { private final ILogger logger; + // Flag that indicates if the connection is accepted on this member (server-side) + // See also TcpServerAcceptor and TcpServerConnector + private final boolean acceptorSide; + private final int connectionId; private final ServerContext serverContext; private Address remoteAddress; + private UUID remoteUuid; + private TcpServerConnectionErrorHandler errorHandler; private volatile String connectionType = NONE; @@ -89,13 +96,16 @@ public class TcpServerConnection implements ServerConnection { public TcpServerConnection(TcpServerConnectionManager connectionManager, ConnectionLifecycleListener lifecycleListener, int connectionId, - Channel channel) { + Channel channel, + boolean acceptorSide + ) { this.connectionId = connectionId; this.connectionManager = connectionManager; this.lifecycleListener = lifecycleListener; this.serverContext = connectionManager.getServer().getContext(); this.logger = serverContext.getLoggingService().getLogger(TcpServerConnection.class); this.channel = channel; + this.acceptorSide = acceptorSide; this.attributeMap = channel.attributeMap(); attributeMap.put(ServerConnection.class, this); } @@ -175,10 +185,25 @@ public Address getRemoteAddress() { return remoteAddress; } + @Override public void setRemoteAddress(Address remoteAddress) { this.remoteAddress = remoteAddress; } + @Override + public UUID getRemoteUuid() { + return remoteUuid; + } + + @Override + public void setRemoteUuid(UUID remoteUuid) { + this.remoteUuid = remoteUuid; + } + + public boolean isAcceptorSide() { + return acceptorSide; + } + public void setErrorHandler(TcpServerConnectionErrorHandler errorHandler) { this.errorHandler = errorHandler; } @@ -237,6 +262,7 @@ public void close(String reason, Throwable cause) { .addParameter("reason", reason) .addParameter("cause", cause) .addParameter("remoteAddress", remoteAddress) + .addParameter("remoteUuid", remoteUuid) .log(); logClose(); @@ -328,6 +354,7 @@ public String toString() { + ", " + channel.localSocketAddress() + "->" + channel.remoteSocketAddress() + ", qualifier=" + connectionManager.getEndpointQualifier() + ", endpoint=" + remoteAddress + + ", remoteUuid=" + remoteUuid + ", alive=" + alive + ", connectionType=" + connectionType + ", planeIndex=" + planeIndex diff --git a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnectionErrorHandler.java b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnectionErrorHandler.java index 92cfabad531b..2bf522e84354 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnectionErrorHandler.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnectionErrorHandler.java @@ -26,14 +26,14 @@ class TcpServerConnectionErrorHandler { private final ILogger logger; private final ServerContext serverContext; - private final Address endPoint; + private final Address endpointAddress; private final long minInterval; private final int maxFaults; private int faults; private long lastFaultTime; - TcpServerConnectionErrorHandler(ServerContext serverContext, Address endPoint) { - this.endPoint = endPoint; + TcpServerConnectionErrorHandler(ServerContext serverContext, Address endpointAddress) { + this.endpointAddress = endpointAddress; this.serverContext = serverContext; this.minInterval = serverContext.getConnectionMonitorInterval(); this.maxFaults = serverContext.getConnectionMonitorMaxFaults(); @@ -41,22 +41,22 @@ class TcpServerConnectionErrorHandler { } synchronized void onError(Throwable cause) { - String errorMessage = "An error occurred on connection to " + endPoint + getCauseDescription(cause); + String errorMessage = "An error occurred on connection to " + endpointAddress + getCauseDescription(cause); logger.finest(errorMessage); final long now = currentTimeMillis(); final long last = lastFaultTime; if (now - last > minInterval) { if (faults++ >= maxFaults) { - String removeEndpointMessage = "Removing connection to endpoint " + endPoint + getCauseDescription(cause); + String removeEndpointMessage = "Removing connection to endpoint " + endpointAddress + getCauseDescription(cause); logger.warning(removeEndpointMessage); - serverContext.removeEndpoint(endPoint); + serverContext.removeEndpoint(endpointAddress); } lastFaultTime = now; } } synchronized TcpServerConnectionErrorHandler reset() { - String resetMessage = "Resetting connection monitor for endpoint " + endPoint; + String resetMessage = "Resetting connection monitor for endpoint " + endpointAddress; logger.finest(resetMessage); faults = 0; lastFaultTime = 0L; diff --git a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnectionManager.java b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnectionManager.java index 2af76e148a6c..88bbdefaad87 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnectionManager.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnectionManager.java @@ -35,11 +35,16 @@ import com.hazelcast.internal.util.ExceptionUtil; import com.hazelcast.internal.util.executor.StripedRunnable; +import javax.annotation.Nonnull; import java.io.IOException; import java.nio.channels.SocketChannel; +import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -48,6 +53,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Function; +import java.util.stream.Collectors; import static com.hazelcast.internal.metrics.MetricDescriptorConstants.TCP_DISCRIMINATOR_BINDADDRESS; import static com.hazelcast.internal.metrics.MetricDescriptorConstants.TCP_DISCRIMINATOR_ENDPOINT; @@ -64,7 +70,6 @@ import static com.hazelcast.internal.nio.IOUtil.close; import static com.hazelcast.internal.nio.IOUtil.setChannelOptions; import static com.hazelcast.internal.util.Preconditions.checkNotNull; -import static java.lang.Math.abs; import static java.util.Arrays.stream; import static java.util.Collections.unmodifiableSet; @@ -77,15 +82,17 @@ public class TcpServerConnectionManager extends TcpServerConnectionManagerBase private final Function channelInitializerFn; private final TcpServerConnector connector; private final TcpServerControl serverControl; - private final AtomicInteger connectionIdGen = new AtomicInteger(); - TcpServerConnectionManager(TcpServer server, - EndpointConfig endpointConfig, - Function channelInitializerFn, - ServerContext serverContext, - Set supportedProtocolTypes) { - super(server, endpointConfig); + TcpServerConnectionManager( + TcpServer server, + EndpointConfig endpointConfig, + LocalAddressRegistry addressRegistry, + Function channelInitializerFn, + ServerContext serverContext, + Set supportedProtocolTypes + ) { + super(server, endpointConfig, addressRegistry); this.channelInitializerFn = channelInitializerFn; this.connector = new TcpServerConnector(this); this.serverControl = new TcpServerControl(this, serverContext, logger, supportedProtocolTypes); @@ -97,6 +104,7 @@ public TcpServer getServer() { } @Override + @Nonnull public Collection getConnections() { return unmodifiableSet(connections); } @@ -113,28 +121,43 @@ public synchronized void accept(Packet packet) { } @Override - public ServerConnection get(Address address, int streamId) { - return getPlane(streamId).getConnection(address); + public ServerConnection get(@Nonnull Address address, int streamId) { + UUID uuid = addressRegistry.uuidOf(address); + return uuid != null ? getPlane(streamId).getConnection(uuid) : null; + } + + @Override + @Nonnull + public List getAllConnections(@Nonnull Address address) { + UUID uuid = addressRegistry.uuidOf(address); + return uuid != null + ? Arrays.stream(planes).map(plane -> plane.getConnection(uuid)).filter(x -> x != null) + .collect(Collectors.toList()) + : Collections.emptyList(); } @Override - public ServerConnection getOrConnect(Address address, int streamId) { + public ServerConnection getOrConnect(@Nonnull Address address, int streamId) { return getOrConnect(address, false, streamId); } @Override - public ServerConnection getOrConnect(final Address address, final boolean silent, int streamId) { + public ServerConnection getOrConnect(@Nonnull final Address address, final boolean silent, int streamId) { Plane plane = getPlane(streamId); - TcpServerConnection connection = plane.getConnection(address); + TcpServerConnection connection = null; + UUID uuid = addressRegistry.uuidOf(address); + if (uuid != null) { + connection = plane.getConnection(uuid); + } if (connection == null && server.isLive()) { final AtomicBoolean isNotYetInProgress = new AtomicBoolean(); - plane.addConnectionInProgressIfAbsent(address, key -> { + plane.addConnectionInProgressIfAbsent(address, ignored -> { isNotYetInProgress.set(true); return connector.asyncConnect(address, silent, plane.index); }); if (isNotYetInProgress.get()) { if (logger.isFineEnabled()) { - logger.fine("Connection to: " + address + " streamId:" + streamId + " is not yet progress"); + logger.fine("Connection to: " + address + " streamId:" + streamId + " is not yet in progress"); } } else { if (logger.isFineEnabled()) { @@ -146,31 +169,43 @@ public ServerConnection getOrConnect(final Address address, final boolean silent } @Override - public synchronized boolean register(final Address remoteAddress, final ServerConnection c, int planeIndex) { + public synchronized boolean register( + Address primaryAddress, + Address targetAddress, + Collection
remoteAddressAliases, + UUID remoteUuid, + final ServerConnection c, + int planeIndex + ) { Plane plane = planes[planeIndex]; TcpServerConnection connection = (TcpServerConnection) c; try { - if (remoteAddress.equals(serverContext.getThisAddress())) { - return false; - } - if (!connection.isAlive()) { if (logger.isFinestEnabled()) { - logger.finest(connection + " to " + remoteAddress + " is not registered since connection is not active."); + logger.finest(connection + " to " + remoteUuid + " is not registered since connection is not active."); } return false; } - Address currentRemoteAddress = connection.getRemoteAddress(); - if (currentRemoteAddress != null && !currentRemoteAddress.equals(remoteAddress)) { - throw new IllegalArgumentException(connection + " has already a different remoteAddress than: " + remoteAddress); - } - connection.setRemoteAddress(remoteAddress); + connection.setRemoteAddress(primaryAddress); + connection.setRemoteUuid(remoteUuid); if (!connection.isClient()) { - connection.setErrorHandler(getErrorHandler(remoteAddress, plane.index).reset()); + connection.setErrorHandler(getErrorHandler(primaryAddress, plane.index).reset()); } - plane.putConnection(remoteAddress, connection); + + registerAddresses(remoteUuid, primaryAddress, targetAddress, remoteAddressAliases); + + // handle error cases after registering the addresses to avoid the later failing connections + // occur because target addresses are not registered. + + // handle self connection + if (remoteUuid.equals(serverContext.getThisUuid())) { + connection.close("Connecting to self!", null); + return false; + } + + plane.putConnection(remoteUuid, connection); serverContext.getEventService().executeEventCallback(new StripedRunnable() { @Override @@ -180,24 +215,16 @@ public void run() { @Override public int getKey() { - return remoteAddress.hashCode(); + return primaryAddress.hashCode(); } }); + return true; } finally { - plane.removeConnectionInProgress(remoteAddress); - } - } - - public Plane getPlane(int streamId) { - int planeIndex; - if (streamId == -1 || streamId == Integer.MIN_VALUE) { - planeIndex = 0; - } else { - planeIndex = abs(streamId) % planeCount; + if (targetAddress != null) { + plane.removeConnectionInProgress(targetAddress); + } } - - return planes[planeIndex]; } public synchronized void reset(boolean cleanListeners) { @@ -213,6 +240,7 @@ public synchronized void reset(boolean cleanListeners) { stream(planes).forEach(plane -> plane.errorHandlers.clear()); connections.clear(); + addressRegistry.reset(); if (cleanListeners) { connectionListeners.clear(); @@ -220,10 +248,10 @@ public synchronized void reset(boolean cleanListeners) { } @Override - public boolean transmit(Packet packet, Address target, int streamId) { + public boolean transmit(Packet packet, Address targetAddress, int streamId) { checkNotNull(packet, "packet can't be null"); - checkNotNull(target, "target can't be null"); - return send(packet, target, null, streamId); + checkNotNull(targetAddress, "targetAddress can't be null"); + return send(packet, targetAddress, null, streamId); } @Override @@ -256,14 +284,14 @@ void failedConnection(Address address, int planeIndex, Throwable t, boolean sile } } - synchronized TcpServerConnection newConnection(Channel channel, Address remoteAddress) { + synchronized TcpServerConnection newConnection(Channel channel, Address remoteAddress, boolean acceptorSide) { try { if (!server.isLive()) { throw new IllegalStateException("connection manager is not live!"); } TcpServerConnection connection = new TcpServerConnection(this, connectionLifecycleListener, - connectionIdGen.incrementAndGet(), channel); + connectionIdGen.incrementAndGet(), channel, acceptorSide); connection.setRemoteAddress(remoteAddress); connections.add(connection); @@ -329,8 +357,8 @@ public void provideDynamicMetrics(MetricDescriptor descriptor, MetricsCollection int clientCount = 0; int textCount = 0; for (Plane plane : planes) { - for (Map.Entry entry : plane.connections()) { - Address bindAddress = entry.getKey(); + for (Map.Entry entry : plane.connections()) { + UUID uuid = entry.getKey(); TcpServerConnection connection = entry.getValue(); if (connection.isClient()) { clientCount++; @@ -343,7 +371,7 @@ public void provideDynamicMetrics(MetricDescriptor descriptor, MetricsCollection if (connection.getRemoteAddress() != null) { context.collect(rootDescriptor .copy() - .withDiscriminator(TCP_DISCRIMINATOR_BINDADDRESS, bindAddress.toString()) + .withDiscriminator(TCP_DISCRIMINATOR_BINDADDRESS, uuid.toString()) .withTag(TCP_TAG_ENDPOINT, connection.getRemoteAddress().toString()), connection); } } @@ -356,12 +384,12 @@ public void provideDynamicMetrics(MetricDescriptor descriptor, MetricsCollection } @Override - public boolean blockOnConnect(Address address, long millis, int streamId) throws InterruptedException { + public boolean blockOnConnect(Address address, long timeoutMillis, int streamId) throws InterruptedException { Plane plane = getPlane(streamId); try { - Future future = plane.getconnectionInProgress(address); + Future future = plane.getConnectionInProgress(address); if (future != null) { - future.get(millis, TimeUnit.MILLISECONDS); + future.get(timeoutMillis, TimeUnit.MILLISECONDS); return future.isDone(); } else { // connection-in-progress has come and gone, diff --git a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnectionManagerBase.java b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnectionManagerBase.java index 291a340d5cd7..c62207851035 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnectionManagerBase.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnectionManagerBase.java @@ -25,6 +25,7 @@ import com.hazelcast.internal.nio.ConnectionListener; import com.hazelcast.internal.nio.Packet; import com.hazelcast.internal.server.NetworkStats; +import com.hazelcast.internal.server.ServerConnection; import com.hazelcast.internal.server.ServerConnectionManager; import com.hazelcast.internal.server.ServerContext; import com.hazelcast.internal.util.ConstructorFunction; @@ -34,10 +35,12 @@ import com.hazelcast.logging.ILogger; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.Future; @@ -56,6 +59,7 @@ import static com.hazelcast.internal.util.ConcurrencyUtil.getOrPutIfAbsent; import static com.hazelcast.internal.util.counters.MwCounter.newMwCounter; import static com.hazelcast.spi.properties.ClusterProperty.CHANNEL_COUNT; +import static java.lang.Math.abs; import static java.util.Collections.newSetFromMap; /** @@ -78,6 +82,7 @@ abstract class TcpServerConnectionManagerBase implements ServerConnectionManager protected final NetworkStatsImpl networkStats; protected final EndpointConfig endpointConfig; protected final EndpointQualifier endpointQualifier; + protected final LocalAddressRegistry addressRegistry; final Plane[] planes; final int planeCount; @@ -94,12 +99,16 @@ abstract class TcpServerConnectionManagerBase implements ServerConnectionManager private final ConstructorFunction errorHandlerConstructor; - TcpServerConnectionManagerBase(TcpServer tcpServer, EndpointConfig endpointConfig) { + TcpServerConnectionManagerBase(TcpServer tcpServer, EndpointConfig endpointConfig, LocalAddressRegistry addressRegistry) { this.server = tcpServer; - this.errorHandlerConstructor = endpoint -> new TcpServerConnectionErrorHandler(tcpServer.getContext(), endpoint); + this.errorHandlerConstructor = endpoint -> new TcpServerConnectionErrorHandler( + tcpServer.getContext(), + endpoint + ); this.serverContext = tcpServer.getContext(); this.endpointConfig = endpointConfig; this.endpointQualifier = endpointConfig != null ? endpointConfig.getQualifier() : null; + this.addressRegistry = addressRegistry; this.logger = serverContext.getLoggingService().getLogger(TcpServerConnectionManager.class); this.networkStats = endpointQualifier == null ? null : new NetworkStatsImpl(); this.planeCount = serverContext.properties().getInteger(CHANNEL_COUNT); @@ -124,22 +133,18 @@ static class Plane { final int index; private final Map> connectionsInProgress = new ConcurrentHashMap<>(); - private final ConcurrentHashMap connectionMap = new ConcurrentHashMap<>(100); + private final ConcurrentHashMap connectionMap = new ConcurrentHashMap<>(100); Plane(int index) { this.index = index; } - TcpServerConnection getConnection(Address address) { - return connectionMap.get(address); + TcpServerConnection getConnection(UUID uuid) { + return connectionMap.get(uuid); } - void putConnection(Address address, TcpServerConnection connection) { - connectionMap.put(address, connection); - } - - void putConnectionIfAbsent(Address address, TcpServerConnection connection) { - connectionMap.putIfAbsent(address, connection); + void putConnection(UUID uuid, TcpServerConnection connection) { + connectionMap.put(uuid, connection); } void removeConnection(TcpServerConnection connection) { @@ -177,7 +182,7 @@ public void clearConnections() { connectionMap.clear(); } - public Set> connections() { + public Set> connections() { return Collections.unmodifiableSet(connectionMap.entrySet()); } @@ -189,7 +194,7 @@ public boolean hasConnectionInProgress(Address address) { return connectionsInProgress.containsKey(address); } - public Future getconnectionInProgress(Address address) { + public Future getConnectionInProgress(Address address) { return connectionsInProgress.get(address); } @@ -215,13 +220,13 @@ public int connectionsInProgressCount() { private final class SendTask implements Runnable { private final Packet packet; - private final Address target; + private final Address targetAddress; private final int streamId; private volatile int retries; - private SendTask(Packet packet, Address target, int streamId) { + private SendTask(Packet packet, Address targetAddress, int streamId) { this.packet = packet; - this.target = target; + this.targetAddress = targetAddress; this.streamId = streamId; } @@ -230,9 +235,9 @@ private SendTask(Packet packet, Address target, int streamId) { public void run() { retries++; if (logger.isFinestEnabled()) { - logger.finest("Retrying[" + retries + "] packet send operation to: " + target); + logger.finest("Retrying[" + retries + "] packet send operation to: " + targetAddress); } - send(packet, target, this, streamId); + send(packet, targetAddress, this, streamId); } } @@ -274,6 +279,10 @@ public void onConnectionClose(TcpServerConnection connection, Throwable cause, b if (planeIndex > -1) { plane = planes[connection.getPlaneIndex()]; plane.removeConnection(connection); + UUID remoteUuid = connection.getRemoteUuid(); + if (remoteUuid != null) { + addressRegistry.tryRemoveRegistration(remoteUuid, connection.getRemoteAddress()); + } fireConnectionRemovedEvent(connection, remoteAddress); } else { // it might be the case that the connection was closed quickly enough @@ -305,8 +314,18 @@ public void onConnectionClose(TcpServerConnection connection, Throwable cause, b } } + protected ServerConnection get(UUID uuid, int streamId) { + return uuid != null ? getPlane(streamId).getConnection(uuid) : null; + } + protected boolean send(Packet packet, Address target, SendTask sendTask, int streamId) { - Connection connection = get(target, streamId); + UUID targetUuid = addressRegistry.uuidOf(target); + if (targetUuid == serverContext.getThisUuid()) { + logger.warning("Packet send task is rejected. Target is this node! Target[uuid=" + targetUuid + + ", address=" + target + "]"); + return false; + } + Connection connection = get(targetUuid, streamId); if (connection != null) { return connection.write(packet); } @@ -338,6 +357,17 @@ protected TcpServerConnectionErrorHandler getErrorHandler(Address endpoint, int return getErrorHandler(endpoint, errorHandlers); } + protected Plane getPlane(int streamId) { + int planeIndex; + if (streamId == -1 || streamId == Integer.MIN_VALUE) { + planeIndex = 0; + } else { + planeIndex = abs(streamId) % planeCount; + } + + return planes[planeIndex]; + } + private TcpServerConnectionErrorHandler getErrorHandler( Address endpoint, ConcurrentHashMap errorHandlers @@ -378,4 +408,21 @@ void onConnectionClose(TcpServerConnection connection) { bytesSentOnClosed.inc(connection.getChannel().bytesWritten()); } } + + /** + * Registers the given addresses to the address registry + */ + protected void registerAddresses(UUID remoteUuid, Address primaryAddress, Address targetAddress, + Collection
remoteAddressAliases) { + LinkedAddresses addressesToRegister = LinkedAddresses.getResolvedAddresses(primaryAddress); + if (targetAddress != null) { + addressesToRegister.addAllResolvedAddresses(targetAddress); + } + if (remoteAddressAliases != null) { + for (Address remoteAddressAlias : remoteAddressAliases) { + addressesToRegister.addAllResolvedAddresses(remoteAddressAlias); + } + } + addressRegistry.register(remoteUuid, addressesToRegister); + } } diff --git a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnector.java b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnector.java index c80b398636bb..383ec353d07b 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnector.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerConnector.java @@ -199,7 +199,7 @@ private void tryToConnect(InetSocketAddress socketAddress, int timeout) throws E serverContext.interceptSocket(connectionManager.getEndpointQualifier(), socketChannel.socket(), false); - connection = connectionManager.newConnection(channel, remoteAddress); + connection = connectionManager.newConnection(channel, remoteAddress, false); new SendMemberHandshakeTask(logger, serverContext, connection, remoteAddress, true, planeIndex, planeCount).run(); } catch (Exception e) { diff --git a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerContext.java b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerContext.java index 07ccdfa7dc17..87f7490e9c7e 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerContext.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerContext.java @@ -126,6 +126,11 @@ public Address getThisAddress() { return node.getThisAddress(); } + @Override + public UUID getThisUuid() { + return node.getThisUuid(); + } + @Override public Map getThisAddresses() { return nodeEngine.getLocalMember().getAddressMap(); @@ -174,9 +179,9 @@ public TextCommandService getTextCommandService() { } @Override - public void removeEndpoint(final Address endPoint) { + public void removeEndpoint(Address endpointAddress) { nodeEngine.getExecutionService().execute(ExecutionService.IO_EXECUTOR, - () -> node.clusterService.suspectAddressIfNotConnected(endPoint)); + () -> node.clusterService.suspectAddressIfNotConnected(endpointAddress)); } @Override @@ -213,7 +218,8 @@ public void onFailedConnection(final Address address) { @Override public void shouldConnectTo(Address address) { - if (node.getThisAddress().equals(address)) { + UUID memberUuid = node.getLocalAddressRegistry().uuidOf(address); + if (memberUuid != null && memberUuid.equals(node.getThisUuid())) { throw new RuntimeException("Connecting to self! " + address); } } @@ -344,9 +350,4 @@ public MemcacheProtocolConfig getMemcacheProtocolConfig() { public AuditlogService getAuditLogService() { return node.getNodeExtension().getAuditlogService(); } - - @Override - public UUID getUuid() { - return node.getThisUuid(); - } } diff --git a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerControl.java b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerControl.java index ddf5165002c6..c0a99e9627d4 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerControl.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/server/tcp/TcpServerControl.java @@ -20,7 +20,6 @@ import com.hazelcast.instance.EndpointQualifier; import com.hazelcast.instance.ProtocolType; import com.hazelcast.internal.cluster.impl.MemberHandshake; -import com.hazelcast.internal.nio.Connection; import com.hazelcast.internal.nio.ConnectionType; import com.hazelcast.internal.nio.Packet; import com.hazelcast.internal.server.ServerContext; @@ -32,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.logging.Level; import static com.hazelcast.spi.properties.ClusterProperty.CHANNEL_COUNT; @@ -88,7 +88,13 @@ private synchronized void process(TcpServerConnection connection, MemberHandshak } Map> remoteAddressesPerProtocolType = handshake.getLocalAddresses(); - List
allAliases = new ArrayList
(); + List
allAliases = new ArrayList<>(); + // if we support member protocol in the corresponding cm, we want to give priority to member + // public address in address registration phase + if (supportedProtocolTypes.contains(ProtocolType.MEMBER) + && remoteAddressesPerProtocolType.containsKey(ProtocolType.MEMBER)) { + allAliases.addAll(remoteAddressesPerProtocolType.remove(ProtocolType.MEMBER)); + } for (Map.Entry> remoteAddresses : remoteAddressesPerProtocolType.entrySet()) { if (supportedProtocolTypes.contains(remoteAddresses.getKey())) { allAliases.addAll(remoteAddresses.getValue()); @@ -129,73 +135,54 @@ private synchronized void process(TcpServerConnection connection, MemberHandshak * without any spoofing or other validation checks. * When executed on the connection initiator side, the connection is registered on the remote address * with which it was registered in {@link TcpServerConnectionManager#planes}, - * ignoring the {@code remoteEndpoint} argument. + * ignoring the {@code primaryAddress} argument. * * @param connection the connection that send the handshake - * @param remoteEndpoint the address of the remote endpoint + * @param remoteEndpointAddress the address of the remote endpoint * @param remoteAddressAliases alias addresses as provided by the remote endpoint, under which the connection * will be registered. These are the public addresses configured on the remote. */ - @SuppressWarnings({"checkstyle:npathcomplexity"}) + @SuppressWarnings({"checkstyle:cyclomaticcomplexity", "checkstyle:npathcomplexity"}) @SuppressFBWarnings("RV_RETURN_VALUE_OF_PUTIFABSENT_IGNORED") private synchronized void process0(TcpServerConnection connection, - Address remoteEndpoint, + Address remoteEndpointAddress, Collection
remoteAddressAliases, MemberHandshake handshake) { - final Address remoteAddress = new Address(connection.getRemoteSocketAddress()); - if (connectionManager.planes[handshake.getPlaneIndex()].hasConnectionInProgress(remoteAddress)) { - // this is the connection initiator side --> register the connection under the address that was requested - remoteEndpoint = remoteAddress; - } - if (remoteEndpoint == null) { + UUID remoteUuid = handshake.getUuid(); + Address primaryAddress = remoteEndpointAddress; + if (primaryAddress == null) { if (remoteAddressAliases == null) { throw new IllegalStateException("Remote endpoint and remote address aliases cannot be both null"); } else { - // let it fail if no remoteEndpoint and no aliases are defined - remoteEndpoint = remoteAddressAliases.iterator().next(); + // let it fail if no primaryAddress and no aliases are defined + primaryAddress = remoteAddressAliases.iterator().next(); } } - connection.setRemoteAddress(remoteEndpoint); - serverContext.onSuccessfulConnection(remoteEndpoint); + // On the acceptor side, this target address is null, on the connector side, + // it may not be the same as the connected address in some conditions. + Address targetAddress = connection.getRemoteAddress(); + Address connectedAddress = new Address(connection.getRemoteSocketAddress()); + remoteAddressAliases.add(connectedAddress); + + connection.setRemoteAddress(primaryAddress); + serverContext.onSuccessfulConnection(primaryAddress); if (handshake.isReply()) { - new SendMemberHandshakeTask(logger, serverContext, connection, remoteEndpoint, false, + new SendMemberHandshakeTask(logger, serverContext, connection, primaryAddress, false, handshake.getPlaneIndex(), handshake.getPlaneCount()).run(); } - if (checkAlreadyConnected(connection, remoteEndpoint, handshake.getPlaneIndex())) { - return; - } - if (logger.isLoggable(Level.FINEST)) { - logger.finest("Registering connection " + connection + " to address " + remoteEndpoint + logger.finest("Registering connection " + connection + " to address " + primaryAddress + " planeIndex:" + handshake.getPlaneIndex()); } - boolean registered = connectionManager.register(remoteEndpoint, connection, handshake.getPlaneIndex()); - - if (remoteAddressAliases != null && registered) { - for (Address remoteAddressAlias : remoteAddressAliases) { - if (logger.isLoggable(Level.FINEST)) { - logger.finest("Registering connection " + connection + " to address alias " + remoteAddressAlias - + " planeIndex:" + handshake.getPlaneIndex()); - } - connectionManager.planes[handshake.getPlaneIndex()].putConnectionIfAbsent(remoteAddressAlias, connection); - } - } - } - private boolean checkAlreadyConnected(TcpServerConnection connection, Address remoteEndPoint, int planeIndex) { - Connection existingConnection = connectionManager.planes[planeIndex].getConnection(remoteEndPoint); - if (existingConnection != null && existingConnection.isAlive()) { - if (existingConnection != connection) { - if (logger.isFinestEnabled()) { - logger.finest(existingConnection + " is already bound to " + remoteEndPoint - + ", new one is " + connection + " planeIndex:" + planeIndex); - } - // todo probably it's already in activeConnections (ConnectTask , AcceptorIOThread) - connectionManager.connections.add(connection); - } - return true; - } - return false; + connectionManager.register( + primaryAddress, + targetAddress, + remoteAddressAliases, + remoteUuid, + connection, + handshake.getPlaneIndex() + ); } } diff --git a/hazelcast/src/main/java/com/hazelcast/spi/impl/operationservice/impl/OperationRunnerImpl.java b/hazelcast/src/main/java/com/hazelcast/spi/impl/operationservice/impl/OperationRunnerImpl.java index a064ed7ebf65..ba0e45ef39fb 100644 --- a/hazelcast/src/main/java/com/hazelcast/spi/impl/operationservice/impl/OperationRunnerImpl.java +++ b/hazelcast/src/main/java/com/hazelcast/spi/impl/operationservice/impl/OperationRunnerImpl.java @@ -20,7 +20,6 @@ import com.hazelcast.cluster.Address; import com.hazelcast.cluster.ClusterState; import com.hazelcast.cluster.Member; -import com.hazelcast.cluster.impl.MemberImpl; import com.hazelcast.core.HazelcastException; import com.hazelcast.core.HazelcastInstanceNotActiveException; import com.hazelcast.instance.impl.Node; @@ -68,6 +67,7 @@ import com.hazelcast.splitbrainprotection.impl.SplitBrainProtectionServiceImpl; import java.io.IOException; +import java.util.UUID; import java.util.concurrent.ConcurrentMap; import java.util.logging.Level; @@ -453,6 +453,7 @@ public boolean run(Packet packet) throws Exception { ServerConnection connection = packet.getConn(); Address caller = connection.getRemoteAddress(); + UUID callerUuid = connection.getRemoteUuid(); Operation op = null; try { Object object = nodeEngine.toObject(packet); @@ -460,7 +461,7 @@ public boolean run(Packet packet) throws Exception { op.setNodeEngine(nodeEngine); setCallerAddress(op, caller); setConnection(op, connection); - setCallerUuidIfNotSet(caller, op); + setCallerUuidIfNotSet(op, callerUuid); setOperationResponseHandler(op); if (!ensureValidMember(op)) { @@ -525,14 +526,12 @@ private boolean ensureValidMember(Operation op) { return false; } - private void setCallerUuidIfNotSet(Address caller, Operation op) { + private void setCallerUuidIfNotSet(Operation op, UUID callerUuid) { if (op.getCallerUuid() != null) { return; - } - MemberImpl callerMember = node.clusterService.getMember(caller); - if (callerMember != null) { - op.setCallerUuid(callerMember.getUuid()); + if (callerUuid != null) { + op.setCallerUuid(callerUuid); } } diff --git a/hazelcast/src/test/java/com/hazelcast/client/test/TestClientRegistry.java b/hazelcast/src/test/java/com/hazelcast/client/test/TestClientRegistry.java index f6139a9d303a..e100fc1c500f 100644 --- a/hazelcast/src/test/java/com/hazelcast/client/test/TestClientRegistry.java +++ b/hazelcast/src/test/java/com/hazelcast/client/test/TestClientRegistry.java @@ -40,6 +40,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.UnknownHostException; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -84,7 +85,7 @@ public ClientConnectionManager createConnectionManager(HazelcastClientInstanceIm class MockTcpClientConnectionManager extends TcpClientConnectionManager { - private final ConcurrentHashMap addressBlockMap = new ConcurrentHashMap(); + private final ConcurrentHashMap addressBlockMap = new ConcurrentHashMap<>(); private final HazelcastClientInstanceImpl client; private final String host; @@ -111,19 +112,27 @@ protected void stopNetworking() { } @Override - protected TcpClientConnection createSocketConnection(Address address) { + protected TcpClientConnection createSocketConnection(Address remoteAddress) { checkClientActive(); try { - HazelcastInstance instance = nodeRegistry.getInstance(address); + HazelcastInstance instance = nodeRegistry.getInstance(remoteAddress); + UUID remoteUuid = nodeRegistry.uuidOf(remoteAddress); if (instance == null) { - throw new IOException("Can not connect to " + address + ": instance does not exist"); + throw new IOException("Can not connect to " + remoteAddress + ": instance does not exist"); } Address localAddress = new Address(host, ports.incrementAndGet()); - LockPair lockPair = getLockPair(address); - - MockedTcpClientConnection connection = new MockedTcpClientConnection(client, connectionIdGen.incrementAndGet(), - getNodeEngineImpl(instance), address, localAddress, lockPair); - LOGGER.info("Created connection to endpoint: " + address + ", connection: " + connection); + LockPair lockPair = getLockPair(remoteAddress); + + MockedTcpClientConnection connection = new MockedTcpClientConnection( + client, + connectionIdGen.incrementAndGet(), + getNodeEngineImpl(instance), + localAddress, + remoteAddress, + remoteUuid, + lockPair + ); + LOGGER.info("Created connection to endpoint: " + remoteAddress + ", connection: " + connection); return connection; } catch (Exception e) { throw rethrow(e); @@ -178,8 +187,10 @@ void unblockTo(Address address) { private class MockedTcpClientConnection extends TcpClientConnection { - private final Address remoteAddress; + // the bind address of client private final Address localAddress; + // the remote address that belongs to server side of the connection + private final Address remoteAddress; private final TwoWayBlockableExecutor executor; private final MockedServerConnection serverConnection; private final String connectionType; @@ -187,15 +198,29 @@ private class MockedTcpClientConnection extends TcpClientConnection { private volatile long lastReadTime; private volatile long lastWriteTime; - MockedTcpClientConnection(HazelcastClientInstanceImpl client, - int connectionId, NodeEngineImpl serverNodeEngine, Address address, Address localAddress, - LockPair lockPair) { + MockedTcpClientConnection( + HazelcastClientInstanceImpl client, + int connectionId, + NodeEngineImpl serverNodeEngine, + Address localAddress, + Address remoteAddress, + UUID serverUuid, + LockPair lockPair + ) { super(client, connectionId); - this.remoteAddress = address; this.localAddress = localAddress; + this.remoteAddress = remoteAddress; this.executor = new TwoWayBlockableExecutor(lockPair); - this.serverConnection = new MockedServerConnection(connectionId, remoteAddress, - localAddress, serverNodeEngine, this); + this.serverConnection = new MockedServerConnection( + connectionId, + remoteAddress, + localAddress, + serverUuid, + null, + null, + serverNodeEngine, + this + ); this.connectionType = client.getProperties().getBoolean(MC_CLIENT_MODE_PROP) ? ConnectionType.MC_JAVA_CLIENT : ConnectionType.JAVA_CLIENT; } @@ -329,10 +354,18 @@ private class MockedServerConnection extends MockServerConnection { private volatile long lastReadTimeMillis; private volatile long lastWriteTimeMillis; - MockedServerConnection(int connectionId, Address localEndpoint, - Address remoteEndpoint, NodeEngineImpl nodeEngine, - MockedTcpClientConnection responseConnection) { - super(localEndpoint, remoteEndpoint, nodeEngine); + MockedServerConnection( + int connectionId, + Address localEndpointAddress, + Address remoteEndpointAddress, + UUID localEndpointUuid, + UUID remoteEndpointUuid, + NodeEngineImpl localNodeEngine, + NodeEngineImpl remoteNodeEngine, + MockedTcpClientConnection responseConnection + ) { + super(localEndpointAddress, remoteEndpointAddress, localEndpointUuid, remoteEndpointUuid, + localNodeEngine, remoteNodeEngine); this.responseConnection = responseConnection; this.connectionId = connectionId; lastReadTimeMillis = System.currentTimeMillis(); diff --git a/hazelcast/src/test/java/com/hazelcast/cluster/TcpIpHostnameJoinTest.java b/hazelcast/src/test/java/com/hazelcast/cluster/TcpIpHostnameJoinTest.java new file mode 100644 index 000000000000..08943088956a --- /dev/null +++ b/hazelcast/src/test/java/com/hazelcast/cluster/TcpIpHostnameJoinTest.java @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2008-2021, Hazelcast, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hazelcast.cluster; + +import com.hazelcast.config.Config; +import com.hazelcast.config.JoinConfig; +import com.hazelcast.config.NetworkConfig; +import com.hazelcast.config.PartitionGroupConfig; +import com.hazelcast.config.TcpIpConfig; +import com.hazelcast.config.security.RealmConfig; +import com.hazelcast.core.Hazelcast; +import com.hazelcast.core.HazelcastInstance; +import com.hazelcast.instance.impl.HazelcastInstanceFactory; +import com.hazelcast.spi.properties.ClusterProperty; +import com.hazelcast.test.ChangeLoggingRule; +import com.hazelcast.test.HazelcastSerialClassRunner; +import com.hazelcast.test.annotation.QuickTest; +import org.junit.After; +import org.junit.Assume; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +import static com.hazelcast.jet.impl.util.ExceptionUtil.rethrow; + +// TODO [ufuk]: Make it nightly +@RunWith(HazelcastSerialClassRunner.class) +@Category(QuickTest.class) +public class TcpIpHostnameJoinTest extends AbstractJoinTest { + @ClassRule + public static ChangeLoggingRule changeLoggingRule = new ChangeLoggingRule("log4j2-trace-hostname-join.xml"); + + + private static final String HOSTNAME1; + private static final String HOSTNAME2; + + static { + try { + HOSTNAME1 = "localhost"; + HOSTNAME2 = InetAddress.getLocalHost().getHostName(); + Assume.assumeFalse(HOSTNAME1.equals(HOSTNAME2)); + } catch (UnknownHostException e) { + throw rethrow(e); + } + } + + @Before + @After + public void killAllHazelcastInstances() { + HazelcastInstanceFactory.terminateAll(); + } + + @Test + public void test_whenNoExplicitPortConfigured() throws Exception { + Config config = new Config(); + + NetworkConfig networkConfig = config.getNetworkConfig(); + JoinConfig join = networkConfig.getJoin(); + TcpIpConfig tcpIpConfig = join.getTcpIpConfig(); + tcpIpConfig.setEnabled(true); + tcpIpConfig.addMember(HOSTNAME1); + + testJoin(config); + } + + @Test + public void test_whenExplicitPortConfigured() throws Exception { + Config config = new Config(); + + NetworkConfig networkConfig = config.getNetworkConfig(); + JoinConfig join = networkConfig.getJoin(); + TcpIpConfig tcpIpConfig = join.getTcpIpConfig(); + tcpIpConfig.setEnabled(true); + tcpIpConfig.addMember(HOSTNAME1 + ":5701"); + tcpIpConfig.addMember(HOSTNAME1 + ":5702"); + + testJoin(config); + } + + @Test + public void test_whenExplicitPortConfiguredMixedHostnames() throws Exception { + Config config = new Config(); + + NetworkConfig networkConfig = config.getNetworkConfig(); + JoinConfig join = networkConfig.getJoin(); + TcpIpConfig tcpIpConfig = join.getTcpIpConfig(); + tcpIpConfig.setEnabled(true); + tcpIpConfig.addMember(HOSTNAME1 + ":5701"); + tcpIpConfig.addMember(HOSTNAME2 + ":5702"); + + testJoin(config); + } + + @Test + public void test_whenDifferentBuildNumber() { + Config config = new Config(); + NetworkConfig networkConfig = config.getNetworkConfig(); + JoinConfig join = networkConfig.getJoin(); + join.getTcpIpConfig().setEnabled(true); + join.getTcpIpConfig().addMember(HOSTNAME1); + + testJoin_With_DifferentBuildNumber(config); + } + + @Test + public void test_whenHostUnresolvable() { + Config config = new Config(); + NetworkConfig networkConfig = config.getNetworkConfig(); + JoinConfig join = networkConfig.getJoin(); + TcpIpConfig tcpIpConfig = join.getTcpIpConfig(); + tcpIpConfig.setEnabled(true); + tcpIpConfig.addMember(HOSTNAME1); + tcpIpConfig.addMember("nonexistinghost"); + + HazelcastInstance hz = Hazelcast.newHazelcastInstance(config); + assertClusterSize(1, hz); + } + + + @Test + public void test_whenIncompatibleClusterNameMixedHostnames() { + Config config1 = new Config(); + config1.setProperty(ClusterProperty.WAIT_SECONDS_BEFORE_JOIN.getName(), "0"); + config1.setProperty(ClusterProperty.MAX_JOIN_SECONDS.getName(), "3"); + config1.setClusterName("cluster1"); + config1.getNetworkConfig().getJoin().getTcpIpConfig() + .setEnabled(true).setConnectionTimeoutSeconds(3).addMember(HOSTNAME1); + + Config config2 = new Config(); + config2.setProperty(ClusterProperty.WAIT_SECONDS_BEFORE_JOIN.getName(), "0"); + config2.setProperty(ClusterProperty.MAX_JOIN_SECONDS.getName(), "3"); + config2.setClusterName("cluster2"); + config2.getNetworkConfig().getJoin().getTcpIpConfig() + .setEnabled(true).setConnectionTimeoutSeconds(3).addMember(HOSTNAME2); + + assertIndependentClusters(config1, config2); + } + + @Test + public void test_whenSameClusterNamesButDifferentPasswordMixedHostnames() { + Config config1 = new Config(); + config1.setProperty(ClusterProperty.WAIT_SECONDS_BEFORE_JOIN.getName(), "0"); + config1.setProperty(ClusterProperty.MAX_JOIN_SECONDS.getName(), "3"); + config1.getSecurityConfig().setMemberRealmConfig("m1", + new RealmConfig().setUsernamePasswordIdentityConfig("foo", "Here")); + config1.getNetworkConfig().getJoin().getTcpIpConfig().setEnabled(true).setConnectionTimeoutSeconds(3) + .addMember(HOSTNAME1); + + Config config2 = new Config(); + config2.setProperty(ClusterProperty.WAIT_SECONDS_BEFORE_JOIN.getName(), "0"); + config2.setProperty(ClusterProperty.MAX_JOIN_SECONDS.getName(), "3"); + config2.getSecurityConfig().setMemberRealmConfig("m1", + new RealmConfig().setUsernamePasswordIdentityConfig("foo", "There")); + config2.getNetworkConfig().getJoin().getTcpIpConfig().setEnabled(true).setConnectionTimeoutSeconds(3) + .addMember(HOSTNAME2); + + HazelcastInstance hz1 = Hazelcast.newHazelcastInstance(config1); + HazelcastInstance hz2 = Hazelcast.newHazelcastInstance(config2); + + assertClusterSize(2, hz1); + assertClusterSize(2, hz2); + } + + @Test + public void test_whenIncompatiblePartitionGroupsMixedHostnames() { + Config config1 = new Config(); + config1.setProperty(ClusterProperty.WAIT_SECONDS_BEFORE_JOIN.getName(), "0"); + config1.setProperty(ClusterProperty.MAX_JOIN_SECONDS.getName(), "3"); + config1.getNetworkConfig().getJoin().getTcpIpConfig() + .setEnabled(true).setConnectionTimeoutSeconds(3).addMember(HOSTNAME1); + config1.getPartitionGroupConfig().setEnabled(true) + .setGroupType(PartitionGroupConfig.MemberGroupType.CUSTOM); + + Config config2 = new Config(); + config2.setProperty(ClusterProperty.WAIT_SECONDS_BEFORE_JOIN.getName(), "0"); + config2.setProperty(ClusterProperty.MAX_JOIN_SECONDS.getName(), "3"); + config2.getNetworkConfig().getJoin().getTcpIpConfig() + .setEnabled(true).setConnectionTimeoutSeconds(3).addMember(HOSTNAME2); + config2.getPartitionGroupConfig().setEnabled(true) + .setGroupType(PartitionGroupConfig.MemberGroupType.HOST_AWARE); + assertIncompatible(config1, config2); + } + + @Test + public void test_whenIncompatibleJoiners() { + Config config1 = new Config(); + config1.setProperty(ClusterProperty.WAIT_SECONDS_BEFORE_JOIN.getName(), "0"); + config1.setProperty(ClusterProperty.MAX_JOIN_SECONDS.getName(), "3"); + config1.getNetworkConfig().getJoin().getMulticastConfig().setEnabled(true).setMulticastTimeoutSeconds(3); + config1.getNetworkConfig().getJoin().getTcpIpConfig().setEnabled(false); + + Config config2 = new Config(); + config2.setProperty(ClusterProperty.WAIT_SECONDS_BEFORE_JOIN.getName(), "0"); + config2.setProperty(ClusterProperty.MAX_JOIN_SECONDS.getName(), "3"); + config2.getNetworkConfig().getJoin().getTcpIpConfig().setConnectionTimeoutSeconds(3) + .setEnabled(true).addMember(HOSTNAME1); + + assertIncompatible(config1, config2); + } +} diff --git a/hazelcast/src/test/java/com/hazelcast/instance/FirewallingNodeContext.java b/hazelcast/src/test/java/com/hazelcast/instance/FirewallingNodeContext.java index 0b87fd9c9482..62d32fbef4d8 100644 --- a/hazelcast/src/test/java/com/hazelcast/instance/FirewallingNodeContext.java +++ b/hazelcast/src/test/java/com/hazelcast/instance/FirewallingNodeContext.java @@ -18,8 +18,8 @@ import com.hazelcast.instance.impl.DefaultNodeContext; import com.hazelcast.instance.impl.Node; +import com.hazelcast.internal.server.tcp.LocalAddressRegistry; import com.hazelcast.internal.server.tcp.ServerSocketRegistry; -import com.hazelcast.cluster.Address; import com.hazelcast.internal.server.Server; import com.hazelcast.internal.server.FirewallingServer; @@ -27,14 +27,14 @@ /** * Creates a {@link DefaultNodeContext} with wrapping its ConnectionManager - * with a {@link FirewallingConnectionManager}. + * with a {@link FirewallingServer.FirewallingServerConnectionManager}. */ public class FirewallingNodeContext extends DefaultNodeContext { @Override - public Server createServer(Node node, ServerSocketRegistry registry) { - Server server = super.createServer(node, registry); - return new FirewallingServer(server, Collections.
emptySet()); + public Server createServer(Node node, ServerSocketRegistry registry, LocalAddressRegistry addressRegistry) { + Server server = super.createServer(node, registry, addressRegistry); + return new FirewallingServer(server, Collections.emptySet()); } } diff --git a/hazelcast/src/test/java/com/hazelcast/instance/StaticMemberNodeContext.java b/hazelcast/src/test/java/com/hazelcast/instance/StaticMemberNodeContext.java index d8b563851572..5119e604df03 100644 --- a/hazelcast/src/test/java/com/hazelcast/instance/StaticMemberNodeContext.java +++ b/hazelcast/src/test/java/com/hazelcast/instance/StaticMemberNodeContext.java @@ -22,6 +22,7 @@ import com.hazelcast.instance.impl.Node; import com.hazelcast.instance.impl.NodeContext; import com.hazelcast.instance.impl.NodeExtension; +import com.hazelcast.internal.server.tcp.LocalAddressRegistry; import com.hazelcast.internal.server.tcp.ServerSocketRegistry; import com.hazelcast.cluster.Address; import com.hazelcast.internal.server.Server; @@ -63,7 +64,7 @@ public Joiner createJoiner(Node node) { } @Override - public Server createServer(Node node, ServerSocketRegistry registry) { - return delegate.createServer(node, registry); + public Server createServer(Node node, ServerSocketRegistry registry, LocalAddressRegistry addressRegistry) { + return delegate.createServer(node, registry, addressRegistry); } } diff --git a/hazelcast/src/test/java/com/hazelcast/instance/TestNodeContext.java b/hazelcast/src/test/java/com/hazelcast/instance/TestNodeContext.java index 3e9adb88eb4c..39b8ba5713ee 100644 --- a/hazelcast/src/test/java/com/hazelcast/instance/TestNodeContext.java +++ b/hazelcast/src/test/java/com/hazelcast/instance/TestNodeContext.java @@ -26,11 +26,12 @@ import com.hazelcast.internal.cluster.Joiner; import com.hazelcast.internal.dynamicconfig.ClusterWideConfigurationService; import com.hazelcast.internal.dynamicconfig.DynamicConfigListener; +import com.hazelcast.internal.server.tcp.LocalAddressRegistry; +import com.hazelcast.internal.server.tcp.ServerSocketRegistry; import com.hazelcast.internal.memory.DefaultMemoryStats; import com.hazelcast.internal.serialization.impl.DefaultSerializationServiceBuilder; import com.hazelcast.internal.server.Server; import com.hazelcast.internal.server.ServerConnectionManager; -import com.hazelcast.internal.server.tcp.ServerSocketRegistry; import com.hazelcast.internal.util.UuidUtil; import com.hazelcast.map.impl.MapService; import com.hazelcast.spi.impl.NodeEngine; @@ -104,7 +105,7 @@ public Joiner createJoiner(Node node) { } @Override - public Server createServer(Node node, ServerSocketRegistry registry) { + public Server createServer(Node node, ServerSocketRegistry registry, LocalAddressRegistry addressRegistry) { return server; } @@ -117,7 +118,7 @@ static class TestAddressPicker implements AddressPicker { } @Override - public void pickAddress() throws Exception { + public void pickAddress() { } @Override @@ -135,6 +136,11 @@ public Map getPublicAddressMap() { return Collections.singletonMap(EndpointQualifier.MEMBER, address); } + @Override + public Map getBindAddressMap() { + return Collections.singletonMap(EndpointQualifier.MEMBER, address); + } + @Override public ServerSocketChannel getServerSocketChannel(EndpointQualifier qualifier) { return null; diff --git a/hazelcast/src/test/java/com/hazelcast/instance/impl/AbstractOutOfMemoryHandlerTest.java b/hazelcast/src/test/java/com/hazelcast/instance/impl/AbstractOutOfMemoryHandlerTest.java index 1ac57c6318e4..aac936486da1 100644 --- a/hazelcast/src/test/java/com/hazelcast/instance/impl/AbstractOutOfMemoryHandlerTest.java +++ b/hazelcast/src/test/java/com/hazelcast/instance/impl/AbstractOutOfMemoryHandlerTest.java @@ -33,7 +33,9 @@ import javax.annotation.Nonnull; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; public abstract class AbstractOutOfMemoryHandlerTest extends HazelcastTestSupport { @@ -87,22 +89,35 @@ public Server getServer() { } @Override - public ServerConnection get(Address address, int streamId) { + public ServerConnection get(@Nonnull Address address, int streamId) { return null; } @Override - public ServerConnection getOrConnect(Address address, int streamId) { + @Nonnull + public List getAllConnections(@Nonnull Address address) { + return Collections.emptyList(); + } + + @Override + public ServerConnection getOrConnect(@Nonnull Address address, int streamId) { return null; } @Override - public ServerConnection getOrConnect(Address address, boolean silent, int streamId) { + public ServerConnection getOrConnect(@Nonnull Address address, boolean silent, int streamId) { return null; } @Override - public boolean register(Address remoteAddress, ServerConnection connection, int streamId) { + public boolean register( + Address remoteAddress, + Address targetAddress, + Collection
remoteAddressAliases, + UUID remoteUuid, + ServerConnection connection, + int streamId + ) { return false; } diff --git a/hazelcast/src/test/java/com/hazelcast/internal/server/DroppingServerConnection.java b/hazelcast/src/test/java/com/hazelcast/internal/server/DroppingServerConnection.java index 49c17fd5e94c..7eb7a005ea63 100644 --- a/hazelcast/src/test/java/com/hazelcast/internal/server/DroppingServerConnection.java +++ b/hazelcast/src/test/java/com/hazelcast/internal/server/DroppingServerConnection.java @@ -26,6 +26,7 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -33,15 +34,21 @@ class DroppingServerConnection implements ServerConnection { private final Address remoteAddress; + private final UUID remoteUuid; private final long timestamp = Clock.currentTimeMillis(); private final ServerConnectionManager connectionManager; private final ConnectionLifecycleListener lifecycleListener; private final AtomicBoolean isAlive = new AtomicBoolean(true); private final ConcurrentMap attributeMap = new ConcurrentHashMap(); - DroppingServerConnection(ConnectionLifecycleListener lifecycleListener, Address remoteAddress, - ServerConnectionManager connectionManager) { + DroppingServerConnection( + ConnectionLifecycleListener lifecycleListener, + Address remoteAddress, + UUID remoteUuid, + ServerConnectionManager connectionManager + ) { this.remoteAddress = remoteAddress; + this.remoteUuid = remoteUuid; this.connectionManager = connectionManager; this.lifecycleListener = lifecycleListener; } @@ -122,6 +129,15 @@ public InetSocketAddress getRemoteSocketAddress() { public void setRemoteAddress(Address remoteAddress) { } + @Override + public void setRemoteUuid(UUID remoteUuid) { + } + + @Override + public UUID getRemoteUuid() { + return remoteUuid; + } + @Override public Address getRemoteAddress() { return remoteAddress; diff --git a/hazelcast/src/test/java/com/hazelcast/internal/server/FirewallingServer.java b/hazelcast/src/test/java/com/hazelcast/internal/server/FirewallingServer.java index 6e445ec70988..c98a5213f8e9 100644 --- a/hazelcast/src/test/java/com/hazelcast/internal/server/FirewallingServer.java +++ b/hazelcast/src/test/java/com/hazelcast/internal/server/FirewallingServer.java @@ -28,13 +28,16 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import java.util.stream.Collectors; import static com.hazelcast.instance.EndpointQualifier.MEMBER; import static com.hazelcast.internal.util.Preconditions.checkNotNull; @@ -57,7 +60,7 @@ public class FirewallingServer private final Server delegate; private final Consumer packetConsumer; - private AtomicReference connectionManagerRef = new AtomicReference(null); + private final AtomicReference connectionManagerRef = new AtomicReference<>(null); @SuppressWarnings("unchecked") public FirewallingServer(Server delegate, Set
initiallyBlockedAddresses) { @@ -268,28 +271,42 @@ Collection getConnections() { return delegate.getConnections(); } + @Override - public ServerConnection get(Address address, int streamId) { + public ServerConnection get(@Nonnull Address address, int streamId) { return wrap(delegate.get(address, streamId)); } + @Override + @Nonnull + public List getAllConnections(@Nonnull Address address) { + return delegate.getAllConnections(address).stream().map(this::wrap).collect(Collectors.toList()); + } + private ServerConnection wrap(ServerConnection c) { return c == null ? null : new FirewallingConnection(c, this); } @Override - public synchronized ServerConnection getOrConnect(Address address, boolean silent, int streamId) { + public synchronized ServerConnection getOrConnect(@Nonnull Address address, boolean silent, int streamId) { return wrap(delegate.getOrConnect(address, streamId)); } @Override - public synchronized ServerConnection getOrConnect(Address address, int streamId) { + public synchronized ServerConnection getOrConnect(@Nonnull Address address, int streamId) { return wrap(delegate.getOrConnect(address, streamId)); } @Override - public boolean register(Address remoteAddress, ServerConnection connection, int streamId) { - return delegate.register(remoteAddress, connection, streamId); + public boolean register( + Address remoteAddress, + Address targetAddress, + Collection
remoteAddressAliases, + UUID remoteUuid, + ServerConnection connection, + int streamId + ) { + return delegate.register(remoteAddress, targetAddress, remoteAddressAliases, remoteUuid, connection, streamId); } @Override @@ -362,6 +379,16 @@ public void setRemoteAddress(Address remoteAddress) { delegate.setRemoteAddress(remoteAddress); } + @Override + public UUID getRemoteUuid() { + return delegate.getRemoteUuid(); + } + + @Override + public void setRemoteUuid(UUID remoteUuid) { + delegate.setRemoteUuid(remoteUuid); + } + @Override public InetAddress getInetAddress() { return delegate.getInetAddress(); diff --git a/hazelcast/src/test/java/com/hazelcast/internal/server/MockServerContext.java b/hazelcast/src/test/java/com/hazelcast/internal/server/MockServerContext.java index 203b7e705700..3c68ec277dcc 100644 --- a/hazelcast/src/test/java/com/hazelcast/internal/server/MockServerContext.java +++ b/hazelcast/src/test/java/com/hazelcast/internal/server/MockServerContext.java @@ -57,7 +57,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; import java.util.concurrent.FutureTask; -import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; import static com.hazelcast.internal.nio.Packet.Type.SERVER_CONTROL; @@ -68,14 +67,14 @@ public class MockServerContext implements ServerContext { public final ServerSocketChannel serverSocketChannel; public final Address thisAddress; + public final UUID thisUuid; public final InternalSerializationService serializationService; public final LoggingServiceImpl loggingService; - public final ConcurrentHashMap payloads = new ConcurrentHashMap(); private final HazelcastProperties properties; public volatile Consumer packetConsumer; private final ILogger logger; - public MockServerContext(int port) throws Exception { + public MockServerContext(int port, UUID memberUuid) throws Exception { loggingService = new LoggingServiceImpl("somegroup", "log4j2", BuildInfoProvider.getBuildInfo(), true, null); logger = loggingService.getLogger(MockServerContext.class); serverSocketChannel = ServerSocketChannel.open(); @@ -84,6 +83,7 @@ public MockServerContext(int port) throws Exception { serverSocket.setSoTimeout(1000); serverSocket.bind(new InetSocketAddress("0.0.0.0", port)); thisAddress = new Address("127.0.0.1", port); + this.thisUuid = memberUuid; this.serializationService = new DefaultSerializationServiceBuilder() .addDataSerializableFactory(TestDataFactory.FACTORY_ID, new TestDataFactory()) .build(); @@ -119,6 +119,11 @@ public Address getThisAddress() { return thisAddress; } + @Override + public UUID getThisUuid() { + return thisUuid; + } + @Override public Map getThisAddresses() { Map addressMap = new HashMap(); @@ -396,8 +401,4 @@ public AuditlogService getAuditLogService() { return NoOpAuditlogService.INSTANCE; } - @Override - public UUID getUuid() { - return null; - } } diff --git a/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/LocalAddressRegistryIntegrationTest.java b/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/LocalAddressRegistryIntegrationTest.java new file mode 100644 index 000000000000..e34793b5cb3f --- /dev/null +++ b/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/LocalAddressRegistryIntegrationTest.java @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2008-2021, Hazelcast, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hazelcast.internal.server.tcp; + +import com.hazelcast.cluster.Address; +import com.hazelcast.config.AdvancedNetworkConfig; +import com.hazelcast.config.Config; +import com.hazelcast.config.JoinConfig; +import com.hazelcast.config.ServerSocketEndpointConfig; +import com.hazelcast.core.Hazelcast; +import com.hazelcast.core.HazelcastInstance; +import com.hazelcast.instance.EndpointQualifier; +import com.hazelcast.instance.impl.Node; +import com.hazelcast.internal.nio.Connection; +import com.hazelcast.map.IMap; +import com.hazelcast.partition.Partition; +import com.hazelcast.partition.PartitionService; +import com.hazelcast.test.HazelcastSerialClassRunner; +import com.hazelcast.test.HazelcastTestSupport; +import com.hazelcast.test.annotation.NightlyTest; +import org.junit.After; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.UUID; + +import static com.hazelcast.internal.util.ExceptionUtil.rethrow; +import static com.hazelcast.test.Accessors.getNode; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +@RunWith(HazelcastSerialClassRunner.class) +@Category(NightlyTest.class) +public class LocalAddressRegistryIntegrationTest extends HazelcastTestSupport { + + // MEMBER & WAN & CLIENT addresses of the connection initiator + private static final Address INITIATOR_MEMBER_ADDRESS; + private static final Address INITIATOR_WAN_ADDRESS; + private static final Address INITIATOR_CLIENT_ADDRESS; + + // server-side member addresses + private static final Address SERVER_MEMBER_ADDRESS; + private static final Address SERVER_CLIENT_ADDRESS; + private static final Address SERVER_WAN_ADDRESS; + + static { + try { + INITIATOR_MEMBER_ADDRESS = new Address("127.0.0.1", 5702); + INITIATOR_CLIENT_ADDRESS = new Address("127.0.0.1", 6001); + INITIATOR_WAN_ADDRESS = new Address("127.0.0.1", 9000); + + SERVER_MEMBER_ADDRESS = new Address("127.0.0.1", 5701); + SERVER_CLIENT_ADDRESS = new Address("127.0.0.1", 6000); + SERVER_WAN_ADDRESS = new Address("127.0.0.1", 10000); + } catch (Exception e) { + throw rethrow(e); + } + } + + @After + public void tearDown() { + Hazelcast.shutdownAll(); + } + + @Test + public void whenSomeConnectionClosedBetweenMembers_registeredAddresses_should_not_be_cleaned_up() { + int timeoutSecs = 10; + int tcpChannelsPerConnection = 3; + System.setProperty("tcp.channels.per.connection", String.valueOf(tcpChannelsPerConnection)); + + // create members + HazelcastInstance serverMember = Hazelcast.newHazelcastInstance(createConfigForServer()); + HazelcastInstance initiatorMember = Hazelcast.newHazelcastInstance(createConfigForInitiator()); + UUID initiatorMemberUuid = initiatorMember.getCluster().getLocalMember().getUuid(); + assertClusterSize(2, serverMember, initiatorMember); + + createConnectionsOnEachPlane(serverMember); + Node serverNode = getNode(serverMember); + TcpServerConnectionManager connectionManager = (TcpServerConnectionManager) serverNode.getServer() + .getConnectionManager(EndpointQualifier.MEMBER); + assertTrueEventually(() -> + assertGreaterOrEquals( + "The number of connections must be greater than the number of channels.", + connectionManager.getConnections().size(), + tcpChannelsPerConnection + ), timeoutSecs); + + LinkedAddresses registeredAddressesOfInitiatorMember = serverNode + .getLocalAddressRegistry() + .linkedAddressesOf(initiatorMemberUuid); + assertNotNull(registeredAddressesOfInitiatorMember); + assertContains(registeredAddressesOfInitiatorMember.getAllAddresses(), INITIATOR_MEMBER_ADDRESS); + int previousNoConnections = connectionManager.getConnections().size(); + closeRandomConnection(new ArrayList<>(connectionManager.getConnections())); + assertTrueEventually(() -> + assertEquals( + previousNoConnections - 1, + connectionManager.getConnections().size() + ), timeoutSecs); + + LinkedAddresses registeredAddressesAfterConnectionClose = serverNode + .getLocalAddressRegistry() + .linkedAddressesOf(initiatorMemberUuid); + assertNotNull(registeredAddressesAfterConnectionClose); + assertContains(registeredAddressesAfterConnectionClose.getAllAddresses(), INITIATOR_MEMBER_ADDRESS); + + waitAllForSafeState(serverMember, initiatorMember); + } + + @Test + public void whenAllConnectionClosedBetweenMembers_registeredAddresses_should_be_cleaned_up() { + int timeoutSecs = 10; + int tcpChannelsPerConnection = 3; + System.setProperty("tcp.channels.per.connection", String.valueOf(tcpChannelsPerConnection)); + + // create members + HazelcastInstance serverMember = Hazelcast.newHazelcastInstance(createConfigForServer()); + HazelcastInstance initiatorMember = Hazelcast.newHazelcastInstance(createConfigForInitiator()); + UUID iniatiatorMemberUuid = initiatorMember.getCluster().getLocalMember().getUuid(); + assertClusterSize(2, serverMember, initiatorMember); + createConnectionsOnEachPlane(serverMember); + + Node serverNode = getNode(serverMember); + TcpServerConnectionManager connectionManager = (TcpServerConnectionManager) serverNode.getServer() + .getConnectionManager(EndpointQualifier.MEMBER); + assertTrueEventually(() -> + assertGreaterOrEquals( + "The number of connections must be greater than the number of channels.", + connectionManager.getConnections().size(), + tcpChannelsPerConnection + ), timeoutSecs); + + LinkedAddresses registeredAddressesOfInitiatorMember = serverNode + .getLocalAddressRegistry() + .linkedAddressesOf(iniatiatorMemberUuid); + + assertNotNull(registeredAddressesOfInitiatorMember); + assertContains(registeredAddressesOfInitiatorMember.getAllAddresses(), INITIATOR_MEMBER_ADDRESS); + initiatorMember.shutdown(); + assertTrueEventually(() -> + assertEquals( + 0, + connectionManager.getConnections().size() + ), timeoutSecs); + LinkedAddresses registeredAddressesAfterAllConnectionsClosed = serverNode + .getLocalAddressRegistry() + .linkedAddressesOf(iniatiatorMemberUuid); + assertNull(registeredAddressesAfterAllConnectionsClosed); + + } + + + private Config createConfigForServer() { + Config config = smallInstanceConfig(); + AdvancedNetworkConfig advancedNetworkConfig = config.getAdvancedNetworkConfig(); + JoinConfig advancedJoinConfig = advancedNetworkConfig.getJoin(); + advancedJoinConfig.getTcpIpConfig().setEnabled(true); + ServerSocketEndpointConfig memberServerSocketConfig = new ServerSocketEndpointConfig() + .setPort(SERVER_MEMBER_ADDRESS.getPort()); + memberServerSocketConfig.getInterfaces().addInterface(SERVER_MEMBER_ADDRESS.getHost()); + ServerSocketEndpointConfig clientServerSocketConfig = new ServerSocketEndpointConfig() + .setPort(SERVER_CLIENT_ADDRESS.getPort()); + clientServerSocketConfig.getInterfaces().addInterface(SERVER_CLIENT_ADDRESS.getHost()); + ServerSocketEndpointConfig wanServerSocketConfig = new ServerSocketEndpointConfig() + .setName("wan") + .setPort(SERVER_WAN_ADDRESS.getPort()); + wanServerSocketConfig.getInterfaces().addInterface(SERVER_WAN_ADDRESS.getHost()); + + memberServerSocketConfig.getInterfaces().addInterface("127.0.0.1"); + advancedNetworkConfig.setEnabled(true) + .setMemberEndpointConfig(memberServerSocketConfig) + .setClientEndpointConfig(clientServerSocketConfig) + .addWanEndpointConfig(wanServerSocketConfig); + return config; + } + + private Config createConfigForInitiator() { + Config config = smallInstanceConfig(); + AdvancedNetworkConfig advancedNetworkConfig = config.getAdvancedNetworkConfig(); + JoinConfig advancedJoinConfig = advancedNetworkConfig.getJoin(); + advancedJoinConfig.getTcpIpConfig().setEnabled(true).addMember( + SERVER_MEMBER_ADDRESS.getHost() + ":" + SERVER_MEMBER_ADDRESS.getPort() + ); + + ServerSocketEndpointConfig memberServerSocketConfig = new ServerSocketEndpointConfig() + .setPort(INITIATOR_MEMBER_ADDRESS.getPort()); + memberServerSocketConfig.getInterfaces().addInterface(INITIATOR_MEMBER_ADDRESS.getHost()); + ServerSocketEndpointConfig clientServerSocketConfig = new ServerSocketEndpointConfig() + .setPort(INITIATOR_CLIENT_ADDRESS.getPort()); + ServerSocketEndpointConfig wanServerSocketConfig = new ServerSocketEndpointConfig() + .setName("wan") + .setPort(SERVER_WAN_ADDRESS.getPort()); + wanServerSocketConfig.getInterfaces().addInterface(INITIATOR_WAN_ADDRESS.getHost()); + + memberServerSocketConfig.getInterfaces().addInterface("127.0.0.1"); + advancedNetworkConfig.setEnabled(true) + .setMemberEndpointConfig(memberServerSocketConfig) + .setClientEndpointConfig(clientServerSocketConfig) + .addWanEndpointConfig(wanServerSocketConfig); + + return config; + } + + private void createConnectionsOnEachPlane(HazelcastInstance hz) { + IMap dummy = hz.getMap(randomMapName()); + hz.getPartitionService().getPartitions().forEach( + partition -> { + String key = randomKeyNameOwnedByPartition(hz, partition.getPartitionId()); + dummy.put(key, key); + } + ); + dummy.destroy(); + } + + private void closeRandomConnection(List connections) { + Random random = new Random(); + int randomIdx = random.nextInt(connections.size()); + connections.get(randomIdx).close("Failure is injected", null); + } + + private String randomKeyNameOwnedByPartition(HazelcastInstance hz, int partitionId) { + PartitionService partitionService = hz.getPartitionService(); + while (true) { + String name = randomString(); + Partition partition = partitionService.getPartition(name); + if (partition.getPartitionId() == partitionId) { + return name; + } + } + } +} diff --git a/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/LocalAddressRegistryTest.java b/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/LocalAddressRegistryTest.java new file mode 100644 index 000000000000..bcc815e0ff8e --- /dev/null +++ b/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/LocalAddressRegistryTest.java @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2008-2021, Hazelcast, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hazelcast.internal.server.tcp; + +import com.hazelcast.cluster.Address; +import com.hazelcast.internal.util.UuidUtil; +import com.hazelcast.logging.ILogger; +import com.hazelcast.logging.Logger; +import com.hazelcast.test.HazelcastParallelClassRunner; +import com.hazelcast.test.HazelcastTestSupport; +import com.hazelcast.test.annotation.ParallelJVMTest; +import com.hazelcast.test.annotation.QuickTest; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.net.UnknownHostException; +import java.util.Set; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +@RunWith(HazelcastParallelClassRunner.class) +@Category({QuickTest.class, ParallelJVMTest.class}) +public class LocalAddressRegistryTest extends HazelcastTestSupport { + + private final ILogger logger = Logger.getLogger(LocalAddressRegistry.class); + private LocalAddressRegistry addressRegistry; + + @Before + public void setUp() { + addressRegistry = new LocalAddressRegistry(logger); + } + + @Test + public void whenRegistrationCountEqualsToRemovalCount_addressesShouldBeRemoved() throws UnknownHostException { + UUID memberUuid = UuidUtil.newUnsecureUUID(); + addressRegistry.register(memberUuid, sampleAddresses()); + assertExpectedAddressesRegistered(memberUuid, sampleAddresses().getAllAddresses()); + addressRegistry.tryRemoveRegistration(memberUuid, sampleAddresses().getPrimaryAddress()); + assertUUID_And_AddressesRemoved(memberUuid, sampleAddresses().getAllAddresses()); + } + + @Test + public void whenRegistrationCountEqualsToRemovalCount_addressesShouldBeRemoved2() throws UnknownHostException { + int count = 100; + UUID memberUuid = UuidUtil.newUnsecureUUID(); + for (int i = 0; i < count; i++) { + addressRegistry.register(memberUuid, sampleAddresses()); + } + assertExpectedAddressesRegistered(memberUuid, sampleAddresses().getAllAddresses()); + for (int i = 0; i < count; i++) { + addressRegistry.tryRemoveRegistration(memberUuid, sampleAddresses().getPrimaryAddress()); + } + assertUUID_And_AddressesRemoved(memberUuid, sampleAddresses().getAllAddresses()); + } + + @Test + public void whenRegistrationCountNotEqualToRemovalCount_addressesShouldBeRemoved2() throws UnknownHostException { + int count = 100; + UUID memberUuid = UuidUtil.newUnsecureUUID(); + for (int i = 0; i < count; i++) { + addressRegistry.register(memberUuid, sampleAddresses()); + } + assertExpectedAddressesRegistered(memberUuid, sampleAddresses().getAllAddresses()); + for (int i = 0; i < count - 1; i++) { + addressRegistry.tryRemoveRegistration(memberUuid, sampleAddresses().getPrimaryAddress()); + } + assertExpectedAddressesRegistered(memberUuid, sampleAddresses().getAllAddresses()); + } + + @Test + public void whenDisjointAddressSetRegisteredToSameUUID_oldAddressesShouldBeRemoved() throws UnknownHostException { + UUID memberUuid = UuidUtil.newUnsecureUUID(); + addressRegistry.register(memberUuid, sampleAddresses()); + assertExpectedAddressesRegistered(memberUuid, sampleAddresses().getAllAddresses()); + // disjoint address set is registered to the same uuid + addressRegistry.register(memberUuid, sampleAddresses2()); + assertExpectedAddressesRegistered(memberUuid, sampleAddresses2().getAllAddresses()); + assertAddressesRemoved(memberUuid, sampleAddresses().getAllAddresses()); + + // remove registration attempts with the old registered addresses should fail. + // try to remove twice in case we can't remove it because of the registration count + addressRegistry.tryRemoveRegistration(memberUuid, sampleAddresses().getPrimaryAddress()); + addressRegistry.tryRemoveRegistration(memberUuid, sampleAddresses().getPrimaryAddress()); + assertExpectedAddressesRegistered(memberUuid, sampleAddresses2().getAllAddresses()); + + // try removal with the last addresses + addressRegistry.tryRemoveRegistration(memberUuid, sampleAddresses2().getPrimaryAddress()); + assertUUID_And_AddressesRemoved(memberUuid, sampleAddresses2().getAllAddresses()); + } + + @Test + public void whenDisjointAddressSetRegisteredToSameUUID_oldAddressesShouldBeRemoved2() throws UnknownHostException { + int registerCount = 100; + UUID memberUuid = UuidUtil.newUnsecureUUID(); + for (int i = 0; i < registerCount; i++) { + addressRegistry.register(memberUuid, sampleAddresses()); + } + assertExpectedAddressesRegistered(memberUuid, sampleAddresses().getAllAddresses()); + + // disjoint address set is registered to the same uuid + for (int i = 0; i < registerCount; i++) { + addressRegistry.register(memberUuid, sampleAddresses2()); + } + assertExpectedAddressesRegistered(memberUuid, sampleAddresses2().getAllAddresses()); + assertAddressesRemoved(memberUuid, sampleAddresses().getAllAddresses()); + + // remove registration attempts with the old registered addresses should fail. + // try to remove 2x times in case we can't remove it because of the registration count + for (int i = 0; i < 2 * registerCount; i++) { + addressRegistry.tryRemoveRegistration(memberUuid, sampleAddresses().getPrimaryAddress()); + } + // show that removal requests on stale items do not remove the newly registered entries + assertExpectedAddressesRegistered(memberUuid, sampleAddresses2().getAllAddresses()); + + // try removal with the second set of addresses + for (int i = 0; i < registerCount; i++) { + addressRegistry.tryRemoveRegistration(memberUuid, sampleAddresses2().getPrimaryAddress()); + } + assertUUID_And_AddressesRemoved(memberUuid, sampleAddresses2().getAllAddresses()); + } + + + @Test + public void test_whenDisjointAddressSetRegisteredToSameUUID_and_oldAddressSetRegisteredToDifferentUuid() + throws UnknownHostException { + UUID firstMemberUuid = UuidUtil.newUnsecureUUID(); + UUID secondMemberUuid = UuidUtil.newUnsecureUUID(); + addressRegistry.register(firstMemberUuid, sampleAddresses()); + assertExpectedAddressesRegistered(firstMemberUuid, sampleAddresses().getAllAddresses()); + // same addresses with new uuid + addressRegistry.register(secondMemberUuid, sampleAddresses()); + assertExpectedAddressesRegistered(secondMemberUuid, sampleAddresses().getAllAddresses()); + // old uuid with new addresses + addressRegistry.register(firstMemberUuid, sampleAddresses2()); + assertExpectedAddressesRegistered(firstMemberUuid, sampleAddresses2().getAllAddresses()); + assertAddressesRemoved(firstMemberUuid, sampleAddresses().getAllAddresses()); + + addressRegistry.tryRemoveRegistration(firstMemberUuid, sampleAddresses().getPrimaryAddress()); + addressRegistry.tryRemoveRegistration(firstMemberUuid, sampleAddresses().getPrimaryAddress()); + // show that removal requests on stale items do not remove the newly registered entries + assertExpectedAddressesRegistered(firstMemberUuid, sampleAddresses2().getAllAddresses()); + + addressRegistry.tryRemoveRegistration(firstMemberUuid, sampleAddresses2().getPrimaryAddress()); + addressRegistry.tryRemoveRegistration(secondMemberUuid, sampleAddresses().getPrimaryAddress()); + + assertUUID_And_AddressesRemoved(firstMemberUuid, sampleAddresses2().getAllAddresses()); + assertUUID_And_AddressesRemoved(secondMemberUuid, sampleAddresses().getAllAddresses()); + } + + private void assertExpectedAddressesRegistered(UUID memberUuid, Set
addresses) { + for (Address address : addresses) { + assertEquals(memberUuid, addressRegistry.uuidOf(address)); + } + LinkedAddresses linkedAddresses = addressRegistry.linkedAddressesOf(memberUuid); + assertNotNull(linkedAddresses); + assertNotNull(linkedAddresses.getAllAddresses()); + assertTrue(linkedAddresses.getAllAddresses().containsAll(addresses)); + } + + private void assertAddressesRemoved(UUID memberUuid, Set
removedAddresses) { + for (Address address : removedAddresses) { + UUID correspondingMemberUuid = addressRegistry.uuidOf(address); + assertTrue(correspondingMemberUuid == null || correspondingMemberUuid != memberUuid); + } + LinkedAddresses linkedAddresses = addressRegistry.linkedAddressesOf(memberUuid); + if (linkedAddresses != null) { + for (Address address : removedAddresses) { + assertNotContains(linkedAddresses.getAllAddresses(), address); + } + } + } + + private void assertUUID_And_AddressesRemoved(UUID memberUuid, Set
addresses) { + for (Address address : addresses) { + assertNull(addressRegistry.uuidOf(address)); + } + LinkedAddresses linkedAddresses = addressRegistry.linkedAddressesOf(memberUuid); + assertNull(linkedAddresses); + } + + private static LinkedAddresses sampleAddresses() throws UnknownHostException { + LinkedAddresses addresses = new LinkedAddresses(new Address("localhost", 5701)); + addresses.addAllResolvedAddresses(new Address("127.0.0.1", 5701)); + return addresses; + } + + private static LinkedAddresses sampleAddresses2() throws UnknownHostException { + LinkedAddresses addresses = new LinkedAddresses(new Address("localhost", 5702)); + addresses.addAllResolvedAddresses(new Address("127.0.0.1", 5702)); + return addresses; + } +} diff --git a/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/TcpMetricValuesTest.java b/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/TcpMetricValuesTest.java new file mode 100644 index 000000000000..a13dc9f845d1 --- /dev/null +++ b/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/TcpMetricValuesTest.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2008-2021, Hazelcast, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hazelcast.internal.server.tcp; + +import com.hazelcast.config.Config; +import com.hazelcast.config.JoinConfig; +import com.hazelcast.config.NetworkConfig; +import com.hazelcast.core.Hazelcast; +import com.hazelcast.core.HazelcastInstance; +import com.hazelcast.instance.impl.HazelcastInstanceFactory; +import com.hazelcast.internal.metrics.MetricDescriptor; +import com.hazelcast.internal.metrics.impl.CapturingCollector; +import com.hazelcast.test.HazelcastSerialClassRunner; +import com.hazelcast.test.annotation.SlowTest; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import static com.hazelcast.internal.metrics.MetricDescriptorConstants.TCP_DISCRIMINATOR_BINDADDRESS; +import static com.hazelcast.internal.metrics.MetricDescriptorConstants.TCP_DISCRIMINATOR_ENDPOINT; +import static com.hazelcast.internal.metrics.MetricDescriptorConstants.TCP_METRIC_ENDPOINT_MANAGER_ACTIVE_COUNT; +import static com.hazelcast.internal.metrics.MetricDescriptorConstants.TCP_METRIC_ENDPOINT_MANAGER_COUNT; +import static com.hazelcast.internal.metrics.MetricDescriptorConstants.TCP_PREFIX_CONNECTION; +import static com.hazelcast.internal.metrics.ProbeUnit.COUNT; +import static com.hazelcast.internal.metrics.impl.DefaultMetricDescriptorSupplier.DEFAULT_DESCRIPTOR_SUPPLIER; +import static com.hazelcast.test.Accessors.getNodeEngineImpl; +import static com.hazelcast.test.HazelcastTestSupport.assertClusterSize; +import static com.hazelcast.test.HazelcastTestSupport.assertTrueEventually; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +@RunWith(HazelcastSerialClassRunner.class) +@Category(SlowTest.class) +@Ignore("https://github.com/hazelcast/hazelcast/issues/18877") +public class TcpMetricValuesTest { + + @Before + @After + public void after() throws IOException { + HazelcastInstanceFactory.terminateAll(); + } + + @Test + public void membersDefinedViaIps() { + test("127.0.0.1:5701", "127.0.0.1:5702", "127.0.0.1:5703"); + } + + @Test + public void membersDefinedViaHostname() { + test("localhost:5701", "localhost:5702", "localhost:5703"); + } + + private void test(String... members) { + int timeoutSecs = 10; + // start 3 member cluster + final HazelcastInstance hz1 = Hazelcast.newHazelcastInstance(getConfig(members)); + final HazelcastInstance hz2 = Hazelcast.newHazelcastInstance(getConfig(members)); + final HazelcastInstance hz3 = Hazelcast.newHazelcastInstance(getConfig(members)); + assertClusterSize(3, hz1, hz2, hz3); + assertTrueEventually(() -> { + CapturingCollector collector = new CapturingCollector(); + getNodeEngineImpl(hz1).getMetricsRegistry().collect(collector); + + verifyMetricValue(collector, metricDescriptor(TCP_METRIC_ENDPOINT_MANAGER_COUNT), 2); + verifyMetricValue(collector, metricDescriptor(TCP_METRIC_ENDPOINT_MANAGER_ACTIVE_COUNT), 2); + verifyNoOfCollectedMetrics(collector, TCP_DISCRIMINATOR_ENDPOINT, 2); + verifyNoOfCollectedMetrics(collector, TCP_DISCRIMINATOR_BINDADDRESS, 2); + }, timeoutSecs); + } + + private void verifyMetricValue( + CapturingCollector collector, + MetricDescriptor expectedDescriptor, + long expected + ) { + CapturingCollector.Capture capture = collector.captures().get(expectedDescriptor); + assertNotNull(capture); + assertEquals(expectedDescriptor.toString(), 1, capture.hits()); + long actual = (long) capture.singleCapturedValue(); + assertEquals(expected, actual); + } + + private void verifyNoOfCollectedMetrics( + CapturingCollector collector, + String discriminator, + long expected + ) { + Map captures = collector.captures().entrySet().stream() + .filter(e -> Objects.equals(e.getKey().prefix(), TCP_PREFIX_CONNECTION)) + .filter(e -> Objects.equals(e.getKey().discriminator(), discriminator)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + int actual = captures.size(); + assertEquals(String.format("Actual value of %d for %s is not equal to %d", + actual, discriminator, expected), expected, actual); + } + + private MetricDescriptor metricDescriptor(String metric) { + return DEFAULT_DESCRIPTOR_SUPPLIER.get() + .withPrefix(TCP_PREFIX_CONNECTION) + .withMetric(metric) + .withUnit(COUNT); + } + + private Config getConfig(String... members) { + Config config = new Config(); + NetworkConfig networkConfig = config.getNetworkConfig(); + JoinConfig join = networkConfig.getJoin(); + join.getMulticastConfig().setEnabled(false); + join.getTcpIpConfig().setEnabled(true); + for (String member : members) { + join.getTcpIpConfig().addMember(member); + } + return config; + } +} diff --git a/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/TcpServerConnection_AbstractTest.java b/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/TcpServerConnection_AbstractTest.java index 99e4ce058d12..b6ddaf3abae2 100644 --- a/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/TcpServerConnection_AbstractTest.java +++ b/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/TcpServerConnection_AbstractTest.java @@ -27,6 +27,7 @@ import com.hazelcast.internal.server.NetworkingFactory; import com.hazelcast.internal.server.ServerConnection; import com.hazelcast.internal.server.TestDataFactory; +import com.hazelcast.internal.util.UuidUtil; import com.hazelcast.logging.ILogger; import com.hazelcast.logging.impl.LoggingServiceImpl; import com.hazelcast.cluster.Address; @@ -123,7 +124,7 @@ protected TcpServer newMockTcpServer(MetricsRegistry metricsRegistry) throws Exc MockServerContext serverContext = null; while (serverContext == null) { try { - serverContext = new MockServerContext(portNumber++); + serverContext = new MockServerContext(portNumber++, UuidUtil.newUnsecureUUID()); } catch (IOException e) { if (portNumber >= PORT_NUMBER_UPPER_LIMIT) { throw e; @@ -132,10 +133,12 @@ protected TcpServer newMockTcpServer(MetricsRegistry metricsRegistry) throws Exc } ServerSocketRegistry registry = new ServerSocketRegistry(singletonMap(MEMBER, serverContext.serverSocketChannel), true); + LocalAddressRegistry addressRegistry = new LocalAddressRegistry(logger); MockServerContext finalServiceContext = serverContext; return new TcpServer(null, serverContext, registry, + addressRegistry, metricsRegistry, networkingFactory.create(serverContext, metricsRegistry), qualifier -> new UnifiedChannelInitializer(finalServiceContext)); diff --git a/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/TcpServerControlTest.java b/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/TcpServerControlTest.java index eb245f0c1ffb..ff0a043b1f5d 100644 --- a/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/TcpServerControlTest.java +++ b/hazelcast/src/test/java/com/hazelcast/internal/server/tcp/TcpServerControlTest.java @@ -29,6 +29,7 @@ import com.hazelcast.internal.nio.ConnectionType; import com.hazelcast.internal.nio.Packet; import com.hazelcast.internal.serialization.InternalSerializationService; +import com.hazelcast.internal.util.UuidUtil; import com.hazelcast.test.HazelcastParallelParametersRunnerFactory; import com.hazelcast.test.HazelcastParametrizedRunner; import com.hazelcast.test.TestAwareInstanceFactory; @@ -59,10 +60,12 @@ import static com.hazelcast.internal.util.ExceptionUtil.rethrow; import static com.hazelcast.test.Accessors.getNode; import static com.hazelcast.test.Accessors.getSerializationService; +import static com.hazelcast.test.HazelcastTestSupport.assertTrueEventually; import static com.hazelcast.test.HazelcastTestSupport.smallInstanceConfig; import static com.hazelcast.test.starter.ReflectionUtils.getFieldValueReflectively; -import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -85,6 +88,8 @@ public class TcpServerControlTest { private static final Address SERVER_CLIENT_ADDRESS; private static final Address SERVER_WAN_ADDRESS; + // The uuid of member + private static final UUID MEMBER_UUID; static { try { INITIATOR_MEMBER_ADDRESS = new Address("127.0.0.1", 5702); @@ -94,6 +99,7 @@ public class TcpServerControlTest { SERVER_MEMBER_ADDRESS = new Address("127.0.0.1", 5701); SERVER_CLIENT_ADDRESS = new Address("127.0.0.1", 6000); SERVER_WAN_ADDRESS = new Address("127.0.0.1", 10000); + MEMBER_UUID = UuidUtil.newUnsecureUUID(); } catch (Exception e) { throw rethrow(e); } @@ -106,7 +112,7 @@ public class TcpServerControlTest { @Parameter(1) public String protocolIdentifier; - // connection type of TcpIpConnection for which MemberHandshake is processed + // connection type of TcpServerConnection for which MemberHandshake is processed @Parameter(2) public String connectionType; @@ -119,15 +125,17 @@ public class TcpServerControlTest { @Parameter(4) public boolean reply; - // addresses on which the TcpIpConnection is expected to be registered in the connectionsMap + // addresses on which the TcpServerConnection is expected to be registered in the address registry @Parameter(5) public List
expectedAddresses; private final TestAwareInstanceFactory factory = new TestAwareInstanceFactory(); - private final UUID uuid = UUID.randomUUID(); private InternalSerializationService serializationService; private TcpServerControl tcpServerControl; + private LocalAddressRegistry addressRegistry; + private ConnectionLifecycleListener lifecycleListener; + private TcpServerConnection connection; // mocks private Channel channel; @@ -144,7 +152,7 @@ public static List parameters() { new Object[]{ProtocolType.MEMBER, null, ConnectionType.MEMBER, localAddresses_memberWan(), false, singletonList(INITIATOR_MEMBER_ADDRESS)}, // when protocol type not supported by BindHandler, nothing is registered - new Object[]{ProtocolType.CLIENT, null, null, localAddresses_memberWan(), false, emptyList()}, + new Object[]{ProtocolType.CLIENT, null, null, localAddresses_memberWan(), false, singletonList(INITIATOR_CLIENT_SOCKET_ADDRESS)}, // when protocol type is WAN, initiator address is always registered new Object[]{WAN, "wan", ConnectionType.MEMBER, localAddresses_memberOnly(), false, singletonList(INITIATOR_CLIENT_SOCKET_ADDRESS)}, @@ -164,10 +172,11 @@ public void setup() throws IllegalAccessException { HazelcastInstance hz = factory.newHazelcastInstance(createConfig()); serializationService = getSerializationService(hz); Node node = getNode(hz); - connectionManager = TcpServerConnectionManager.class.cast( - node.getServer().getConnectionManager(EndpointQualifier.resolve(protocolType, protocolIdentifier))); + connectionManager = (TcpServerConnectionManager) node.getServer() + .getConnectionManager(EndpointQualifier.resolve(protocolType, protocolIdentifier)); tcpServerControl = getFieldValueReflectively(connectionManager, "serverControl"); - + lifecycleListener = getFieldValueReflectively(connectionManager, "connectionLifecycleListener"); + addressRegistry = node.getLocalAddressRegistry(); // setup mock channel & socket Socket socket = mock(Socket.class); when(socket.getRemoteSocketAddress()).thenReturn(CLIENT_SOCKET_ADDRESS); @@ -188,35 +197,70 @@ public void tearDown() { public void process() { tcpServerControl.process(memberHandshakeMessage()); assertExpectedAddressesRegistered(); + assertMemberConnectionRegistered(); + assertTrueEventually(() -> + assertEquals( + 0, + connectionManager.getConnections().size() + ), 5); + connection.close("close connection", null); + assertAddressesCleanedUp(); } - private void assertExpectedAddressesRegistered() { + private void assertMemberConnectionRegistered() { TcpServerConnectionManager.Plane[] planes = connectionManager.planes; try { - for (Address address : expectedAddresses) { - boolean found = false; - for (TcpServerConnectionManager.Plane plane : planes) { - if (plane.getConnection((address)) != null) { - found = true; - break; - } + // check connection is found for this member uuid + boolean found = false; + for (TcpServerConnectionManager.Plane plane : planes) { + if (plane.getConnection(MEMBER_UUID) != null) { + found = true; + break; } - - assertTrue("Address " + address + " not found", found); } + assertTrue("Connection for the member uuid=" + MEMBER_UUID + " not found", found); } catch (AssertionError error) { // dump complete connections map + System.err.println("Connection for member uuid=" + MEMBER_UUID + " is expected to be registered " + + "but connections map only contains: " + connectionManager.connections); + throw error; + } + + } + private void assertAddressesCleanedUp() { + assertNull(addressRegistry.linkedAddressesOf(MEMBER_UUID)); + for (Address address : expectedAddresses) { + UUID memberUuid = addressRegistry.uuidOf(address); + assertNull(memberUuid); + } + } - System.err.println("Expected " + expectedAddresses + " but connections map contained: " + connectionManager.connections); + private void assertExpectedAddressesRegistered() { + try { + for (Address address : expectedAddresses) { + UUID memberUuid = addressRegistry.uuidOf(address); + assertEquals(MEMBER_UUID, memberUuid); + } + } catch (AssertionError error) { + LinkedAddresses linkedAddresses = addressRegistry.linkedAddressesOf(MEMBER_UUID); + if (linkedAddresses != null) { + // dump complete address registry + System.err.println("Expected " + expectedAddresses + " for the member uuid=" + MEMBER_UUID + + ", but the addresses registered in the address registry as belonging to this member uuid: " + + linkedAddresses.getAllAddresses()); + } else { + System.err.println("We cannot find any addresses registered for the given member UUID: " + MEMBER_UUID + + " See the dump of address registry:" + addressRegistry); + } throw error; } } private Packet memberHandshakeMessage() { - MemberHandshake handshake = new MemberHandshake(SCHEMA_VERSION_2, localAddresses, new Address(CLIENT_SOCKET_ADDRESS), reply, uuid); + MemberHandshake handshake = new MemberHandshake(SCHEMA_VERSION_2, localAddresses, new Address(CLIENT_SOCKET_ADDRESS), reply, MEMBER_UUID); Packet packet = new Packet(serializationService.toBytes(handshake)); - TcpServerConnection connection = new TcpServerConnection(connectionManager, mock(ConnectionLifecycleListener.class), 1, channel); + connection = new TcpServerConnection(connectionManager, lifecycleListener, 1, channel, false); if (connectionType != null) { connection.setConnectionType(connectionType); } @@ -261,4 +305,8 @@ private Config createConfig() { .addWanEndpointConfig(wanServerSocketConfig); return config; } + + public void assertEmpty(Map map) { + assertEquals("expecting an empty map, but the map is:" + map, 0, map.size()); + } } diff --git a/hazelcast/src/test/java/com/hazelcast/map/impl/operation/MapFetchIndexOperationTest.java b/hazelcast/src/test/java/com/hazelcast/map/impl/operation/MapFetchIndexOperationTest.java index da96d0f4aea6..cebd853bf0ed 100644 --- a/hazelcast/src/test/java/com/hazelcast/map/impl/operation/MapFetchIndexOperationTest.java +++ b/hazelcast/src/test/java/com/hazelcast/map/impl/operation/MapFetchIndexOperationTest.java @@ -32,6 +32,7 @@ import com.hazelcast.internal.cluster.Joiner; import com.hazelcast.internal.iteration.IndexIterationPointer; import com.hazelcast.internal.server.Server; +import com.hazelcast.internal.server.tcp.LocalAddressRegistry; import com.hazelcast.internal.server.tcp.ServerSocketRegistry; import com.hazelcast.internal.util.collection.PartitionIdSet; import com.hazelcast.map.IMap; @@ -632,8 +633,8 @@ public Joiner createJoiner(Node node) { } @Override - public Server createServer(Node node, ServerSocketRegistry registry) { - return delegate.createServer(node, registry); + public Server createServer(Node node, ServerSocketRegistry registry, LocalAddressRegistry addressRegistry) { + return delegate.createServer(node, registry, addressRegistry); } } diff --git a/hazelcast/src/test/java/com/hazelcast/spi/impl/operationservice/impl/Invocation_ServerConnectionManagerTest.java b/hazelcast/src/test/java/com/hazelcast/spi/impl/operationservice/impl/Invocation_ServerConnectionManagerTest.java index 0f461d0f66a5..a66391767882 100644 --- a/hazelcast/src/test/java/com/hazelcast/spi/impl/operationservice/impl/Invocation_ServerConnectionManagerTest.java +++ b/hazelcast/src/test/java/com/hazelcast/spi/impl/operationservice/impl/Invocation_ServerConnectionManagerTest.java @@ -44,6 +44,8 @@ import javax.annotation.Nonnull; import java.util.Collection; import java.util.Collections; +import java.util.List; +import java.util.UUID; import java.util.function.Predicate; import static com.hazelcast.test.Accessors.getNodeEngineImpl; @@ -97,22 +99,35 @@ public int connectionCount(Predicate predicate) { } @Override - public boolean register(Address remoteAddress, ServerConnection connection, int streamId) { + public boolean register( + Address remoteAddress, + Address targetAddress, + Collection
remoteAddressAliases, + UUID remoteUuid, + ServerConnection connection, + int streamId + ) { return false; } @Override - public ServerConnection get(Address address, int streamId) { + public ServerConnection get(@Nonnull Address address, int streamId) { return null; } @Override - public ServerConnection getOrConnect(Address address , int streamId) { + @Nonnull + public List getAllConnections(@Nonnull Address address) { + return Collections.emptyList(); + } + + @Override + public ServerConnection getOrConnect(@Nonnull Address address , int streamId) { throw new UnsupportedOperationException(EXPECTED_MSG); } @Override - public ServerConnection getOrConnect(Address address, boolean silent, int streamId) { + public ServerConnection getOrConnect(@Nonnull Address address, boolean silent, int streamId) { throw new UnsupportedOperationException(EXPECTED_MSG); } diff --git a/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/MockNodeContext.java b/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/MockNodeContext.java index 221fa014f39d..8f71d47a978c 100644 --- a/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/MockNodeContext.java +++ b/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/MockNodeContext.java @@ -24,6 +24,7 @@ import com.hazelcast.instance.impl.NodeContext; import com.hazelcast.instance.impl.NodeExtension; import com.hazelcast.instance.impl.NodeExtensionFactory; +import com.hazelcast.internal.server.tcp.LocalAddressRegistry; import com.hazelcast.internal.server.tcp.ServerSocketRegistry; import com.hazelcast.cluster.Address; import com.hazelcast.internal.server.Server; @@ -78,7 +79,7 @@ public Joiner createJoiner(Node node) { } @Override - public Server createServer(Node node, ServerSocketRegistry serverSocketRegistry) { + public Server createServer(Node node, ServerSocketRegistry serverSocketRegistry, LocalAddressRegistry addressRegistry) { TcpServerContext serverContext = new TcpServerContext(node, node.nodeEngine); MockServer mockNetworkingService = new MockServer(serverContext, node, registry); return new FirewallingServer(mockNetworkingService, initiallyBlockedAddresses); diff --git a/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/MockServer.java b/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/MockServer.java index abd53f78a5d5..ef53e362241d 100644 --- a/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/MockServer.java +++ b/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/MockServer.java @@ -30,6 +30,8 @@ import com.hazelcast.internal.server.ServerConnection; import com.hazelcast.internal.server.ServerConnectionManager; import com.hazelcast.internal.server.ServerContext; +import com.hazelcast.internal.server.tcp.LinkedAddresses; +import com.hazelcast.internal.server.tcp.LocalAddressRegistry; import com.hazelcast.internal.util.concurrent.ThreadFactoryImpl; import com.hazelcast.internal.util.executor.StripedRunnable; import com.hazelcast.logging.ILogger; @@ -37,8 +39,11 @@ import javax.annotation.Nonnull; import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArraySet; @@ -58,8 +63,9 @@ class MockServer implements Server { private static final int RETRY_NUMBER = 5; private static final int DELAY_FACTOR = 100; - private final ConcurrentMap connectionMap = new ConcurrentHashMap<>(10); + private final ConcurrentMap connectionMap = new ConcurrentHashMap<>(10); private final TestNodeRegistry nodeRegistry; + private final LocalAddressRegistry addressRegistry; private final Node node; private final ScheduledExecutorService scheduler; private final ServerContext serverContext; @@ -72,13 +78,14 @@ class MockServer implements Server { this.serverContext = serverContext; this.nodeRegistry = testNodeRegistry; this.node = node; + this.addressRegistry = node.getLocalAddressRegistry(); this.connectionManager = new MockServerConnectionManager(this); this.scheduler = new ScheduledThreadPoolExecutor(4, new ThreadFactoryImpl(createThreadPoolName(serverContext.getHazelcastName(), "MockConnectionManager"))); this.logger = serverContext.getLoggingService().getLogger(MockServer.class); } - static class MockServerConnectionManager + class MockServerConnectionManager implements ServerConnectionManager { private final MockServer server; @@ -95,20 +102,35 @@ public Server getServer() { } @Override - public MockServerConnection get(Address address, int stream) { - return server.connectionMap.get(address); + public ServerConnection get(@Nonnull Address address, int streamId) { + UUID memberUuid = server.nodeRegistry.uuidOf(address); + return memberUuid != null ? get(memberUuid, 0) : null; + } + + public MockServerConnection get(UUID memberUuid, int streamId) { + return server.connectionMap.get(memberUuid); } @Override - public MockServerConnection getOrConnect(Address address, int stream) { - MockServerConnection conn = server.connectionMap.get(address); + @Nonnull + public List getAllConnections(@Nonnull Address address) { + UUID memberUuid = server.nodeRegistry.uuidOf(address); + if (memberUuid == null) { + return Collections.emptyList(); + } + ServerConnection conn = get(memberUuid, 0); + return conn != null ? Collections.singletonList(conn) : Collections.emptyList(); + } + + @Override + public MockServerConnection getOrConnect(@Nonnull Address address, int stream) { + MockServerConnection conn = server.connectionMap.get(server.nodeRegistry.uuidOf(address)); if (conn != null && conn.isAlive()) { return conn; } if (!server.live) { return null; } - Node targetNode = server.nodeRegistry.getNode(address); if (targetNode == null || isTargetLeft(targetNode)) { suspectAddress(address); @@ -122,14 +144,10 @@ public MockServerConnection getOrConnect(Address address, int stream) { public void accept(Packet packet) { } - private void suspectAddress(final Address address) { + private void suspectAddress(Address endpointAddress) { // see ServerContext#removeEndpoint() server.node.getNodeEngine().getExecutionService().execute(ExecutionService.IO_EXECUTOR, - () -> server.node.getClusterService().suspectAddressIfNotConnected(address)); - } - - public static boolean isTargetLeft(Node targetNode) { - return !targetNode.isRunning() && !targetNode.getClusterService().isJoined(); + () -> server.node.getClusterService().suspectAddressIfNotConnected(endpointAddress)); } private synchronized MockServerConnection createConnection(Node targetNode) { @@ -138,42 +156,73 @@ private synchronized MockServerConnection createConnection(Node targetNode) { } Node node = server.node; - Address local = node.getThisAddress(); - Address remote = targetNode.getThisAddress(); - - MockServerConnection thisConnection = new MockServerConnection(lifecycleListener, remote, local, - node.getNodeEngine(), targetNode.getServer().getConnectionManager(EndpointQualifier.MEMBER)); - - MockServerConnection remoteConnection = new MockServerConnection(lifecycleListener, local, remote, - targetNode.getNodeEngine(), node.getServer().getConnectionManager(EndpointQualifier.MEMBER)); - - remoteConnection.localConnection = thisConnection; - thisConnection.localConnection = remoteConnection; - - if (!remoteConnection.isAlive()) { + Address localAddress = node.getThisAddress(); + Address remoteAddress = targetNode.getThisAddress(); + + UUID localMemberUuid = node.getThisUuid(); + UUID remoteMemberUuid = targetNode.getThisUuid(); + + MockServerConnection connectionFromLocalToRemote = new MockServerConnection( + lifecycleListener, + localAddress, + remoteAddress, + localMemberUuid, + remoteMemberUuid, + node.getNodeEngine(), + targetNode.getNodeEngine(), + node.getServer().getConnectionManager(EndpointQualifier.MEMBER) + ); + + MockServerConnection connectionFromRemoteToLocal = new MockServerConnection( + lifecycleListener, + remoteAddress, + localAddress, + remoteMemberUuid, + localMemberUuid, + targetNode.getNodeEngine(), + node.getNodeEngine(), + targetNode.getServer().getConnectionManager(EndpointQualifier.MEMBER) + ); + + connectionFromRemoteToLocal.localConnection = connectionFromLocalToRemote; + connectionFromLocalToRemote.localConnection = connectionFromRemoteToLocal; + + if (!connectionFromRemoteToLocal.isAlive()) { // targetNode is not alive anymore. - suspectAddress(remote); + suspectAddress(remoteAddress); return null; } - server.connectionMap.put(remote, remoteConnection); - server.logger.info("Created connection to endpoint: " + remote + ", connection: " + remoteConnection); + addressRegistry.register(remoteMemberUuid, LinkedAddresses.getResolvedAddresses(remoteAddress)); + LocalAddressRegistry remoteAddressRegistry = targetNode.getLocalAddressRegistry(); + remoteAddressRegistry.register(localMemberUuid, LinkedAddresses.getResolvedAddresses(localAddress)); + + server.connectionMap.put(remoteMemberUuid, connectionFromLocalToRemote); + server.logger.info("Created connection to endpoint: " + remoteAddress + "-" + remoteMemberUuid + ", connection: " + + connectionFromLocalToRemote); - if (!remoteConnection.isAlive()) { + if (!connectionFromLocalToRemote.isAlive()) { // If connection is not alive after inserting it into connection map, // that means remote node is being stopping during connection creation. - suspectAddress(remote); + suspectAddress(remoteAddress); } - return remoteConnection; + return connectionFromLocalToRemote; } @Override - public MockServerConnection getOrConnect(Address address, boolean silent, int stream) { + public MockServerConnection getOrConnect(@Nonnull Address address, boolean silent, int stream) { return getOrConnect(address, stream); } @Override - public synchronized boolean register(final Address remoteAddress, final ServerConnection c, int streamId) { + public synchronized boolean register( + Address remoteAddress, + Address targetAddress, + Collection
remoteAddressAliases, + UUID remoteUuid, + ServerConnection c, + int streamId + ) { MockServerConnection connection = (MockServerConnection) c; if (!server.live) { throw new IllegalStateException("connection manager is not live!"); @@ -183,7 +232,20 @@ public synchronized boolean register(final Address remoteAddress, final ServerCo } connection.setLifecycleListener(lifecycleListener); - server.connectionMap.put(remoteAddress, connection); + connection.setRemoteAddress(remoteAddress); + connection.setRemoteUuid(remoteUuid); + server.connectionMap.put(remoteUuid, connection); + LinkedAddresses addressesToRegister = LinkedAddresses.getResolvedAddresses(remoteAddress); + if (targetAddress != null) { + addressesToRegister.addAllResolvedAddresses(targetAddress); + } + if (remoteAddressAliases != null) { + for (Address remoteAddressAlias : remoteAddressAliases) { + addressesToRegister.addAllResolvedAddresses(remoteAddressAlias); + } + } + addressRegistry.register(remoteUuid, addressesToRegister); + server.serverContext.getEventService().executeEventCallback(new StripedRunnable() { @Override public void run() { @@ -205,7 +267,7 @@ public void addConnectionListener(ConnectionListener connectionListener) { connectionListeners.add(connectionListener); } - private void fireConnectionRemovedEvent(final MockServerConnection connection, final Address endPoint) { + private void fireConnectionRemovedEvent(final MockServerConnection connection, UUID endpointUuid) { if (server.live) { server.serverContext.getEventService().executeEventCallback(new StripedRunnable() { @Override @@ -215,7 +277,7 @@ public void run() { @Override public int getKey() { - return endPoint.hashCode(); + return endpointUuid.hashCode(); } }); } @@ -235,23 +297,30 @@ public int connectionCount(Predicate predicate) { * Retries sending packet maximum 5 times until connection to target becomes available. */ @Override - public boolean transmit(Packet packet, Address target, int streamId) { - return send(packet, target, null); + public boolean transmit(Packet packet, Address targetAddress, int streamId) { + return transmit(packet, server.nodeRegistry.uuidOf(targetAddress), streamId); } - private boolean send(Packet packet, Address target, SendTask sendTask) { - MockServerConnection connection = get(target, 0); + /** + * Retries sending packet maximum 5 times until connection to target becomes available. + */ + public boolean transmit(Packet packet, UUID targetUuid, int streamId) { + return send(packet, targetUuid, null); + } + + private boolean send(Packet packet, UUID targetUuid, SendTask sendTask) { + MockServerConnection connection = get(targetUuid, 0); if (connection != null) { return connection.write(packet); } if (sendTask == null) { - sendTask = new SendTask(packet, target); + sendTask = new SendTask(packet, targetUuid); } int retries = sendTask.retries.get(); if (retries < RETRY_NUMBER && server.serverContext.isNodeActive()) { - getOrConnect(target, true); + getOrConnect(server.nodeRegistry.addressOf(targetUuid), true); // TODO: Caution: may break the order guarantee of the packets sent from the same thread! try { server.scheduler.schedule(sendTask, (retries + 1) * DELAY_FACTOR, TimeUnit.MILLISECONDS); @@ -260,7 +329,7 @@ private boolean send(Packet packet, Address target, SendTask sendTask) { throw e; } if (server.logger.isFinestEnabled()) { - server.logger.finest("Packet send task is rejected. Packet cannot be sent to " + target); + server.logger.finest("Packet send task is rejected. Packet cannot be sent to " + targetUuid); } } return true; @@ -278,8 +347,8 @@ private class MockConnLifecycleListener @Override public void onConnectionClose(MockServerConnection connection, Throwable t, boolean silent) { - final Address endPoint = connection.getRemoteAddress(); - if (!server.connectionMap.remove(endPoint, connection)) { + UUID endpointUuid = connection.getRemoteUuid(); + if (!server.connectionMap.remove(endpointUuid, connection)) { return; } @@ -288,13 +357,14 @@ public void onConnectionClose(MockServerConnection connection, Throwable t, bool // so we pass in null. Once they are changed to use the parameter, we should be notified // and this parameter can be changed Connection remoteConnection = server.getConnectionManager(null) - .get(connection.localAddress); + .get(connection.getRemoteAddress(), 0); if (remoteConnection != null) { remoteConnection.close("Connection closed by the other side", null); } - MockServerConnectionManager.this.server.logger.info("Removed connection to endpoint: " + endPoint + ", connection: " + connection); - fireConnectionRemovedEvent(connection, endPoint); + MockServerConnectionManager.this.server.logger.info("Removed connection to endpoint: " + endpointUuid + + ", connection: " + connection); + fireConnectionRemovedEvent(connection, endpointUuid); } } @@ -304,24 +374,24 @@ private final class SendTask implements Runnable { private final AtomicInteger retries = new AtomicInteger(); private final Packet packet; - private final Address target; + private final UUID targetUuid; - private SendTask(Packet packet, Address target) { + private SendTask(Packet packet, UUID targetUuid) { this.packet = packet; - this.target = target; + this.targetUuid = targetUuid; } @Override public void run() { int actualRetries = retries.incrementAndGet(); if (server.logger.isFinestEnabled()) { - server.logger.finest("Retrying[" + actualRetries + "] packet send operation to: " + target); + server.logger.finest("Retrying[" + actualRetries + "] packet send operation to: " + targetUuid); } - send(packet, target, this); + send(packet, targetUuid, this); } } - private static class MockNetworkStats implements NetworkStats { + private class MockNetworkStats implements NetworkStats { @Override public long getBytesReceived() { @@ -335,6 +405,10 @@ public long getBytesSent() { } } + public LocalAddressRegistry getAddressRegistry() { + return addressRegistry; + } + @Override public ServerContext getContext() { return serverContext; @@ -403,6 +477,10 @@ public synchronized void stop() { } } + public static boolean isTargetLeft(Node targetNode) { + return !targetNode.isRunning() && !targetNode.getClusterService().isJoined(); + } + @Override public synchronized void shutdown() { stop(); diff --git a/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/MockServerConnection.java b/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/MockServerConnection.java index 43c6e95927e3..53eeb367d743 100644 --- a/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/MockServerConnection.java +++ b/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/MockServerConnection.java @@ -30,17 +30,19 @@ import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; import static com.hazelcast.internal.util.ExceptionUtil.rethrow; -import static com.hazelcast.test.mocknetwork.MockServer.MockServerConnectionManager.isTargetLeft; +import static com.hazelcast.test.mocknetwork.MockServer.isTargetLeft; import static org.junit.Assert.assertNotNull; public class MockServerConnection implements ServerConnection { protected final Address localAddress; + protected final NodeEngineImpl localNodeEngine; protected final NodeEngineImpl remoteNodeEngine; volatile MockServerConnection localConnection; @@ -51,21 +53,40 @@ public class MockServerConnection implements ServerConnection { private final Address remoteAddress; + private final UUID localUuid; + private UUID remoteUuid; + private final ServerConnectionManager connectionManager; private final ConcurrentMap attributeMap = new ConcurrentHashMap(); - public MockServerConnection(Address localAddress, - Address remoteAddress, NodeEngineImpl remoteNodeEngine) { - this(null, localAddress, remoteAddress, remoteNodeEngine, null); - } - - public MockServerConnection(ConnectionLifecycleListener lifecycleListener, Address localAddress, - Address remoteAddress, NodeEngineImpl remoteNodeEngine, - ServerConnectionManager localConnectionManager) { + public MockServerConnection( + Address localAddress, + Address remoteAddress, + UUID localUuid, + UUID remoteUuid, + NodeEngineImpl localNodeEngine, + NodeEngineImpl remoteNodeEngine + ) { + this(null, localAddress, remoteAddress, localUuid, remoteUuid, localNodeEngine, remoteNodeEngine, null); + } + + public MockServerConnection( + ConnectionLifecycleListener lifecycleListener, + Address localAddress, + Address remoteAddress, + UUID localUuid, + UUID remoteUuid, + NodeEngineImpl localNodeEngine, + NodeEngineImpl remoteNodeEngine, + ServerConnectionManager localConnectionManager + ) { this.lifecycleListener = lifecycleListener; this.localAddress = localAddress; this.remoteAddress = remoteAddress; + this.localUuid = localUuid; + this.remoteUuid = remoteUuid; + this.localNodeEngine = localNodeEngine; this.remoteNodeEngine = remoteNodeEngine; this.connectionManager = localConnectionManager; } @@ -99,6 +120,11 @@ public Address getRemoteAddress() { return remoteAddress; } + @Override + public UUID getRemoteUuid() { + return remoteUuid; + } + public InetAddress getInetAddress() { try { return localAddress.getInetAddress(); @@ -155,7 +181,11 @@ public void close(String msg, Throwable cause) { if (!alive.compareAndSet(true, false)) { return; } - + if (localNodeEngine != null) { + localNodeEngine.getNode() + .getLocalAddressRegistry() + .tryRemoveRegistration(remoteUuid, remoteAddress); + } if (localConnection != null) { //this is a member-to-member connection localConnection.close(msg, cause); @@ -195,6 +225,11 @@ public InetSocketAddress getRemoteSocketAddress() { public void setRemoteAddress(Address remoteAddress) { } + @Override + public void setRemoteUuid(UUID remoteUuid) { + this.remoteUuid = remoteUuid; + } + @Override public boolean isAlive() { return alive.get() && !isTargetLeft(remoteNodeEngine.getNode()); @@ -203,8 +238,8 @@ public boolean isAlive() { @Override public String toString() { return "MockConnection{" - + "localEndpoint=" + localAddress - + ", remoteEndpoint=" + remoteAddress + + "localEndpoint=[address=" + localAddress + ", uuid=" + localUuid + "]" + + ", remoteEndpoint=[address=" + remoteAddress + ", uuid=" + remoteUuid + "]" + ", alive=" + isAlive() + '}'; } } diff --git a/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/StaticAddressPicker.java b/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/StaticAddressPicker.java index e78a49c4f502..cd9afcc13d51 100644 --- a/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/StaticAddressPicker.java +++ b/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/StaticAddressPicker.java @@ -54,6 +54,13 @@ public Map getPublicAddressMap() { return publicAddressMap; } + @Override + public Map getBindAddressMap() { + HashMap bindAddressMap = new HashMap<>(); + bindAddressMap.put(MEMBER, thisAddress); + return bindAddressMap; + } + @Override public ServerSocketChannel getServerSocketChannel(EndpointQualifier qualifier) { return null; diff --git a/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/TestNodeRegistry.java b/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/TestNodeRegistry.java index c6962002a59e..fcb087c6aed0 100644 --- a/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/TestNodeRegistry.java +++ b/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/TestNodeRegistry.java @@ -32,7 +32,9 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -105,6 +107,20 @@ public Collection getAllHazelcastInstances() { return all; } + public UUID uuidOf(Address address) { + Node node = nodes.get(address); + return node != null ? node.getThisUuid() : null; + } + + public Address addressOf(UUID memberUuid) { + Optional
memberAddress = nodes.entrySet() + .stream() + .filter(entry -> entry.getValue().getThisUuid() == memberUuid) + .map(Map.Entry::getKey) + .findAny(); + return memberAddress.orElse(null); + } + public void shutdown() { shutdown(false); } diff --git a/hazelcast/src/test/java/com/hazelcast/wan/WanServiceMockingNodeContext.java b/hazelcast/src/test/java/com/hazelcast/wan/WanServiceMockingNodeContext.java index c5536b3df097..ab4cfe0fc7ae 100644 --- a/hazelcast/src/test/java/com/hazelcast/wan/WanServiceMockingNodeContext.java +++ b/hazelcast/src/test/java/com/hazelcast/wan/WanServiceMockingNodeContext.java @@ -21,6 +21,7 @@ import com.hazelcast.instance.impl.NodeContext; import com.hazelcast.instance.impl.NodeExtension; import com.hazelcast.internal.cluster.Joiner; +import com.hazelcast.internal.server.tcp.LocalAddressRegistry; import com.hazelcast.internal.server.tcp.ServerSocketRegistry; import com.hazelcast.internal.server.Server; @@ -53,7 +54,7 @@ public Joiner createJoiner(Node node) { } @Override - public Server createServer(Node node, ServerSocketRegistry serverSocketRegistry) { - return this.nodeContextDelegate.createServer(node, serverSocketRegistry); + public Server createServer(Node node, ServerSocketRegistry serverSocketRegistry, LocalAddressRegistry addressRegistry) { + return this.nodeContextDelegate.createServer(node, serverSocketRegistry, addressRegistry); } } diff --git a/hazelcast/src/test/resources/log4j2-trace-hostname-join.xml b/hazelcast/src/test/resources/log4j2-trace-hostname-join.xml new file mode 100644 index 000000000000..ad152dc598c7 --- /dev/null +++ b/hazelcast/src/test/resources/log4j2-trace-hostname-join.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + +