Skip to content

Commit

Permalink
Add SplitBrainHandler blacklist check regression test for TcpIpJoiner
Browse files Browse the repository at this point in the history
To conduct this test thoroughly, it was necessary to extract part of
the `TcpIpJoiner#searchForOtherClusters` function into a separate
function `TcpIpJoiner#getFilteredPossibleAddresses` - this function
reduces the results of `AbstractJoiner#getPossibleAddresses` (applying
filters such as removing known and blacklisted addresses), and does
not change any functionality, it just exposes this list more easily.
  • Loading branch information
JamesHazelcast committed Jun 21, 2023
1 parent bd60822 commit 9d9dc2a
Show file tree
Hide file tree
Showing 2 changed files with 64 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -465,7 +465,7 @@ public void onMemberAdded(Member member) {
Boolean previousBlacklistStatus = blacklistedAddresses.remove(member.getAddress());
if (previousBlacklistStatus != null) {
logger.info(member.getAddress() + " is removed from the " + (previousBlacklistStatus ? "permanent " : " ")
+ "blacklist due to successfully joining the cluster ");
+ "blacklist due to successfully joining the cluster");
}
}
}
Expand All @@ -480,14 +480,13 @@ protected void addTemporaryMemberAddress(Address memberAddress) {
knownMemberAddresses.put(memberAddress, Clock.currentTimeMillis());
}

@Override
public void searchForOtherClusters() {
public Collection<Address> getFilteredPossibleAddresses() {
final Collection<Address> possibleAddresses;
try {
possibleAddresses = getPossibleAddresses();
} catch (Throwable e) {
logger.severe(e);
return;
return Collections.emptyList();
}

// Remove known addresses from possibleAddresses
Expand All @@ -513,6 +512,12 @@ public void searchForOtherClusters() {
.map(Map.Entry::getKey)
.forEach(possibleAddresses::remove);

return possibleAddresses;
}

@Override
public void searchForOtherClusters() {
final Collection<Address> possibleAddresses = getFilteredPossibleAddresses();
if (possibleAddresses.isEmpty()) {
return;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,16 @@
import com.hazelcast.config.JoinConfig;
import com.hazelcast.config.ListenerConfig;
import com.hazelcast.config.NetworkConfig;
import com.hazelcast.config.TcpIpConfig;
import com.hazelcast.core.Hazelcast;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.LifecycleEvent;
import com.hazelcast.core.LifecycleEvent.LifecycleState;
import com.hazelcast.core.LifecycleListener;
import com.hazelcast.instance.FirewallingNodeContext;
import com.hazelcast.instance.impl.HazelcastInstanceFactory;
import com.hazelcast.internal.cluster.Joiner;
import com.hazelcast.internal.cluster.impl.TcpIpJoiner;
import com.hazelcast.internal.util.Clock;
import com.hazelcast.internal.util.RuntimeAvailableProcessors;
import com.hazelcast.spi.merge.PassThroughMergePolicy;
Expand Down Expand Up @@ -750,6 +753,58 @@ public void stateChanged(LifecycleEvent event) {
assertMasterAddress(getAddress(hz3), hz1, hz2, hz3);
}

@Test
// https://github.com/hazelcast/hazelcast/pull/24830
public void testBlacklistsConsideredAndRecovered_WhenUsingTcpIpJoiner() {
Config configA = createSimpleTcpIpConfig("cluster_A");
Config configB = createSimpleTcpIpConfig("cluster_B");

// Start 2 clusters, with 2 members each
HazelcastInstance a1 = Hazelcast.newHazelcastInstance(configA);
HazelcastInstance a2 = Hazelcast.newHazelcastInstance(configA);
HazelcastInstance b1 = Hazelcast.newHazelcastInstance(configB);
HazelcastInstance b2 = Hazelcast.newHazelcastInstance(configB);

// Fetch the port of a2
Address a2Address = getNode(a2).getThisAddress();

// Confirm that A2 is blacklisted by B1 after a SplitBrainHandler run (due to cluster mismatch)
Joiner joinerB1 = getNode(b1).getJoiner();
assertTrueEventually("Node A2 has not been blacklisted by Node B1!",
() -> assertTrue(joinerB1.isBlacklisted(a2Address)), 6);

// Confirm that A2's address is not returned in the list of filtered addresses used in SplitBrainHandler
assertFalse(((TcpIpJoiner) joinerB1).getFilteredPossibleAddresses().contains(a2Address));

// Shutdown A2, and start B3, which should be on the same address/port
a2.shutdown();
HazelcastInstance b3 = Hazelcast.newHazelcastInstance(configB);

// Confirm b3 is running on the address/port previously occupied by a2
Address b3Address = getNode(b3).getThisAddress();
assertEquals(a2Address, b3Address);

// Confirm B3 joins the cluster with B1/B2
assertClusterSizeEventually(3, b1, b2, b3);

// Confirm that B3's address is not blacklisted on B1, allowing SplitBrainHandler checks
assertTrueEventually("Node B3 is still blacklisted by Node B1!",
() -> assertFalse(joinerB1.isBlacklisted(b3Address)), 6);
}

private Config createSimpleTcpIpConfig(String clusterName) {
Config config = new Config();
config.setClusterName(clusterName);
TcpIpConfig tcpIpConfig = config.getNetworkConfig().getJoin().getTcpIpConfig();
tcpIpConfig.setEnabled(true);
tcpIpConfig.setMembers(List.of("127.0.0.1"));
config.getNetworkConfig().setPortAutoIncrement(true);
config.getNetworkConfig().getJoin().setTcpIpConfig(tcpIpConfig);
config.setProperty("hazelcast.merge.first.run.delay.seconds", "3");
config.setProperty("hazelcast.merge.next.run.delay.seconds", "3");
return config;
}

public static class MergedEventLifeCycleListener implements LifecycleListener {

private final CountDownLatch mergeLatch;
Expand Down

0 comments on commit 9d9dc2a

Please sign in to comment.