Skip to content

Commit

Permalink
Completed unit tests for zone expansion
Browse files Browse the repository at this point in the history 
Added zone expansion unit tests to AbstractZonedRebalanceTest. This required tweaking the helper methods in RebalanceUtils and ClusterTestUtils.
  • Loading branch information
@jayjwylie
jayjwylie committed Jun 20, 2013
1 parent d344725 commit 10901f7
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 32 deletions.
23 changes: 12 additions & 11 deletions src/java/voldemort/utils/RebalanceUtils.java
View file
Expand Up @@ -421,6 +421,7 @@ public static void validateClusterNodeState(final Cluster subsetCluster,
}
}

// TODO: Can getInterimCluster and getClusterWithNewNodes be merged?
/**
* Given the current cluster and final cluster, generates an interim cluster
* with empty new nodes (and zones).
Expand Down Expand Up @@ -663,17 +664,17 @@ public static Map<Integer, Set<Pair<Integer, Integer>>> getNodeIdToAllPartitions
if((replicaPartitionList.size() % zonesWithPartitions != 0)
|| ((replicaPartitionList.size() / zonesWithPartitions) != (storeDef.getReplicationFactor() / cluster.getNumberOfZones()))) {

throw new VoldemortException("Number of replicas returned ("
+ replicaPartitionList.size()
+ ") does not make sense given the replication factor ("
+ storeDef.getReplicationFactor()
+ ") and that there are "
+ cluster.getNumberOfZones()
+ " zones of which "
+ zonesWithPartitions
+ " have partitions (and of which "
+ (cluster.getNumberOfZones() - zonesWithPartitions)
+ " are empty).");
// For zone expansion & shrinking, this warning is expected
// in some cases. For other use cases (shuffling, cluster
// expansion), this warning indicates that something
// is wrong between the clusters and store defs.
logger.warn("Number of replicas returned (" + replicaPartitionList.size()
+ ") does not make sense given the replication factor ("
+ storeDef.getReplicationFactor() + ") and that there are "
+ cluster.getNumberOfZones() + " zones of which "
+ zonesWithPartitions + " have partitions (and of which "
+ (cluster.getNumberOfZones() - zonesWithPartitions)
+ " are empty).");
}

int replicaType = 0;
Expand Down
15 changes: 15 additions & 0 deletions test/common/voldemort/ClusterTestUtils.java
View file
Expand Up @@ -669,4 +669,19 @@ public static RebalanceKit getRebalanceKit(String bootstrapUrl,
finalCluster);
}

public static RebalanceKit getRebalanceKit(String bootstrapUrl,
boolean stealerBased,
Cluster finalCluster,
List<StoreDefinition> finalStoreDefs) {
RebalanceController rebalanceController = new RebalanceController(bootstrapUrl,
RebalanceController.MAX_PARALLEL_REBALANCING,
RebalanceController.MAX_TRIES_REBALANCING,
RebalanceController.REBALANCING_CLIENT_TIMEOUT_SEC,
stealerBased);
RebalancePlan rebalancePlan = rebalanceController.getPlan(finalCluster,
finalStoreDefs,
RebalancePlan.BATCH_SIZE);
return new RebalanceKit(rebalanceController, rebalancePlan);
}

}
65 changes: 44 additions & 21 deletions test/unit/voldemort/client/rebalance/AbstractZonedRebalanceTest.java
View file
Expand Up @@ -240,39 +240,57 @@ public void tearDown() {
// TODO: The tests based on this method are susceptible to TOCTOU
// BindException issue since findFreePorts is used to determine the ports
// for localhost:PORT of each node.
/**
* Scripts the execution of a specific type of zoned rebalance test: sets up
* cluster based on cCluster plus any new nodes/zones in fCluster,
* rebalances to fCluster, verifies rebalance was correct.
*
* @param testTag For pretty printing
* @param cCluster current cluster
* @param fCluster final cluster
* @param cStoresXml XML file with current stores xml
* @param fStoresXml Unused parameter. Included for symmetry in method
* declaration.
* @param cStoreDefs store defs for current cluster (from on cStoresXml)
* @param fStoreDefs store defs for final cluster.
* @throws Exception
*/
public void testZonedRebalance(String testTag,
Cluster cCluster,
Cluster fCluster,
String storesXml,
List<StoreDefinition> storeDefs) throws Exception {
String cStoresXml,
String fStoresXml,
List<StoreDefinition> cStoreDefs,
List<StoreDefinition> fStoreDefs) throws Exception {
logger.info("Starting " + testTag);
// Hacky work around of TOCTOU bind Exception issues. Each test that
// invokes this method brings servers up & down on the same ports.
// invokes this method brings servers up & down on the same ports. The
// OS seems to need a rest between subsequent tests...
Thread.sleep(TimeUnit.SECONDS.toMillis(2));
try {
Cluster currentCluster = cCluster;
Cluster targetCluster = fCluster;
Cluster interimCluster = RebalanceUtils.getInterimCluster(cCluster, fCluster);

// start all the servers
List<Integer> serverList = new ArrayList<Integer>(currentCluster.getNodeIds());
List<Integer> serverList = new ArrayList<Integer>(interimCluster.getNodeIds());
Map<String, String> configProps = new HashMap<String, String>();
configProps.put("admin.max.threads", "5");
currentCluster = startServers(currentCluster, storesXml, serverList, configProps);
interimCluster = startServers(interimCluster, cStoresXml, serverList, configProps);

String bootstrapUrl = getBootstrapUrl(currentCluster, 0);
String bootstrapUrl = getBootstrapUrl(interimCluster, 0);
boolean stealerBased = !useDonorBased;
ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl,
stealerBased,
targetCluster);
fCluster,
fStoreDefs);

try {
for(StoreDefinition storeDef: storeDefs) {
populateData(currentCluster, storeDef);
for(StoreDefinition storeDef: cStoreDefs) {
populateData(cCluster, storeDef);
}

rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, serverList);

checkConsistentMetadata(targetCluster, serverList);
checkConsistentMetadata(fCluster, serverList);
} finally {
// stop servers
stopServer(serverList);
Expand All @@ -283,6 +301,14 @@ public void testZonedRebalance(String testTag,
}
}

public void testZonedRebalance(String testTag,
Cluster cCluster,
Cluster fCluster,
String storesXml,
List<StoreDefinition> storeDefs) throws Exception {
testZonedRebalance(testTag, cCluster, fCluster, storesXml, storesXml, storeDefs, storeDefs);
}

@Test(timeout = 600000)
public void testNoopZZ() throws Exception {
testZonedRebalance("TestNoopZZ", zzCurrent, zzCurrent, zzStoresXml, zzStores);
Expand All @@ -293,11 +319,10 @@ public void testShuffleZZ() throws Exception {
testZonedRebalance("TestShuffleZZ", zzCurrent, zzShuffle, zzStoresXml, zzStores);
}

// TODO: Ideally, zzCurrent would be passed instead of zzClusterExpansionNN.
@Test(timeout = 600000)
public void testClusterExpansionZZ() throws Exception {
public void testClusterExpansion() throws Exception {
testZonedRebalance("TestClusterExpansionZZ",
zzClusterExpansionNN,
zzCurrent,
zzClusterExpansionPP,
zzStoresXml,
zzStores);
Expand All @@ -313,25 +338,23 @@ public void testShuffleZZZ() throws Exception {
testZonedRebalance("TestShuffleZZZ", zzzCurrent, zzzShuffle, zzzStoresXml, zzzStores);
}

// TODO: Ideally, zzzCurrent would be passed instead of
// zzzClusterExpansionNNN.
@Test(timeout = 600000)
public void testClusterExpansionZZZ() throws Exception {
testZonedRebalance("TestClusterExpansionZZZ",
zzzClusterExpansionNNN,
zzzCurrent,
zzzClusterExpansionPPP,
zzzStoresXml,
zzzStores);
}

// TODO: Pass in zzCurrent and zzzZoneExpansionXXP after atomic metadata
// update is in place.
@Test(timeout = 600000)
public void testZoneExpansionZZ2ZZZ() throws Exception {
testZonedRebalance("TestZoneExpansionZZ2ZZZ",
zzeZoneExpansion,
zzCurrent,
zzzZoneExpansionXXP,
zzStoresXml,
zzzStoresXml,
zzStores,
zzzStores);
}

Expand Down

0 comments on commit 10901f7

Please sign in to comment.