Skip to content

Commit

Permalink
Refactoring. Renamed RebalanceClusterUtils->RepartitionUtils and move…
Browse files Browse the repository at this point in the history 
…d some tests into new file RepartitionUtilsTest.
  • Loading branch information
@jayjwylie
jayjwylie committed Jun 20, 2013
1 parent 92c1139 commit ce5a73e
Show file tree
Hide file tree
Showing 5 changed files with 284 additions and 243 deletions.
34 changes: 17 additions & 17 deletions src/java/voldemort/tools/RepartitionCLI.java
View file
Expand Up @@ -30,7 +30,7 @@
import voldemort.cluster.Cluster;
import voldemort.store.StoreDefinition;
import voldemort.utils.CmdUtils;
import voldemort.utils.RebalanceClusterUtils;
import voldemort.utils.RepartitionUtils;
import voldemort.utils.Utils;
import voldemort.xml.ClusterMapper;
import voldemort.xml.StoreDefinitionsMapper;
Expand Down Expand Up @@ -250,22 +250,22 @@ public static void main(String[] args) throws Exception {
printUsageAndDie("Provided arguments for generate greedy swaps but did not enable the feature");
}

RebalanceClusterUtils.balanceTargetCluster(currentCluster,
currentStoreDefs,
targetCluster,
targetStoreDefs,
outputDir,
attempts,
disableNodeBalancing,
disableZoneBalancing,
enableRandomSwaps,
randomSwapAttempts,
randomSwapSuccesses,
enableGreedySwaps,
greedySwapAttempts,
greedyMaxPartitionsPerNode,
greedyMaxPartitionsPerZone,
maxContiguousPartitionsPerZone);
RepartitionUtils.repartition(currentCluster,
currentStoreDefs,
targetCluster,
targetStoreDefs,
outputDir,
attempts,
disableNodeBalancing,
disableZoneBalancing,
enableRandomSwaps,
randomSwapAttempts,
randomSwapSuccesses,
enableGreedySwaps,
greedySwapAttempts,
greedyMaxPartitionsPerNode,
greedyMaxPartitionsPerZone,
maxContiguousPartitionsPerZone);

}
}
49 changes: 23 additions & 26 deletions ...oldemort/utils/RebalanceClusterUtils.java → ...ava/voldemort/utils/RepartitionUtils.java
View file
Expand Up @@ -37,16 +37,13 @@
import com.google.common.collect.Maps;

/**
* RebalanceClusterUtils provides functions that balance the distribution of
* RepartitionUtils provides functions that balance the distribution of
* partitions across a cluster.
*
*/
public class RebalanceClusterUtils {
public class RepartitionUtils {

// TODO: (refactor) Rename to "RepartitionUtils" and rename
// balanceTargetCluster to "repartition".

private static Logger logger = Logger.getLogger(RebalanceClusterUtils.class);
private static Logger logger = Logger.getLogger(RepartitionUtils.class);

/**
* Outputs an optimized cluster based on the existing cluster and the new
Expand All @@ -70,22 +67,22 @@ public class RebalanceClusterUtils {
* @param greedySwapMaxPartitionsPerZone See RebalanceCLI.
* @param maxContiguousPartitionsPerZone See RebalanceCLI.
*/
public static void balanceTargetCluster(final Cluster currentCluster,
final List<StoreDefinition> currentStoreDefs,
final Cluster targetCluster,
final List<StoreDefinition> targetStoreDefs,
final String outputDir,
final int attempts,
final boolean disableNodeBalancing,
final boolean disableZoneBalancing,
final boolean enableRandomSwaps,
final int randomSwapAttempts,
final int randomSwapSuccesses,
final boolean enableGreedySwaps,
final int greedySwapAttempts,
final int greedySwapMaxPartitionsPerNode,
final int greedySwapMaxPartitionsPerZone,
final int maxContiguousPartitionsPerZone) {
public static void repartition(final Cluster currentCluster,
final List<StoreDefinition> currentStoreDefs,
final Cluster targetCluster,
final List<StoreDefinition> targetStoreDefs,
final String outputDir,
final int attempts,
final boolean disableNodeBalancing,
final boolean disableZoneBalancing,
final boolean enableRandomSwaps,
final int randomSwapAttempts,
final int randomSwapSuccesses,
final boolean enableGreedySwaps,
final int greedySwapAttempts,
final int greedySwapMaxPartitionsPerNode,
final int greedySwapMaxPartitionsPerZone,
final int maxContiguousPartitionsPerZone) {
PartitionBalance partitionBalance = new ClusterInstance(currentCluster, currentStoreDefs).getPartitionBalance();
dumpAnalysisToFile(outputDir,
RebalanceUtils.initialClusterFileName + ".analysis",
Expand Down Expand Up @@ -621,18 +618,18 @@ public static Cluster randomShufflePartitions(final Cluster targetCluster,
Cluster returnCluster = ClusterUtils.copyCluster(targetCluster);

double currentUtility = new ClusterInstance(returnCluster, storeDefs).getPartitionBalance()
.getUtility();
.getUtility();

int successes = 0;
for(int i = 0; i < randomSwapAttempts; i++) {
Collections.shuffle(zoneIds, new Random(System.currentTimeMillis()));
for(Integer zoneId: zoneIds) {
Cluster shuffleResults = swapRandomPartitionsWithinZone(returnCluster, zoneId);
double nextUtility = new ClusterInstance(shuffleResults, storeDefs).getPartitionBalance()
.getUtility();
.getUtility();
if(nextUtility < currentUtility) {
System.out.println("Swap improved max-min ratio: " + currentUtility
+ " -> " + nextUtility + " (improvement " + successes
System.out.println("Swap improved max-min ratio: " + currentUtility + " -> "
+ nextUtility + " (improvement " + successes
+ " on swap attempt " + i + " in zone " + zoneId + ")");
successes++;
returnCluster = shuffleResults;
Expand Down
3 changes: 3 additions & 0 deletions test/unit/voldemort/utils/ClusterInstanceTest.java
View file
Expand Up @@ -62,6 +62,9 @@
*/
public class ClusterInstanceTest {

// TODO: Rename class/file to PartitionBalanceTest. Change tests to directly
// use PartitionBalance rather than go through ClusterInstance.

public List<StoreDefinition> getZZ111StoreDefs() {

List<StoreDefinition> storeDefs = new LinkedList<StoreDefinition>();
Expand Down
200 changes: 0 additions & 200 deletions test/unit/voldemort/utils/RebalanceUtilsTest.java
View file
Expand Up @@ -79,204 +79,4 @@ public void testGetClusterWithNewNodes() {

}

// TODO: Move to a new class : RepartitionUtilsTest
public void testRemoveItemsToSplitListEvenly() {
// input of size 5
List<Integer> input = new ArrayList<Integer>();
System.out.println("Input of size 5");
for(int i = 0; i < 5; ++i) {
input.add(i);
}

List<Integer> output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 1);
assertEquals(output.size(), 2);
assertEquals(output.get(0), new Integer(1));
assertEquals(output.get(1), new Integer(3));
System.out.println("1 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 2);
assertEquals(output.size(), 1);
assertEquals(output.get(0), new Integer(2));
System.out.println("2 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 3);
assertEquals(output.size(), 1);
assertEquals(output.get(0), new Integer(2));
System.out.println("3 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 4);
assertEquals(output.size(), 1);
assertEquals(output.get(0), new Integer(2));
System.out.println("4 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 5);
assertEquals(output.size(), 0);
System.out.println("5 : " + output);

// input of size 10
input.clear();
System.out.println("Input of size 10");
for(int i = 0; i < 10; ++i) {
input.add(i);
}

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 1);
assertEquals(output.size(), 5);
assertEquals(output.get(0), new Integer(1));
assertEquals(output.get(4), new Integer(9));
System.out.println("1 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 2);
assertEquals(output.size(), 3);
assertEquals(output.get(0), new Integer(2));
assertEquals(output.get(2), new Integer(8));
System.out.println("2 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 3);
assertEquals(output.size(), 2);
assertEquals(output.get(0), new Integer(3));
assertEquals(output.get(1), new Integer(7));
System.out.println("3 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 4);
assertEquals(output.size(), 2);
assertEquals(output.get(0), new Integer(3));
assertEquals(output.get(1), new Integer(7));
System.out.println("4 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 5);
assertEquals(output.size(), 1);
assertEquals(output.get(0), new Integer(5));
System.out.println("5 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 6);
assertEquals(output.size(), 1);
assertEquals(output.get(0), new Integer(5));
System.out.println("6 : " + output);

// input of size 20
input.clear();
System.out.println("Input of size 20");
for(int i = 0; i < 20; ++i) {
input.add(i);
}

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 1);
assertEquals(output.size(), 10);
assertEquals(output.get(0), new Integer(1));
assertEquals(output.get(9), new Integer(19));
System.out.println("1 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 2);
assertEquals(output.size(), 6);
assertEquals(output.get(0), new Integer(2));
assertEquals(output.get(5), new Integer(17));
System.out.println("2 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 3);
assertEquals(output.size(), 5);
assertEquals(output.get(0), new Integer(3));
assertEquals(output.get(4), new Integer(17));
System.out.println("3 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 4);
assertEquals(output.size(), 4);
assertEquals(output.get(0), new Integer(4));
assertEquals(output.get(3), new Integer(16));
System.out.println("4 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 5);
assertEquals(output.size(), 3);
assertEquals(output.get(0), new Integer(5));
assertEquals(output.get(2), new Integer(15));
System.out.println("5 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 6);
assertEquals(output.size(), 2);
assertEquals(output.get(0), new Integer(6));
assertEquals(output.get(1), new Integer(13));
System.out.println("6 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 7);
assertEquals(output.size(), 2);
assertEquals(output.get(0), new Integer(6));
assertEquals(output.get(1), new Integer(13));
System.out.println("7 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 9);
assertEquals(output.size(), 2);
assertEquals(output.get(0), new Integer(6));
assertEquals(output.get(1), new Integer(13));
System.out.println("9 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 10);
assertEquals(output.size(), 1);
assertEquals(output.get(0), new Integer(10));
System.out.println("10 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 11);
assertEquals(output.size(), 1);
assertEquals(output.get(0), new Integer(10));
System.out.println("11 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 19);
assertEquals(output.size(), 1);
assertEquals(output.get(0), new Integer(10));
System.out.println("19 : " + output);

output = RebalanceClusterUtils.removeItemsToSplitListEvenly(input, 20);
assertEquals(output.size(), 0);
System.out.println("20 : " + output);
}

// TODO: Move to a new class : RepartitionUtilsTest
public void testPeanutButterList() {

List<Integer> pbList;

pbList = RebalanceClusterUtils.peanutButterList(4, 4);
assertEquals(pbList.size(), 4);
assertEquals(pbList.get(0), new Integer(1));
assertEquals(pbList.get(1), new Integer(1));
assertEquals(pbList.get(2), new Integer(1));
assertEquals(pbList.get(3), new Integer(1));

pbList = RebalanceClusterUtils.peanutButterList(4, 6);
assertEquals(pbList.size(), 4);
assertEquals(pbList.get(0), new Integer(2));
assertEquals(pbList.get(1), new Integer(2));
assertEquals(pbList.get(2), new Integer(1));
assertEquals(pbList.get(3), new Integer(1));

pbList = RebalanceClusterUtils.peanutButterList(4, 3);
assertEquals(pbList.size(), 4);
assertEquals(pbList.get(0), new Integer(1));
assertEquals(pbList.get(1), new Integer(1));
assertEquals(pbList.get(2), new Integer(1));
assertEquals(pbList.get(3), new Integer(0));

pbList = RebalanceClusterUtils.peanutButterList(4, 0);
assertEquals(pbList.size(), 4);
assertEquals(pbList.get(0), new Integer(0));
assertEquals(pbList.get(1), new Integer(0));
assertEquals(pbList.get(2), new Integer(0));
assertEquals(pbList.get(3), new Integer(0));

boolean caught = false;
try {
pbList = RebalanceClusterUtils.peanutButterList(0, 10);
} catch(IllegalArgumentException iae) {
caught = true;
}
assertTrue(caught);

caught = false;
try {
pbList = RebalanceClusterUtils.peanutButterList(4, -5);
} catch(IllegalArgumentException iae) {
caught = true;
}
assertTrue(caught);
}

}

0 comments on commit ce5a73e

Please sign in to comment.