Skip to content

Commit

Permalink
Add an option for choosing zone id for shuffle
Browse files Browse the repository at this point in the history 
Add a new option that lets the user choose if they want the
shuffle to operate only on one of the zone ids.
  • Loading branch information
@voldemort @jayjwylie
voldemort authored and jayjwylie committed Jun 20, 2013
1 parent c1fb3aa commit ba371fc
Show file tree
Hide file tree
Showing 4 changed files with 40 additions and 2 deletions.
14 changes: 12 additions & 2 deletions src/java/voldemort/tools/Repartitioner.java
View file
Expand Up @@ -129,6 +129,7 @@ public class Repartitioner {
* @param enableRandomSwaps Enables random swap optimization.
* @param randomSwapAttempts
* @param randomSwapSuccesses
* @param randomSwapZoneIds
* @param enableGreedySwaps Enables greedy swap optimization.
* @param greedyZoneIds
* @param greedySwapAttempts
Expand All @@ -150,6 +151,7 @@ public static Cluster repartition(final Cluster currentCluster,
final boolean enableRandomSwaps,
final int randomSwapAttempts,
final int randomSwapSuccesses,
final List<Integer> randomSwapZoneIds,
final boolean enableGreedySwaps,
final List<Integer> greedyZoneIds,
final int greedySwapAttempts,
Expand Down Expand Up @@ -182,6 +184,7 @@ public static Cluster repartition(final Cluster currentCluster,
nextCandidateCluster = randomShufflePartitions(nextCandidateCluster,
randomSwapAttempts,
randomSwapSuccesses,
randomSwapZoneIds,
finalStoreDefs);
}
if(enableGreedySwaps) {
Expand Down Expand Up @@ -628,10 +631,17 @@ public static Cluster swapRandomPartitionsWithinZone(final Cluster nextCandidate
public static Cluster randomShufflePartitions(final Cluster nextCandidateCluster,
final int randomSwapAttempts,
final int randomSwapSuccesses,
final List<Integer> randomSwapZoneIds,
List<StoreDefinition> storeDefs) {
List<Integer> zoneIds = new ArrayList<Integer>(nextCandidateCluster.getZoneIds());
Cluster returnCluster = ClusterUtils.copyCluster(nextCandidateCluster);

List<Integer> zoneIds = null;
if (randomSwapZoneIds.isEmpty()) {
zoneIds = new ArrayList<Integer>(nextCandidateCluster.getZoneIds());
} else {
zoneIds = new ArrayList<Integer>(randomSwapZoneIds);
}

Cluster returnCluster = ClusterUtils.copyCluster(nextCandidateCluster);
double currentUtility = new PartitionBalance(returnCluster, storeDefs).getUtility();

int successes = 0;
Expand Down
11 changes: 11 additions & 0 deletions src/java/voldemort/tools/RepartitionerCLI.java
View file
Expand Up @@ -18,6 +18,7 @@

import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Set;

Expand Down Expand Up @@ -102,6 +103,12 @@ private static void setupParser() {
.withRequiredArg()
.ofType(Integer.class)
.describedAs("num-successes");
parser.accepts("random-swap-zoneids",
"Comma separated zone ids that you want to shuffle. [Default:Shuffle all zones.]")
.withRequiredArg()
.describedAs("zoneids-to-shuffle")
.withValuesSeparatedBy(',')
.ofType(Integer.class);
parser.accepts("enable-greedy-swaps",
"Enable attempts to improve balance by greedily swapping (random) partitions within a zone. [Default: disabled]");
parser.accepts("greedy-swap-attempts",
Expand Down Expand Up @@ -150,6 +157,7 @@ private static void printUsage() {
help.append(" --enable-random-swaps [ Attempt to randomly swap partitions to improve balance ] \n");
help.append(" --random-swap-attempts num-attempts [ Number of random swaps to attempt in hopes of improving balance ] \n");
help.append(" --random-swap-successes num-successes [ Stop after num-successes successful random swap atttempts ] \n");
help.append(" --random-swap-zoneids zoneId(s) [Only swaps partitions within the specified zone(s)] \n");
help.append(" --enable-greedy-swaps [ Attempt to greedily (randomly) swap partitions to improve balance. Greedily/randomly means sample many swaps for each node and choose best swap. ] \n");
help.append(" --greedy-swap-attempts num-attempts [ Number of greedy swap passes to attempt. Each pass can be fairly expensive. ] \n");
help.append(" --greedy-max-partitions-per-node num-partitions [ num-partitions per node to consider in each greedy pass. Partitions selected randomly from each node. ] \n");
Expand Down Expand Up @@ -238,6 +246,8 @@ public static void main(String[] args) throws Exception {
int randomSwapSuccesses = CmdUtils.valueOf(options,
"random-swap-successes",
Repartitioner.DEFAULT_RANDOM_SWAP_SUCCESSES);
List<Integer> randomSwapZoneIds = CmdUtils.valuesOf(options, "random-swap-zoneids",
Collections.<Integer>emptyList());
boolean enableGreedySwaps = options.has("enable-greedy-swaps");
int greedySwapAttempts = CmdUtils.valueOf(options,
"greedy-swap-attempts",
Expand Down Expand Up @@ -280,6 +290,7 @@ public static void main(String[] args) throws Exception {
enableRandomSwaps,
randomSwapAttempts,
randomSwapSuccesses,
randomSwapZoneIds,
enableGreedySwaps,
greedyZoneIds,
greedySwapAttempts,
Expand Down
9 changes: 9 additions & 0 deletions src/java/voldemort/utils/CmdUtils.java
View file
Expand Up @@ -2,6 +2,7 @@

import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import joptsimple.OptionParser;
Expand Down Expand Up @@ -49,6 +50,14 @@ public static <T> T valueOf(OptionSet options, OptionSpec<T> opt, T defaultValue
return defaultValue;
}

@SuppressWarnings("unchecked")
public static <T> List<T> valuesOf(OptionSet options, String opt, List<T> defaultList) {
if (options.has(opt) && options.valuesOf(opt) != null)
return (List<T>) options.valuesOf(opt);
else
return defaultList;
}

public static void croakIfMissing(OptionParser parser, OptionSet options, String... required) {
Set<String> missing = CmdUtils.missing(options, required);
if(missing.size() > 0) {
Expand Down
8 changes: 8 additions & 0 deletions test/unit/voldemort/tools/RepartitionerTest.java
View file
Expand Up @@ -112,6 +112,7 @@ public void verifyBalanceZoneAndNode(Cluster currentCluster,
false,
0,
0,
null,
false,
null,
0,
Expand Down Expand Up @@ -156,6 +157,7 @@ public void verifyBalanceNodesNotZones(Cluster currentCluster,
false,
0,
0,
null,
false,
null,
0,
Expand Down Expand Up @@ -197,6 +199,7 @@ public void verifyRepartitionNoop(Cluster currentCluster,
false,
0,
0,
null,
false,
null,
0,
Expand Down Expand Up @@ -237,6 +240,7 @@ public void verifyRandomSwapsImproveBalance(Cluster currentCluster,
enableRandomSwaps,
swapAttempts,
swapSuccesses,
null,
false,
null,
0,
Expand Down Expand Up @@ -287,6 +291,7 @@ public void verifyGreedySwapsImproveBalance(Cluster currentCluster,
false,
0,
0,
null,
enableGreedySwaps,
greedyZones,
swapAttempts,
Expand Down Expand Up @@ -316,6 +321,7 @@ public void verifyGreedySwapsImproveBalance(Cluster currentCluster,
false,
0,
0,
null,
enableGreedySwaps,
greedyZones,
swapAttempts,
Expand Down Expand Up @@ -345,6 +351,7 @@ public void verifyGreedySwapsImproveBalance(Cluster currentCluster,
false,
0,
0,
null,
enableGreedySwaps,
greedyZones,
swapAttempts,
Expand Down Expand Up @@ -479,6 +486,7 @@ public void decontigRepartition(Cluster currentCluster, List<StoreDefinition> cu
false,
0,
0,
null,
false,
null,
0,
Expand Down

0 comments on commit ba371fc

Please sign in to comment.