Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Added simulation mode to migrate partition tool

  • Loading branch information...
commit 0c9fec94020e2fadedda033fc264da26717b58d0 1 parent 9169b88
@rsumbaly rsumbaly authored afeinberg committed
View
71 src/java/voldemort/client/rebalance/MigratePartitions.java
@@ -45,6 +45,28 @@
private final HashMap<Integer, List<RebalancePartitionsInfo>> donorNodePlans;
private final HashMap<Integer, Versioned<String>> donorStates;
private final boolean transitionToNormal;
+ private boolean simulation = false;
+
+ public MigratePartitions(Cluster currentCluster,
+ Cluster targetCluster,
+ List<StoreDefinition> currentStoreDefs,
+ List<StoreDefinition> targetStoreDefs,
+ AdminClient adminClient,
+ VoldemortConfig voldemortConfig,
+ List<Integer> stealerNodeIds,
+ boolean transitionToNormal,
+ boolean simulation) {
+
+ this(currentCluster,
+ targetCluster,
+ currentStoreDefs,
+ targetStoreDefs,
+ adminClient,
+ voldemortConfig,
+ stealerNodeIds,
+ transitionToNormal);
+ this.simulation = simulation;
+ }
/**
*
@@ -78,6 +100,8 @@ public MigratePartitions(Cluster currentCluster,
false,
null);
+ logger.info("Rebalance cluster plan => " + plan);
+
this.stealerNodePlans = plan.getRebalancingTaskQueuePerNode();
if(this.stealerNodeIds == null) {
this.stealerNodeIds = Lists.newArrayList(stealerNodePlans.keySet());
@@ -105,8 +129,12 @@ public MigratePartitions(Cluster currentCluster,
this.donorStates = Maps.newHashMap();
for(int donorNodeId: donorNodePlans.keySet()) {
- logger.info("Plan for donor node id " + donorNodeId + " - "
- + donorNodePlans.get(donorNodeId));
+ logger.info("Plan for donor node id " + donorNodeId + " => ");
+ List<RebalancePartitionsInfo> list = donorNodePlans.get(donorNodeId);
+ for(RebalancePartitionsInfo stealInfo: list) {
+ logger.info(stealInfo);
+ }
+ logger.info("===============================================");
}
}
@@ -125,14 +153,17 @@ public MigratePartitions(Cluster currentCluster,
public void changeToGrandfather() {
for(int donorNodeId: donorNodePlans.keySet()) {
logger.info("Transitioning " + donorNodeId + " to grandfathering state");
- Versioned<String> serverState = adminClient.updateGrandfatherMetadata(donorNodeId,
- donorNodePlans.get(donorNodeId));
- if(!VoldemortState.valueOf(serverState.getValue())
- .equals(MetadataStore.VoldemortState.GRANDFATHERING_SERVER)) {
- throw new VoldemortException("Node " + donorNodeId
- + " is not in normal state to perform grandfathering");
+ if(!simulation) {
+ Versioned<String> serverState = adminClient.updateGrandfatherMetadata(donorNodeId,
+ donorNodePlans.get(donorNodeId));
+ if(!VoldemortState.valueOf(serverState.getValue())
+ .equals(MetadataStore.VoldemortState.GRANDFATHERING_SERVER)) {
+ throw new VoldemortException("Node "
+ + donorNodeId
+ + " is not in normal state to perform grandfathering");
+ }
+ donorStates.put(donorNodeId, serverState);
}
- donorStates.put(donorNodeId, serverState);
logger.info("Successfully transitioned " + donorNodeId + " to grandfathering state");
}
}
@@ -191,13 +222,14 @@ public void migrate() {
HashMap<Integer, Integer> nodeIdToRequestId = Maps.newHashMap();
for(RebalancePartitionsInfo r: partitionInfo) {
- logger.info("-- Started migration for donor node id " + r.getDonorId());
- nodeIdToRequestId.put(r.getDonorId(),
- adminClient.migratePartitions(r.getDonorId(),
- stealerNodeId,
- storeName,
- r.getPartitionList(),
- null));
+ logger.info("-- Started migration for donor node id " + r);
+ if(!simulation)
+ nodeIdToRequestId.put(r.getDonorId(),
+ adminClient.migratePartitions(r.getDonorId(),
+ stealerNodeId,
+ storeName,
+ r.getPartitionList(),
+ null));
}
@@ -211,7 +243,7 @@ public void migrate() {
logger.info("-- Completed migration for donor node id " + nodeId);
}
}
-
+ logger.info("===============================================");
}
} finally {
@@ -245,6 +277,7 @@ public static void main(String[] args) throws IOException {
.withValuesSeparatedBy(',');
parser.accepts("transition-to-normal",
"At the end of migration do we want to transition back to normal state? [Default-false]");
+ parser.accepts("simulation", "Run the full process as simulation");
OptionSet options = parser.parse(args);
@@ -268,6 +301,7 @@ public static void main(String[] args) throws IOException {
String currentStoresFile = (String) options.valueOf("stores-xml");
String targetStoresFile = currentStoresFile;
boolean transitionToNormal = options.has("transition-to-normal");
+ boolean simulation = options.has("simulation");
if(options.has("target-stores-xml")) {
targetStoresFile = (String) options.valueOf("target-stores-xml");
@@ -304,7 +338,8 @@ public static void main(String[] args) throws IOException {
adminClient,
voldemortConfig,
stealerNodeIds,
- transitionToNormal);
+ transitionToNormal,
+ simulation);
migratePartitions.migrate();
} catch(Exception e) {
View
4 src/java/voldemort/store/grandfather/GrandfatheringStore.java
@@ -110,7 +110,7 @@ public void run() {
null);
} catch(Exception e) {
logger.warn("Failed to put DELETE operation on " + getName()
- + " to node " + futureNodeId + " to slop store");
+ + " to node " + futureNodeId + " to slop store", e);
}
}
} catch(Exception e) {
@@ -167,7 +167,7 @@ public void run() {
null);
} catch(Exception e) {
logger.warn("Failed to put PUT operation on " + getName()
- + " to node " + futureNodeId + " to slop store");
+ + " to node " + futureNodeId + " to slop store", e);
}
}
} catch(Exception e) {
View
39 src/java/voldemort/utils/KeyDistributionGenerator.java
@@ -63,20 +63,24 @@ public static void main(String args[]) throws IOException {
List<StoreDefinition> storeDef = new StoreDefinitionsMapper().readStoreList(new File(storesXml));
// Print distribution for every store
-
for(StoreDefinition def: storeDef) {
- HashMap<Integer, Integer> storeDistribution = generateDistribution(cluster,
- def,
- numKeys);
+ HashMap<Integer, Double> storeDistribution = generateDistribution(cluster, def, numKeys);
System.out.println("For Store " + def.getName());
printDistribution(storeDistribution);
+ System.out.println("Std dev - " + getStdDeviation(storeDistribution));
System.out.println("=========================");
}
}
- public static HashMap<Integer, Integer> generateDistribution(Cluster cluster,
- StoreDefinition storeDef,
- int numKeys) {
+ /**
+ * @param cluster The cluster metadata
+ * @param storeDef The store definition metadata
+ * @param numKeys Number of keys used to generate distribution
+ * @return Map of node id to their corresponding %age distribution
+ */
+ public static HashMap<Integer, Double> generateDistribution(Cluster cluster,
+ StoreDefinition storeDef,
+ int numKeys) {
RoutingStrategyFactory factory = new RoutingStrategyFactory();
RoutingStrategy strategy = factory.updateRoutingStrategy(storeDef, cluster);
@@ -94,28 +98,33 @@ public static void main(String args[]) throws IOException {
total++;
}
}
- HashMap<Integer, Integer> finalDistribution = Maps.newHashMap();
+ HashMap<Integer, Double> finalDistribution = Maps.newHashMap();
for(int nodeId: requestRouting.keySet()) {
- finalDistribution.put(nodeId,
- new Integer((int) ((requestRouting.get(nodeId) * 100) / total)));
+ finalDistribution.put(nodeId, new Double((requestRouting.get(nodeId) * 100.0) / total));
}
return finalDistribution;
}
- public static void printDistribution(HashMap<Integer, Integer> distribution) {
+ public static void printDistribution(HashMap<Integer, Double> distribution) {
for(int nodeId: distribution.keySet()) {
System.out.println("Node " + nodeId + " - " + distribution.get(nodeId));
}
}
- public static double getStdDeviation(HashMap<Integer, Integer> distribution) {
- long sum = 0, squareSum = 0;
- for(int num: distribution.values()) {
+ public static double getStdDeviation(HashMap<Integer, Double> distribution) {
+ HashMap<Integer, Double> offBy = Maps.newHashMap();
+ int numberOfNodes = distribution.keySet().size();
+ double distributionPerNode = 100.0 / numberOfNodes;
+
+ for(Integer nodeId: distribution.keySet()) {
+ offBy.put(nodeId, new Double(distributionPerNode - distribution.get(nodeId)));
+ }
+ double sum = 0, squareSum = 0;
+ for(double num: offBy.values()) {
squareSum += num * num;
sum += num;
}
double mean = sum / distribution.size();
return Math.sqrt(squareSum / distribution.size() - mean * mean);
}
-
}
Please sign in to comment.
Something went wrong with that request. Please try again.