Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merged ReadOnly store changes

  • Loading branch information...
commit 1860e5b1dcd4eac62de452f4cb484f1a1d231cbf 2 parents ad0a6b2 + 475b823
@rsumbaly authored
View
15 src/java/voldemort/store/readonly/ReadOnlyStorageEngine.java
@@ -66,6 +66,7 @@
private volatile ChunkedFileSet fileSet;
private volatile boolean isOpen;
private int deleteBackupMs = 0;
+ private long lastSwapped;
/**
* Create an instance of the store
@@ -161,7 +162,8 @@ public void open(File versionDir) {
Utils.symlink(versionDir.getAbsolutePath(), storeDir.getAbsolutePath() + File.separator
+ "latest");
this.fileSet = new ChunkedFileSet(versionDir, routingStrategy, nodeId);
- isOpen = true;
+ this.lastSwapped = System.currentTimeMillis();
+ this.isOpen = true;
} finally {
fileModificationLock.writeLock().unlock();
}
@@ -216,6 +218,17 @@ public ReadOnlyStorageFormat getReadOnlyStorageFormat() {
}
/**
+ * Time since last time the store was swapped
+ *
+ * @return Time in milliseconds since the store was swapped
+ */
+ @JmxGetter(name = "lastSwapped", description = "Time in milliseconds since the store was swapped")
+ public long getLastSwapped() {
+ long timeSinceLastSwap = System.currentTimeMillis() - lastSwapped;
+ return timeSinceLastSwap > 0 ? timeSinceLastSwap : 0;
+ }
+
+ /**
* Close the store.
*/
public void close() throws VoldemortException {
View
119 src/java/voldemort/utils/KeyDistributionGenerator.java
@@ -5,6 +5,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Set;
+import java.util.Map.Entry;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
@@ -12,6 +13,7 @@
import voldemort.cluster.Node;
import voldemort.routing.RoutingStrategy;
import voldemort.routing.RoutingStrategyFactory;
+import voldemort.routing.RoutingStrategyType;
import voldemort.store.StoreDefinition;
import voldemort.xml.ClusterMapper;
import voldemort.xml.StoreDefinitionsMapper;
@@ -60,19 +62,117 @@ public static void main(String args[]) throws IOException {
}
Cluster cluster = new ClusterMapper().readCluster(new File(clusterXml));
- List<StoreDefinition> storeDef = new StoreDefinitionsMapper().readStoreList(new File(storesXml));
+ List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new File(storesXml));
// Print distribution for every store
- for(StoreDefinition def: storeDef) {
+ for(StoreDefinition def: storeDefs) {
HashMap<Integer, Double> storeDistribution = generateDistribution(cluster, def, numKeys);
System.out.println("For Store " + def.getName());
printDistribution(storeDistribution);
System.out.println("Std dev - " + getStdDeviation(storeDistribution));
System.out.println("=========================");
}
+ HashMap<Integer, Double> overallDistribution = generateOverallDistribution(cluster,
+ storeDefs,
+ numKeys);
+ System.out.println("Overall distribution ");
+ printDistribution(overallDistribution);
+ System.out.println("Std dev - " + getStdDeviation(overallDistribution));
+ }
+
+ public static HashMap<Integer, Double> generateOverallDistribution(Cluster cluster,
+ List<StoreDefinition> storeDefs,
+ int numKeys) {
+ return generateOverallDistributionWithUniqueStores(cluster,
+ getUniqueStoreDefinitionsWithCounts(storeDefs),
+ numKeys);
+ }
+
+ public static HashMap<Integer, Double> generateOverallDistributionWithUniqueStores(Cluster cluster,
+ HashMap<StoreDefinition, Integer> uniqueStoreDefsWithCount,
+ int numKeys) {
+ HashMap<Integer, Double> overallDistributionCount = Maps.newHashMap();
+ for(Node node: cluster.getNodes()) {
+ overallDistributionCount.put(node.getId(), 0.0);
+ }
+
+ int totalStores = 0;
+ for(Entry<StoreDefinition, Integer> entry: uniqueStoreDefsWithCount.entrySet()) {
+ HashMap<Integer, Double> nodeDistribution = generateDistribution(cluster,
+ entry.getKey(),
+ numKeys);
+ for(int nodeId: nodeDistribution.keySet()) {
+ overallDistributionCount.put(nodeId,
+ overallDistributionCount.get(nodeId)
+ + (nodeDistribution.get(nodeId) * entry.getValue()));
+ }
+ totalStores += entry.getValue();
+ }
+
+ // Normalize it
+ for(int nodeId: overallDistributionCount.keySet()) {
+ overallDistributionCount.put(nodeId, overallDistributionCount.get(nodeId)
+ / (totalStores * 100.0) * 100.0);
+ }
+ return overallDistributionCount;
+ }
+
+ public static HashMap<StoreDefinition, Integer> getUniqueStoreDefinitionsWithCounts(List<StoreDefinition> storeDefs) {
+
+ HashMap<StoreDefinition, Integer> uniqueStoreDefs = Maps.newHashMap();
+ for(StoreDefinition storeDef: storeDefs) {
+ if(uniqueStoreDefs.isEmpty()) {
+ uniqueStoreDefs.put(storeDef, 1);
+ } else {
+ StoreDefinition sameStore = null;
+
+ // Go over all the other stores to find if this is unique
+ for(StoreDefinition uniqueStoreDef: uniqueStoreDefs.keySet()) {
+ if(uniqueStoreDef.getReplicationFactor() == storeDef.getReplicationFactor()
+ && uniqueStoreDef.getRoutingStrategyType()
+ .compareTo(storeDef.getRoutingStrategyType()) == 0) {
+
+ // Further check for the zone routing case
+ if(uniqueStoreDef.getRoutingStrategyType()
+ .compareTo(RoutingStrategyType.ZONE_STRATEGY) == 0) {
+ boolean zonesSame = true;
+ for(int zoneId: uniqueStoreDef.getZoneReplicationFactor().keySet()) {
+ if(storeDef.getZoneReplicationFactor().get(zoneId) == null
+ || storeDef.getZoneReplicationFactor().get(zoneId) != uniqueStoreDef.getZoneReplicationFactor()
+ .get(zoneId)) {
+ zonesSame = false;
+ break;
+ }
+ }
+ if(zonesSame) {
+ sameStore = uniqueStoreDef;
+ }
+ } else {
+ sameStore = uniqueStoreDef;
+ }
+
+ if(sameStore != null) {
+ // Bump up the count
+ int currentCount = uniqueStoreDefs.get(sameStore);
+ uniqueStoreDefs.put(sameStore, currentCount + 1);
+ break;
+ }
+ }
+ }
+
+ if(sameStore == null) {
+ // New store
+ uniqueStoreDefs.put(storeDef, 1);
+ }
+ }
+ }
+
+ return uniqueStoreDefs;
}
/**
+ * Generates distribution for a specific store definition
+ *
* @param cluster The cluster metadata
* @param storeDef The store definition metadata
* @param numKeys Number of keys used to generate distribution
@@ -112,12 +212,23 @@ public static void printDistribution(HashMap<Integer, Double> distribution) {
}
public static double getStdDeviation(HashMap<Integer, Double> distribution) {
- HashMap<Integer, Double> offBy = Maps.newHashMap();
+ HashMap<Integer, Double> expectedDistribution = Maps.newHashMap();
int numberOfNodes = distribution.keySet().size();
double distributionPerNode = 100.0 / numberOfNodes;
+ for(int nodeId = 0; nodeId < numberOfNodes; nodeId++) {
+ expectedDistribution.put(nodeId, distributionPerNode);
+ }
+ return getStdDeviation(distribution, expectedDistribution);
+ }
+
+ public static double getStdDeviation(HashMap<Integer, Double> distribution,
+ HashMap<Integer, Double> expectedDistribution) {
+ HashMap<Integer, Double> offBy = Maps.newHashMap();
+
for(Integer nodeId: distribution.keySet()) {
- offBy.put(nodeId, new Double(distributionPerNode - distribution.get(nodeId)));
+ offBy.put(nodeId, new Double(expectedDistribution.get(nodeId)
+ - distribution.get(nodeId)));
}
double sum = 0, squareSum = 0;
for(double num: offBy.values()) {
View
1  src/java/voldemort/utils/RebalanceUtils.java
@@ -216,6 +216,7 @@ public static Node updateNode(Node node, List<Integer> partitionsList) {
node.getHttpPort(),
node.getSocketPort(),
node.getAdminPort(),
+ node.getZoneId(),
partitionsList);
}
Please sign in to comment.
Something went wrong with that request. Please try again.