Permalink
Browse files

Consolidating haskeys logic with getall

  • Loading branch information...
1 parent be70d7a commit 83fcac4f47ee0513909114ff88beb932ff30cce8 @rsumbaly committed Sep 22, 2012
Showing with 583 additions and 907 deletions.
  1. +7 −3 src/java/voldemort/client/ClientConfig.java
  2. +24 −9 src/java/voldemort/client/TimeoutConfig.java
  3. +5 −3 src/java/voldemort/server/VoldemortConfig.java
  4. +0 −85 src/java/voldemort/store/routed/HasKeysPipelineData.java
  5. +5 −8 src/java/voldemort/store/routed/{GetAllPipelineData.java → MultiKeysPipelineData.java}
  6. +24 −23 src/java/voldemort/store/routed/PipelineRoutedStore.java
  7. +4 −4 src/java/voldemort/store/routed/action/GetAllReadRepair.java
  8. +0 −157 src/java/voldemort/store/routed/action/HasKeysConfigureNodes.java
  9. +12 −14 src/java/voldemort/store/routed/action/{GetAllConfigureNodes.java → MultiKeysConfigureNodes.java}
  10. +21 −131 src/java/voldemort/store/routed/action/PerformParallelGetAllRequests.java
  11. +20 −125 src/java/voldemort/store/routed/action/PerformParallelHasKeysRequests.java
  12. +162 −0 src/java/voldemort/store/routed/action/PerformParallelMultiKeysRequests.java
  13. +27 −163 src/java/voldemort/store/routed/action/PerformSerialGetAllRequests.java
  14. +29 −152 src/java/voldemort/store/routed/action/PerformSerialHasKeysRequests.java
  15. +197 −0 src/java/voldemort/store/routed/action/PerformSerialMultiKeysRequests.java
  16. +1 −0 test/integration/voldemort/performance/ClientConnectionStressTest.java
  17. +1 −0 test/unit/voldemort/store/routed/GetallNodeReachTest.java
  18. +1 −1 test/unit/voldemort/store/routed/HintedHandoffTest.java
  19. +1 −0 test/unit/voldemort/store/routed/ReadRepairerTest.java
  20. +18 −6 test/unit/voldemort/store/routed/RoutedStoreTest.java
  21. +24 −23 test/unit/voldemort/store/routed/action/GetAllConfigureNodesTest.java
@@ -56,7 +56,7 @@
private volatile boolean socketKeepAlive = false;
private volatile int selectors = 8;
private volatile long routingTimeoutMs = 15000;
- private volatile TimeoutConfig timeoutConfig = new TimeoutConfig(routingTimeoutMs, false);
+ private volatile TimeoutConfig timeoutConfig = new TimeoutConfig(routingTimeoutMs, false, false);
private volatile int socketBufferSize = 64 * 1024;
private volatile SerializerFactory serializerFactory = new DefaultSerializerFactory();
private volatile List<String> bootstrapUrls = null;
@@ -99,6 +99,7 @@ public ClientConfig() {}
public static final String GET_VERSIONS_ROUTING_TIMEOUT_MS_PROPERTY = "getversions_routing_timeout_ms";
public static final String DELETE_ROUTING_TIMEOUT_MS_PROPERTY = "delete_routing_timeout_ms";
public static final String ALLOW_PARTIAL_GETALLS_PROPERTY = "allow_partial_getalls";
+ public static final String ALLOW_PARTIAL_HASKEYS_PROPERTY = "allow_partial_haskeys";
public static final String NODE_BANNAGE_MS_PROPERTY = "node_bannage_ms";
public static final String SOCKET_BUFFER_SIZE_PROPERTY = "socket_buffer_size";
public static final String SERIALIZER_FACTORY_CLASS_PROPERTY = "serializer_factory_class";
@@ -183,7 +184,7 @@ private void setProperties(Properties properties) {
this.setRoutingTimeout(props.getInt(ROUTING_TIMEOUT_MS_PROPERTY), TimeUnit.MILLISECONDS);
// By default, make all the timeouts equal to routing timeout
- timeoutConfig = new TimeoutConfig(routingTimeoutMs, false);
+ timeoutConfig = new TimeoutConfig(routingTimeoutMs, false, false);
if(props.containsKey(GETALL_ROUTING_TIMEOUT_MS_PROPERTY))
timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE,
@@ -210,7 +211,10 @@ private void setProperties(Properties properties) {
props.getInt(DELETE_ROUTING_TIMEOUT_MS_PROPERTY));
if(props.containsKey(ALLOW_PARTIAL_GETALLS_PROPERTY))
- timeoutConfig.setPartialGetAllAllowed(props.getBoolean(ALLOW_PARTIAL_GETALLS_PROPERTY));
+ timeoutConfig.setPartialGetAllsAllowed(props.getBoolean(ALLOW_PARTIAL_GETALLS_PROPERTY));
+
+ if(props.containsKey(ALLOW_PARTIAL_HASKEYS_PROPERTY))
+ timeoutConfig.setPartialHasKeysAllowed(props.getBoolean(ALLOW_PARTIAL_HASKEYS_PROPERTY));
if(props.containsKey(SOCKET_BUFFER_SIZE_PROPERTY))
this.setSocketBufferSize(props.getInt(SOCKET_BUFFER_SIZE_PROPERTY));
@@ -12,16 +12,21 @@
private HashMap<Byte, Long> timeoutMap;
- private boolean partialGetAllAllowed;
+ private boolean partialGetAllsAllowed;
- public TimeoutConfig(long globalTimeout, boolean allowPartialGetAlls) {
+ private boolean partialHasKeysAllowed;
+
+ public TimeoutConfig(long globalTimeout,
+ boolean partialGetAllsAllowed,
+ boolean partialHasKeysAllowed) {
this(globalTimeout,
globalTimeout,
globalTimeout,
globalTimeout,
globalTimeout,
globalTimeout,
- allowPartialGetAlls);
+ partialGetAllsAllowed,
+ partialHasKeysAllowed);
}
public TimeoutConfig(long getTimeout,
@@ -30,15 +35,17 @@ public TimeoutConfig(long getTimeout,
long getAllTimeout,
long getVersionsTimeout,
long hasKeysTimeout,
- boolean allowPartialGetAlls) {
+ boolean partialGetAllsAllowed,
+ boolean partialHasKeysAllowed) {
timeoutMap = new HashMap<Byte, Long>();
timeoutMap.put(VoldemortOpCode.GET_OP_CODE, getTimeout);
timeoutMap.put(VoldemortOpCode.PUT_OP_CODE, putTimeout);
timeoutMap.put(VoldemortOpCode.DELETE_OP_CODE, deleteTimeout);
timeoutMap.put(VoldemortOpCode.GET_ALL_OP_CODE, getAllTimeout);
timeoutMap.put(VoldemortOpCode.GET_VERSION_OP_CODE, getVersionsTimeout);
timeoutMap.put(VoldemortOpCode.HAS_KEYS_OP_CODE, hasKeysTimeout);
- setPartialGetAllAllowed(allowPartialGetAlls);
+ setPartialGetAllsAllowed(partialGetAllsAllowed);
+ setPartialHasKeysAllowed(partialHasKeysAllowed);
}
public long getOperationTimeout(Byte opCode) {
@@ -50,12 +57,20 @@ public void setOperationTimeout(Byte opCode, long timeoutMs) {
timeoutMap.put(opCode, timeoutMs);
}
- public boolean isPartialGetAllAllowed() {
- return partialGetAllAllowed;
+ public boolean isPartialGetAllsAllowed() {
+ return partialGetAllsAllowed;
+ }
+
+ public void setPartialGetAllsAllowed(boolean partialGetAllsAllowed) {
+ this.partialGetAllsAllowed = partialGetAllsAllowed;
+ }
+
+ public boolean isPartialHasKeysAllowed() {
+ return partialHasKeysAllowed;
}
- public void setPartialGetAllAllowed(boolean allowPartialGetAlls) {
- this.partialGetAllAllowed = allowPartialGetAlls;
+ public void setPartialHasKeysAllowed(boolean partialHasKeysAllowed) {
+ this.partialHasKeysAllowed = partialHasKeysAllowed;
}
}
@@ -285,7 +285,7 @@ public VoldemortConfig(Props props) {
this.clientMaxConnectionsPerNode = props.getInt("client.max.connections.per.node", 50);
this.clientConnectionTimeoutMs = props.getInt("client.connection.timeout.ms", 500);
this.clientRoutingTimeoutMs = props.getInt("client.routing.timeout.ms", 15000);
- this.clientTimeoutConfig = new TimeoutConfig(this.clientRoutingTimeoutMs, false);
+ this.clientTimeoutConfig = new TimeoutConfig(this.clientRoutingTimeoutMs, false, false);
this.clientTimeoutConfig.setOperationTimeout(VoldemortOpCode.GET_OP_CODE,
props.getInt("client.routing.get.timeout.ms",
this.clientRoutingTimeoutMs));
@@ -301,8 +301,10 @@ public VoldemortConfig(Props props) {
this.clientTimeoutConfig.setOperationTimeout(VoldemortOpCode.DELETE_OP_CODE,
props.getInt("client.routing.delete.timeout.ms",
this.clientRoutingTimeoutMs));
- this.clientTimeoutConfig.setPartialGetAllAllowed(props.getBoolean("client.routing.allow.partial.getall",
- false));
+ this.clientTimeoutConfig.setPartialGetAllsAllowed(props.getBoolean("client.routing.allow.partial.getall",
+ false));
+ this.clientTimeoutConfig.setPartialHasKeysAllowed(props.getBoolean("client.routing.allow.partial.haskeys",
+ false));
this.clientMaxThreads = props.getInt("client.max.threads", 500);
this.clientThreadIdleMs = props.getInt("client.thread.idle.ms", 100000);
this.clientMaxQueuedRequests = props.getInt("client.max.queued.requests", 1000);
@@ -1,85 +0,0 @@
-package voldemort.store.routed;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.mutable.MutableInt;
-
-import voldemort.cluster.Node;
-import voldemort.utils.ByteArray;
-
-/**
- * This is used only by the "get all" operation as it includes data specific
- * only to that operation.
- */
-
-public class HasKeysPipelineData extends PipelineData<Iterable<ByteArray>, Map<ByteArray, Boolean>> {
-
- private final Map<ByteArray, Boolean> result;
-
- // Keys for each node needed to satisfy storeDef.getPreferredReads() if
- // no failures.
- private Map<Node, List<ByteArray>> nodeToKeysMap;
-
- // Keep track of nodes per key that might be needed if there are
- // failures during getAll
- private Map<ByteArray, List<Node>> keyToExtraNodesMap;
-
- private final Map<ByteArray, MutableInt> keyToSuccessCount;
-
- private final Map<ByteArray, HashSet<Integer>> keyToZoneResponses;
-
- private Integer zonesRequired;
-
- public HasKeysPipelineData() {
- this.result = new HashMap<ByteArray, Boolean>();
- this.keyToSuccessCount = new HashMap<ByteArray, MutableInt>();
- this.keyToZoneResponses = new HashMap<ByteArray, HashSet<Integer>>();
- }
-
- public Map<ByteArray, HashSet<Integer>> getKeyToZoneResponse() {
- return this.keyToZoneResponses;
- }
-
- public Map<ByteArray, Boolean> getResult() {
- return result;
- }
-
- public Map<Node, List<ByteArray>> getNodeToKeysMap() {
- return nodeToKeysMap;
- }
-
- public void setNodeToKeysMap(Map<Node, List<ByteArray>> nodeToKeysMap) {
- this.nodeToKeysMap = nodeToKeysMap;
- }
-
- public Map<ByteArray, List<Node>> getKeyToExtraNodesMap() {
- return keyToExtraNodesMap;
- }
-
- public void setKeyToExtraNodesMap(Map<ByteArray, List<Node>> keyToExtraNodesMap) {
- this.keyToExtraNodesMap = keyToExtraNodesMap;
- }
-
- public MutableInt getSuccessCount(ByteArray key) {
- MutableInt value = keyToSuccessCount.get(key);
-
- if(value == null) {
- value = new MutableInt(0);
- keyToSuccessCount.put(key, value);
- }
-
- return value;
- }
-
- public void setZonesRequired(Integer zonesRequired) {
- this.zonesRequired = zonesRequired;
- }
-
- public Integer getZonesRequired() {
- return this.zonesRequired;
- }
-
-}
@@ -25,17 +25,14 @@
import voldemort.cluster.Node;
import voldemort.utils.ByteArray;
-import voldemort.versioning.Versioned;
/**
* This is used only by the "get all" operation as it includes data specific
* only to that operation.
*/
+public class MultiKeysPipelineData<T> extends PipelineData<Iterable<ByteArray>, Map<ByteArray, T>> {
-public class GetAllPipelineData extends
- PipelineData<Iterable<ByteArray>, Map<ByteArray, List<Versioned<byte[]>>>> {
-
- private final Map<ByteArray, List<Versioned<byte[]>>> result;
+ private final Map<ByteArray, T> result;
// Keys for each node needed to satisfy storeDef.getPreferredReads() if
// no failures.
@@ -53,8 +50,8 @@
private Integer zonesRequired;
- public GetAllPipelineData() {
- this.result = new HashMap<ByteArray, List<Versioned<byte[]>>>();
+ public MultiKeysPipelineData() {
+ this.result = new HashMap<ByteArray, T>();
this.keyToSuccessCount = new HashMap<ByteArray, MutableInt>();
this.keyToZoneResponses = new HashMap<ByteArray, HashSet<Integer>>();
}
@@ -63,7 +60,7 @@ public GetAllPipelineData() {
return this.keyToZoneResponses;
}
- public Map<ByteArray, List<Versioned<byte[]>>> getResult() {
+ public Map<ByteArray, T> getResult() {
return result;
}
@@ -37,10 +37,9 @@
import voldemort.store.routed.Pipeline.Event;
import voldemort.store.routed.Pipeline.Operation;
import voldemort.store.routed.action.ConfigureNodes;
-import voldemort.store.routed.action.GetAllConfigureNodes;
import voldemort.store.routed.action.GetAllReadRepair;
-import voldemort.store.routed.action.HasKeysConfigureNodes;
import voldemort.store.routed.action.IncrementClock;
+import voldemort.store.routed.action.MultiKeysConfigureNodes;
import voldemort.store.routed.action.PerformDeleteHintedHandoff;
import voldemort.store.routed.action.PerformParallelDeleteRequests;
import voldemort.store.routed.action.PerformParallelGetAllRequests;
@@ -295,7 +294,7 @@ private String formatNodeValuesFromGet(List<Response<ByteArray, List<Versioned<b
boolean allowReadRepair = repairReads && (transforms == null || transforms.size() == 0);
- GetAllPipelineData pipelineData = new GetAllPipelineData();
+ MultiKeysPipelineData<List<Versioned<byte[]>>> pipelineData = new MultiKeysPipelineData<List<Versioned<byte[]>>>();
if(zoneRoutingEnabled)
pipelineData.setZonesRequired(storeDef.getZoneCountReads());
else
@@ -306,15 +305,15 @@ private String formatNodeValuesFromGet(List<Response<ByteArray, List<Versioned<b
timeoutConfig.getOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE),
TimeUnit.MILLISECONDS);
pipeline.addEventAction(Event.STARTED,
- new GetAllConfigureNodes(pipelineData,
- Event.CONFIGURED,
- failureDetector,
- storeDef.getPreferredReads(),
- storeDef.getRequiredReads(),
- routingStrategy,
- keys,
- transforms,
- clientZone));
+ new MultiKeysConfigureNodes<List<Versioned<byte[]>>>(pipelineData,
+ Event.CONFIGURED,
+ failureDetector,
+ storeDef.getPreferredReads(),
+ storeDef.getRequiredReads(),
+ routingStrategy,
+ keys,
+ transforms,
+ clientZone));
pipeline.addEventAction(Event.CONFIGURED,
new PerformParallelGetAllRequests(pipelineData,
Event.INSUFFICIENT_SUCCESSES,
@@ -330,7 +329,7 @@ private String formatNodeValuesFromGet(List<Response<ByteArray, List<Versioned<b
innerStores,
storeDef.getPreferredReads(),
storeDef.getRequiredReads(),
- timeoutConfig.isPartialGetAllAllowed()));
+ timeoutConfig.isPartialGetAllsAllowed()));
if(allowReadRepair)
pipeline.addEventAction(Event.RESPONSES_RECEIVED,
@@ -384,7 +383,7 @@ private String formatNodeValuesFromGet(List<Response<ByteArray, List<Versioned<b
startTimeNs = System.nanoTime();
}
- HasKeysPipelineData pipelineData = new HasKeysPipelineData();
+ MultiKeysPipelineData<Boolean> pipelineData = new MultiKeysPipelineData<Boolean>();
if(zoneRoutingEnabled)
pipelineData.setZonesRequired(storeDef.getZoneCountReads());
else
@@ -395,14 +394,15 @@ private String formatNodeValuesFromGet(List<Response<ByteArray, List<Versioned<b
timeoutConfig.getOperationTimeout(VoldemortOpCode.HAS_KEYS_OP_CODE),
TimeUnit.MILLISECONDS);
pipeline.addEventAction(Event.STARTED,
- new HasKeysConfigureNodes(pipelineData,
- Event.CONFIGURED,
- failureDetector,
- storeDef.getPreferredReads(),
- storeDef.getRequiredReads(),
- routingStrategy,
- keys,
- clientZone));
+ new MultiKeysConfigureNodes<Boolean>(pipelineData,
+ Event.CONFIGURED,
+ failureDetector,
+ storeDef.getPreferredReads(),
+ storeDef.getRequiredReads(),
+ routingStrategy,
+ keys,
+ null,
+ clientZone));
pipeline.addEventAction(Event.CONFIGURED,
new PerformParallelHasKeysRequests(pipelineData,
Event.INSUFFICIENT_SUCCESSES,
@@ -418,7 +418,8 @@ private String formatNodeValuesFromGet(List<Response<ByteArray, List<Versioned<b
failureDetector,
innerStores,
storeDef.getPreferredReads(),
- storeDef.getRequiredReads()));
+ storeDef.getRequiredReads(),
+ timeoutConfig.isPartialHasKeysAllowed()));
pipeline.addEvent(Event.STARTED);
@@ -21,18 +21,18 @@
import java.util.Map;
import voldemort.store.nonblockingstore.NonblockingStore;
-import voldemort.store.routed.GetAllPipelineData;
+import voldemort.store.routed.MultiKeysPipelineData;
+import voldemort.store.routed.Pipeline.Event;
import voldemort.store.routed.ReadRepairer;
import voldemort.store.routed.Response;
-import voldemort.store.routed.Pipeline.Event;
import voldemort.utils.ByteArray;
import voldemort.versioning.Versioned;
public class GetAllReadRepair
extends
- AbstractReadRepair<Iterable<ByteArray>, Map<ByteArray, List<Versioned<byte[]>>>, GetAllPipelineData> {
+ AbstractReadRepair<Iterable<ByteArray>, Map<ByteArray, List<Versioned<byte[]>>>, MultiKeysPipelineData<List<Versioned<byte[]>>>> {
- public GetAllReadRepair(GetAllPipelineData pipelineData,
+ public GetAllReadRepair(MultiKeysPipelineData<List<Versioned<byte[]>>> pipelineData,
Event completeEvent,
int preferred,
long timeoutMs,
Oops, something went wrong.

0 comments on commit 83fcac4

Please sign in to comment.