Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Moved SlowStorageEngine into tests/integration.

  • Loading branch information...
commit 0c2bd874547175aa227088efc52902388fe69c56 1 parent ad3d0cc
Jay Wylie jayjwylie authored
32 src/java/voldemort/client/TimeoutConfig.java
View
@@ -1,26 +1,20 @@
package voldemort.client;
-import java.util.HashMap;
-
-import voldemort.common.VoldemortOpCode;
+import voldemort.utils.OpTimeMap;
/**
- * Encapsulates the timeouts for various voldemort operations
+ * Encapsulates the timeouts, in ms, for various Voldemort operations
*
*/
public class TimeoutConfig {
- private HashMap<Byte, Long> timeoutMap;
+ private OpTimeMap timeoutMap;
private boolean partialGetAllAllowed;
public TimeoutConfig(long globalTimeout, boolean allowPartialGetAlls) {
- this(globalTimeout,
- globalTimeout,
- globalTimeout,
- globalTimeout,
- globalTimeout,
- allowPartialGetAlls);
+ timeoutMap = new OpTimeMap(globalTimeout);
+ setPartialGetAllAllowed(allowPartialGetAlls);
}
public TimeoutConfig(long getTimeout,
@@ -29,22 +23,20 @@ public TimeoutConfig(long getTimeout,
long getAllTimeout,
long getVersionsTimeout,
boolean allowPartialGetAlls) {
- timeoutMap = new HashMap<Byte, Long>();
- timeoutMap.put(VoldemortOpCode.GET_OP_CODE, getTimeout);
- timeoutMap.put(VoldemortOpCode.PUT_OP_CODE, putTimeout);
- timeoutMap.put(VoldemortOpCode.DELETE_OP_CODE, deleteTimeout);
- timeoutMap.put(VoldemortOpCode.GET_ALL_OP_CODE, getAllTimeout);
- timeoutMap.put(VoldemortOpCode.GET_VERSION_OP_CODE, getVersionsTimeout);
+ timeoutMap = new OpTimeMap(getTimeout,
+ putTimeout,
+ deleteTimeout,
+ getAllTimeout,
+ getVersionsTimeout);
setPartialGetAllAllowed(allowPartialGetAlls);
}
public long getOperationTimeout(Byte opCode) {
- assert timeoutMap.containsKey(opCode);
- return timeoutMap.get(opCode);
+ return timeoutMap.getOpTime(opCode);
}
public void setOperationTimeout(Byte opCode, long timeoutMs) {
- timeoutMap.put(opCode, timeoutMs);
+ timeoutMap.setOpTime(opCode, timeoutMs);
}
public boolean isPartialGetAllAllowed() {
54 src/java/voldemort/server/VoldemortConfig.java
View
@@ -30,11 +30,11 @@
import voldemort.store.bdb.BdbStorageConfiguration;
import voldemort.store.memory.CacheStorageConfiguration;
import voldemort.store.memory.InMemoryStorageConfiguration;
-import voldemort.store.memory.SlowStorageEngine;
import voldemort.store.mysql.MysqlStorageConfiguration;
import voldemort.store.readonly.BinarySearchStrategy;
import voldemort.store.readonly.ReadOnlyStorageConfiguration;
import voldemort.utils.ConfigurationException;
+import voldemort.utils.OpTimeMap;
import voldemort.utils.Props;
import voldemort.utils.Time;
import voldemort.utils.UndefinedPropertyException;
@@ -104,8 +104,8 @@
private long reportingIntervalBytes;
private int fetcherBufferSize;
- private SlowStorageEngine.OperationDelays slowQueueingDelays;
- private SlowStorageEngine.OperationDelays slowConcurrentDelays;
+ private OpTimeMap testingSlowQueueingDelays;
+ private OpTimeMap testingSlowConcurrentDelays;
private int coreThreads;
private int maxThreads;
@@ -256,19 +256,33 @@ public VoldemortConfig(Props props) {
this.mysqlPort = props.getInt("mysql.port", 3306);
this.mysqlDatabaseName = props.getString("mysql.database", "voldemort");
- this.slowQueueingDelays = new SlowStorageEngine.OperationDelays();
- this.slowQueueingDelays.getMs = props.getInt("slow.queueing.get.ms", 0);
- this.slowQueueingDelays.getVersionsMs = props.getInt("slow.queueing.getversions.ms", 0);
- this.slowQueueingDelays.getAllMs = props.getInt("slow.queueing.getall.ms", 0);
- this.slowQueueingDelays.putMs = props.getInt("slow.queueing.put.ms", 0);
- this.slowQueueingDelays.deleteMs = props.getInt("slow.queueing.delete.ms", 0);
-
- this.slowConcurrentDelays = new SlowStorageEngine.OperationDelays();
- this.slowConcurrentDelays.getMs = props.getInt("slow.concurrent.get.ms", 0);
- this.slowConcurrentDelays.getVersionsMs = props.getInt("slow.concurrent.getversions.ms", 0);
- this.slowConcurrentDelays.getAllMs = props.getInt("slow.concurrent.getall.ms", 0);
- this.slowConcurrentDelays.putMs = props.getInt("slow.concurrent.put.ms", 0);
- this.slowConcurrentDelays.deleteMs = props.getInt("slow.concurrent.delete.ms", 0);
+ this.testingSlowQueueingDelays = new OpTimeMap(0);
+ this.testingSlowQueueingDelays.setOpTime(VoldemortOpCode.GET_OP_CODE,
+ props.getInt("testing.slow.queueing.get.ms", 0));
+ this.testingSlowQueueingDelays.setOpTime(VoldemortOpCode.GET_ALL_OP_CODE,
+ props.getInt("testing.slow.queueing.getall.ms", 0));
+ this.testingSlowQueueingDelays.setOpTime(VoldemortOpCode.GET_VERSION_OP_CODE,
+ props.getInt("testing.slow.queueing.getversions.ms",
+ 0));
+ this.testingSlowQueueingDelays.setOpTime(VoldemortOpCode.PUT_OP_CODE,
+ props.getInt("testing.slow.queueing.put.ms", 0));
+ this.testingSlowQueueingDelays.setOpTime(VoldemortOpCode.DELETE_OP_CODE,
+ props.getInt("testing.slow.queueing.delete.ms", 0));
+
+ this.testingSlowConcurrentDelays = new OpTimeMap(0);
+ this.testingSlowConcurrentDelays.setOpTime(VoldemortOpCode.GET_OP_CODE,
+ props.getInt("testing.slow.concurrent.get.ms", 0));
+ this.testingSlowConcurrentDelays.setOpTime(VoldemortOpCode.GET_ALL_OP_CODE,
+ props.getInt("testing.slow.concurrent.getall.ms",
+ 0));
+ this.testingSlowConcurrentDelays.setOpTime(VoldemortOpCode.GET_VERSION_OP_CODE,
+ props.getInt("testing.slow.concurrent.getversions.ms",
+ 0));
+ this.testingSlowConcurrentDelays.setOpTime(VoldemortOpCode.PUT_OP_CODE,
+ props.getInt("testing.slow.concurrent.put.ms", 0));
+ this.testingSlowConcurrentDelays.setOpTime(VoldemortOpCode.DELETE_OP_CODE,
+ props.getInt("testing.slow.concurrent.delete.ms",
+ 0));
this.maxThreads = props.getInt("max.threads", 100);
this.coreThreads = props.getInt("core.threads", Math.max(1, maxThreads / 2));
@@ -1565,11 +1579,11 @@ public void setEnableJmxClusterName(boolean enableJmxClusterName) {
this.enableJmxClusterName = enableJmxClusterName;
}
- public SlowStorageEngine.OperationDelays getSlowQueueingDelays() {
- return this.slowQueueingDelays;
+ public OpTimeMap testingGetSlowQueueingDelays() {
+ return this.testingSlowQueueingDelays;
}
- public SlowStorageEngine.OperationDelays getSlowConcurrentDelays() {
- return this.slowConcurrentDelays;
+ public OpTimeMap testingGetSlowConcurrentDelays() {
+ return this.testingSlowConcurrentDelays;
}
}
4 src/java/voldemort/store/socket/clientrequest/ClientRequestExecutorPool.java
View
@@ -43,6 +43,7 @@
import voldemort.utils.pool.KeyedResourcePool;
import voldemort.utils.pool.QueuedKeyedResourcePool;
import voldemort.utils.pool.ResourcePoolConfig;
+import voldemort.utils.pool.ResourceRequest;
/**
* A pool of {@link ClientRequestExecutor} keyed off the
@@ -231,8 +232,7 @@ public ClientSocketStats getStats() {
* Wrap up an asynchronous request and actually issue it once a
* SocketDestination is checked out.
*/
- private class AsyncRequest<T> implements
- QueuedKeyedResourcePool.ResourceRequest<ClientRequestExecutor> {
+ private class AsyncRequest<T> implements ResourceRequest<ClientRequestExecutor> {
private final SocketDestination destination;
public final ClientRequest<T> delegate;
26 src/java/voldemort/utils/pool/QueuedKeyedResourcePool.java
View
@@ -33,23 +33,6 @@
*/
public class QueuedKeyedResourcePool<K, V> extends KeyedResourcePool<K, V> {
- public interface ResourceRequest<V> {
-
- // Invoked with checked out resource; resource guaranteed to be
- // not-null.
- void useResource(V resource);
-
- // Invoked sometime after deadline. Will never invoke useResource.
- void handleTimeout();
-
- // Invoked upon resource pool exception. Will never invoke useResource.
- void handleException(Exception e);
-
- // Returns deadline (in nanoseconds), after which handleTimeout()
- // should be invoked.
- long getDeadlineNs();
- }
-
private static final Logger logger = Logger.getLogger(QueuedKeyedResourcePool.class.getName());
private final ConcurrentMap<K, Queue<ResourceRequest<V>>> requestQueueMap;
@@ -121,7 +104,7 @@ public void requestResource(K key, ResourceRequest<V> resourceRequest) {
try {
resource = attemptCheckout(resourcePool);
} catch(Exception e) {
- super.destroyResource(key, resourcePool, resource);
+ destroyResource(key, resourcePool, resource);
resourceRequest.handleException(e);
}
if(resource != null) {
@@ -175,7 +158,7 @@ private boolean processQueue(K key) throws Exception {
attemptGrow(key, resourcePool);
resource = attemptCheckout(resourcePool);
} catch(Exception e) {
- super.destroyResource(key, resourcePool, resource);
+ destroyResource(key, resourcePool, resource);
}
if(resource == null) {
return false;
@@ -184,7 +167,8 @@ private boolean processQueue(K key) throws Exception {
// With resource in hand, process the resource requests
ResourceRequest<V> resourceRequest = getNextUnexpiredResourceRequest(requestQueue);
if(resourceRequest == null) {
- // Did not use the resource!
+ // Did not use the resource! Directly check in via super to avoid
+ // circular call to processQueue().
super.checkin(key, resource);
return false;
}
@@ -203,7 +187,7 @@ private boolean processQueue(K key) throws Exception {
public void checkin(K key, V resource) throws Exception {
super.checkin(key, resource);
// NB: Blocking checkout calls may get checked in resource before
- // processQueue.
+ // processQueue() attempts checkout.
while(processQueue(key)) {}
}
2  test/integration/voldemort/nonblocking/E2ENonblockingCheckoutTest.java
View
@@ -47,7 +47,7 @@
import voldemort.store.StoreDefinitionBuilder;
import voldemort.store.bdb.BdbStorageConfiguration;
import voldemort.store.memory.InMemoryStorageConfiguration;
-import voldemort.store.memory.SlowStorageConfiguration;
+import voldemort.store.slow.SlowStorageConfiguration;
import voldemort.store.socket.SocketStoreFactory;
import voldemort.store.socket.clientrequest.ClientRequestExecutorPool;
import voldemort.versioning.ObsoleteVersionException;
12 ...demort/store/memory/SlowStorageConfiguration.java → ...oldemort/store/slow/SlowStorageConfiguration.java
View
@@ -13,8 +13,7 @@
* License for the specific language governing permissions and limitations under
* the License.
*/
-
-package voldemort.store.memory;
+package voldemort.store.slow;
import voldemort.VoldemortException;
import voldemort.server.VoldemortConfig;
@@ -22,6 +21,7 @@
import voldemort.store.StorageEngine;
import voldemort.store.StoreDefinition;
import voldemort.utils.ByteArray;
+import voldemort.utils.OpTimeMap;
/**
* A storage engine that wraps InMemoryStorageEngine with delays.
@@ -41,12 +41,12 @@ public SlowStorageConfiguration(VoldemortConfig config) {
public StorageEngine<ByteArray, byte[], byte[]> getStore(StoreDefinition storeDef) {
if(voldemortConfig != null) {
return new SlowStorageEngine<ByteArray, byte[], byte[]>(storeDef.getName(),
- this.voldemortConfig.getSlowQueueingDelays(),
- this.voldemortConfig.getSlowConcurrentDelays());
+ this.voldemortConfig.testingGetSlowQueueingDelays(),
+ this.voldemortConfig.testingGetSlowConcurrentDelays());
}
return new SlowStorageEngine<ByteArray, byte[], byte[]>(storeDef.getName(),
- new SlowStorageEngine.OperationDelays(),
- new SlowStorageEngine.OperationDelays());
+ new OpTimeMap(0),
+ new OpTimeMap(0));
}
public String getType() {
76 ...ava/voldemort/store/memory/SlowStorageEngine.java → ...ation/voldemort/store/slow/SlowStorageEngine.java
View
@@ -14,16 +14,19 @@
* the License.
*/
-package voldemort.store.memory;
+package voldemort.store.slow;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import voldemort.VoldemortException;
+import voldemort.common.VoldemortOpCode;
import voldemort.store.StorageEngine;
import voldemort.store.StoreCapabilityType;
+import voldemort.store.memory.InMemoryStorageEngine;
import voldemort.utils.ClosableIterator;
+import voldemort.utils.OpTimeMap;
import voldemort.utils.Pair;
import voldemort.versioning.Version;
import voldemort.versioning.Versioned;
@@ -47,46 +50,15 @@
*/
public class SlowStorageEngine<K, V, T> implements StorageEngine<K, V, T> {
- public static class OperationDelays {
-
- public long getMs;
- public long getVersionsMs;
- public long getAllMs;
- public long putMs;
- public long deleteMs;
-
- public OperationDelays() {
- this.getMs = 0;
- this.getVersionsMs = 0;
- this.getAllMs = 0;
- this.putMs = 0;
- this.deleteMs = 0;
- }
-
- public OperationDelays(long getMs,
- long getVersionsMs,
- long getAllMs,
- long putMs,
- long deleteMs) {
- this.getMs = getMs;
- this.getVersionsMs = getVersionsMs;
- this.getAllMs = getAllMs;
- this.putMs = putMs;
- this.deleteMs = deleteMs;
- }
- }
-
private final InMemoryStorageEngine<K, V, T> imStore;
- private final OperationDelays queueingDelays;
- private final OperationDelays concurrentDelays;
+ private final OpTimeMap queueingDelays;
+ private final OpTimeMap concurrentDelays;
public SlowStorageEngine(String name) {
- this(name, new OperationDelays(), new OperationDelays());
+ this(name, new OpTimeMap(0), new OpTimeMap(0));
}
- public SlowStorageEngine(String name,
- OperationDelays queueingDelays,
- OperationDelays concurrentDelays) {
+ public SlowStorageEngine(String name, OpTimeMap queueingDelays, OpTimeMap concurrentDelays) {
imStore = new InMemoryStorageEngine<K, V, T>(name);
this.queueingDelays = queueingDelays;
this.concurrentDelays = concurrentDelays;
@@ -108,48 +80,40 @@ private void concurrentSleep(long ms) {
}
}
+ private void delayByOp(byte opCode) {
+ if(queueingDelays.getOpTime(opCode) > 0)
+ queueingSleep(queueingDelays.getOpTime(opCode));
+ if(concurrentDelays.getOpTime(opCode) > 0)
+ concurrentSleep(concurrentDelays.getOpTime(opCode));
+ }
+
public boolean delete(K key) {
return delete(key, null);
}
public boolean delete(K key, Version version) {
- if(queueingDelays.deleteMs > 0)
- queueingSleep(queueingDelays.deleteMs);
- if(concurrentDelays.deleteMs > 0)
- concurrentSleep(concurrentDelays.deleteMs);
+ delayByOp(VoldemortOpCode.DELETE_OP_CODE);
return imStore.delete(key, version);
}
public List<Version> getVersions(K key) {
- if(queueingDelays.getVersionsMs > 0)
- queueingSleep(queueingDelays.getVersionsMs);
- if(concurrentDelays.getVersionsMs > 0)
- concurrentSleep(concurrentDelays.getVersionsMs);
+ delayByOp(VoldemortOpCode.GET_VERSION_OP_CODE);
return imStore.getVersions(key);
}
public List<Versioned<V>> get(K key, T transform) throws VoldemortException {
- if(queueingDelays.getMs > 0)
- queueingSleep(queueingDelays.getMs);
- if(concurrentDelays.getMs > 0)
- concurrentSleep(concurrentDelays.getMs);
+ delayByOp(VoldemortOpCode.GET_OP_CODE);
return imStore.get(key, transform);
}
public Map<K, List<Versioned<V>>> getAll(Iterable<K> keys, Map<K, T> transforms)
throws VoldemortException {
- if(queueingDelays.getAllMs > 0)
- queueingSleep(queueingDelays.getAllMs);
- if(concurrentDelays.getAllMs > 0)
- concurrentSleep(concurrentDelays.getAllMs);
+ delayByOp(VoldemortOpCode.GET_ALL_OP_CODE);
return imStore.getAll(keys, transforms);
}
public void put(K key, Versioned<V> value, T transforms) throws VoldemortException {
- if(queueingDelays.putMs > 0)
- queueingSleep(queueingDelays.putMs);
- if(concurrentDelays.putMs > 0)
- concurrentSleep(concurrentDelays.putMs);
+ delayByOp(VoldemortOpCode.PUT_OP_CODE);
imStore.put(key, value, transforms);
}
22 test/unit/voldemort/store/memory/SlowStorageEngineTest.java
View
@@ -29,8 +29,10 @@
import voldemort.common.VoldemortOpCode;
import voldemort.store.AbstractStorageEngineTest;
import voldemort.store.StorageEngine;
+import voldemort.store.slow.SlowStorageEngine;
import voldemort.utils.ByteArray;
import voldemort.utils.ByteUtils;
+import voldemort.utils.OpTimeMap;
import voldemort.utils.pool.KeyedResourcePool;
import voldemort.versioning.ObsoleteVersionException;
import voldemort.versioning.VectorClock;
@@ -62,16 +64,8 @@ public void setUp() throws Exception {
super.setUp();
// Do not change the magic constants in the next two constructors! The
// unit tests assert on specific delays occurring.
- SlowStorageEngine.OperationDelays queued = new SlowStorageEngine.OperationDelays(10,
- 20,
- 30,
- 40,
- 50);
- SlowStorageEngine.OperationDelays concurrent = new SlowStorageEngine.OperationDelays(50,
- 40,
- 30,
- 20,
- 10);
+ OpTimeMap queued = new OpTimeMap(10, 20, 30, 40, 50);
+ OpTimeMap concurrent = new OpTimeMap(50, 40, 30, 20, 10);
this.store = new SlowStorageEngine<ByteArray, byte[], byte[]>("test", queued, concurrent);
}
@@ -262,16 +256,16 @@ public void testEachOpTypeRepeated() {
expectedTimeMs = (5 * 10) + 50;
break;
case VoldemortOpCode.GET_VERSION_OP_CODE:
- expectedTimeMs = (5 * 20) + 40;
+ expectedTimeMs = (5 * 50) + 10;
break;
case VoldemortOpCode.GET_ALL_OP_CODE:
- expectedTimeMs = (5 * 30) + 30;
+ expectedTimeMs = (5 * 40) + 20;
break;
case VoldemortOpCode.PUT_OP_CODE:
- expectedTimeMs = (5 * 40) + 20;
+ expectedTimeMs = (5 * 20) + 40;
break;
case VoldemortOpCode.DELETE_OP_CODE:
- expectedTimeMs = (5 * 50) + 10;
+ expectedTimeMs = (5 * 30) + 30;
break;
}
String details = "(maxTimeMs: " + maxTimeMs + ", " + expectedTimeMs + ")";
3  test/unit/voldemort/utils/pool/QueuedKeyedResourcePoolTest.java
View
@@ -359,8 +359,7 @@ public void run() {
}
}
- protected static class TestResourceRequest implements
- QueuedKeyedResourcePool.ResourceRequest<TestResource> {
+ protected static class TestResourceRequest implements ResourceRequest<TestResource> {
private AtomicBoolean usedResource;
private AtomicBoolean handledTimeout;
Please sign in to comment.
Something went wrong with that request. Please try again.