Permalink
Browse files

Partition scans - more tests, typo fixes

  • Loading branch information...
1 parent 4c1064e commit fbe56d23036bac844632dd51415cf6acd69eb820 @vinothchandar vinothchandar committed Sep 15, 2012
@@ -55,6 +55,7 @@
import voldemort.common.service.ServiceType;
import voldemort.routing.RoutingStrategy;
import voldemort.routing.RoutingStrategyFactory;
+import voldemort.routing.RoutingStrategyType;
import voldemort.serialization.SerializerDefinition;
import voldemort.serialization.avro.versioned.SchemaEvolutionValidator;
import voldemort.server.RequestRoutingType;
@@ -270,7 +271,7 @@ protected void startInner() {
null,
null,
null,
- null,
+ RoutingStrategyType.CONSISTENT_STRATEGY,
0,
null,
0,
@@ -92,7 +92,7 @@ public PartitionPrefixedBdbStorageEngine(String name,
public List<Versioned<byte[]>> get(ByteArray key, byte[] transforms)
throws PersistenceFailureException {
StoreUtils.assertValidKey(key);
- short partition = routingStrategy.getPartitionList(key.get()).get(0).shortValue();
+ int partition = routingStrategy.getPartitionList(key.get()).get(0);
ByteArray prefixedKey = new ByteArray(StoreBinaryFormat.makePrefixedKey(key.get(),
partition));
return super.get(prefixedKey, transforms);
@@ -103,7 +103,7 @@ public void put(ByteArray key, Versioned<byte[]> value, byte[] transforms)
throws PersistenceFailureException {
StoreUtils.assertValidKey(key);
- short partition = routingStrategy.getPartitionList(key.get()).get(0).shortValue();
+ int partition = routingStrategy.getPartitionList(key.get()).get(0);
ByteArray prefixedKey = new ByteArray(StoreBinaryFormat.makePrefixedKey(key.get(),
partition));
super.put(prefixedKey, value, transforms);
@@ -113,7 +113,7 @@ public void put(ByteArray key, Versioned<byte[]> value, byte[] transforms)
public boolean delete(ByteArray key, Version version) throws PersistenceFailureException {
StoreUtils.assertValidKey(key);
- short partition = routingStrategy.getPartitionList(key.get()).get(0).shortValue();
+ int partition = routingStrategy.getPartitionList(key.get()).get(0);
ByteArray prefixedKey = new ByteArray(StoreBinaryFormat.makePrefixedKey(key.get(),
partition));
return super.delete(prefixedKey, version);
@@ -90,6 +90,10 @@ public static String getSingleStoreWithZonesXml() {
return readString("config/single-store-with-zones.xml");
}
+ public static String getTwoStoresWithZonesXml() {
+ return readString("config/two-stores-with-zones.xml");
+ }
+
public static Cluster getTenNodeCluster() {
return new ClusterMapper().readCluster(new StringReader(getTenNodeClusterXml()));
}
@@ -0,0 +1,50 @@
+<?xml version="1.0"?>
+<stores>
+ <store>
+ <name>cstore</name>
+ <persistence>bdb</persistence>
+ <routing>client</routing>
+ <routing-strategy>consistent-routing</routing-strategy>
+ <replication-factor>3</replication-factor>
+ <preferred-reads>3</preferred-reads>
+ <required-reads>3</required-reads>
+ <preferred-writes>3</preferred-writes>
+ <required-writes>2</required-writes>
+ <zone-count-reads>1</zone-count-reads>
+ <zone-count-writes>1</zone-count-writes>
+ <key-serializer>
+ <type>string</type>
+ <schema-info>UTF-8</schema-info>
+ </key-serializer>
+ <value-serializer>
+ <type>string</type>
+ <schema-info>UTF-8</schema-info>
+ </value-serializer>
+ </store>
+ <store>
+ <name>zstore</name>
+ <persistence>bdb</persistence>
+ <routing>client</routing>
+ <routing-strategy>zone-routing</routing-strategy>
+ <replication-factor>3</replication-factor>
+ <zone-replication-factor>
+ <replication-factor zone-id="0">1</replication-factor>
+ <replication-factor zone-id="1">1</replication-factor>
+ <replication-factor zone-id="2">1</replication-factor>
+ </zone-replication-factor>
+ <preferred-reads>3</preferred-reads>
+ <required-reads>3</required-reads>
+ <preferred-writes>3</preferred-writes>
+ <required-writes>2</required-writes>
+ <zone-count-reads>1</zone-count-reads>
+ <zone-count-writes>1</zone-count-writes>
+ <key-serializer>
+ <type>string</type>
+ <schema-info>UTF-8</schema-info>
+ </key-serializer>
+ <value-serializer>
+ <type>string</type>
+ <schema-info>UTF-8</schema-info>
+ </value-serializer>
+ </store>
+</stores>
@@ -119,7 +119,6 @@ public void setUp() throws IOException {
partitionToKeysMap.put(partition, new HashSet<String>());
partitionToKeysMap.get(partition).add(key);
voldClient.put(key, "value" + i);
- System.out.println(key);
}
}
@@ -4,17 +4,23 @@
import static junit.framework.Assert.fail;
import java.io.File;
+import java.io.StringReader;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import org.apache.commons.io.FileDeleteStrategy;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import voldemort.ServerTestUtils;
import voldemort.TestUtils;
+import voldemort.VoldemortTestConstants;
+import voldemort.cluster.Cluster;
import voldemort.routing.RoutingStrategy;
+import voldemort.routing.RoutingStrategyFactory;
import voldemort.server.VoldemortConfig;
import voldemort.store.StoreBinaryFormat;
import voldemort.store.StoreDefinition;
@@ -25,6 +31,7 @@
import voldemort.utils.Props;
import voldemort.versioning.Versioned;
import voldemort.xml.ClusterMapper;
+import voldemort.xml.StoreDefinitionsMapper;
/**
* Tests for the BDB storage engine prefixing the partition id to the keys, to
@@ -81,6 +88,52 @@ public void testPartitionToByteArrayConversion() {
assertEquals(0, ByteUtils.compare(key, StoreBinaryFormat.extractKey(prefixedkey)));
}
+ @Test
+ public void testHashConsistencyAcrossRoutingStrategies() {
+ // check that as long as the cluster.xml is the same, a key will hash to
+ // the same partition, immaterial of whether it is zone or consistent
+ // routing strategy
+
+ StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
+ List<StoreDefinition> storeDefs = mapper.readStoreList(new StringReader(VoldemortTestConstants.getTwoStoresWithZonesXml()));
+
+ StoreDefinition consistentStore = storeDefs.get(0);
+ StoreDefinition zoneStore = storeDefs.get(1);
+
+ assertEquals(consistentStore.getName(), "cstore");
+ assertEquals(zoneStore.getName(), "zstore");
+
+ Cluster cluster = VoldemortTestConstants.getEightNodeClusterWithZones();
+ RoutingStrategy cStrategy = new RoutingStrategyFactory().updateRoutingStrategy(consistentStore,
+ cluster);
+ RoutingStrategy zStrategy = new RoutingStrategyFactory().updateRoutingStrategy(zoneStore,
+ cluster);
+
+ BdbStorageEngine cPrefixedBdbStore = (BdbStorageEngine) bdbStorage.getStore(consistentStore,
+ cStrategy);
+
+ BdbStorageEngine zPrefixedBdbStore = (BdbStorageEngine) bdbStorage.getStore(zoneStore,
+ zStrategy);
+
+ HashMap<ByteArray, byte[]> kvpairs = ServerTestUtils.createRandomKeyValuePairs(10000);
+ for(ByteArray key: kvpairs.keySet()) {
+ assertEquals(cStrategy.getPartitionList(key.get()).get(0),
+ zStrategy.getPartitionList(key.get()).get(0));
+
+ cPrefixedBdbStore.put(key, new Versioned<byte[]>(kvpairs.get(key)), null);
+ zPrefixedBdbStore.put(key, new Versioned<byte[]>(kvpairs.get(key)), null);
+ }
+
+ for(ByteArray key: kvpairs.keySet()) {
+ assertEquals("Values read back does not match up",
+ 0,
+ ByteUtils.compare(cPrefixedBdbStore.get(key, null).get(0).getValue(),
+ zPrefixedBdbStore.get(key, null).get(0).getValue()));
+ }
+ cPrefixedBdbStore.close();
+ zPrefixedBdbStore.close();
+ }
+
private Set<String> getKeys(ClosableIterator<ByteArray> itr) {
HashSet<String> keySet = new HashSet<String>();
while(itr.hasNext()) {
@@ -139,7 +192,7 @@ public void testPartitionScan() {
assertEquals(partitionToKeysMap.get(p).size(), keys.size());
assertEquals(partitionToKeysMap.get(p), keys);
- // verfiy values
+ // verify values
keys = getEntries(prefixedBdbStore.entries(p));
assertEquals(partitionToKeysMap.get(p).size(), keys.size());
assertEquals(partitionToKeysMap.get(p), keys);

0 comments on commit fbe56d2

Please sign in to comment.