Partition scans - more tests, typo fixes

src/java/voldemort/server/storage/StorageService.java

@@ -55,6 +55,7 @@
 import voldemort.common.service.ServiceType;
 import voldemort.routing.RoutingStrategy;
 import voldemort.routing.RoutingStrategyFactory;
+import voldemort.routing.RoutingStrategyType;
 import voldemort.serialization.SerializerDefinition;
 import voldemort.serialization.avro.versioned.SchemaEvolutionValidator;
 import voldemort.server.RequestRoutingType;
@@ -270,7 +271,7 @@ protected void startInner() {
                                                                       null,
                                                                       null,
                                                                       null,
-                                                                      null,
+                                                                      RoutingStrategyType.CONSISTENT_STRATEGY,
                                                                       0,
                                                                       null,
                                                                       0,

src/java/voldemort/store/bdb/PartitionPrefixedBdbStorageEngine.java

@@ -92,7 +92,7 @@ public ClosableIterator<ByteArray> keys(int partition) {
     public List<Versioned<byte[]>> get(ByteArray key, byte[] transforms)
             throws PersistenceFailureException {
         StoreUtils.assertValidKey(key);
-        short partition = routingStrategy.getPartitionList(key.get()).get(0).shortValue();
+        int partition = routingStrategy.getPartitionList(key.get()).get(0);
         ByteArray prefixedKey = new ByteArray(StoreBinaryFormat.makePrefixedKey(key.get(),
                                                                                 partition));
         return super.get(prefixedKey, transforms);
@@ -103,7 +103,7 @@ public void put(ByteArray key, Versioned<byte[]> value, byte[] transforms)
             throws PersistenceFailureException {
 
         StoreUtils.assertValidKey(key);
-        short partition = routingStrategy.getPartitionList(key.get()).get(0).shortValue();
+        int partition = routingStrategy.getPartitionList(key.get()).get(0);
         ByteArray prefixedKey = new ByteArray(StoreBinaryFormat.makePrefixedKey(key.get(),
                                                                                 partition));
         super.put(prefixedKey, value, transforms);
@@ -113,7 +113,7 @@ public void put(ByteArray key, Versioned<byte[]> value, byte[] transforms)
     public boolean delete(ByteArray key, Version version) throws PersistenceFailureException {
 
         StoreUtils.assertValidKey(key);
-        short partition = routingStrategy.getPartitionList(key.get()).get(0).shortValue();
+        int partition = routingStrategy.getPartitionList(key.get()).get(0);
         ByteArray prefixedKey = new ByteArray(StoreBinaryFormat.makePrefixedKey(key.get(),
                                                                                 partition));
         return super.delete(prefixedKey, version);

test/common/voldemort/VoldemortTestConstants.java

@@ -90,6 +90,10 @@ public static String getSingleStoreWithZonesXml() {
         return readString("config/single-store-with-zones.xml");
     }
 
+    public static String getTwoStoresWithZonesXml() {
+        return readString("config/two-stores-with-zones.xml");
+    }
+
     public static Cluster getTenNodeCluster() {
         return new ClusterMapper().readCluster(new StringReader(getTenNodeClusterXml()));
     }

test/common/voldemort/config/two-stores-with-zones.xml

@@ -0,0 +1,50 @@
+<?xml version="1.0"?>
+<stores>
+  <store>
+    <name>cstore</name>
+    <persistence>bdb</persistence>
+    <routing>client</routing>
+    <routing-strategy>consistent-routing</routing-strategy>
+    <replication-factor>3</replication-factor>
+    <preferred-reads>3</preferred-reads>
+    <required-reads>3</required-reads>
+    <preferred-writes>3</preferred-writes>
+    <required-writes>2</required-writes>
+    <zone-count-reads>1</zone-count-reads>
+    <zone-count-writes>1</zone-count-writes>
+    <key-serializer>
+      <type>string</type>
+      <schema-info>UTF-8</schema-info>
+    </key-serializer>
+    <value-serializer>
+      <type>string</type>
+      <schema-info>UTF-8</schema-info>
+    </value-serializer>
+  </store>
+  <store>
+    <name>zstore</name>
+    <persistence>bdb</persistence>
+    <routing>client</routing>
+    <routing-strategy>zone-routing</routing-strategy>
+    <replication-factor>3</replication-factor>
+    <zone-replication-factor>
+    <replication-factor zone-id="0">1</replication-factor>
+        <replication-factor zone-id="1">1</replication-factor>
+        <replication-factor zone-id="2">1</replication-factor>
+    </zone-replication-factor>
+    <preferred-reads>3</preferred-reads>
+    <required-reads>3</required-reads>
+    <preferred-writes>3</preferred-writes>
+    <required-writes>2</required-writes>
+    <zone-count-reads>1</zone-count-reads>
+    <zone-count-writes>1</zone-count-writes>
+    <key-serializer>
+      <type>string</type>
+      <schema-info>UTF-8</schema-info>
+    </key-serializer>
+    <value-serializer>
+      <type>string</type>
+      <schema-info>UTF-8</schema-info>
+    </value-serializer>
+  </store>
+</stores>

test/unit/voldemort/client/AdminFetchTest.java

@@ -119,7 +119,6 @@ public void setUp() throws IOException {
                 partitionToKeysMap.put(partition, new HashSet<String>());
             partitionToKeysMap.get(partition).add(key);
             voldClient.put(key, "value" + i);
-            System.out.println(key);
         }
     }
 

test/unit/voldemort/store/bdb/PartitionPrefixedBdbStorageEngineTest.java

@@ -4,17 +4,23 @@
 import static junit.framework.Assert.fail;
 
 import java.io.File;
+import java.io.StringReader;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.io.FileDeleteStrategy;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
+import voldemort.ServerTestUtils;
 import voldemort.TestUtils;
+import voldemort.VoldemortTestConstants;
+import voldemort.cluster.Cluster;
 import voldemort.routing.RoutingStrategy;
+import voldemort.routing.RoutingStrategyFactory;
 import voldemort.server.VoldemortConfig;
 import voldemort.store.StoreBinaryFormat;
 import voldemort.store.StoreDefinition;
@@ -25,6 +31,7 @@
 import voldemort.utils.Props;
 import voldemort.versioning.Versioned;
 import voldemort.xml.ClusterMapper;
+import voldemort.xml.StoreDefinitionsMapper;
 
 /**
  * Tests for the BDB storage engine prefixing the partition id to the keys, to
@@ -81,6 +88,52 @@ public void testPartitionToByteArrayConversion() {
         assertEquals(0, ByteUtils.compare(key, StoreBinaryFormat.extractKey(prefixedkey)));
     }
 
+    @Test
+    public void testHashConsistencyAcrossRoutingStrategies() {
+        // check that as long as the cluster.xml is the same, a key will hash to
+        // the same partition, immaterial of whether it is zone or consistent
+        // routing strategy
+
+        StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
+        List<StoreDefinition> storeDefs = mapper.readStoreList(new StringReader(VoldemortTestConstants.getTwoStoresWithZonesXml()));
+
+        StoreDefinition consistentStore = storeDefs.get(0);
+        StoreDefinition zoneStore = storeDefs.get(1);
+
+        assertEquals(consistentStore.getName(), "cstore");
+        assertEquals(zoneStore.getName(), "zstore");
+
+        Cluster cluster = VoldemortTestConstants.getEightNodeClusterWithZones();
+        RoutingStrategy cStrategy = new RoutingStrategyFactory().updateRoutingStrategy(consistentStore,
+                                                                                       cluster);
+        RoutingStrategy zStrategy = new RoutingStrategyFactory().updateRoutingStrategy(zoneStore,
+                                                                                       cluster);
+
+        BdbStorageEngine cPrefixedBdbStore = (BdbStorageEngine) bdbStorage.getStore(consistentStore,
+                                                                                    cStrategy);
+
+        BdbStorageEngine zPrefixedBdbStore = (BdbStorageEngine) bdbStorage.getStore(zoneStore,
+                                                                                    zStrategy);
+
+        HashMap<ByteArray, byte[]> kvpairs = ServerTestUtils.createRandomKeyValuePairs(10000);
+        for(ByteArray key: kvpairs.keySet()) {
+            assertEquals(cStrategy.getPartitionList(key.get()).get(0),
+                         zStrategy.getPartitionList(key.get()).get(0));
+
+            cPrefixedBdbStore.put(key, new Versioned<byte[]>(kvpairs.get(key)), null);
+            zPrefixedBdbStore.put(key, new Versioned<byte[]>(kvpairs.get(key)), null);
+        }
+
+        for(ByteArray key: kvpairs.keySet()) {
+            assertEquals("Values read back does not match up",
+                         0,
+                         ByteUtils.compare(cPrefixedBdbStore.get(key, null).get(0).getValue(),
+                                           zPrefixedBdbStore.get(key, null).get(0).getValue()));
+        }
+        cPrefixedBdbStore.close();
+        zPrefixedBdbStore.close();
+    }
+
     private Set<String> getKeys(ClosableIterator<ByteArray> itr) {
         HashSet<String> keySet = new HashSet<String>();
         while(itr.hasNext()) {
@@ -139,7 +192,7 @@ public void testPartitionScan() {
                 assertEquals(partitionToKeysMap.get(p).size(), keys.size());
                 assertEquals(partitionToKeysMap.get(p), keys);
 
-                // verfiy values
+                // verify values
                 keys = getEntries(prefixedBdbStore.entries(p));
                 assertEquals(partitionToKeysMap.get(p).size(), keys.size());
                 assertEquals(partitionToKeysMap.get(p), keys);