Adding unit test for fork lift tool

src/java/voldemort/utils/AbstractConsistencyFixer.java

@@ -35,6 +35,16 @@
 
 import com.google.common.collect.Lists;
 
+/**
+ * Base class that contains logic to read the state of a key on a cluster and
+ * resolve it using read repair
+ * 
+ * NOTE : For now, there is only one subclass extending this class, to perform
+ * Consistency fixing by writing the resolved versions back to the cluster. Any
+ * future tool that needs similar functionality can extend this class and
+ * implement logic to do whatever it wants to do with the resolved versions.
+ * 
+ */
 abstract class AbstractConsistencyFixer {
 
     private static final Logger logger = Logger.getLogger(AbstractConsistencyFixer.class);

src/java/voldemort/utils/ClusterForkLiftTool.java

@@ -81,6 +81,29 @@
  * forklift window. Of course, after the forklift window, the destination
  * cluster resumes normal operation.
  * 
+ * 3) For now, we will fallback to fetching the key from the primary replica,
+ * fetch the values out manually, resolve and write it back. PitFalls : primary
+ * somehow does not have the key.
+ * 
+ * Two scenarios.
+ * 
+ * 1) Key active after double writes: the situation is the result of slop not
+ * propagating to the primary. But double writes would write the key back to
+ * destination cluster anyway. We are good.
+ * 
+ * 2) Key inactive after double writes: This indicates a problem elsewhere. This
+ * is a base guarantee voldemort should offer.
+ * 
+ * 4) Zoned <-> Non Zoned forklift implications.
+ * 
+ * When forklifting data from a non-zoned to zoned cluster, both destination
+ * zones will be populated with data, by simply running the tool once with the
+ * respective bootstrap urls. If you need to forklift data from zoned to
+ * non-zoned clusters (i.e your replication between datacenters is not handled
+ * by Voldemort), then you need to run the tool twice for each destination
+ * non-zoned cluster. Zoned -> Zoned and Non-Zoned -> Non-Zoned forklifts are
+ * trivial.
+ * 
  */
 public class ClusterForkLiftTool implements Runnable {
 
@@ -113,10 +136,10 @@ public ClusterForkLiftTool(String srcBootstrapUrl,
                                               new ClientConfig());
 
         // set up streaming client to the destination cluster
-        Props property = new Props();
-        property.put("streaming.platform.bootstrapURL", dstBootstrapUrl);
-        property.put("streaming.platform.throttle.qps", maxPutsPerSecond);
-        StreamingClientConfig config = new StreamingClientConfig(property);
+        Props props = new Props();
+        props.put("streaming.platform.bootstrapURL", dstBootstrapUrl);
+        props.put("streaming.platform.throttle.qps", maxPutsPerSecond);
+        StreamingClientConfig config = new StreamingClientConfig(props);
         this.dstStreamingClient = new StreamingClient(config);
 
         // determine and verify final list of stores to be forklifted over
@@ -173,6 +196,12 @@ private HashMap<String, StoreDefinition> checkStoresOnBothSides() {
         return srcStoreDefMap;
     }
 
+    /**
+     * TODO this base class can potentially provide some framework of execution
+     * for the subclasses, to yield a better objected oriented design (progress
+     * tracking etc)
+     * 
+     */
     abstract class SinglePartitionForkLiftTask {
 
         protected int partitionId;
@@ -209,21 +238,6 @@ class SinglePartitionGloballyResolvingForkLiftTask extends SinglePartitionForkLi
             super(storeInstance, partitionId, latch);
         }
 
-        /**
-         * For now, we will fallback to fetching the key from the primary
-         * replica, fetch the values out manually, resolve and write it back.
-         * PitFalls : primary somehow does not have the key.
-         * 
-         * Two scenarios.
-         * 
-         * 1) Key active after double writes: the situation is the result of
-         * slop not propagating to the primary. But double writes would write
-         * the key back to destination cluster anyway. We are good.
-         * 
-         * 2) Key inactive after double writes: This indicates a problem
-         * elsewhere. This is a base guarantee voldemort should offer.
-         * 
-         */
         public void run() {
             String storeName = this.storeInstance.getStoreDefinition().getName();
             long entriesForkLifted = 0;
@@ -455,8 +469,6 @@ public Object call() throws Exception {
             }
             srcAdminClient.close();
             dstStreamingClient.getAdminClient().close();
-            // TODO cleanly shut down the threadpool
-            System.exit(0);
         }
     }
 
@@ -570,5 +582,7 @@ public static void main(String[] args) throws Exception {
                                                                    partitions,
                                                                    options.has("global-resolution"));
         forkLiftTool.run();
+        // TODO cleanly shut down the hanging threadpool
+        System.exit(0);
     }
 }

test/common/voldemort/config/two-stores-replicated.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0"?>
+<stores>
+  <store>
+    <name>test</name>
+    <persistence>bdb</persistence>
+    <routing>client</routing>
+    <replication-factor>2</replication-factor>
+    <preferred-reads>1</preferred-reads>
+    <required-reads>1</required-reads>
+    <preferred-writes>1</preferred-writes>
+    <required-writes>1</required-writes>
+    <key-serializer>
+      <type>string</type>
+      <schema-info>UTF-8</schema-info>
+    </key-serializer>
+    <value-serializer>
+      <type>string</type>
+      <schema-info>UTF-8</schema-info>
+    </value-serializer>
+  </store>
+  <store>
+    <name>best</name>
+    <persistence>bdb</persistence>
+    <routing>client</routing>
+    <replication-factor>3</replication-factor>
+    <preferred-reads>2</preferred-reads>
+    <required-reads>2</required-reads>
+    <preferred-writes>2</preferred-writes>
+    <required-writes>2</required-writes>
+    <key-serializer>
+      <type>string</type>
+      <schema-info>UTF-8</schema-info>
+    </key-serializer>
+    <value-serializer>
+      <type>string</type>
+      <schema-info>UTF-8</schema-info>
+    </value-serializer>
+  </store>
+</stores>