merged conflicted sources

.classpath

@@ -51,8 +51,11 @@
 	<classpathentry kind="lib" path="lib/libthrift-0.5.0.jar"/>
 	<classpathentry kind="lib" path="lib/compress-lzf-0.9.1.jar"/>
 	<classpathentry kind="lib" path="lib/snappy-0.2.jar"/>
-	<classpathentry kind="lib" path="lib/httpclient-4.1.2.jar" />
-	<classpathentry kind="lib" path="lib/httpcore-4.1.2.jar" />
+	<classpathentry kind="lib" path="lib/httpclient-4.1.2.jar"/>
+	<classpathentry kind="lib" path="lib/httpcore-4.1.2.jar"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
+	<classpathentry kind="lib" path="lib/joda-time-1.6.jar"/>
+	<classpathentry kind="lib" path="lib/mail-1.4.1.jar"/>
+	<classpathentry kind="lib" path="lib/azkaban-common-0.05.jar"/>
 	<classpathentry kind="output" path="classes"/>
 </classpath>

.gitignore

@@ -13,3 +13,4 @@ server.state
 .version
 .temp
 .idea
+data/

.settings/org.eclipse.jdt.core.prefs

@@ -1,4 +1,4 @@
-#Fri Dec 30 14:37:10 PST 2011
+#Thu Aug 30 10:43:57 PDT 2012
 eclipse.preferences.version=1
 org.eclipse.jdt.core.codeComplete.argumentPrefixes=
 org.eclipse.jdt.core.codeComplete.argumentSuffixes=
@@ -11,9 +11,9 @@ org.eclipse.jdt.core.codeComplete.staticFieldSuffixes=
 org.eclipse.jdt.core.codeComplete.staticFinalFieldPrefixes=
 org.eclipse.jdt.core.codeComplete.staticFinalFieldSuffixes=
 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.5
+org.eclipse.jdt.core.compiler.compliance=1.6
 org.eclipse.jdt.core.compiler.debug.lineNumber=generate
 org.eclipse.jdt.core.compiler.debug.localVariable=generate
 org.eclipse.jdt.core.compiler.debug.sourceFile=generate
@@ -77,7 +77,7 @@ org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disa
 org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning
 org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning
 org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning
-org.eclipse.jdt.core.compiler.source=1.5
+org.eclipse.jdt.core.compiler.source=1.6
 org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
 org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=82
 org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=82

META-INF/MANIFEST.MF

@@ -0,0 +1,7 @@
+Manifest-Version: 1.0
+Ant-Version: Apache Ant 1.7.1
+Created-By: 20.2-b06 (Sun Microsystems Inc.)
+Implementation-Title: Voldemort
+Implementation-Version: 1.0.0
+Implementation-Vendor: LinkedIn
+

bin/generate_cluster_xml.py

@@ -1,42 +1,90 @@
+#!/usr/bin/python
+
 import sys
 import random
+import argparse
+
+# Get a random seed
+rseed = int(random.randint(00000000001,99999999999))
+
+# Setup and argument parser
+parser = argparse.ArgumentParser(description='Build a voldemort cluster.xml.')
+# Add supported arguments
+parser.add_argument('-N', '--name', type=str, default='voldemort', dest='name',
+                    help='the name you want to give the cluster')
+parser.add_argument('-n', '--nodes', type=int, default=2, dest='nodes',
+                    help='the number of nodes in the cluster')
+parser.add_argument('-p', '--partitions', type=int, default=300,
+                    dest='partitions', help='number of partitions per node')
+parser.add_argument('-s', '--socket-port', type=int, default=6666,
+                    dest='sock_port', help='socket port number')
+parser.add_argument('-a', '--admin-port', type=int, default=6667,
+                    dest='admin_port', help='admin port number')
+parser.add_argument('-H', '--http-port', type=int, default=6665,
+                    dest='http_port', help='http port number')
+genType = parser.add_mutually_exclusive_group()
+genType.add_argument('-S', '--seed', type=int, default=rseed, dest='seed',
+                    help='seed for randomizing partition distribution')
+genType.add_argument('-l', '--loops', type=int, default=1000, dest='loops',
+                    help='loop n times, using a different random seed every \
+                          time (Note: not currently supported)')
+parser.add_argument('-z', '--zones', type=int, dest='zones',
+                    help='if using zones, the number of zones you will have\
+                          (Note: you must add your own <zone> fields \
+                          manually)')
+
+# Parse arguments
+args = parser.parse_args()
+
+# Check args
+if args.zones:
+  zones = args.zones
+  if (args.nodes % zones) != 0:
+    print "Number of nodes must be evenly divisible by number of zones"
+    sys.exit(1)
+
+# Store arguments
+nodes = args.nodes
+partitions = args.partitions
+name = args.name
+http_port = args.http_port
+sock_port = args.sock_port
+admin_port = args.admin_port
+seed = args.seed
+
+# Generate the full list of partition IDs
+part_ids = range(nodes * partitions)
+# Generate full list of zone IDs
+if args.zones:
+  zone_ids = range(zones)
+  zone_id = 0
+
+# Shuffle up the partitions
+random.seed(seed)
+random.shuffle(part_ids)
+
+# Printing cluster.xml
+print "<!-- Partition distribution generated using seed [%d] -->" % seed
+print "<cluster>"
+print "  <name>%s</name>" % name
+
+for i in xrange(nodes):
+  node_partitions = ", ".join(str(p) for p in sorted(part_ids[i*partitions:(i+1)*partitions]))
+
+  print "  <server>"
+  print "    <id>%d</id>" % i
+  print "    <host>host%d</host>" % i
+  print "    <http-port>%d</http-port>" % http_port
+  print "    <socket-port>%d</socket-port>" % sock_port
+  print "    <admin-port>%d</admin-port>" % admin_port
+  print "    <partitions>%s</partitions>" % node_partitions
+  # If zones are being used, assign a zone-id
+  if args.zones:
+    print "    <zone-id>%d</zone-id>" % zone_id
+    if zone_id == (zones - 1):
+      zone_id = 0
+    else:
+      zone_id += 1
+  print "  </server>"
 
-if len(sys.argv) != 3:
-    print >> sys.stderr, "USAGE: python generate_partitions.py <nodes_file> <partitions_per_node>"
-    sys.exit()
-
-FORMAT_WIDTH = 10
-
-nodes = 0
-for line in open(sys.argv[1],'r'):
-	nodes+=1
-
-partitions = int(sys.argv[2])
-
-ids = range(nodes * partitions)
-
-# use known seed so this is repeatable
-random.seed(92873498274)
-random.shuffle(ids)
-
-print '<cluster>'
-print '<name>prodcluster</name>'
-id = 0
-for host in open(sys.argv[1],'r'):
-    print '<server>'
-    print "  <id>%d</id>" % id
-    print "  <host>%s</host>" % host.strip()
-    print '  <http-port>8081</http-port>'
-    print '  <socket-port>6666</socket-port>'
-    print '  <partitions>',
-    node_ids = sorted(ids[id*partitions:(id+1)*partitions])
-    for j in xrange(len(node_ids)):
-        print str(node_ids[j]) + ',',
-        if j % FORMAT_WIDTH == FORMAT_WIDTH - 1:
-            print '    ',
-    print '  </partitions>'
-    print '</server>'
-    id += 1
-print '</cluster>'
-
-
+print "</cluster>"

bin/repeat-junit-test.sh

@@ -0,0 +1,93 @@
+#!/bin/bash -e
+
+# Copyright 2012 LinkedIn, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+usage()  {
+    echo
+    echo "Usage:"
+    echo "bin/repeat-junit-test.sh test_file num_times"
+    echo 
+    cat <<EOF 
+Invoke bin/repeat-junit-test.sh from the root of a Voldemort
+checkout. bin/repeat-junit-test.sh invokes 'ant junit-test' num_times
+for test test_name.
+
+The argument num_times must be an integer. The argument test_name must
+be a class name suitable for 'ant junit-test'. I.e., a fully qualified
+java class name. Remember, the class name does not include the .java
+extension. An example test_name is voldemort.utils.ServerTestUtilsTest.
+
+The pretty html junit output that ends up in dist/junit-single-report
+on a single invocation of 'ant junit-test' is collected in a temp
+directory. This circumvents the normal behavior of ant in which
+dist/junit-single-report is overwritten with each invocation of 'ant
+junit-test'.
+
+bin/repeat-junit-test.sh is useful to run after adding a new test
+case, or when trying to reproduce intermittent failures of a specific
+test.
+EOF
+}
+
+if [ $# != 2 ]; then
+    echo "ERROR: Incorrect number of arguments: $# provided, 2 needed." >&2
+    usage
+    exit 1
+fi
+
+TESTNAME=$1
+# Hackish test that makes sure some java file exists for given
+# testname. No guarantee that junit-test can run the specified test,
+# but at least protects against typos.
+FILENAME=`echo $TESTNAME | sed 's/.*\.//g'`.java
+FINDFILE=`find . -name "$FILENAME" | wc -l`
+if [[ $FINDFILE == 0 ]]
+then
+   echo "ERROR: Did not find an appropriate file (with name $FILENAME), given test name $TESTNAME." >&2
+   usage
+   exit 1
+fi
+
+
+NUMTIMES=$2
+if [[ ! $NUMTIMES == +([0-9]) ]]
+then
+   echo "ERROR: argument num_times is not an integer: $NUMTIMES." >&2
+   usage
+   exit 1
+fi
+
+TMPDIR=`mktemp -d -p '/tmp/'`
+
+for ((i=1;i<=$NUMTIMES;i++)); do 
+    echo
+    echo "STARTING ITERATION $i"
+    echo
+
+    # Run junit-test and capture stdout to .out and stderr to .err
+    junitiout="$TMPDIR/TEST-$TESTNAME-$i.out"
+    junitierr="$TMPDIR/TEST-$TESTNAME-$i.err"
+    ant junit-test -Dtest.name=$TESTNAME > >(tee $junitiout) 2> >(tee $junitierr >&2)
+
+    # Collect results
+    junitidir="$TMPDIR/junit-single-report-$TESTNAME-$i"
+    echo
+    echo "COLLECTING RESULTS OF ITERATION $i IN $junitidir"
+    cp -r dist/junit-single-reports $junitidir
+    mv $junitiout $junitidir
+    mv $junitierr $junitidir
+done
+
+

bin/repeat-junit.sh

@@ -0,0 +1,75 @@
+#!/bin/bash -e
+
+# Copyright 2012 LinkedIn, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+usage()  {
+    echo
+    echo "Usage:"
+    echo "bin/repeat-junit.sh num_times"
+    echo 
+    cat <<EOF 
+Invoke bin/repeat-junit.sh from the root of a Voldemort
+checkout. bin/repeat-junit.sh invokes 'ant junit' num_times. 
+
+The argument num_times must be an integer. 
+
+The pretty html junit output that ends up in dist/junit-reports on a
+single invocation of 'ant junit' is collected in a temp
+directory. This circumvents the normal behavior of ant in which
+dist/junit-reports is overwritten with each invocation of 'ant
+junit'. 
+
+bin/repeat-junit.sh is useful to run after making some substantial
+changes, or when trying to track down intermittent failures (that
+occur more on your local box then on a Hudson test machine...).
+EOF
+}
+
+if [ $# != 1 ]; then
+    echo "ERROR: Incorrect number of arguments: $# provided, 1 needed." >&2
+    usage
+    exit 1
+fi
+
+NUMTIMES=$1
+if [[ ! $NUMTIMES == +([0-9]) ]]
+then
+   echo "ERROR: argument num_times is not an integer: $NUMTIMES." >&2
+   usage
+   exit 1
+fi
+
+TMPDIR=`mktemp -d -p '/tmp/'`
+
+for ((i=1;i<=$NUMTIMES;i++)); do 
+    echo
+    echo "STARTING ITERATION $i"
+    echo
+
+    # Run junit and capture stdout to .out and stderr to .err
+    junitiout="$TMPDIR/junit-$i.out"
+    junitierr="$TMPDIR/junit-$i.err"
+    ant junit > >(tee $junitiout) 2> >(tee $junitierr >&2)
+
+    # Collect results
+    junitidir="$TMPDIR/junit-reports-$i"
+    echo
+    echo "COLLECTING RESULTS OF ITERATION $i IN $junitidir"
+    cp -r dist/junit-reports $junitidir
+    mv $junitiout $junitidir
+    mv $junitierr $junitidir
+done
+
+

build.properties

@@ -9,10 +9,15 @@ classes.dir=dist/classes
 resources.dir=dist/resources
 commontestsrc.dir=test/common
 unittestsrc.dir=test/unit
+longtestsrc.dir=test/long
 inttestsrc.dir=test/integration
 testclasses.dir=dist/testclasses
 testreport.dir=dist/junit-reports
 testhtml.dir=dist/junit-reports/html
+singletestreport.dir=dist/junit-single-reports
+singletesthtml.dir=dist/junit-single-reports/html
+longtestreport.dir=dist/junit-long-reports
+longtesthtml.dir=dist/junit-long-reports/html
 
 ## Contrib
 contrib.root.dir=contrib
@@ -34,4 +39,4 @@ tomcat.manager.password=tomcat
 tomcat.context=/voldemort
 
 ## Release
-curr.release=0.90.1
+curr.release=1.0.0