Permalink
Browse files

Fixing failing contrib tests.

  • Loading branch information...
1 parent 1af7f0c commit 8afb67ca6d505c9e48ea40a43d849ecde7f7d6b9 @bbansal bbansal committed Mar 10, 2009
View
@@ -170,7 +170,8 @@
<formatter type="brief" usefile="false"/>
<batchtest fork="yes" todir="${testreport.dir}">
<fileset dir="${contrib.classes.dir}">
- <include name="**/Test*.class"/>
+ <include name="**/*Test.class"/>
+ <exclude name="**/*PerformanceTest.class"/>
</fileset>
</batchtest>
</junit>
@@ -197,6 +198,9 @@
</junitreport>
</target>
+ <target name="junit-all" depends="junit, contrib-junit" description="Run All junit tests including contrib.">
+ </target>
+
<target name="war" depends="build" description="Build server war file">
<war destfile="${dist.dir}/${name}.war" webxml="web.xml" basedir="${classes.dir}">
<classes dir="${classes.dir}"/>
@@ -52,7 +52,7 @@
* @author bbansal
*
*/
-public class TestReadOnlyBatchIndexer extends TestCase {
+public class ReadOnlyBatchIndexerTest extends TestCase {
@SuppressWarnings("unchecked")
public void testCSVFileBatchIndexer() throws Exception {
@@ -84,7 +84,7 @@ public void testCSVFileBatchIndexer() throws Exception {
Valueserializer);
// query all keys and check for value
- BufferedReader reader = new BufferedReader(new FileReader(new File("contrib/test/common/test-data/usersCSV.txt")));
+ BufferedReader reader = new BufferedReader(new FileReader(new File("contrib/common/test-data/usersCSV.txt")));
String line;
while(null != (line = reader.readLine())) {
@@ -125,17 +125,15 @@ public Object getValueBytes(LongWritable key, Text value) {
class TextBatchIndexer extends ReadOnlyBatchIndexer {
- @Override
public void configure(JobConf conf) {
conf.set("job.name", "testCSVBatchIndexer");
- conf.set("voldemort.cluster.local.filePath",
- "contrib/test/common/config/nine-node-cluster.xml");
- conf.set("voldemort.store.local.filePath", "contrib/test/common/config/stores.xml");
+ conf.set("voldemort.cluster.local.filePath", "contrib/common/config/nine-node-cluster.xml");
+ conf.set("voldemort.store.local.filePath", "contrib/common/config/stores.xml");
conf.set("voldemort.store.name", "users");
// set inset/outset path
- FileInputFormat.addInputPaths(conf, "contrib/test/common/test-data/usersCSV.txt");
+ FileInputFormat.addInputPaths(conf, "contrib/common/test-data/usersCSV.txt");
FileOutputFormat.setOutputPath(conf, new Path("contrib/batch-indexer/temp-output/text"));
conf.setMapperClass(TextBatchMapper.class);
@@ -26,10 +26,7 @@
import org.apache.commons.io.FileDeleteStrategy;
import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Mapper;
import voldemort.ServerTestUtils;
import voldemort.TestUtils;
@@ -45,13 +42,13 @@
import voldemort.utils.ByteArray;
import voldemort.xml.ClusterMapper;
-public class TestReadOnlyHadoopSwapper extends TestCase {
+public class ReadOnlyHadoopSwapperTest extends TestCase {
private static final int TEST_SIZE = 500;
private static final String baseDir = TestUtils.createTempDir().getAbsolutePath();
- private static final String clusterFile = "contrib/test/common/config/two-node-cluster.xml";
- private static final String storerFile = "contrib/test/common/config/testSwapStore.xml";
+ private static final String clusterFile = "contrib/common/config/two-node-cluster.xml";
+ private static final String storerFile = "contrib/common/config/testSwapStore.xml";
private static final String storeName = "swapTestStore";
VoldemortServer server1;
@@ -66,7 +63,10 @@ public void setUp() throws Exception {
// store cleanly
String indexDir = makeReadOnlyIndex(1, 1000);
- VoldemortConfig config = ServerTestUtils.createServerConfig(0, baseDir, clusterFile, storerFile);
+ VoldemortConfig config = ServerTestUtils.createServerConfig(0,
+ baseDir,
+ clusterFile,
+ storerFile);
server1 = new VoldemortServer(config);
// copy read-only index before starting
FileUtils.copyFile(new File(indexDir, "0.index"),
@@ -99,8 +99,8 @@ private String makeReadOnlyIndex(int minKey, int maxKey) throws Exception {
entryMap.put("key" + i, "value-" + i);
}
- Cluster cluster = new ClusterMapper().readCluster(new FileReader(new File("contrib/test/common/config/two-node-cluster.xml")));
- return TestUtils.createReadOnlyIndex(cluster, entryMap, baseDir);
+ Cluster cluster = new ClusterMapper().readCluster(new FileReader(new File("contrib/common/config/two-node-cluster.xml")));
+ return ReadOnlySwapperTestUtils.createReadOnlyIndex(cluster, entryMap, baseDir);
}
public void testswap() throws Throwable {
@@ -137,7 +137,7 @@ public void configure(JobConf conf) {
}
@Override
- public Class<? extends Mapper<LongWritable, Text, Text, Text>> getSwapperMapperClass() {
+ public Class<? extends SwapperMapper> getSwapperMapperClass() {
return SwapperMapper.class;
}
};
@@ -41,14 +41,14 @@
import voldemort.utils.Props;
import voldemort.xml.ClusterMapper;
-public class TestReadOnlySimpleSwapper extends TestCase {
+public class ReadOnlySimpleSwapperTest extends TestCase {
private static final int TEST_SIZE = 500;
private static final String baseDir = TestUtils.createTempDir().getAbsolutePath();
- private static final String clusterFile = "contrib/test/common/config/two-node-cluster.xml";
- private static final String storerFile = "contrib/test/common/config/testSwapStore.xml";
- private static final String storeName = "swap-store";
+ private static final String clusterFile = "contrib/common/config/two-node-cluster.xml";
+ private static final String storerFile = "contrib/common/config/testSwapStore.xml";
+ private static final String storeName = "swapTestStore";
VoldemortServer server1;
VoldemortServer server2;
@@ -59,15 +59,15 @@ public void setUp() throws Exception {
FileDeleteStrategy.FORCE.delete(new File(baseDir));
String indexDir = makeReadOnlyIndex(1, 1000);
- server1 = startServer(2, indexDir);
+ server1 = startServer(0, indexDir);
server2 = startServer(1, indexDir);
}
private VoldemortServer startServer(int nodeId, String indexDir) throws Exception {
VoldemortConfig config = ServerTestUtils.createServerConfig(nodeId,
- baseDir,
- clusterFile,
- storerFile);
+ baseDir,
+ clusterFile,
+ storerFile);
VoldemortServer server = new VoldemortServer(config);
// copy read-only index before starting
FileUtils.copyFile(new File(indexDir, nodeId + ".index"),
@@ -92,8 +92,8 @@ private String makeReadOnlyIndex(int minKey, int maxKey) throws Exception {
entryMap.put("key" + i, "value-" + i);
}
- Cluster cluster = new ClusterMapper().readCluster(new FileReader(new File("contrib/test/common/config/two-node-cluster.xml")));
- return TestUtils.createReadOnlyIndex(cluster, entryMap, baseDir);
+ Cluster cluster = new ClusterMapper().readCluster(new FileReader(new File("contrib/common/config/two-node-cluster.xml")));
+ return ReadOnlySwapperTestUtils.createReadOnlyIndex(cluster, entryMap, baseDir);
}
public void testswap() throws Throwable {
@@ -110,7 +110,6 @@ public void testswap() throws Throwable {
ByteArray key = new ByteArray(serializer.toBytes("key" + i));
byte[] value = serializer.toBytes("value" + i);
- System.out.println("checking key:" + i);
assertEquals("either store1 or store2 will have the key:'key-" + i + "'",
true,
store1.get(key).size() > 0 || store2.get(key).size() > 0);
@@ -0,0 +1,73 @@
+package test.voldemort.contrib.batchswapper;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.util.Map;
+
+import voldemort.client.RoutingTier;
+import voldemort.cluster.Cluster;
+import voldemort.routing.ConsistentRoutingStrategy;
+import voldemort.routing.RoutingStrategy;
+import voldemort.serialization.SerializerDefinition;
+import voldemort.serialization.json.JsonReader;
+import voldemort.store.StorageEngineType;
+import voldemort.store.StoreDefinition;
+import voldemort.store.readonly.JsonStoreBuilder;
+
+public class ReadOnlySwapperTestUtils {
+
+ /**
+ *
+ * @param cluster
+ * @param data
+ * @param baseDir
+ * @param TEST_SIZE
+ * @return the directory where the index is created
+ * @throws Exception
+ */
+ public static String createReadOnlyIndex(Cluster cluster,
+ Map<String, String> data,
+ String baseDir) throws Exception {
+ // write data to file
+ File dataFile = File.createTempFile("test", ".txt");
+ dataFile.deleteOnExit();
+ BufferedWriter writer = new BufferedWriter(new FileWriter(dataFile));
+ for(Map.Entry<String, String> entry: data.entrySet())
+ writer.write("\"" + entry.getKey() + "\"\t\"" + entry.getValue() + "\"\n");
+ writer.close();
+ BufferedReader reader = new BufferedReader(new FileReader(dataFile));
+ JsonReader jsonReader = new JsonReader(reader);
+
+ SerializerDefinition serDef = new SerializerDefinition("json", "'string'");
+ StoreDefinition storeDef = new StoreDefinition("test",
+ StorageEngineType.READONLY,
+ serDef,
+ serDef,
+ RoutingTier.CLIENT,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1);
+ RoutingStrategy router = new ConsistentRoutingStrategy(cluster.getNodes(), 1);
+
+ // make a temp dir
+ File dataDir = new File(baseDir + File.separatorChar + "read-only-temp-index-"
+ + new Integer((int) (Math.random() * 1000)));
+ // build and open store
+ JsonStoreBuilder storeBuilder = new JsonStoreBuilder(jsonReader,
+ cluster,
+ storeDef,
+ router,
+ dataDir,
+ 100,
+ 1);
+ storeBuilder.build();
+
+ return dataDir.getAbsolutePath();
+ }
+}
@@ -9,14 +9,11 @@
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobConfigurable;
-import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.util.Tool;
import org.apache.log4j.Logger;
@@ -128,5 +125,5 @@ private void moveMetaDatatoHDFS(JobConf conf, Path clusterFile) throws IOExcepti
*/
public abstract void configure(JobConf conf);
- public abstract Class<? extends Mapper<LongWritable, Text, Text, Text>> getSwapperMapperClass();
+ public abstract Class<? extends AbstractSwapperMapper> getSwapperMapperClass();
}
@@ -22,7 +22,7 @@
import voldemort.contrib.utils.ContribUtils;
-public class TestContribUtils extends TestCase {
+public class ContribUtilsTest extends TestCase {
public void testGetFileFromPathList() {
Path path1 = new Path("simpleFileName");

0 comments on commit 8afb67c

Please sign in to comment.