AVRO-521. Out of memory and other issues with Junit tests for mapreduce

git-svn-id: https://svn.apache.org/repos/asf/hadoop/avro/trunk@937307 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -27,6 +27,8 @@ Avro 1.4.0 (unreleased)
 
     AVRO-520. Refactor C++ validation code. (sbanacho)
 
+    AVRO-521. Out of memory and other issues with Junit tests for mapreduce (thiru)
+
   BUG FIXES
     AVRO-461. Skipping primitives in the ruby side (jmhodges)
 

lang/java/build.xml

@@ -340,6 +340,7 @@
              printsummary="withOutAndErr"
              haltonfailure="no"
              fork="yes" forkMode="once"
+             maxmemory="128m"
              errorProperty="tests.failed" failureProperty="tests.failed">
         <sysproperty key="test.count" value="${test.count}"/>
         <sysproperty key="test.dir" value="@{test.dir}"/>

lang/java/src/test/java/org/apache/avro/mapred/TestWordCountGeneric.java

@@ -18,8 +18,6 @@
 
 package org.apache.avro.mapred;
 
-import junit.framework.TestCase;
-
 import java.io.IOException;
 import java.util.StringTokenizer;
 
@@ -33,8 +31,9 @@
 import org.apache.avro.util.Utf8;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.avro.generic.GenericData;
+import org.junit.Test;
 
-public class TestWordCountGeneric extends TestCase {
+public class TestWordCountGeneric {
 
   private static GenericRecord newWordCount(String word, int count) {
     GenericRecord value = new GenericData.Record(WordCount.SCHEMA$);
@@ -75,27 +74,34 @@ public void close() throws IOException {
 
   }
 
+  @Test
+  @SuppressWarnings("deprecation")
   public void testJob() throws Exception {
-    WordCountUtil.writeLinesFile();
-
+    String dir = System.getProperty("test.dir", ".") + "/mapred";
+    Path outputPath = new Path(dir + "/out");
     JobConf job = new JobConf();
-    job.setJobName("wordcount");
-
-    AvroJob.setInputGeneric(job, Schema.create(Schema.Type.STRING));
-    AvroJob.setOutputGeneric(job, WordCount.SCHEMA$);
-
-    job.setMapperClass(MapImpl.class);        
-    job.setCombinerClass(ReduceImpl.class);
-    job.setReducerClass(ReduceImpl.class);
-
-    String dir = System.getProperty("test.dir",".")+"/mapred";
-    FileInputFormat.setInputPaths(job, new Path(dir+"/in"));
-    FileOutputFormat.setOutputPath(job, new Path(dir+"/out"));
-    FileOutputFormat.setCompressOutput(job, true);
-
-    JobClient.runJob(job);
-
-    WordCountUtil.validateCountsFile();
+    try {
+      WordCountUtil.writeLinesFile();
+
+      job.setJobName("wordcount");
+
+      AvroJob.setInputGeneric(job, Schema.create(Schema.Type.STRING));
+      AvroJob.setOutputGeneric(job, WordCount.SCHEMA$);
+
+      job.setMapperClass(MapImpl.class);        
+      job.setCombinerClass(ReduceImpl.class);
+      job.setReducerClass(ReduceImpl.class);
+
+      FileInputFormat.setInputPaths(job, new Path(dir + "/in"));
+      FileOutputFormat.setOutputPath(job, outputPath);
+      FileOutputFormat.setCompressOutput(job, true);
+
+      JobClient.runJob(job);
+
+      WordCountUtil.validateCountsFile();
+    } finally {
+      outputPath.getFileSystem(job).delete(outputPath);
+    }
   }
 
 }

lang/java/src/test/java/org/apache/avro/mapred/TestWordCountSpecific.java

@@ -18,8 +18,6 @@
 
 package org.apache.avro.mapred;
 
-import junit.framework.TestCase;
-
 import java.io.IOException;
 import java.util.StringTokenizer;
 
@@ -31,8 +29,9 @@
 
 import org.apache.avro.Schema;
 import org.apache.avro.util.Utf8;
+import org.junit.Test;
 
-public class TestWordCountSpecific extends TestCase {
+public class TestWordCountSpecific {
 
   private static WordCount newWordCount(String word, int count) {
     WordCount value = new WordCount();
@@ -42,6 +41,7 @@ private static WordCount newWordCount(String word, int count) {
   }
 
   public static class MapImpl extends AvroMapper<Utf8, WordCount> {
+    @Override
     public void map(Utf8 text) throws IOException {
       StringTokenizer tokens = new StringTokenizer(text.toString());
       while (tokens.hasMoreTokens())
@@ -53,6 +53,7 @@ public static class ReduceImpl extends AvroReducer<WordCount, WordCount> {
 
     private WordCount previous;
 
+    @Override
     public void reduce(WordCount current) throws IOException {
       if (current.equals(previous)) {
         previous.count++;
@@ -63,34 +64,43 @@ public void reduce(WordCount current) throws IOException {
       }
     }
 
+    @Override
     public void close() throws IOException {
       if (previous != null)
         collect(previous);
     }
 
   }
 
+  @Test
+  @SuppressWarnings("deprecation")
   public void testJob() throws Exception {
-    WordCountUtil.writeLinesFile();
-
     JobConf job = new JobConf();
-    job.setJobName("wordcount");
-
-    AvroJob.setInputSpecific(job, Schema.create(Schema.Type.STRING));
-    AvroJob.setOutputSpecific(job, WordCount.SCHEMA$);
-
-    job.setMapperClass(MapImpl.class);        
-    job.setCombinerClass(ReduceImpl.class);
-    job.setReducerClass(ReduceImpl.class);
-
-    String dir = System.getProperty("test.dir",".")+"/mapred";
-    FileInputFormat.setInputPaths(job, new Path(dir+"/in"));
-    FileOutputFormat.setOutputPath(job, new Path(dir+"/out"));
-    FileOutputFormat.setCompressOutput(job, true);
-
-    JobClient.runJob(job);
-
-    WordCountUtil.validateCountsFile();
+    String dir = System.getProperty("test.dir", ".") + "/mapred";
+    Path outputPath = new Path(dir + "/out");
+
+    try {
+      WordCountUtil.writeLinesFile();
+
+      job.setJobName("wordcount");
+
+      AvroJob.setInputSpecific(job, Schema.create(Schema.Type.STRING));
+      AvroJob.setOutputSpecific(job, WordCount.SCHEMA$);
+
+      job.setMapperClass(MapImpl.class);        
+      job.setCombinerClass(ReduceImpl.class);
+      job.setReducerClass(ReduceImpl.class);
+
+      FileInputFormat.setInputPaths(job, new Path(dir + "/in"));
+      FileOutputFormat.setOutputPath(job, outputPath);
+      FileOutputFormat.setCompressOutput(job, true);
+
+      JobClient.runJob(job);
+
+      WordCountUtil.validateCountsFile();
+    } finally {
+      outputPath.getFileSystem(job).delete(outputPath);
+    }
 
   }