fixed line lengths and changed java test

YanTangZhai · Aug 14, 2014 · df8e528 · df8e528
1 parent 9a313d5
commit df8e528
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 10 deletions.
diff --git a/core/src/main/scala/org/apache/spark/input/FixedLengthBinaryInputFormat.scala b/core/src/main/scala/org/apache/spark/input/FixedLengthBinaryInputFormat.scala
@@ -42,7 +42,8 @@ private[spark] object FixedLengthBinaryInputFormat {
 
 }
 
-private[spark] class FixedLengthBinaryInputFormat extends FileInputFormat[LongWritable, BytesWritable] {
+private[spark] class FixedLengthBinaryInputFormat
+  extends FileInputFormat[LongWritable, BytesWritable] {
 
 
   /**

diff --git a/core/src/main/scala/org/apache/spark/input/FixedLengthBinaryRecordReader.scala b/core/src/main/scala/org/apache/spark/input/FixedLengthBinaryRecordReader.scala
@@ -37,7 +37,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileSplit
  * VALUE = the record itself (BytesWritable)
  *
  */
-private[spark] class FixedLengthBinaryRecordReader extends RecordReader[LongWritable, BytesWritable] {
+private[spark] class FixedLengthBinaryRecordReader
+  extends RecordReader[LongWritable, BytesWritable] {
 
   override def close() {
     if (fileInputStream != null) {

diff --git a/core/src/test/java/org/apache/spark/JavaAPISuite.java b/core/src/test/java/org/apache/spark/JavaAPISuite.java
@@ -18,6 +18,8 @@
 package org.apache.spark;
 
 import java.io.*;
+import java.nio.channels.FileChannel;
+import java.nio.ByteBuffer;
 import java.net.URI;
 import java.util.*;
 
@@ -840,21 +842,22 @@ public Tuple2<Integer, String> call(Tuple2<IntWritable, Text> pair) {
     public void binaryFiles() throws Exception {
         // Reusing the wholeText files example
         byte[] content1 = "spark is easy to use.\n".getBytes("utf-8");
-        byte[] content2 = "spark is also easy to use.\n".getBytes("utf-8");
+
 
         String tempDirName = tempDir.getAbsolutePath();
         File file1 = new File(tempDirName + "/part-00000");
-        Files.write(content1, file1);
-        File file2 = new File(tempDirName + "/part-00001");
-        Files.write(content2, file2);
+
+        FileOutputStream fos1 = new FileOutputStream(file1);
+
+        FileChannel channel1 = fos1.getChannel();
+        ByteBuffer bbuf = java.nio.ByteBuffer.wrap(content1);
+        channel1.write(bbuf);
+
 
         JavaPairRDD<String, byte[]> readRDD = sc.binaryFiles(tempDirName,3);
         List<Tuple2<String, byte[]>> result = readRDD.collect();
         for (Tuple2<String, byte[]> res : result) {
-            if (res._1()==file1.toString())
-                Assert.assertArrayEquals(content1,res._2());
-            else
-               Assert.assertArrayEquals(content2,res._2());
+            Assert.assertArrayEquals(content1, res._2());
         }
     }