fixing missing encapsulated token scenario

uwescience · Aug 16, 2016 · 44a1de1 · 44a1de1
1 parent 52252c2
commit 44a1de1
Showing 1 changed file with 14 additions and 5 deletions.
diff --git a/src/edu/washington/escience/myria/operator/CSVFileScanFragment.java b/src/edu/washington/escience/myria/operator/CSVFileScanFragment.java
@@ -179,30 +179,36 @@ public CSVFileScanFragment(
   @Override
   protected TupleBatch fetchNextReady() throws IOException, DbException {
     long lineNumberBegin = lineNumber;
+    boolean unfinishedEncapsulatedTokenRow = false;
 
     while ((buffer.numTuples() < TupleBatch.BATCH_SIZE) && !flagAsIncomplete) {
       lineNumber++;
       if (parser.isClosed()) {
         break;
       }
 
+      if (unfinishedEncapsulatedTokenRow) {
+        onLastRow = true;
+      }
+
       try {
         if (!onLastRow) {
           record = iterator.next();
         }
       } catch (Exception e) {
-        /*
-         * FIX ME: If we hit an exception for a malformed row (in case of quotes for example), we read more bytes by
-         * marking this as the final line
-         */
-        onLastRow = true;
       }
 
       try {
         if (!iterator.hasNext()) {
           onLastRow = true;
         }
       } catch (Exception e) {
+        /*
+         * FIX ME: Only mark if we hit an exception for a malformed row (in case of quotes for example)
+         */
+        if (e.getMessage().contains("EOF reached before encapsulated token finished")) {
+          unfinishedEncapsulatedTokenRow = true;
+        }
       }
 
       if (record != null) {
@@ -261,6 +267,9 @@ record = iterator.next();
                       0);
               iterator = parser.iterator();
               record = iterator.next();
+              if (unfinishedEncapsulatedTokenRow) {
+                record = iterator.next();
+              }
               finishedReadingLastRow = true;
             } else {
               byteOverlap *= 2;