Skip to content

Commit

Permalink
Fix for #11 UnsupportedOperationException from SplitInput
Browse files Browse the repository at this point in the history
  • Loading branch information
drewfarris committed Aug 16, 2013
1 parent 7b22adc commit 9867d52
Showing 1 changed file with 14 additions and 6 deletions.
20 changes: 14 additions & 6 deletions src/main/java/com/tamingtext/util/SplitInput.java
Expand Up @@ -26,7 +26,8 @@
import java.io.Writer;
import java.nio.charset.Charset;
import java.util.BitSet;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;

import org.apache.commons.cli2.CommandLine;
import org.apache.commons.cli2.Group;
Expand Down Expand Up @@ -353,16 +354,19 @@ else if (fs.getFileStatus(inputFile).isDir()) {
BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(inputFile), charset));
Writer trainingWriter = new OutputStreamWriter(fs.create(trainingOutputFile), charset);
Writer testWriter = new OutputStreamWriter(fs.create(testOutputFile), charset);
Set<Writer> writers = new HashSet<Writer>();
writers.add(trainingWriter);
writers.add(testWriter);

int pos = 0;
int trainCount = 0;
int testCount = 0;

String line;
Writer writer;
while ((line = reader.readLine()) != null) {
pos++;

Writer writer;
if (testRandomSelectionPct > 0) { // Randomly choose
writer = randomSel.get(pos) ? testWriter : trainingWriter;
} else { // Choose based on location
Expand All @@ -384,9 +388,8 @@ else if (fs.getFileStatus(inputFile).isDir()) {
writer.write(line);
writer.write('\n');
}

IOUtils.close(Collections.singleton(trainingWriter));
IOUtils.close(Collections.singleton(testWriter));

IOUtils.close(writers);

log.info("file: {}, input: {} train: {}, test: {} starting at {}",
new Object[] {inputFile.getName(), lineCount, trainCount, testCount, testSplitStart});
Expand Down Expand Up @@ -580,7 +583,12 @@ public static int countLines(FileSystem fs, Path inputFile, Charset charset) thr
lineCount++;
}
} finally {
IOUtils.close(Collections.singleton(countReader));
try {
countReader.close();
}
catch (IOException ex) {
log.warn("Could not close line count reader", ex);
}
}

return lineCount;
Expand Down

0 comments on commit 9867d52

Please sign in to comment.