Permalink
Browse files

Fixed read/copy on input streams reading sometimes less than expected.

  • Loading branch information...
luccioman committed Jul 11, 2017
1 parent 23bda13 commit f369679d1cfc1cbd989fe7efbb3803c01c19f60d
Showing with 107 additions and 20 deletions.
  1. +11 −12 source/net/yacy/kelondro/util/FileUtils.java
  2. +96 −8 test/java/net/yacy/kelondro/util/FileUtilsTest.java
@@ -110,10 +110,17 @@ public static long copy(final InputStream source, final OutputStream dest, final
int c;
long total = 0;
while ( (c = source.read(buffer, 0, chunkSize)) > 0 ) {
long remaining;
if(count > 0) {
remaining = count;
} else {
remaining = Long.MAX_VALUE;
}
while ( (c = source.read(buffer, 0, remaining < chunkSize ? (int)remaining : chunkSize)) > 0 ) {
dest.write(buffer, 0, c);
dest.flush();
total += c;
remaining -= c;
if ( count > 0 && count == total) {
break;
@@ -383,19 +390,11 @@ public static void copy(final byte[] source, final File dest) throws IOException
* @throws NullPointerException when source parameter is null
*/
public static byte[] read(final InputStream source, final int count) throws IOException {
if ( count > 0 ) {
final byte[] b = new byte[count];
final int c = source.read(b, 0, count);
assert c == count : "count = " + count + ", c = " + c;
if ( c != count ) {
final byte[] bb = new byte[c];
System.arraycopy(b, 0, bb, 0, c);
return bb;
}
return b;
if(count == 0) {
return new byte[0];
}
final ByteArrayOutputStream baos = new ByteArrayOutputStream(512);
copy(source, baos);
copy(source, baos, count);
baos.close();
return baos.toByteArray();
}
@@ -23,18 +23,51 @@
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
/**
* Unit tests for the {@link FileUtils} class.
*
*/
public class FileUtilsTest {
@BeforeClass
public static void beforeAll() {
/* Disable assertions here : success/failure of these tests should not rely on assertions that are likely to be disabled at runtime */
FileUtilsTest.class.getClassLoader().setDefaultAssertionStatus(false);
}
/**
* A test stream reading each time less than desired bytes.
* Simulates what can occur for example on real-world HTTP streams.
*/
private class LowerReadThanExpectedInputStream extends FilterInputStream {
protected LowerReadThanExpectedInputStream(InputStream in) {
super(in);
}
/**
* Reads less than specified len bytes
*/
@Override
public int read(byte[] b, int off, int len) throws IOException {
if(len > 10 ) {
return super.read(b, off, len - 10);
}
return super.read(b, off, len);
}
}
/**
* Copy stream : normal case
@@ -102,16 +135,71 @@ public void testPartialCopyInputStreamOutputStream() throws IOException {
for(int i = 0; i < sourceBytes.length; i++) {
sourceBytes[i] = (byte)(i % Byte.MAX_VALUE);
}
InputStream source = new ByteArrayInputStream(sourceBytes);
ByteArrayOutputStream dest = new ByteArrayOutputStream();
final int COUNT = 8;
try {
final int COUNT = 1200;
try (InputStream source = new ByteArrayInputStream(sourceBytes);
ByteArrayOutputStream dest = new ByteArrayOutputStream();) {
FileUtils.copy(source, dest, COUNT);
} finally {
source.close();
dest.close();
Assert.assertEquals(COUNT, dest.size());
}
/* Copy from a stream reading less than desired bytes (can occurs for example on real-world HTTP streams) */
try (InputStream bufferedSource = new ByteArrayInputStream(sourceBytes);
InputStream source = new LowerReadThanExpectedInputStream(bufferedSource);
ByteArrayOutputStream dest = new ByteArrayOutputStream();) {
FileUtils.copy(source, dest, COUNT);
Assert.assertEquals(COUNT, dest.size());
}
Assert.assertEquals(COUNT, dest.size());
}
/**
* Test reading n bytes in a stream
* @throws IOException when a read/write error occurred
*/
@Test
public void testReadInputStream() throws IOException {
/* Fill an input stream with more bytes than FileUtils.DEFAULT_BUFFER_SIZE */
byte[] sourceBytes = new byte[2000];
for(int i = 0; i < sourceBytes.length; i++) {
sourceBytes[i] = (byte)(i % Byte.MAX_VALUE);
}
/* Read all*/
try(InputStream source = new ByteArrayInputStream(sourceBytes);) {
Assert.assertEquals(sourceBytes.length, FileUtils.read(source, -1).length);
}
/* Read zero */
try(InputStream source = new ByteArrayInputStream(sourceBytes);) {
Assert.assertEquals(0, FileUtils.read(source, 0).length);
}
/* Read only one */
try(InputStream source = new ByteArrayInputStream(sourceBytes);) {
Assert.assertEquals(1, FileUtils.read(source, 1).length);
}
/* Read half */
try(InputStream source = new ByteArrayInputStream(sourceBytes);) {
Assert.assertEquals(sourceBytes.length / 2, FileUtils.read(source, sourceBytes.length / 2).length);
}
/* Read half on a stream reading each time less than desired bytes (can occurs for example on real-world HTTP streams) */
try(InputStream bufferedStream = new ByteArrayInputStream(sourceBytes);
InputStream source = new LowerReadThanExpectedInputStream(bufferedStream);) {
Assert.assertEquals(sourceBytes.length / 2, FileUtils.read(source, sourceBytes.length / 2).length);
}
/* Read exactly source bytes count */
try(InputStream source = new ByteArrayInputStream(sourceBytes);) {
Assert.assertEquals(sourceBytes.length, FileUtils.read(source, sourceBytes.length).length);
}
/* Trying to read more than source bytes count */
try(InputStream source = new ByteArrayInputStream(sourceBytes);) {
Assert.assertEquals(sourceBytes.length, FileUtils.read(source, sourceBytes.length + 10).length);
}
}
}

0 comments on commit f369679

Please sign in to comment.