Skip to content

Commit

Permalink
Some bytes may be lost with NonBlockingReaderInputStream, fixes #238
Browse files Browse the repository at this point in the history
  • Loading branch information
@gnodet
gnodet committed Mar 7, 2018
1 parent ae62ec8 commit 88010aa
Show file tree
Hide file tree
Showing 2 changed files with 51 additions and 62 deletions.
84 changes: 24 additions & 60 deletions terminal/src/main/java/org/jline/utils/NonBlocking.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,7 @@
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CoderResult;
import java.nio.charset.CodingErrorAction;
import java.nio.charset.MalformedInputException;
import java.nio.charset.UnmappableCharacterException;

public class NonBlocking {

Expand Down Expand Up @@ -74,23 +71,19 @@ private static class NonBlockingReaderInputStream extends NonBlockingInputStream
private final CharBuffer chars;

private NonBlockingReaderInputStream(NonBlockingReader reader, Charset charset) {
this(reader, charset, 4);
}

private NonBlockingReaderInputStream(NonBlockingReader reader, Charset charset, int bufferSize) {
this.reader = reader;
this.encoder = charset.newEncoder()
.onUnmappableCharacter(CodingErrorAction.REPLACE)
.onMalformedInput(CodingErrorAction.REPLACE);
this.bytes = ByteBuffer.allocate((int) Math.ceil(bufferSize * encoder.maxBytesPerChar()));
this.chars = CharBuffer.allocate(bufferSize);
this.bytes = ByteBuffer.allocate(4);
this.chars = CharBuffer.allocate(2);
// No input available after initialization
this.bytes.limit(0);
this.chars.limit(0);
}

@Override
public int available() throws IOException {
public int available() {
return (int) (reader.available() * this.encoder.averageBytesPerChar())
+ bytes.remaining();
}
Expand All @@ -113,34 +106,16 @@ public int read(long timeout, boolean isPeek) throws IOException {
return EOF;
}
if (c >= 0) {
if (!chars.hasRemaining()) {
chars.position(0);
chars.limit(0);
}
int l = chars.limit();
chars.array()[chars.arrayOffset() + l] = (char) c;
chars.limit(l + 1);
int p = bytes.position();
l = bytes.limit();
bytes.position(l);
bytes.limit(bytes.capacity());
CoderResult result = encoder.encode(chars, bytes, false);
l = bytes.position();
bytes.position(p);
bytes.limit(l);
if (result.isUnderflow()) {
if (chars.limit() == chars.capacity()) {
chars.compact();
chars.limit(chars.position());
chars.position(0);
}
} else if (result.isOverflow()) {
if (bytes.limit() == bytes.capacity()) {
bytes.compact();
bytes.limit(bytes.position());
bytes.position(0);
}
} else if (result.isMalformed()) {
throw new MalformedInputException(result.length());
} else if (result.isUnmappable()) {
throw new UnmappableCharacterException(result.length());
}
bytes.clear();
encoder.encode(chars, bytes, false);
bytes.flip();
}
if (!isInfinite) {
timeout -= System.currentTimeMillis() - start;
Expand All @@ -161,7 +136,7 @@ public int read(long timeout, boolean isPeek) throws IOException {

private static class NonBlockingInputStreamReader extends NonBlockingReader {

private final NonBlockingInputStream nbis;
private final NonBlockingInputStream input;
private final CharsetDecoder decoder;
private final ByteBuffer bytes;
private final CharBuffer chars;
Expand All @@ -170,15 +145,14 @@ public NonBlockingInputStreamReader(NonBlockingInputStream inputStream, Charset
this(inputStream,
(encoding != null ? encoding : Charset.defaultCharset()).newDecoder()
.onMalformedInput(CodingErrorAction.REPLACE)
.onUnmappableCharacter(CodingErrorAction.REPLACE),
4);
.onUnmappableCharacter(CodingErrorAction.REPLACE));
}

public NonBlockingInputStreamReader(NonBlockingInputStream nbis, CharsetDecoder decoder, int bufferSize) {
this.nbis = nbis;
public NonBlockingInputStreamReader(NonBlockingInputStream input, CharsetDecoder decoder) {
this.input = input;
this.decoder = decoder;
this.bytes = ByteBuffer.allocate((int) Math.ceil(bufferSize / decoder.maxCharsPerByte()));
this.chars = CharBuffer.allocate(bufferSize);
this.bytes = ByteBuffer.allocate(4);
this.chars = CharBuffer.allocate(2);
this.bytes.limit(0);
this.chars.limit(0);
}
Expand All @@ -191,32 +165,21 @@ protected int read(long timeout, boolean isPeek) throws IOException {
if (!isInfinite) {
start = System.currentTimeMillis();
}
int b = nbis.read(timeout);
int b = input.read(timeout);
if (b == EOF) {
return EOF;
}
if (b >= 0) {
if (bytes.capacity() - bytes.limit() < 1) {
bytes.compact();
bytes.limit(bytes.position());
if (!bytes.hasRemaining()) {
bytes.position(0);
}
if (chars.capacity() - chars.limit() < 2) {
chars.compact();
chars.limit(chars.position());
chars.position(0);
bytes.limit(0);
}
int l = bytes.limit();
bytes.array()[bytes.arrayOffset() + l] = (byte) b;
bytes.limit(l + 1);
int p = chars.position();
l = chars.limit();
chars.position(l);
chars.limit(chars.capacity());
chars.clear();
decoder.decode(bytes, chars, false);
l = chars.position();
chars.position(p);
chars.limit(l);
chars.flip();
}

if (!isInfinite) {
Expand All @@ -236,12 +199,13 @@ protected int read(long timeout, boolean isPeek) throws IOException {

@Override
public void shutdown() {
nbis.shutdown();
input.shutdown();
}

@Override
public void close() throws IOException {
nbis.close();
input.close();
}
}

}
29 changes: 27 additions & 2 deletions terminal/src/test/java/org/jline/utils/NonBlockingTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,32 @@ public void testNonBlockingPumpReader() throws IOException {
public void testNonBlockStreamOnReader() throws IOException {
NonBlockingPumpReader reader = NonBlocking.nonBlockingPumpReader();
NonBlockingInputStream is = NonBlocking.nonBlockingStream(reader, StandardCharsets.UTF_8);
reader.getWriter().write("a");
assertEquals('a', is.read(1000L));

String s = "aaaaaaaaaaaaa";
byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
reader.getWriter().write(s);
for(int i = 0; i < bytes.length; i++) {
int b = is.read(100L);
assertEquals("Mismatch at " + i, bytes[i], b);
}
assertEquals(NonBlockingInputStream.READ_EXPIRED, is.read(100));

s = "aaaaaaaaa中";
bytes = s.getBytes(StandardCharsets.UTF_8);
reader.getWriter().write(s);
for(int i = 0; i < bytes.length; i++) {
int b = is.read(100L);
assertEquals("Mismatch at " + i, bytes[i], b);
}
assertEquals(NonBlockingInputStream.READ_EXPIRED, is.read(100));

s = "aaaaaaaaa\uD801\uDC37";
bytes = s.getBytes(StandardCharsets.UTF_8);
reader.getWriter().write(s);
for(int i = 0; i < bytes.length; i++) {
int b = is.read(100L);
assertEquals("Mismatch at " + i, bytes[i], b);
}
assertEquals(NonBlockingInputStream.READ_EXPIRED, is.read(100));
}
}

0 comments on commit 88010aa

Please sign in to comment.