From 21629669245f9d7a6d96174edf3b8055c26a7a96 Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Mon, 6 Apr 2020 13:14:34 +0200
Subject: [PATCH 1/3] Use InputStreamReader for serial UTF8 decoder
The implementation is much more straightforward.
It should also solve a JDK incompatiblity:
java.lang.NoSuchMethodError: java.nio.ByteBuffer.flip()Ljava/nio/ByteBuffer;
at processing.app.Serial.serialEvent(Serial.java:185)
at jssc.SerialPort$LinuxEventThread.run(SerialPort.java:1299)
See #8903
---
arduino-core/src/processing/app/Serial.java | 85 ++++++++-------------
1 file changed, 30 insertions(+), 55 deletions(-)
diff --git a/arduino-core/src/processing/app/Serial.java b/arduino-core/src/processing/app/Serial.java
index edc5e8f0c0f..75958b2d25f 100644
--- a/arduino-core/src/processing/app/Serial.java
+++ b/arduino-core/src/processing/app/Serial.java
@@ -22,23 +22,22 @@
package processing.app;
-import jssc.SerialPort;
-import jssc.SerialPortEvent;
-import jssc.SerialPortEventListener;
-import jssc.SerialPortException;
+import static processing.app.I18n.format;
+import static processing.app.I18n.tr;
import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
+import java.io.InputStreamReader;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
import java.nio.charset.Charset;
-import java.nio.charset.CharsetDecoder;
-import java.nio.charset.CodingErrorAction;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
-import static processing.app.I18n.format;
-import static processing.app.I18n.tr;
+import jssc.SerialPort;
+import jssc.SerialPortEvent;
+import jssc.SerialPortEventListener;
+import jssc.SerialPortException;
public class Serial implements SerialPortEventListener {
@@ -53,11 +52,8 @@ public class Serial implements SerialPortEventListener {
private SerialPort port;
- private CharsetDecoder bytesToStrings;
- private static final int IN_BUFFER_CAPACITY = 128;
- private static final int OUT_BUFFER_CAPACITY = 128;
- private ByteBuffer inFromSerial = ByteBuffer.allocate(IN_BUFFER_CAPACITY);
- private CharBuffer outToMessage = CharBuffer.allocate(OUT_BUFFER_CAPACITY);
+ private PipedOutputStream decoderInRaw;
+ private InputStreamReader decoderOutputUTF8;
public Serial() throws SerialException {
this(PreferencesData.get("serial.port"),
@@ -189,42 +185,18 @@ public synchronized void serialEvent(SerialPortEvent serialEvent) {
public void processSerialEvent(byte[] buf) {
int next = 0;
- // This uses a CharsetDecoder to convert from bytes to UTF-8 in
- // a streaming fashion (i.e. where characters might be split
- // over multiple reads). This needs the data to be in a
- // ByteBuffer (inFromSerial, which we also use to store leftover
- // incomplete characters for the nexst run) and produces a
- // CharBuffer (outToMessage), which we then convert to char[] to
- // pass onwards.
- // Note that these buffers switch from input to output mode
- // using flip/compact/clear
- while (next < buf.length || inFromSerial.position() > 0) {
- do {
- // This might be 0 when all data was already read from buf
- // (but then there will be data in inFromSerial left to
- // decode).
- int copyNow = Math.min(buf.length - next, inFromSerial.remaining());
- inFromSerial.put(buf, next, copyNow);
- next += copyNow;
-
- inFromSerial.flip();
- bytesToStrings.decode(inFromSerial, outToMessage, false);
- inFromSerial.compact();
-
- // When there are multi-byte characters, outToMessage might
- // still have room, so add more bytes if we have any.
- } while (next < buf.length && outToMessage.hasRemaining());
-
- // If no output was produced, the input only contained
- // incomplete characters, so we're done processing
- if (outToMessage.position() == 0)
- break;
-
- outToMessage.flip();
- char[] chars = new char[outToMessage.remaining()];
- outToMessage.get(chars);
- message(chars, chars.length);
- outToMessage.clear();
+ int max = buf.length;
+ char chars[] = new char[512];
+ try {
+ while (next < max) {
+ int w = Integer.min(max - next, 128);
+ decoderInRaw.write(buf, next, w);
+ next += w;
+ int n = decoderOutputUTF8.read(chars);
+ message(chars, n);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
}
}
@@ -295,10 +267,13 @@ public void setRTS(boolean state) {
* before they are handed as Strings to {@Link #message(char[], int)}.
*/
public synchronized void resetDecoding(Charset charset) {
- bytesToStrings = charset.newDecoder()
- .onMalformedInput(CodingErrorAction.REPLACE)
- .onUnmappableCharacter(CodingErrorAction.REPLACE)
- .replaceWith("\u2e2e");
+ try {
+ decoderInRaw = new PipedOutputStream();
+ decoderOutputUTF8 = new InputStreamReader(new PipedInputStream(decoderInRaw), charset);
+ } catch (IOException e) {
+ // Should never happen...
+ e.printStackTrace();
+ }
}
static public List<String> list() {
From 618eef0e0db4a3e0d35318fde4d907b73b3d37ce Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Fri, 19 Jun 2020 16:05:56 +0200
Subject: [PATCH 2/3] Serial UTF-8 decoder now handles blocks of 16Kb at a time
---
arduino-core/src/processing/app/Serial.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/arduino-core/src/processing/app/Serial.java b/arduino-core/src/processing/app/Serial.java
index 75958b2d25f..abeacf93a82 100644
--- a/arduino-core/src/processing/app/Serial.java
+++ b/arduino-core/src/processing/app/Serial.java
@@ -54,6 +54,7 @@ public class Serial implements SerialPortEventListener {
private PipedOutputStream decoderInRaw;
private InputStreamReader decoderOutputUTF8;
+ private final int DECODER_BUFF_SIZE = 16384;
public Serial() throws SerialException {
this(PreferencesData.get("serial.port"),
@@ -186,10 +187,10 @@ public synchronized void serialEvent(SerialPortEvent serialEvent) {
public void processSerialEvent(byte[] buf) {
int next = 0;
int max = buf.length;
- char chars[] = new char[512];
+ char chars[] = new char[DECODER_BUFF_SIZE];
try {
while (next < max) {
- int w = Integer.min(max - next, 128);
+ int w = Integer.min(max - next, chars.length);
decoderInRaw.write(buf, next, w);
next += w;
int n = decoderOutputUTF8.read(chars);
@@ -269,7 +270,8 @@ public void setRTS(boolean state) {
public synchronized void resetDecoding(Charset charset) {
try {
decoderInRaw = new PipedOutputStream();
- decoderOutputUTF8 = new InputStreamReader(new PipedInputStream(decoderInRaw), charset);
+ // add 16 extra bytes to make room for incomplete UTF-8 chars
+ decoderOutputUTF8 = new InputStreamReader(new PipedInputStream(decoderInRaw, DECODER_BUFF_SIZE + 16), charset);
} catch (IOException e) {
// Should never happen...
e.printStackTrace();
From 782a35bf9e4d64b612a21443c7671c53087dfafc Mon Sep 17 00:00:00 2001
From: Cristian Maglie <c.maglie@arduino.cc>
Date: Fri, 19 Jun 2020 16:06:18 +0200
Subject: [PATCH 3/3] Added test for invalid UTF-8 sequences
---
app/test/processing/app/SerialTest.java | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/app/test/processing/app/SerialTest.java b/app/test/processing/app/SerialTest.java
index 63280811e24..f63b8e99404 100644
--- a/app/test/processing/app/SerialTest.java
+++ b/app/test/processing/app/SerialTest.java
@@ -29,6 +29,7 @@
package processing.app;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import org.junit.Test;
@@ -47,6 +48,16 @@ protected void message(char[] chars, int length) {
String output = "";
}
+ @Test
+ public void testSerialUTF8DecoderWithInvalidChars() throws Exception {
+ NullSerial s = new NullSerial();
+ byte[] testdata = new byte[] { '>', (byte) 0xC3, (byte) 0x28, '<' };
+ byte[] expected = new byte[] { '>', (byte) 0xEF, (byte) 0xBF, (byte) 0xBD, (byte) 0x28, '<' };
+ s.processSerialEvent(testdata);
+ byte[] res = s.output.getBytes("UTF-8");
+ assertArrayEquals(expected, res);
+ }
+
@Test
public void testSerialUTF8Decoder() throws Exception {
NullSerial s = new NullSerial();