Use WriteConsoleW to write to Windows console

Code pages in the Windows console are a mess: The default code page on most Windows systems doesn't support Unicode characters, and even changing the code page manually to UTF-8 results in broken output on some systems (e.g. for Chinese characters, see #164). JLine avoids this problem for the input mechanism by using the Windows API to read Unicode input. Consequently, input was usually working correctly, but the input wasn't displayed correctly in the console. Fortunately, a similar way can be applied for console output. The Windows API provides WriteConsoleW as a way to write unicode text to the console window, bypassing all problems with the current ode page. Unicode characters are given directly to the API instead of being encoded and written to standard output. Note: Unicode works only when using Terminal.writer() to write characters. For compatibility reasons, Terminal.output() decodes written bytes using the current code page and then redirects them to the Writer.
jline · Sep 14, 2017 · 96c5e0f · 96c5e0f
1 parent 3fc333b
commit 96c5e0f
Show file tree

Hide file tree

Showing 11 changed files with 240 additions and 105 deletions.
diff --git a/terminal-jansi/src/main/java/org/jline/terminal/impl/jansi/win/JansiWinConsoleWriter.java b/terminal-jansi/src/main/java/org/jline/terminal/impl/jansi/win/JansiWinConsoleWriter.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2002-2017, the original author or authors.
+ *
+ * This software is distributable under the BSD license. See the terms of the
+ * BSD license in the documentation provided with this software.
+ *
+ * http://www.opensource.org/licenses/bsd-license.php
+ */
+package org.jline.terminal.impl.jansi.win;
+
+import org.fusesource.jansi.internal.WindowsSupport;
+
+import java.io.IOException;
+import java.io.Writer;
+
+import static org.fusesource.jansi.internal.Kernel32.GetStdHandle;
+import static org.fusesource.jansi.internal.Kernel32.STD_OUTPUT_HANDLE;
+import static org.fusesource.jansi.internal.Kernel32.WriteConsoleW;
+
+class JansiWinConsoleWriter extends Writer {
+
+    private static final long console = GetStdHandle(STD_OUTPUT_HANDLE);
+
+    @Override
+    public void write(char[] cbuf, int off, int len) throws IOException {
+        char[] text = cbuf;
+        if (off != 0) {
+            text = new char[len];
+            System.arraycopy(cbuf, off, text, 0, len);
+        }
+
+        if (WriteConsoleW(console, text, len, new int[1], 0) == 0) {
+            throw new IOException("Failed to write to console: " + WindowsSupport.getLastErrorMessage());
+        }
+    }
+
+    @Override
+    public void flush() throws IOException {
+    }
+
+    @Override
+    public void close() throws IOException {
+    }
+
+}
diff --git a/terminal-jansi/src/main/java/org/jline/terminal/impl/jansi/win/JansiWinSysTerminal.java b/terminal-jansi/src/main/java/org/jline/terminal/impl/jansi/win/JansiWinSysTerminal.java
@@ -8,9 +8,7 @@
  */
 package org.jline.terminal.impl.jansi.win;
 
-import java.io.BufferedOutputStream;
-import java.io.FileDescriptor;
-import java.io.FileOutputStream;
+import java.io.BufferedWriter;
 import java.io.IOError;
 import java.io.IOException;
 import java.util.function.IntConsumer;
@@ -36,7 +34,7 @@ public JansiWinSysTerminal(String name, boolean nativeSignals) throws IOExceptio
     }
 
     public JansiWinSysTerminal(String name, int codepage, boolean nativeSignals, SignalHandler signalHandler) throws IOException {
-        super(new WindowsAnsiOutputStream(new BufferedOutputStream(new FileOutputStream(FileDescriptor.out))),
+        super(new WindowsAnsiWriter(new BufferedWriter(new JansiWinConsoleWriter())),
               name, codepage, nativeSignals, signalHandler);
     }
 
@@ -45,11 +43,6 @@ protected int getConsoleOutputCP() {
         return Kernel32.GetConsoleOutputCP();
     }
 
-    @Override
-    protected void setConsoleOutputCP(int cp) {
-        Kernel32.SetConsoleOutputCP(cp);
-    }
-
     @Override
     protected int getConsoleMode() {
         return WindowsSupport.getConsoleMode();

diff --git a/...pl/jansi/win/WindowsAnsiOutputStream.java → ...nal/impl/jansi/win/WindowsAnsiWriter.java b/...pl/jansi/win/WindowsAnsiOutputStream.java → ...nal/impl/jansi/win/WindowsAnsiWriter.java
@@ -17,10 +17,10 @@
 
 import org.fusesource.jansi.internal.Kernel32.*;
 import org.fusesource.jansi.internal.WindowsSupport;
-import org.jline.utils.AnsiOutputStream;
+import org.jline.utils.AnsiWriter;
 
 import java.io.IOException;
-import java.io.OutputStream;
+import java.io.Writer;
 
 import static org.fusesource.jansi.internal.Kernel32.*;
 
@@ -32,7 +32,7 @@
  * @author <a href="http://hiramchirino.com">Hiram Chirino</a>
  * @author Joris Kuipers
  */
-public final class WindowsAnsiOutputStream extends AnsiOutputStream {
+public final class WindowsAnsiWriter extends AnsiWriter {
 
     private static final long console = GetStdHandle(STD_OUTPUT_HANDLE);
 
@@ -77,8 +77,8 @@ public final class WindowsAnsiOutputStream extends AnsiOutputStream {
     private short savedX = -1;
     private short savedY = -1;
 
-    public WindowsAnsiOutputStream(OutputStream os) throws IOException {
-        super(os);
+    public WindowsAnsiWriter(Writer out) throws IOException {
+        super(out);
         getConsoleInfo();
         originalColors = info.attributes;
     }

diff --git a/terminal-jna/src/main/java/org/jline/terminal/impl/jna/win/JnaWinConsoleWriter.java b/terminal-jna/src/main/java/org/jline/terminal/impl/jna/win/JnaWinConsoleWriter.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2002-2017, the original author or authors.
+ *
+ * This software is distributable under the BSD license. See the terms of the
+ * BSD license in the documentation provided with this software.
+ *
+ * http://www.opensource.org/licenses/bsd-license.php
+ */
+package org.jline.terminal.impl.jna.win;
+
+import com.sun.jna.LastErrorException;
+import com.sun.jna.Pointer;
+import com.sun.jna.ptr.IntByReference;
+
+import java.io.IOException;
+import java.io.Writer;
+
+class JnaWinConsoleWriter extends Writer {
+
+    private final Pointer consoleHandle;
+
+    JnaWinConsoleWriter(Pointer consoleHandle) {
+        this.consoleHandle = consoleHandle;
+    }
+
+    @Override
+    public synchronized void write(char[] cbuf, int off, int len) throws IOException {
+        char[] text = cbuf;
+        if (off != 0) {
+            text = new char[len];
+            System.arraycopy(cbuf, off, text, 0, len);
+        }
+
+        try {
+            Kernel32.INSTANCE.WriteConsoleW(this.consoleHandle, text, len, new IntByReference(), null);
+        } catch (LastErrorException e) {
+            throw new IOException("Failed to write to console", e);
+        }
+    }
+
+    @Override
+    public void flush() {
+    }
+
+    @Override
+    public void close() {
+    }
+
+}
diff --git a/terminal-jna/src/main/java/org/jline/terminal/impl/jna/win/JnaWinSysTerminal.java b/terminal-jna/src/main/java/org/jline/terminal/impl/jna/win/JnaWinSysTerminal.java
@@ -8,20 +8,16 @@
  */
 package org.jline.terminal.impl.jna.win;
 
-import java.io.BufferedOutputStream;
-import java.io.FileDescriptor;
-import java.io.FileOutputStream;
+import java.io.BufferedWriter;
 import java.io.IOException;
 import java.util.function.IntConsumer;
 
-import com.sun.jna.LastErrorException;
 import com.sun.jna.Pointer;
 import com.sun.jna.ptr.IntByReference;
 import org.jline.terminal.Cursor;
 import org.jline.terminal.Size;
 import org.jline.terminal.impl.AbstractWindowsTerminal;
 import org.jline.utils.InfoCmp;
-import org.jline.utils.Log;
 
 public class JnaWinSysTerminal extends AbstractWindowsTerminal {
 
@@ -35,7 +31,7 @@ public JnaWinSysTerminal(String name, boolean nativeSignals) throws IOException
     }
 
     public JnaWinSysTerminal(String name, int codepage, boolean nativeSignals, SignalHandler signalHandler) throws IOException {
-        super(new WindowsAnsiOutputStream(new BufferedOutputStream(new FileOutputStream(FileDescriptor.out)), consoleOut),
+        super(new WindowsAnsiWriter(new BufferedWriter(new JnaWinConsoleWriter(consoleOut)), consoleOut),
               name, codepage, nativeSignals, signalHandler);
         strings.put(InfoCmp.Capability.key_mouse, "\\E[M");
     }
@@ -45,16 +41,6 @@ protected int getConsoleOutputCP() {
         return Kernel32.INSTANCE.GetConsoleOutputCP();
     }
 
-    @Override
-    protected void setConsoleOutputCP(int cp) {
-        try {
-            Kernel32.INSTANCE.SetConsoleOutputCP(cp);
-        } catch (LastErrorException e) {
-            // Not sure why it throws exceptions, just log at trace
-            Log.trace("Error setting console output code page", e);
-        }
-    }
-
     @Override
     protected int getConsoleMode() {
         IntByReference mode = new IntByReference();

diff --git a/terminal-jna/src/main/java/org/jline/terminal/impl/jna/win/Kernel32.java b/terminal-jna/src/main/java/org/jline/terminal/impl/jna/win/Kernel32.java
@@ -237,6 +237,16 @@ void SetConsoleWindowInfo(Pointer in_hConsoleOutput,
                               boolean in_bAbsolute, SMALL_RECT in_lpConsoleWindow)
             throws LastErrorException;
 
+    // BOOL WINAPI WriteConsole(
+    //  _In_             HANDLE  hConsoleOutput,
+    //  _In_       const VOID    *lpBuffer,
+    //  _In_             DWORD   nNumberOfCharsToWrite,
+    //  _Out_            LPDWORD lpNumberOfCharsWritten,
+    //  _Reserved_       LPVOID  lpReserved
+    // );
+    void WriteConsoleW(Pointer in_hConsoleOutput, char[] in_lpBuffer, int in_nNumberOfCharsToWrite,
+                          IntByReference out_lpNumberOfCharsWritten, Pointer reserved_lpReserved) throws LastErrorException;
+
     // BOOL WINAPI WriteConsoleOutput(
     // _In_ HANDLE hConsoleOutput,
     // _In_ const CHAR_INFO *lpBuffer,

diff --git a/...impl/jna/win/WindowsAnsiOutputStream.java → ...minal/impl/jna/win/WindowsAnsiWriter.java b/...impl/jna/win/WindowsAnsiOutputStream.java → ...minal/impl/jna/win/WindowsAnsiWriter.java
@@ -9,11 +9,11 @@
 package org.jline.terminal.impl.jna.win;
 
 import java.io.IOException;
-import java.io.OutputStream;
+import java.io.Writer;
 
 import com.sun.jna.Pointer;
 import com.sun.jna.ptr.IntByReference;
-import org.jline.utils.AnsiOutputStream;
+import org.jline.utils.AnsiWriter;
 
 import static org.jline.terminal.impl.jna.win.Kernel32.BACKGROUND_BLUE;
 import static org.jline.terminal.impl.jna.win.Kernel32.BACKGROUND_GREEN;
@@ -33,7 +33,7 @@
  * @author <a href="http://hiramchirino.com">Hiram Chirino</a>
  * @author Joris Kuipers
  */
-public final class WindowsAnsiOutputStream extends AnsiOutputStream {
+public final class WindowsAnsiWriter extends AnsiWriter {
 
     private static final short FOREGROUND_BLACK   = 0;
     private static final short FOREGROUND_YELLOW  = (short) (FOREGROUND_RED|FOREGROUND_GREEN);
@@ -80,8 +80,8 @@ public final class WindowsAnsiOutputStream extends AnsiOutputStream {
     private short savedX = -1;
     private short savedY = -1;
 
-    public WindowsAnsiOutputStream(OutputStream os, Pointer console) throws IOException {
-        super(os);
+    public WindowsAnsiWriter(Writer out, Pointer console) throws IOException {
+        super(out);
         this.console = console;
         getConsoleInfo();
         originalColors = info.wAttributes;

diff --git a/terminal/src/main/java/org/jline/terminal/impl/AbstractWindowsTerminal.java b/terminal/src/main/java/org/jline/terminal/impl/AbstractWindowsTerminal.java
@@ -16,18 +16,21 @@
 import org.jline.utils.NonBlockingReader;
 import org.jline.utils.ShutdownHooks;
 import org.jline.utils.Signals;
+import org.jline.utils.WriterOutputStream;
 
+import java.io.BufferedOutputStream;
 import java.io.FilterInputStream;
 import java.io.InputStream;
 import java.io.IOError;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.io.OutputStreamWriter;
 import java.io.PipedInputStream;
 import java.io.PipedOutputStream;
 import java.io.PrintWriter;
 import java.io.StringWriter;
+import java.io.Writer;
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -37,8 +40,6 @@ public abstract class AbstractWindowsTerminal extends AbstractTerminal {
 
     private static final int PIPE_SIZE = 1024;
 
-    private static final String UTF8 = "UTF-8";
-    private static final Charset UTF8_CHARSET = Charset.forName(UTF8);
     private static final int UTF8_CODE_PAGE = 65001;
 
     protected static final int ENABLE_PROCESSED_INPUT = 0x0001;
@@ -58,35 +59,20 @@ public abstract class AbstractWindowsTerminal extends AbstractTerminal {
     protected final ShutdownHooks.Task closer;
     protected final Attributes attributes = new Attributes();
     protected final Thread pump;
-    protected final int consoleOutputCP;
 
     protected MouseTracking tracking = MouseTracking.Off;
     private volatile boolean closing;
 
-    public AbstractWindowsTerminal(OutputStream output, String name, int codepage, boolean nativeSignals, SignalHandler signalHandler) throws IOException {
+    public AbstractWindowsTerminal(Writer writer, String name, int codepage, boolean nativeSignals, SignalHandler signalHandler) throws IOException {
         super(name, TYPE_WINDOWS, signalHandler);
         PipedInputStream input = new PipedInputStream(PIPE_SIZE); // UTF-8 encoded
         this.slaveInputPipe = new PipedOutputStream(input); // UTF-8 encoded
         this.input = new FilterInputStream(input) {}; // UTF-8 encoded
-        this.output = output;
-        if (codepage > 0) {
-            // Find out the console code page and save it
-            this.consoleOutputCP = getConsoleOutputCP();
-            // Try to set the code page
-            if (this.consoleOutputCP != codepage) {
-                setConsoleOutputCP(codepage);
-            }
-        } else {
-            this.consoleOutputCP = 0;
-        }
-        // Whether the above call succeeded or failed, grab the console
-        // code page and find a matching charset to encode
-        String encoding = getConsoleEncoding();
-        if (encoding == null) {
-            encoding = Charset.defaultCharset().name();
-        }
-        this.reader = new NonBlockingReader(getName(), new org.jline.utils.InputStreamReader(input, UTF8_CHARSET));
-        this.writer = new PrintWriter(new OutputStreamWriter(output, encoding));
+        this.reader = new NonBlockingReader(getName(), new org.jline.utils.InputStreamReader(input, StandardCharsets.UTF_8));
+        this.writer = new PrintWriter(writer);
+        // Grab the console code page and find a matching charset to encode
+        Charset charset = getConsoleEncoding(codepage);
+        this.output = new BufferedOutputStream(new WriterOutputStream(writer, charset));
         parseInfoCmp();
         // Attributes
         attributes.setLocalFlag(Attributes.LocalFlag.ISIG, true);
@@ -110,21 +96,24 @@ public AbstractWindowsTerminal(OutputStream output, String name, int codepage, b
         ShutdownHooks.add(closer);
     }
 
-    protected String getConsoleEncoding() {
-        int codepage = getConsoleOutputCP();
+    protected Charset getConsoleEncoding(int codepage) {
+        if (codepage <= 0) {
+            codepage = getConsoleOutputCP();
+        }
+
         //http://docs.oracle.com/javase/6/docs/technotes/guides/intl/encoding.doc.html
         if (codepage == UTF8_CODE_PAGE) {
-            return UTF8;
+            return StandardCharsets.UTF_8;
         }
         String charsetMS = "ms" + codepage;
         if (Charset.isSupported(charsetMS)) {
-            return charsetMS;
+            return Charset.forName(charsetMS);
         }
         String charsetCP = "cp" + codepage;
         if (Charset.isSupported(charsetCP)) {
-            return charsetCP;
+            return Charset.forName(charsetCP);
         }
-        return null;
+        return Charset.defaultCharset();
     }
 
     @Override
@@ -205,9 +194,6 @@ public void close() throws IOException {
         }
         reader.close();
         writer.close();
-        if (consoleOutputCP > 0) {
-            setConsoleOutputCP(consoleOutputCP);
-        }
     }
 
     static final int SHIFT_FLAG = 0x01;
@@ -396,7 +382,7 @@ protected void pump() {
         try {
             while (!closing) {
                 String buf = readConsoleInput();
-                for (byte b : buf.getBytes(UTF8_CHARSET)) {
+                for (byte b : buf.getBytes(StandardCharsets.UTF_8)) {
                     processInputByte(b);
                 }
             }
@@ -449,8 +435,6 @@ public boolean trackMouse(MouseTracking tracking) {
 
     protected abstract int getConsoleOutputCP();
 
-    protected abstract void setConsoleOutputCP(int cp);
-
     protected abstract int getConsoleMode();
 
     protected abstract void setConsoleMode(int mode);