Rewrite resize logic, introduce "direct mode", and tweak scroll scrap…

…ing.

 * Unfreeze the console while changing the buffer size.  Changing the
   buffer size hangs conhost.exe.  See:
    - #31
    - https://wpdev.uservoice.com/forums/266908-command-prompt/suggestions/9941292-conhost-exe-hangs-in-win10-if-setconsolescreenbuff

 * Detect buffer size changes and switch to a "direct mode".  Direct mode
   makes no attempt to track incremental console changes.  Instead, the
   content of the current console window is printed.  This mode is
   intended for full-screen apps that resize the console.

 * Reopen CONOUT$, which detects apps that change the active screen buffer.
   Fixes #34.

 * In the scroll scraping (scrollingScrapeOutput), consider a line changed
   if the new content is truncated relative to the content previously
   output.  Previously, we only compared against the line-buffer up to the
   current console width.  e.g.

   If this:
      |C:\Program|

   turns into:
      |C:\Prog|
      |ram    |

   we previously left |C:\Program| in the line-buffer for the first line
   and did not re-output the first line.

   We *should* reoutput the first line at this point so that, if the line
   scrolls upward, and the terminal is later expanded, we will have
   output an "Erase in Line" CSI command to clear the obscured "ram" text.

   We need to update the line-buffer for the sake of Windows 10 combined
   with terminals like xterm and putty.  On such a terminal, if the
   terminal later widened, Windows 10 will restore the console to the
   first state.  At that point, we need to reoutput the line, because
   xterm and putty do not save and restore truncated line content extending
   past the current terminal width.

agent/Agent.h

@@ -25,15 +25,19 @@
 #include <windows.h>
 #include "EventLoop.h"
 #include "DsrSender.h"
+#include "Coord.h"
+#include "SmallRect.h"
 
 class Win32Console;
 class ConsoleInput;
 class Terminal;
 class ReadBuffer;
 class NamedPipe;
+struct ConsoleScreenBufferInfo;
 
 const int BUFFER_LINE_COUNT = 3000; // TODO: Use something like 9000.
 const int MAX_CONSOLE_WIDTH = 500;
+const int SYNC_MARKER_LEN = 16;
 
 class Agent : public EventLoop, public DsrSender
 {
@@ -61,10 +65,16 @@ class Agent : public EventLoop, public DsrSender
     virtual void onPipeIo(NamedPipe *namedPipe);
 
 private:
-    void markEntireWindowDirty();
+    void markEntireWindowDirty(const SmallRect &windowRect);
     void scanForDirtyLines();
+    void clearBufferLines(int firstRow, int count, WORD attributes);
+    void resizeImpl(const ConsoleScreenBufferInfo &origInfo);
     void resizeWindow(int cols, int rows);
-    void scrapeOutput();
+    void syncConsoleContentAndSize(bool forceResize);
+    void syncConsoleTitle();
+    void directScrapeOutput(const ConsoleScreenBufferInfo &info);
+    void scrollingScrapeOutput(const ConsoleScreenBufferInfo &info);
+    void reopenConsole();
     void freezeConsole();
     void unfreezeConsole();
     void syncMarkerText(CHAR_INFO *output);
@@ -84,6 +94,8 @@ class Agent : public EventLoop, public DsrSender
     int m_syncRow;
     int m_syncCounter;
 
+    bool m_directMode;
+    Coord m_ptySize;
     int m_scrapedLineCount;
     int m_scrolledCount;
     int m_maxBufferedLine;

agent/ConsoleInput.cc

@@ -61,8 +61,8 @@ ConsoleInput::KeyDescriptor ConsoleInput::keyDescriptorTable[] = {
     {   ESC"OF",        VK_END,     0,  0,                      }, // gnome-terminal
 };
 
-ConsoleInput::ConsoleInput(Win32Console *console, DsrSender *dsrSender) :
-    m_console(console),
+ConsoleInput::ConsoleInput(DsrSender *dsrSender) :
+    m_console(new Win32Console),
     m_dsrSender(dsrSender),
     m_dsrSent(false),
     lastWriteTick(0)
@@ -166,6 +166,11 @@ ConsoleInput::ConsoleInput(Win32Console *console, DsrSender *dsrSender) :
     }
 }
 
+ConsoleInput::~ConsoleInput()
+{
+    delete m_console;
+}
+
 void ConsoleInput::writeInput(const std::string &input)
 {
     trace("writeInput: %d bytes", input.size());

agent/ConsoleInput.h

@@ -31,7 +31,8 @@ class DsrSender;
 class ConsoleInput
 {
 public:
-    ConsoleInput(Win32Console *console, DsrSender *dsrSender);
+    ConsoleInput(DsrSender *dsrSender);
+    ~ConsoleInput();
     void writeInput(const std::string &input);
     void flushIncompleteEscapeCode();
 

agent/SmallRect.h

@@ -85,6 +85,18 @@ struct SmallRect : SMALL_RECT
                          std::max(0, y2 - y1 + 1));
     }
 
+    SmallRect ensureLineIncluded(SHORT line) const
+    {
+        const SHORT h = height();
+        if (line < Top) {
+            return SmallRect(Left, line, width(), h);
+        } else if (line > Bottom) {
+            return SmallRect(Left, line - h + 1, width(), h);
+        } else {
+            return *this;
+        }
+    }
+
     SHORT top() const               { return Top;                       }
     SHORT left() const              { return Left;                      }
     SHORT width() const             { return Right - Left + 1;          }

agent/Terminal.cc

@@ -65,8 +65,11 @@ void Terminal::setConsoleMode(int mode)
 
 void Terminal::reset(bool sendClearFirst, int newLine)
 {
-    if (sendClearFirst && !m_consoleMode)
+    if (sendClearFirst && !m_consoleMode) {
+        // 1;1H ==> move cursor to top-left position
+        // 2J   ==> clear the entire screen
         m_output->write(CSI"1;1H"CSI"2J");
+    }
     m_remoteLine = newLine;
     m_cursorHidden = false;
     m_cursorPos = std::pair<int, int>(0, newLine);

agent/Win32Console.cc

@@ -55,12 +55,15 @@ namespace {
 Win32Console::Win32Console() : m_titleWorkBuf(16)
 {
     m_conin = GetStdHandle(STD_INPUT_HANDLE);
-    m_conout = GetStdHandle(STD_OUTPUT_HANDLE);
+    m_conout = CreateFileW(L"CONOUT$",
+                           GENERIC_READ | GENERIC_WRITE,
+                           FILE_SHARE_READ | FILE_SHARE_WRITE,
+                           NULL, OPEN_EXISTING, 0, NULL);
+    ASSERT(m_conout != INVALID_HANDLE_VALUE);
 }
 
 Win32Console::~Win32Console()
 {
-    CloseHandle(m_conin);
     CloseHandle(m_conout);
 }
 
@@ -314,14 +317,12 @@ void Win32Console::dumpConsoleFont(const char *prefix)
     }
 }
 
-void Win32Console::clearLines(int row, int count)
+void Win32Console::clearLines(
+    int row,
+    int count,
+    const ConsoleScreenBufferInfo &info)
 {
     // TODO: error handling
-    CONSOLE_SCREEN_BUFFER_INFO info;
-    memset(&info, 0, sizeof(info));
-    if (!GetConsoleScreenBufferInfo(m_conout, &info)) {
-        trace("GetConsoleScreenBufferInfo failed");
-    }
     const int width = SmallRect(info.srWindow).width();
     DWORD actual = 0;
     if (!FillConsoleOutputCharacterW(
@@ -372,42 +373,6 @@ void Win32Console::moveWindow(const SmallRect &rect)
     }
 }
 
-void Win32Console::reposition(const Coord &newBufferSize,
-                              const SmallRect &newWindowRect)
-{
-    // Windows has one API for resizing the screen buffer and a different one
-    // for resizing the window.  It seems that either API can fail if the
-    // window does not fit on the screen buffer.
-
-    const SmallRect origWindowRect(windowRect());
-    const SmallRect origBufferRect(Coord(), bufferSize());
-
-    ASSERT(!newBufferSize.isEmpty());
-    SmallRect bufferRect(Coord(), newBufferSize);
-    ASSERT(bufferRect.contains(newWindowRect));
-
-    SmallRect tempWindowRect = origWindowRect.intersected(bufferRect);
-    if (tempWindowRect.width() <= 0) {
-        tempWindowRect.setLeft(newBufferSize.X - 1);
-        tempWindowRect.setWidth(1);
-    }
-    if (tempWindowRect.height() <= 0) {
-        tempWindowRect.setTop(newBufferSize.Y - 1);
-        tempWindowRect.setHeight(1);
-    }
-
-    // Alternatively, if we can immediately use the new window size,
-    // do that instead.
-    if (origBufferRect.contains(newWindowRect))
-        tempWindowRect = newWindowRect;
-
-    if (tempWindowRect != origWindowRect)
-        moveWindow(tempWindowRect);
-    resizeBuffer(newBufferSize);
-    if (newWindowRect != tempWindowRect)
-        moveWindow(newWindowRect);
-}
-
 Coord Win32Console::cursorPosition()
 {
     return bufferInfo().dwCursorPosition;

agent/Win32Console.h

@@ -35,15 +35,9 @@ struct ConsoleScreenBufferInfo : CONSOLE_SCREEN_BUFFER_INFO
         memset(this, 0, sizeof(*this));
     }
 
-    Coord bufferSize() const
-    {
-        return dwSize;
-    }
-
-    SmallRect windowRect() const
-    {
-        return srWindow;
-    }
+    Coord bufferSize() const        { return dwSize;    }
+    SmallRect windowRect() const    { return srWindow;  }
+    Coord cursorPosition() const    { return dwCursorPosition; }
 };
 
 class Win32Console
@@ -63,15 +57,14 @@ class Win32Console
     bool setSmallConsoleFontVista();
     void dumpConsoleFont(const char *prefix);
 public:
-    void clearLines(int row, int count);
+    void clearLines(int row, int count, const ConsoleScreenBufferInfo &info);
 
     // Buffer and window sizes.
     ConsoleScreenBufferInfo bufferInfo();
     Coord bufferSize();
     SmallRect windowRect();
     void resizeBuffer(const Coord &size);
     void moveWindow(const SmallRect &rect);
-    void reposition(const Coord &bufferSize, const SmallRect &windowRect);
 
     // Cursor.
     Coord cursorPosition();

misc/BufferResizeTests.cc

@@ -0,0 +1,91 @@
+#include <windows.h>
+#include <cassert>
+
+#include "../shared/DebugClient.cc"
+#include "TestUtil.cc"
+
+void dumpInfoToTrace() {
+    CONSOLE_SCREEN_BUFFER_INFO info;
+    assert(GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &info));
+    trace("win=(%d,%d,%d,%d)",
+        (int)info.srWindow.Left,
+        (int)info.srWindow.Top,
+        (int)info.srWindow.Right,
+        (int)info.srWindow.Bottom);
+    trace("buf=(%d,%d)",
+        (int)info.dwSize.X,
+        (int)info.dwSize.Y);
+    trace("cur=(%d,%d)",
+        (int)info.dwCursorPosition.X,
+        (int)info.dwCursorPosition.Y);
+}
+
+int main(int argc, char *argv[]) {
+    if (argc == 1) {
+        startChildProcess(L"CHILD");
+        return 0;
+    }
+
+    setWindowPos(0, 0, 1, 1);
+
+    if (false) {
+        // Reducing the buffer height can move the window up.
+        setBufferSize(80, 25);
+        setWindowPos(0, 20, 80, 5);
+        Sleep(2000);
+        setBufferSize(80, 10);
+    }
+
+    if (false) {
+        // Reducing the buffer height moves the window up and the buffer
+        // contents up too.
+        setBufferSize(80, 25);
+        setWindowPos(0, 20, 80, 5);
+        setCursorPos(0, 20);
+        printf("TEST1\nTEST2\nTEST3\nTEST4\n");
+        fflush(stdout);
+        Sleep(2000);
+        setBufferSize(80, 10);
+    }
+
+    if (false) {
+        // Reducing the buffer width can move the window left.
+        setBufferSize(80, 25);
+        setWindowPos(40, 0, 40, 25);
+        Sleep(2000);
+        setBufferSize(60, 25);
+    }
+
+    if (false) {
+        // Sometimes the buffer contents are shifted up; sometimes they're
+        // shifted down.  It seems to depend on the cursor position?
+
+        // setBufferSize(80, 25);
+        // setWindowPos(0, 20, 80, 5);
+        // setCursorPos(0, 20);
+        // printf("TESTa\nTESTb\nTESTc\nTESTd\nTESTe");
+        // fflush(stdout);
+        // setCursorPos(0, 0);
+        // printf("TEST1\nTEST2\nTEST3\nTEST4\nTEST5");
+        // fflush(stdout);
+        // setCursorPos(0, 24);
+        // Sleep(5000);
+        // setBufferSize(80, 24);
+
+        setBufferSize(80, 20);
+        setWindowPos(0, 10, 80, 10);
+        setCursorPos(0, 18);
+
+        printf("TEST1\nTEST2");
+        fflush(stdout);
+        setCursorPos(0, 18);
+
+        Sleep(2000);
+        setBufferSize(80, 18);
+    }
+
+    dumpInfoToTrace();
+    Sleep(30000);
+
+    return 0;
+}

misc/ChangeScreenBuffer.cc

@@ -0,0 +1,54 @@
+// A test program for CreateConsoleScreenBuffer / SetConsoleActiveScreenBuffer
+//
+
+#include <windows.h>
+#include <stdio.h>
+#include <conio.h>
+#include <io.h>
+#include <cassert>
+
+#include "../shared/DebugClient.cc"
+#include "TestUtil.cc"
+
+int main()
+{
+    HANDLE origBuffer = GetStdHandle(STD_OUTPUT_HANDLE);
+    HANDLE childBuffer = CreateConsoleScreenBuffer(
+            GENERIC_READ | GENERIC_WRITE,
+            FILE_SHARE_READ | FILE_SHARE_WRITE,
+            NULL, CONSOLE_TEXTMODE_BUFFER, NULL);
+
+    SetConsoleActiveScreenBuffer(childBuffer);
+
+    while (true) {
+        char buf[1024];
+        CONSOLE_SCREEN_BUFFER_INFO info;
+
+        assert(GetConsoleScreenBufferInfo(origBuffer, &info));
+        trace("child.size=(%d,%d)", (int)info.dwSize.X, (int)info.dwSize.Y);
+        trace("child.cursor=(%d,%d)", (int)info.dwCursorPosition.X, (int)info.dwCursorPosition.Y);
+        trace("child.window=(%d,%d,%d,%d)",
+            (int)info.srWindow.Left, (int)info.srWindow.Top,
+            (int)info.srWindow.Right, (int)info.srWindow.Bottom);
+        trace("child.maxSize=(%d,%d)", (int)info.dwMaximumWindowSize.X, (int)info.dwMaximumWindowSize.Y);
+
+        int ch = getch();
+        sprintf(buf, "%02x\n", ch);
+        DWORD actual = 0;
+        WriteFile(childBuffer, buf, strlen(buf), &actual, NULL);
+        if (ch == 0x1b/*ESC*/ || ch == 0x03/*CTRL-C*/)
+            break;
+
+        if (ch == 'b') {
+            setBufferSize(origBuffer, 40, 25);
+        } else if (ch == 'w') {
+            setWindowPos(origBuffer, 1, 1, 38, 23);
+        } else if (ch == 'c') {
+            setCursorPos(origBuffer, 10, 10);
+        }
+    }
+
+    SetConsoleActiveScreenBuffer(origBuffer);
+
+    return 0;
+}

misc/DumpLines.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+import sys
+
+for i in range(1, int(sys.argv[1]) + 1):
+    print i, "X" * 78

misc/SetCursorPos.cc

@@ -0,0 +1,11 @@
+#include <windows.h>
+
+#include "TestUtil.cc"
+#include "../shared/DebugClient.cc"
+
+int main(int argc, char *argv[]) {
+    int col = atoi(argv[1]);
+    int row = atoi(argv[2]);
+    setCursorPos(col, row);
+    return 0;
+}