Add Unicode support in TEditor

It was not that difficult.

README.md

@@ -92,7 +92,7 @@ I'm sorry, the root makefile assumes it is executed from the `project` directory
 * Ncurses-based terminal support.
 * Mouse and key modifiers support on the linux console.
 * Overall better display performance than SET's or Sergio Sigala's ports.
-* Reads UTF-8 input from the terminal and displays UTF-8 text, but still works with 8-bit ASCII characters internally.
+* UTF-8 support both in terminal I/O and the API.
 * Implementation of some Borland C++ RTL functions: `findfirst`, `findnext`, `fnsplit`, `_dos_findfirst`, `_dos_findnext`, `getdisk`, `setdisk`, `getcurdir`, `filelenght`.
 * Accepts both Unix and Windows-style file paths in 'Open File' dialogs.
 * Simple segmentation fault handler that gives you the chance to 'continue running' the application if something goes wrong.
@@ -341,11 +341,9 @@ The functions above depend on the following lower-level functions. You will need
 size_t TText::next(TStringView text);
 size_t TText::prev(TStringView text, size_t index);
 size_t TText::wseek(TStringView text, int count, Boolean incRemainder=True);
-#ifndef __BORLANDC__
 void TText::eat(TScreenCell *cell, size_t n, size_t &width, TStringView text, size_t &bytes);
 void TText::next(TStringView text, size_t &bytes, size_t &width);
 void TText::wseek(TStringView text, size_t &index, size_t &remainder, int count);
-#endif
 ```
 
 For drawing `TScreenCell` buffers directly, the following methods are available:
@@ -433,4 +431,4 @@ Support for creating Unicode-aware views is in place, but some views that are pa
 * `TInputLine` can display and process Unicode text.
 * Word wrapping in `TStaticText` and `THelpViewer` is Unicode-aware.
 * Automatic shortcuts in `TMenuBox` won't work with Unicode text, as shortcuts are still compared against `event.keyDown.charScan.charCode`.
-* `TEditor` assumes a single-byte encoding both when handling input events and when displaying text. So it won't display UTF-8 but at least it has a consistent behaviour.
+* `TEditor` instances (as in the `tvedit` application) are in UTF-8 mode by default. Press `Ctrl+P` to switch back to single-byte mode. This only changes how the document is displayed and the encoding of user input; it does not alter the document.

include/tvision/editors.h

@@ -77,7 +77,8 @@ const int
   cmIndentMode  = 522,
   cmUpdateTitle = 523,
   cmSelectAll   = 524,
-  cmDelWordLeft = 525;
+  cmDelWordLeft = 525,
+  cmEncoding    = 526;
 
 const int
   edOutOfMemory   = 0,
@@ -216,7 +217,7 @@ class TEditor : public TView
     void doUpdate();
     void doSearchReplace();
     void drawLines( int, int, uint );
-    void formatLine(ushort *, uint, int, ushort );
+    void formatLine(TScreenCell *, uint, int, ushort );
     void find();
     uint getMousePtr( TPoint );
     Boolean hasSelection();
@@ -238,6 +239,7 @@ class TEditor : public TView
     void setBufLen( uint );
     void setCurPtr( uint, uchar );
     void startSelect();
+    void toggleEncoding();
     void toggleInsMode();
     void unlock();
     void update( uchar );
@@ -272,6 +274,13 @@ class TEditor : public TView
     const char* eolBytes;
     uint eolSize;
 
+    Boolean encSingleByte;
+    char charsBuf[4];
+    void nextChar( TStringView, uint &P, uint &width );
+    void formatCell( TScreenCell*, uint, uint&, TStringView, uint& , TCellAttribs );
+    TStringView bufChars( uint );
+    TStringView bufPrevChars( uint );
+
     static TEditorDialog _NEAR editorDialog;
     static ushort _NEAR editorFlags;
     static char _NEAR findStr[maxFindStrLen];

include/tvision/scrncell.h

@@ -36,6 +36,11 @@ inline void setChar(TScreenCell &cell, TCellChar ch)
     ((uchar *) &cell)[0] = ch;
 }
 
+inline void setCell(TScreenCell &cell, TCellChar ch, TCellAttribs attr)
+{
+    cell = (attr << 8) | ch;
+}
+
 #else
 
 #include <cstring>
@@ -238,6 +243,14 @@ inline void setChar(TScreenCell &cell, uchar ch)
     cell.extraWidth = 0;
 }
 
+inline void setCell(TScreenCell &cell, TCellChar ch, TCellAttribs attr, uchar extraWidth=0)
+{
+    TScreenCell c = 0;
+    ::setChar(c, ch, extraWidth);
+    ::setAttr(c, attr);
+    cell = c;
+}
+
 #endif // __BORLANDC__
 
 #endif

include/tvision/ttext.h

@@ -136,11 +136,9 @@ class TText {
     static size_t prev(TStringView text, size_t index);
     static size_t wseek(TStringView text, int count, Boolean incRemainder=True);
 
-#ifndef __BORLANDC__
     static void eat(TScreenCell *cell, size_t n, size_t &width, TStringView text, size_t &bytes);
     static void next(TStringView text, size_t &bytes, size_t &width);
     static void wseek(TStringView text, size_t &index, size_t &remainder, int count);
-#endif
 
 };
 
@@ -161,6 +159,31 @@ inline size_t TText::wseek(TStringView text, int count, Boolean)
     return count > 0 ? min(count, text.size()) : 0;
 }
 
+inline void TText::eat( TScreenCell *cell, size_t n, size_t &width,
+                        TStringView text, size_t &bytes )
+{
+    if (n) {
+        ::setChar(*cell, text[0];
+        ++width;
+        ++bytes;
+    }
+}
+
+inline void TText::next(TStringView text, size_t &bytes, size_t &width)
+{
+    if (text.size()) {
+        ++bytes;
+        ++width;
+    }
+}
+
+inline void TText::wseek(TStringView text, size_t &index, size_t &remainder, int count)
+{
+    if (count > 0)
+        index += count;
+    remainder = 0;
+}
+
 #else
 
 inline size_t TText::next(TStringView text)

source/tvision/edits.cpp

@@ -27,7 +27,7 @@ uint TEditor::bufPtr( uint P )
     return P < curPtr ? P : P + gapLen;
 }
 
-void TEditor::formatLine( ushort *DrawBuf,
+void TEditor::formatLine( TScreenCell *DrawBuf,
                           uint P,
                           int Width,
                           ushort Colors
@@ -42,29 +42,28 @@ void TEditor::formatLine( ushort *DrawBuf,
         { uchar(Colors), bufLen }
     };
 
-    uchar Color = uchar(Colors);
     int X = 0;
-
     for (int r = 0; r < 3; ++r)
     {
-        Color = ranges[r].color;
+        uchar Color = ranges[r].color;
         while (P < ranges[r].end && X < Width)
         {
-            uchar Char = bufChar(P++);
+            TStringView chars = bufChars(P);
+            uchar Char = chars[0];
             if (Char == '\r' || Char == '\n')
                 goto fill;
             if (Char == '\t') {
                 do {
-                    DrawBuf[X++] = (Color << 8) | ' ';
+                    ::setCell(DrawBuf[X++], ' ', Color);
                 } while (X%8 != 0 && X < Width);
-            } else {
-                DrawBuf[X++] = (Color << 8) | Char;
-            }
+                ++P;
+            } else
+                formatCell(&DrawBuf[X], Width - X, (uint&) X, chars, P, Color);
         }
     }
 fill:
     while (X < Width)
-        DrawBuf[X++] = (Color << 8) | ' ';
+        ::setCell(DrawBuf[X++], ' ', uchar(Colors));
 }
 
 uint TEditor::lineEnd( uint P )
@@ -103,7 +102,10 @@ uint TEditor::nextChar( uint P )
     {
         if (bufChar(P) == '\r' && bufChar(P + 1) == '\n')
             return P + 2;
-        return P + 1;
+        if (encSingleByte)
+            return P + 1;
+        else
+            return P + TText::next(bufChars(P));
     }
     return bufLen;
 }
@@ -114,7 +116,13 @@ uint TEditor::prevChar( uint P )
     {
         if (bufChar(P - 2) == '\r' && bufChar(P - 1) == '\n')
             return P - 2;
-        return P - 1;
+        if (encSingleByte)
+            return P - 1;
+        else
+        {
+            TStringView t = bufPrevChars(P);
+            return P - TText::prev(t, t.size());
+        }
     }
     return 0;
 }