Handle invalid UTF-8 by converting from current code page to UTF-8

Author: nickgammon

doc.cpp

@@ -1964,6 +1964,7 @@ CString strLine (lpszText, size);
       switch (m_phase)
         {
         case HAVE_ESC:            Phase_ESC (c); continue;
+        case HAVE_UTF8_CHARACTER: Phase_UTF8 (c); continue;
 
         case HAVE_FOREGROUND_256_START:    // these 4 are similar to Phase_ANSI
         case HAVE_FOREGROUND_256_FINISH:
@@ -2107,9 +2108,32 @@ CString strLine (lpszText, size);
         continue;   // back to main loop
         } // end of Pueblo startup
 
-// here when phase is none
+// here when phase is NONE
 
-    cOneCharacterLine [0] = c;    // in case we need to use it
+    // do not display UTF-8 characters until they have completely arrived
+    // check if high-order bit is set
+    if (m_bUTF_8 && (c & 0x80))
+      {
+      m_UTF8Sequence [0] = c;
+      m_UTF8Sequence [1] = 0;  // null terminator
+      m_phase = HAVE_UTF8_CHARACTER;
+
+      // 0x00000080 - 0x000007FF    110 xxxxx 10 xxxxxx
+      if ((c & 0xE0) == 0xC0)
+        m_iUTF8BytesLeft = 1;
+      // 0x00000800 - 0x0000FFFF    1110 xxxx 10 xxxxxx 10 xxxxxx
+      else if ((c & 0xF0) == 0xE0)
+        m_iUTF8BytesLeft = 2;
+      // 0x00010000 - 0x001FFFFF    11110 xxx 10 xxxxxx 10 xxxxxx 10 xxxxxx
+      else if ((c & 0xF8) == 0xF0)
+        m_iUTF8BytesLeft = 3;
+      else
+      // some bogus byte with the high-order bit set
+        OutputBadUTF8characters ();
+      continue;  // we are done for now with this byte
+      }  // end of high-order bit set with UTF-8 enabled
+
+    cOneCharacterLine [0] = c;    // in case we need to use it as a string
 
     switch (c)
       {

doc.h

@@ -96,6 +96,9 @@ enum { NONE,          // normal text
        HAVE_BACKGROUND_256_START,   // received ESC[48;
        HAVE_BACKGROUND_256_FINISH,  // received ESC[48;5;
 
+       // utf-8 modes
+       HAVE_UTF8_CHARACTER,         // received 110 xxxxx, 1110 xxxx, or 11110 xxx
+
 
        // mxp modes
        HAVE_MXP_ELEMENT, // collecting element, eg. < xxxxx >. Starts on <, stops on >
@@ -972,6 +975,9 @@ class CMUSHclientDoc : public CDocument
   long m_iUTF8ErrorCount;               // count of lines with bad UTF8
   long m_iOutputWindowRedrawCount;      // count of times output window redrawn
 
+  unsigned char m_UTF8Sequence [5];     // we collect up to 4 UTF8 bytes here and a null-terminator
+  int  m_iUTF8BytesLeft;                // how many UTF8 bytes to go
+
   long m_iTriggersEvaluatedCount;    // how many triggers we evaluated
   long m_iTriggersMatchedCount;      // how many triggers matched
   long m_iAliasesEvaluatedCount;     // how many aliases we evaluated 
@@ -1383,7 +1389,8 @@ class CMUSHclientDoc : public CDocument
 	void DisplayMsg(LPCTSTR lpszText, int size, const int flags);
   void AddToLine (LPCTSTR lpszText, const int flags);
   void StartNewLine_KeepPreviousStyle (const int flags);
-  void Phase_ESC (const unsigned char c);            
+  void Phase_ESC (const unsigned char c);  
+  void Phase_UTF8 (const unsigned char c);  
   void Phase_ANSI (const unsigned char c);           
   void Phase_IAC (unsigned char & c);            
   void Phase_WILL (const unsigned char c);           
@@ -1545,6 +1552,7 @@ class CMUSHclientDoc : public CDocument
   void WriteToLog (const CString & strText);
   void LogLineInHTMLcolour (POSITION startpos);
   void LogCommand (const char * text);
+  void OutputBadUTF8characters (void);
 
   BOOL OpenSession (void);
   void SetUpOutputWindow (void);

telnet_phases.cpp

@@ -803,3 +803,64 @@ void CMUSHclientDoc::Phase_COMPRESS_WILL (const unsigned char c)
   m_phase = NONE;
   } // end of Phase_COMPRESS_WILL
 
+// in UTF-8 mode, if we get a "bad" UTF-8 character we assume it is standard ANSI
+// and convert it from ANSI into Unicode, and then into UTF-8 and output that instead
+void CMUSHclientDoc::OutputBadUTF8characters (void)
+  {
+  for (int i = 0; m_UTF8Sequence [i]; i++)
+    {
+    // convert ANSI to Unicode:
+    WCHAR sUnicode [1];
+    MultiByteToWideChar (CP_THREAD_ACP, MB_PRECOMPOSED, (const char *) &m_UTF8Sequence [i], 1, sUnicode, 1); 
+    // now convert Unicode to UTF8
+    char sOutput [5];
+    memset (sOutput, 0, sizeof sOutput);  // ensure trailing null
+    WideCharToMultiByte (CP_UTF8, 0, sUnicode, 1, sOutput, sizeof sOutput, NULL, NULL);
+    AddToLine (sOutput, 0); 
+    m_cLastChar = m_UTF8Sequence [i];
+    }
+
+  m_phase = NONE;
+  }  // end of CMUSHclientDoc::OutputBadUTF8characters
+
+// test data: testing \C5\87\C4\A8\C4\86\C4\B6 Gammon and now: \C6 <---
+
+// here when getting second or subsequent bytes of a UTF8 character
+void CMUSHclientDoc::Phase_UTF8 (const unsigned char c)
+  {
+
+  // append to our UTF8 sequence
+  int i = 0;
+  while (m_UTF8Sequence [i])
+    i++;
+  m_UTF8Sequence [i] = c;
+  m_UTF8Sequence [i + 1] = 0;  // null terminator
+
+  if ((c & 0xC0) != 0x80)
+    {
+    OutputBadUTF8characters ();
+    return;
+    }
+
+  // we are waiting for less of them
+  m_iUTF8BytesLeft--;
+
+  // more to go
+  if (m_iUTF8BytesLeft > 0)
+    return;
+
+  // check the sequence
+
+  int erroroffset;
+  int iBad = _pcre_valid_utf ((const unsigned char *) m_UTF8Sequence, strlen ((const char *) m_UTF8Sequence), &erroroffset);
+  if (iBad > 0)
+    {
+    OutputBadUTF8characters ();
+    return;
+    }
+
+  // valid UTF8 sequence, add to line
+  AddToLine ((const char *) m_UTF8Sequence, 0);
+  m_phase = NONE;
+
+  }  // end of CMUSHclientDoc::Phase_UTF8