Fix: Display Unicode correctly in unhandled MXP tags

Fixes #8482

Root cause: Two issues prevented non-ASCII characters from displaying:
1. Tag content was decoded using Latin-1 instead of connection encoding
2. HANDLER_INSERT_ENTITY_LIT mode uses toLatin1() which destroys Unicode

Solution:
- Added getEncoding() interface to TMxpClient/TMxpMudlet
- Use connection encoding to decode tag content bytes
- Changed to HANDLER_INSERT_ENTITY_SYS to preserve Unicode characters
- Added Qt6::Core5Compat dependency for QTextCodec support in tests

Author: mpconley

src/TMxpClient.h

@@ -103,6 +103,10 @@ class TMxpClient
     }
 
     virtual void setCaptionForSendEvent(const QString& caption) { Q_UNUSED(caption) }
+    
+    // Get the encoding used by the connection (for proper decoding of MXP tags)
+    // Default implementation returns UTF-8 for test clients
+    virtual QByteArray getEncoding() const { return QByteArrayLiteral("UTF-8"); }
 
     // Get the console wrap width for layout purposes (e.g., HR tag)
     virtual int getWrapWidth() const { return 80; } // Default fallback

src/TMxpMudlet.cpp

@@ -265,6 +265,11 @@ bool TMxpMudlet::isTagAllowedInMode(const QString& tagName, TMXPMode mode) const
     return openModeTags.contains(tagName);
 }
 
+QByteArray TMxpMudlet::getEncoding() const
+{
+    return mpHost->mTelnet.getEncoding();
+}
+
 int TMxpMudlet::getWrapWidth() const
 {
     // Return the host's configured wrap width, with a sensible minimum

src/TMxpMudlet.h

@@ -143,6 +143,8 @@ class TMxpMudlet : public TMxpClient
 
     QStack<TMxpEvent> mPendingSendEvents;
 
+    // Get the encoding used by the connection
+    QByteArray getEncoding() const override;
     bool shouldLockModeToSecure() const override;
 
 private:

src/TMxpProcessor.cpp

@@ -22,6 +22,7 @@
 
 #include "TMxpProcessor.h"
 #include <QDebug>
+#include <QTextCodec>
 
 bool TMxpProcessor::setMode(const QString& code)
 {
@@ -163,7 +164,7 @@ TMxpProcessingResult TMxpProcessor::processMxpInput(char& ch, bool resolveCustom
     && mMxpTagBuilder.isInsideTag()
     && !mMxpTagBuilder.isQuotedSequence()
     && !mMxpTagBuilder.isInsideComment()) {
-        lastEntityValue = QStringLiteral("<") + QString::fromStdString(mMxpTagBuilder.getRawTagContent());
+        lastEntityValue = qsl("<") + QString::fromStdString(mMxpTagBuilder.getRawTagContent());
         mMxpTagBuilder.resetForNewTag();
         return HANDLER_INSERT_ENTITY_LIT;
     }
@@ -173,6 +174,30 @@ TMxpProcessingResult TMxpProcessor::processMxpInput(char& ch, bool resolveCustom
     }
 
     if (mMxpTagBuilder.hasTag()) {
+        // Save raw tag content before it gets cleared by buildTag()
+        // Note: getRawTagContent() returns content INCLUDING the closing '>'
+        const std::string rawTagBytes = mMxpTagBuilder.getRawTagContent();
+        const QByteArray encoding = mpMxpClient->getEncoding();
+        
+        // Build the tag content string with proper encoding
+        QString rawTagContent = qsl("<");
+        
+        // Decode the raw bytes using the proper encoding
+        if (encoding == qsl("UTF-8")) {
+            rawTagContent += QString::fromStdString(rawTagBytes);
+        } else if (encoding == qsl("ISO 8859-1")) {
+            rawTagContent += QString::fromLatin1(rawTagBytes.c_str(), static_cast<int>(rawTagBytes.length()));
+        } else {
+            // For other encodings (GBK, BIG5, EUC-KR, etc.), use QTextCodec
+            QTextCodec* codec = QTextCodec::codecForName(encoding);
+            if (codec) {
+                rawTagContent += codec->toUnicode(rawTagBytes.c_str(), static_cast<int>(rawTagBytes.length()));
+            } else {
+                // Fallback to UTF-8
+                rawTagContent += QString::fromStdString(rawTagBytes);
+            }
+        }
+        
         QScopedPointer<MxpTag> const tag(mMxpTagBuilder.buildTag());
 
         //        qDebug() << "TAG RECEIVED: " << tag->asString();
@@ -183,9 +208,11 @@ TMxpProcessingResult TMxpProcessor::processMxpInput(char& ch, bool resolveCustom
         TMxpTagHandlerResult const result = mMxpTagProcessor.handleTag(mMxpTagProcessor, *mpMxpClient, tag.get());
 
         // If tag was not handled (not valid MXP and not a custom element), display it as-is
+        // Use HANDLER_INSERT_ENTITY_SYS so the Unicode content is inserted directly
+        // without being reprocessed through toLatin1() which would destroy non-ASCII chars
         if (result == MXP_TAG_NOT_HANDLED) {
-            lastEntityValue = tag->toString();
-            return HANDLER_INSERT_ENTITY_LIT;
+            lastEntityValue = rawTagContent;
+            return HANDLER_INSERT_ENTITY_SYS;
         }
 
         return result == MXP_TAG_COMMIT_LINE ? HANDLER_COMMIT_LINE : HANDLER_NEXT_CHAR;

test/CMakeLists.txt

@@ -13,6 +13,7 @@ ENABLE_TESTING()
 
 find_package(Qt6 6.8.2 REQUIRED
     COMPONENTS Core
+    Core5Compat
     Multimedia
     MultimediaWidgets
     Network
@@ -26,6 +27,7 @@ link_libraries(
     Qt6::Test
     Qt6::Concurrent
     Qt6::Core
+    Qt6::Core5Compat
     Qt6::Network
     Qt6::Multimedia
     Qt6::MultimediaWidgets