Render private-use range unicode chars (except the four reserved by i…

…term). Some fonts use them for fancy special characters and it's nice to support that. The down side is that for normal fonts they'll render a bit uglier as a double-width character, but you shouldn't get them often anyway

ScreenChar.h

@@ -43,8 +43,8 @@
 // copy-pasting.
 #define TAB_FILLER 0xf001
 
-// If a private-use character is received as input, we convert it to this
-// meaningless character.
+// If DWC_SKIP appears in the input, we convert it to this to avoid causing confusion.
+// NOTE: I think this isn't used because DWC_SKIP is caught early and converted to a '?'.
 #define BOGUS_CHAR 0xf002
 
 // Double-width characters have their "real" code in one cell and this code in
@@ -57,15 +57,15 @@
 #define EOL_SOFT 1 // Soft line break (a long line was wrapped)
 #define EOL_DWC  2 // Double-width character wrapped to next line
 
-// The range of private codes we use, with specific instances defined right
+// The range of private codes we use, with specific instances defined
 // above here.
 #define ITERM2_PRIVATE_BEGIN 0xf000
-#define ITERM2_PRIVATE_END 0xf8fe
+#define ITERM2_PRIVATE_END 0xf003
 
 // This is the standard unicode replacement character for when input couldn't
 // be parsed properly but we need to render something there.
-#define UNKNOWN 0xfffd
-#define ONECHAR_UNKNOWN ('#')   // Used for encodings other than utf-8.
+#define UNICODE_REPLACEMENT_CHAR 0xfffd
+#define ONECHAR_UNKNOWN ('?')   // Used for encodings other than utf-8.
 
 // Alternate semantics definitions
 // Default background color
@@ -136,7 +136,7 @@ typedef struct screen_char_t
 // Standard unicode replacement string. Is a double-width character.
 static inline NSString* ReplacementString()
 {
-    const unichar kReplacementCharacter = 0xfffd;
+    const unichar kReplacementCharacter = UNICODE_REPLACEMENT_CHAR;
     return [NSString stringWithCharacters:&kReplacementCharacter length:1];
 }
 

ScreenChar.m

@@ -43,7 +43,7 @@
 
 NSString* ComplexCharToStr(int key)
 {
-    if (key == UNKNOWN) {
+    if (key == UNICODE_REPLACEMENT_CHAR) {
         return ReplacementString();
     }
 
@@ -57,7 +57,7 @@
 
 NSString* CharToStr(unichar code, BOOL isComplex)
 {
-    if (code == UNKNOWN) {
+    if (code == UNICODE_REPLACEMENT_CHAR) {
         return ReplacementString();
     }
 
@@ -70,7 +70,7 @@
 
 int ExpandScreenChar(screen_char_t* sct, unichar* dest) {
     NSString* value = nil;
-    if (sct->code == UNKNOWN) {
+    if (sct->code == UNICODE_REPLACEMENT_CHAR) {
         value = ReplacementString();
     } else if (sct->complexChar) {
         value = ComplexCharToStr(sct->code);
@@ -126,8 +126,8 @@ int GetOrSetComplexChar(NSString* str)
 
 int AppendToComplexChar(int key, unichar codePoint)
 {
-    if (key == UNKNOWN) {
-        return UNKNOWN;
+    if (key == UNICODE_REPLACEMENT_CHAR) {
+        return UNICODE_REPLACEMENT_CHAR;
     }
 
     NSString* str = [complexCharMap objectForKey:[NSNumber numberWithInt:key]];
@@ -145,8 +145,8 @@ int AppendToComplexChar(int key, unichar codePoint)
 
 int BeginComplexChar(unichar initialCodePoint, unichar combiningChar)
 {
-    if (initialCodePoint == UNKNOWN) {
-        return UNKNOWN;
+    if (initialCodePoint == UNICODE_REPLACEMENT_CHAR) {
+        return UNICODE_REPLACEMENT_CHAR;
     }
 
     unichar temp[2];

VT100Screen.m

@@ -144,11 +144,11 @@ void StringToScreenChars(NSString *s,
             lastInitializedChar = j;
         }
 
-        if ((sc[i] >= 0xe000 && sc[i] <= ITERM2_PRIVATE_END) ||
-            sc[i] >= 0xfffd) {
-            // Translate private-use characters into a replacement char.
-            // Unfortunately, the proper replacement char U+fffd is double-width
-            // (at least for some fonts) which screws up formatting.
+        if (sc[i] >= ITERM2_PRIVATE_BEGIN && sc[i] <= ITERM2_PRIVATE_END) {
+            // Translate iTerm2's private-use characters into a "?". Although the replacement
+            // character renders as a double-width char in a single-width char's space and is ugly,
+            // some fonts use dwc's to add extra glyphs. It's kinda sketch, but it's better form to
+            // render what you get than to try to be clever and break such edge cases.
             buf[j].code = '?';
         } else if (sc[i] > 0xa0 && [NSString isDoubleWidthCharacter:sc[i]
                                                            encoding:encoding
@@ -166,10 +166,10 @@ void StringToScreenChars(NSString *s,
 
             buf[j].backgroundColor = bg.backgroundColor;
             buf[j].alternateBackgroundSemantics = bg.alternateBackgroundSemantics;
-        } else if (sc[i] == 0xfeff ||
-                   sc[i] == 0x200b ||
-                   sc[i] == 0x200c ||
-                   sc[i] == 0x200d) {
+        } else if (sc[i] == 0xfeff ||  // zero width no-break space
+                   sc[i] == 0x200b ||  // zero width space
+                   sc[i] == 0x200c ||  // zero width non-joiner
+                   sc[i] == 0x200d) {  // zero width joiner
             j--;
             lastInitializedChar--;
         } else if (IsCombiningMark(sc[i]) || IsLowSurrogate(sc[i])) {
@@ -2702,6 +2702,7 @@ - (void)setString:(NSString *)string ascii:(BOOL)ascii
         NSAssert(buffer[idx].code != DWC_RIGHT, @"DWC cut off");
 
         if (buffer[idx].code == DWC_SKIP) {
+            // I'm pretty sure this can never happen and that this code is just a historical leftover.
             // This is an invalid unicode character that iTerm2 has appropriated
             // for internal use. Change it to something invalid but safe.
             buffer[idx].code = BOGUS_CHAR;