Add a flag to DecodeUnitsInto to detect the case where the buffer ends at a truncated unicode sequence

Only enabled this if the option to do chunked-decoding is enabled.

Author: boingoing

lib/Common/Codex/Utf8Codex.cpp

@@ -97,7 +97,7 @@ namespace utf8
     }
 
     _At_(ptr, _In_reads_(end - ptr) _Post_satisfies_(ptr >= _Old_(ptr) - 1 && ptr <= end))
-    inline char16 DecodeTail(char16 c1, LPCUTF8& ptr, LPCUTF8 end, DecodeOptions& options)
+    inline char16 DecodeTail(char16 c1, LPCUTF8& ptr, LPCUTF8 end, DecodeOptions& options, bool *chunkEndsAtTruncatedSequence)
     {
         char16 ch = 0;
         BYTE c2, c3, c4;
@@ -129,8 +129,15 @@ namespace utf8
             if (ptr >= end)
             {
                 if ((options & doChunkedEncoding) != 0)
+                {
                     // The is a sequence that spans a chunk, push ptr back to the beginning of the sequence.
                     ptr--;
+
+                    if (chunkEndsAtTruncatedSequence)
+                    {
+                        *chunkEndsAtTruncatedSequence = true;
+                    }
+                }
                 return g_chUnknown;
             }
             c2 = *ptr++;
@@ -160,8 +167,16 @@ namespace utf8
             if (ptr + 1 >= end)
             {
                 if ((options & doChunkedEncoding) != 0)
+                {
                     // The is a sequence that spans a chunk, push ptr back to the beginning of the sequence.
                     ptr--;
+
+                    if (chunkEndsAtTruncatedSequence)
+                    {
+                        *chunkEndsAtTruncatedSequence = true;
+                    }
+                }
+
                 return g_chUnknown;
             }
 
@@ -221,9 +236,16 @@ namespace utf8
             if (ptr + 2 >= end)
             {
                 if ((options & doChunkedEncoding) != 0)
+                {
                     // The is a sequence that spans a chunk, push ptr back to the beginning of the sequence.
                     ptr--;
 
+                    if (chunkEndsAtTruncatedSequence)
+                    {
+                        *chunkEndsAtTruncatedSequence = true;
+                    }
+                }
+
                 ch = g_chUnknown;
                 break;
             }
@@ -378,10 +400,15 @@ namespace utf8
     }
 
     _Use_decl_annotations_
-    size_t DecodeUnitsInto(char16 *buffer, LPCUTF8& pbUtf8, LPCUTF8 pbEnd, DecodeOptions options)
+    size_t DecodeUnitsInto(char16 *buffer, LPCUTF8& pbUtf8, LPCUTF8 pbEnd, DecodeOptions options, bool *chunkEndsAtTruncatedSequence)
     {
         DecodeOptions localOptions = options;
 
+        if (chunkEndsAtTruncatedSequence)
+        {
+            *chunkEndsAtTruncatedSequence = false;
+        }
+
         LPCUTF8 p = pbUtf8;
         char16 *dest = buffer;
 
@@ -402,7 +429,7 @@ namespace utf8
         while (p < pbEnd)
         {
             LPCUTF8 s = p;
-            char16 chDest = Decode(p, pbEnd, localOptions);
+            char16 chDest = Decode(p, pbEnd, localOptions, chunkEndsAtTruncatedSequence);
 
             if (s < p)
             {

lib/Common/Codex/Utf8Codex.h

@@ -143,19 +143,19 @@ namespace utf8
 
     // Decode the trail bytes after the UTF8 lead byte c1 but returning 0xFFFD if trail bytes are expected after end.
     _At_(ptr, _In_reads_(end - ptr) _Post_satisfies_(ptr >= _Old_(ptr) - 1 && ptr <= end))
-    char16 DecodeTail(char16 c1, LPCUTF8& ptr, LPCUTF8 end, DecodeOptions& options);
+    char16 DecodeTail(char16 c1, LPCUTF8& ptr, LPCUTF8 end, DecodeOptions& options, bool *chunkEndsAtTruncatedSequence = nullptr);
 
     // Decode the UTF8 sequence into a UTF16 encoding. Code points outside the Unicode base plain will generate
     // surrogate pairs, using the 'doSecondSurrogatePair' option to remember the first word has already been returned.
     // If ptr == end 0x0000 is emitted. If ptr < end but the lead byte of the UTF8 sequence
     // expects trail bytes past end then 0xFFFD are emitted until ptr == end.
     _At_(ptr, _In_reads_(end - ptr) _Post_satisfies_(ptr >= _Old_(ptr) && ptr <= end))
-    inline char16 Decode(LPCUTF8& ptr, LPCUTF8 end, DecodeOptions& options)
+    inline char16 Decode(LPCUTF8& ptr, LPCUTF8 end, DecodeOptions& options, bool *chunkEndsAtTruncatedSequence = nullptr)
     {
         if (ptr >= end) return 0;
         utf8char_t c1 = *ptr++;
         if (c1 < 0x80) return static_cast<char16>(c1);
-        return DecodeTail(c1, ptr, end, options);
+        return DecodeTail(c1, ptr, end, options, chunkEndsAtTruncatedSequence);
     }
 
     // Encode ch into a UTF8 sequence ignoring surrogate pairs (which are encoded as two
@@ -274,7 +274,7 @@ namespace utf8
 
     // Decode cb bytes from ptr to into buffer returning the number of characters converted and written to buffer
     _Ret_range_(0, pbEnd - _Old_(pbUtf8))
-    size_t DecodeUnitsInto(_Out_writes_(pbEnd - pbUtf8) char16 *buffer, LPCUTF8& pbUtf8, LPCUTF8 pbEnd, DecodeOptions options = doDefault);
+    size_t DecodeUnitsInto(_Out_writes_(pbEnd - pbUtf8) char16 *buffer, LPCUTF8& pbUtf8, LPCUTF8 pbEnd, DecodeOptions options = doDefault, bool *chunkEndsAtTruncatedSequence = nullptr);
 
     // Decode cb bytes from ptr to into buffer returning the number of characters converted and written to buffer (excluding the null terminator)
     size_t DecodeUnitsIntoAndNullTerminate(__out_ecount(pbEnd - pbUtf8 + 1) __nullterminated char16 *buffer, LPCUTF8& pbUtf8, LPCUTF8 pbEnd, DecodeOptions options = doDefault);