Synced up the HtmlTokenizer with HtmlKit to get the new optimizations

MimeKit/MimeKit.Android.csproj

@@ -162,6 +162,7 @@
     <Compile Include="IO\Filters\TrailingWhitespaceFilter.cs" />
     <Compile Include="IO\Filters\Unix2DosFilter.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Text\CharBuffer.cs" />
     <Compile Include="Text\FlowedToHtml.cs" />
     <Compile Include="Text\FlowedToText.cs" />
     <Compile Include="Text\HeaderFooterFormat.cs" />

MimeKit/MimeKit.Mac.csproj

@@ -151,6 +151,7 @@
     <Compile Include="IO\Filters\TrailingWhitespaceFilter.cs" />
     <Compile Include="IO\Filters\Unix2DosFilter.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Text\CharBuffer.cs" />
     <Compile Include="Text\FlowedToHtml.cs" />
     <Compile Include="Text\FlowedToText.cs" />
     <Compile Include="Text\HeaderFooterFormat.cs" />

MimeKit/MimeKit.Net45.csproj

@@ -162,6 +162,7 @@
     <Compile Include="IO\Filters\TrailingWhitespaceFilter.cs" />
     <Compile Include="IO\Filters\Unix2DosFilter.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Text\CharBuffer.cs" />
     <Compile Include="Text\FlowedToHtml.cs" />
     <Compile Include="Text\FlowedToText.cs" />
     <Compile Include="Text\HeaderFooterFormat.cs" />

MimeKit/MimeKit.NetStandard.csproj

@@ -158,6 +158,7 @@
     <Compile Include="IO\Filters\TrailingWhitespaceFilter.cs" />
     <Compile Include="IO\Filters\Unix2DosFilter.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Text\CharBuffer.cs" />
     <Compile Include="Text\FlowedToHtml.cs" />
     <Compile Include="Text\FlowedToText.cs" />
     <Compile Include="Text\HeaderFooterFormat.cs" />

MimeKit/MimeKit.TvOS.csproj

@@ -144,6 +144,7 @@
     <Compile Include="IO\Filters\TrailingWhitespaceFilter.cs" />
     <Compile Include="IO\Filters\Unix2DosFilter.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Text\CharBuffer.cs" />
     <Compile Include="Text\FlowedToHtml.cs" />
     <Compile Include="Text\FlowedToText.cs" />
     <Compile Include="Text\HeaderFooterFormat.cs" />

MimeKit/MimeKit.WatchOS.csproj

@@ -150,6 +150,7 @@
     <Compile Include="IO\Filters\TrailingWhitespaceFilter.cs" />
     <Compile Include="IO\Filters\Unix2DosFilter.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Text\CharBuffer.cs" />
     <Compile Include="Text\FlowedToHtml.cs" />
     <Compile Include="Text\FlowedToText.cs" />
     <Compile Include="Text\HeaderFooterFormat.cs" />

MimeKit/MimeKit.iOS.csproj

@@ -161,6 +161,7 @@
     <Compile Include="IO\Filters\TrailingWhitespaceFilter.cs" />
     <Compile Include="IO\Filters\Unix2DosFilter.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Text\CharBuffer.cs" />
     <Compile Include="Text\FlowedToHtml.cs" />
     <Compile Include="Text\FlowedToText.cs" />
     <Compile Include="Text\HeaderFooterFormat.cs" />

MimeKit/MimeKitLite.Android.csproj

@@ -90,6 +90,7 @@
     <Compile Include="IO\Filters\TrailingWhitespaceFilter.cs" />
     <Compile Include="IO\Filters\Unix2DosFilter.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Text\CharBuffer.cs" />
     <Compile Include="Text\FlowedToHtml.cs" />
     <Compile Include="Text\FlowedToText.cs" />
     <Compile Include="Text\HeaderFooterFormat.cs" />

MimeKit/MimeKitLite.Net45.csproj

@@ -85,6 +85,7 @@
     <Compile Include="IO\Filters\TrailingWhitespaceFilter.cs" />
     <Compile Include="IO\Filters\Unix2DosFilter.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Text\CharBuffer.cs" />
     <Compile Include="Text\FlowedToHtml.cs" />
     <Compile Include="Text\FlowedToText.cs" />
     <Compile Include="Text\HeaderFooterFormat.cs" />

MimeKit/MimeKitLite.NetStandard.csproj

@@ -77,6 +77,7 @@
     <Compile Include="IO\Filters\TrailingWhitespaceFilter.cs" />
     <Compile Include="IO\Filters\Unix2DosFilter.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Text\CharBuffer.cs" />
     <Compile Include="Text\FlowedToHtml.cs" />
     <Compile Include="Text\FlowedToText.cs" />
     <Compile Include="Text\HeaderFooterFormat.cs" />

MimeKit/MimeKitLite.iOS.csproj

@@ -85,6 +85,7 @@
     <Compile Include="IO\Filters\TrailingWhitespaceFilter.cs" />
     <Compile Include="IO\Filters\Unix2DosFilter.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Text\CharBuffer.cs" />
     <Compile Include="Text\FlowedToHtml.cs" />
     <Compile Include="Text\FlowedToText.cs" />
     <Compile Include="Text\HeaderFooterFormat.cs" />

MimeKit/Text/CharBuffer.cs

@@ -0,0 +1,93 @@
+//
+// CharBuffer.cs
+//
+// Author: Jeffrey Stedfast <jestedfa@microsoft.com>
+//
+// Copyright (c) 2013-2019 Xamarin Inc. (www.xamarin.com)
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+//
+
+using System;
+using System.Runtime.CompilerServices;
+
+namespace MimeKit.Text {
+	class CharBuffer
+	{
+		char[] buffer;
+
+		public CharBuffer (int capacity)
+		{
+			buffer = new char[capacity];
+		}
+
+		public int Length {
+			[MethodImpl (MethodImplOptions.AggressiveInlining)]
+			get;
+			[MethodImpl (MethodImplOptions.AggressiveInlining)]
+			set;
+		}
+
+		public char this[int index] {
+			[MethodImpl (MethodImplOptions.AggressiveInlining)]
+			get { return buffer[index]; }
+			[MethodImpl (MethodImplOptions.AggressiveInlining)]
+			set { buffer[index] = value; }
+		}
+
+		[MethodImpl (MethodImplOptions.AggressiveInlining)]
+		void EnsureCapacity (int length)
+		{
+			if (length < buffer.Length)
+				return;
+
+			int capacity = buffer.Length << 1;
+			while (capacity <= length)
+				capacity <<= 1;
+
+			Array.Resize (ref buffer, capacity);
+		}
+
+		[MethodImpl (MethodImplOptions.AggressiveInlining)]
+		public void Append (char c)
+		{
+			EnsureCapacity (Length + 1);
+			buffer[Length++] = c;
+		}
+
+		[MethodImpl (MethodImplOptions.AggressiveInlining)]
+		public void Append (string str)
+		{
+			EnsureCapacity (Length + str.Length);
+			str.CopyTo (0, buffer, Length, str.Length);
+			Length += str.Length;
+		}
+
+		[MethodImpl (MethodImplOptions.AggressiveInlining)]
+		public override string ToString ()
+		{
+			return new string (buffer, 0, Length);
+		}
+
+		public static implicit operator string (CharBuffer buffer)
+		{
+			return buffer.ToString ();
+		}
+	}
+}

MimeKit/Text/HtmlTokenizer.cs

@@ -25,7 +25,7 @@
 //
 
 using System.IO;
-using System.Text;
+using System.Runtime.CompilerServices;
 
 namespace MimeKit.Text {
 	/// <summary>
@@ -40,8 +40,8 @@ public class HtmlTokenizer
 		const string CData = "[CDATA[";
 
 		readonly HtmlEntityDecoder entity = new HtmlEntityDecoder ();
-		readonly StringBuilder data = new StringBuilder ();
-		readonly StringBuilder name = new StringBuilder ();
+		readonly CharBuffer data = new CharBuffer (2048);
+		readonly CharBuffer name = new CharBuffer (32);
 		readonly char[] cdata = new char[3];
 		HtmlDocTypeToken doctype;
 		HtmlAttribute attribute;
@@ -231,19 +231,26 @@ protected virtual HtmlAttribute CreateAttribute (string name)
 			return new HtmlAttribute (name);
 		}
 
-		static bool IsAlphaNumeric (char c)
+		[MethodImpl (MethodImplOptions.AggressiveInlining)]
+		static bool IsAlphaNumeric (int c)
 		{
-			return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9');
+			return ((uint) (c - 'A') <= 'Z' - 'A') || ((uint) (c - 'a') <= 'z' - 'a') || ((uint) (c - '0') <= '9' - '0');
 		}
 
-		static bool IsAsciiLetter (char c)
+		[MethodImpl (MethodImplOptions.AggressiveInlining)]
+		static bool IsAsciiLetter (int c)
 		{
-			return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
+			return ((uint) (c - 'A') <= 'Z' - 'A') || ((uint) (c - 'a') <= 'z' - 'a');
 		}
 
-		static char ToLower (char c)
+		[MethodImpl (MethodImplOptions.AggressiveInlining)]
+		static char ToLower (int c)
 		{
-			return (c >= 'A' && c <= 'Z') ? (char) (c + 0x20) : c;
+			// check if the char is within the uppercase range
+			if ((uint) (c - 'A') <= 'Z' - 'A')
+				return (char) (c + 0x20);
+
+			return (char) c;
 		}
 
 		int Peek ()
@@ -299,7 +306,7 @@ HtmlToken EmitCommentToken (string comment, bool bogus = false)
 			return token;
 		}
 
-		HtmlToken EmitCommentToken (StringBuilder comment, bool bogus = false)
+		HtmlToken EmitCommentToken (CharBuffer comment, bool bogus = false)
 		{
 			return EmitCommentToken (comment.ToString (), bogus);
 		}