Bug 1261841 part 2 - Use encoding_rs instead of uconv. r=emk,mystor.

MozReview-Commit-ID: 15Y5GTX98bv

Author: hsivonen

browser/components/migration/360seProfileMigrator.js

@@ -234,7 +234,7 @@ Object.defineProperty(Qihoo360seProfileMigrator.prototype, "sourceProfiles", {
         throw new Error("360 Secure Browser's 'login.ini' file could not be read.");
       }
 
-      let loginIniInUtf8 = copyToTempUTF8File(loginIni, "gbk");
+      let loginIniInUtf8 = copyToTempUTF8File(loginIni, "GBK");
       let loginIniObj = parseINIStrings(loginIniInUtf8);
       try {
         loginIniInUtf8.remove(false);

browser/components/preferences/fonts.xul

@@ -25,15 +25,15 @@
   <prefpane id="FontsDialogPane"
             class="largeDialogContainer"
             helpTopic="prefs-fonts-and-colors">
-  
+
     <preferences id="fontPreferences">
       <preference id="font.language.group"  name="font.language.group"  type="wstring"/>
       <preference id="browser.display.use_document_fonts"
                   name="browser.display.use_document_fonts"
                   type="int"/>
       <preference id="intl.charset.fallback.override" name="intl.charset.fallback.override" type="string"/>
     </preferences>
-    
+
     <stringbundle id="bundlePreferences" src="chrome://browser/locale/preferences/preferences.properties"/>
     <script type="application/javascript" src="chrome://mozapps/content/preferences/fontbuilder.js"/>
     <script type="application/javascript" src="chrome://browser/content/preferences/fonts.js"/>
@@ -79,15 +79,15 @@
           </menupopup>
         </menulist>
       </caption>
-      
+
       <grid>
         <columns>
           <column/>
           <column flex="1"/>
           <column/>
           <column/>
         </columns>
-        
+
         <rows>
           <row>
             <separator class="thin"/>
@@ -107,7 +107,7 @@
               <label value="&size.label;"
                      accesskey="&sizeProportional.accesskey;"
                      control="sizeVar"/>
-            </hbox>  
+            </hbox>
             <menulist id="sizeVar" delayprefsave="true">
               <menupopup>
                 <menuitem value="9" label="9"/>
@@ -164,7 +164,7 @@
               <label value="&size.label;"
                      accesskey="&sizeMonospace.accesskey;"
                      control="sizeMono"/>
-            </hbox>  
+            </hbox>
             <menulist id="sizeMono" delayprefsave="true">
               <menupopup>
                 <menuitem value="9" label="9"/>
@@ -237,7 +237,7 @@
       <separator/>
       <separator class="groove"/>
       <hbox>
-        <checkbox id="useDocumentFonts" 
+        <checkbox id="useDocumentFonts"
                   label="&allowPagesToUseOwn.label;" accesskey="&allowPagesToUseOwn.accesskey;"
                   preference="browser.display.use_document_fonts"
                   onsyncfrompreference="return gFontsDialog.readUseDocumentFonts();"
@@ -260,6 +260,8 @@
             <menuitem label="&languages.customize.Fallback.baltic;"      value="windows-1257"/>
             <menuitem label="&languages.customize.Fallback.ceiso;"       value="ISO-8859-2"/>
             <menuitem label="&languages.customize.Fallback.cewindows;"   value="windows-1250"/>
+            <!-- Using gbk instead of GBK for compat with previously-stored prefs.
+                 The value gets normalized in dom/encoding/FallbackEncoding.cpp. -->
             <menuitem label="&languages.customize.Fallback.simplified;"  value="gbk"/>
             <menuitem label="&languages.customize.Fallback.traditional;" value="Big5"/>
             <menuitem label="&languages.customize.Fallback.cyrillic;"    value="windows-1251"/>

docshell/base/nsDocShell.cpp

@@ -11583,18 +11583,16 @@ nsDocShell::ScrollToAnchor(bool aCurHasRef, bool aNewHasRef,
       NS_ENSURE_TRUE(mContentViewer, NS_ERROR_FAILURE);
       nsIDocument* doc = mContentViewer->GetDocument();
       NS_ENSURE_TRUE(doc, NS_ERROR_FAILURE);
-      const nsACString& aCharset = doc->GetDocumentCharacterSet();
+      const nsACString& charset = doc->GetDocumentCharacterSet();
 
       nsCOMPtr<nsITextToSubURI> textToSubURI =
         do_GetService(NS_ITEXTTOSUBURI_CONTRACTID, &rv);
       NS_ENSURE_SUCCESS(rv, rv);
 
       // Unescape and convert to unicode
-      nsXPIDLString uStr;
+      nsAutoString uStr;
 
-      rv = textToSubURI->UnEscapeAndConvert(PromiseFlatCString(aCharset).get(),
-                                            PromiseFlatCString(aNewHash).get(),
-                                            getter_Copies(uStr));
+      rv = textToSubURI->UnEscapeAndConvert(charset, aNewHash, uStr);
       NS_ENSURE_SUCCESS(rv, rv);
 
       // Ignore return value of GoToAnchor, since it will return an error

dom/base/BodyUtil.cpp

@@ -7,7 +7,7 @@
 #include "nsError.h"
 #include "nsString.h"
 #include "nsIGlobalObject.h"
-#include "nsIUnicodeDecoder.h"
+#include "mozilla/Encoding.h"
 
 #include "nsCharSeparatedTokenizer.h"
 #include "nsDOMString.h"
@@ -17,7 +17,6 @@
 #include "nsStringStream.h"
 
 #include "mozilla/ErrorResult.h"
-#include "mozilla/dom/EncodingUtils.h"
 #include "mozilla/dom/Exceptions.h"
 #include "mozilla/dom/FetchUtil.h"
 #include "mozilla/dom/File.h"
@@ -31,53 +30,6 @@ namespace dom {
 
 namespace {
 
-class StreamDecoder final
-{
-  nsCOMPtr<nsIUnicodeDecoder> mDecoder;
-  nsString mDecoded;
-
-public:
-  StreamDecoder()
-    : mDecoder(EncodingUtils::DecoderForEncoding("UTF-8"))
-  {
-    MOZ_ASSERT(mDecoder);
-  }
-
-  nsresult
-  AppendText(const char* aSrcBuffer, uint32_t aSrcBufferLen)
-  {
-    int32_t destBufferLen;
-    nsresult rv =
-      mDecoder->GetMaxLength(aSrcBuffer, aSrcBufferLen, &destBufferLen);
-    if (NS_WARN_IF(NS_FAILED(rv))) {
-      return rv;
-    }
-
-    if (!mDecoded.SetCapacity(mDecoded.Length() + destBufferLen, fallible)) {
-      return NS_ERROR_OUT_OF_MEMORY;
-    }
-
-    char16_t* destBuffer = mDecoded.BeginWriting() + mDecoded.Length();
-    int32_t totalChars = mDecoded.Length();
-
-    int32_t srcLen = (int32_t) aSrcBufferLen;
-    int32_t outLen = destBufferLen;
-    rv = mDecoder->Convert(aSrcBuffer, &srcLen, destBuffer, &outLen);
-    MOZ_ASSERT(NS_SUCCEEDED(rv));
-
-    totalChars += outLen;
-    mDecoded.SetLength(totalChars);
-
-    return NS_OK;
-  }
-
-  nsString&
-  GetText()
-  {
-    return mDecoded;
-  }
-};
-
 // Reads over a CRLF and positions start after it.
 static bool
 PushOverLine(nsACString::const_iterator& aStart,
@@ -535,13 +487,11 @@ nsresult
 BodyUtil::ConsumeText(uint32_t aInputLength, uint8_t* aInput,
                        nsString& aText)
 {
-  StreamDecoder decoder;
-  nsresult rv = decoder.AppendText(reinterpret_cast<char*>(aInput),
-                                   aInputLength);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
+  nsresult rv =
+    UTF_8_ENCODING->DecodeWithBOMRemoval(MakeSpan(aInput, aInputLength), aText);
+  if (NS_FAILED(rv)) {
     return rv;
   }
-  aText = decoder.GetText();
   return NS_OK;
 }
 

dom/base/EventSource.cpp

@@ -37,22 +37,20 @@
 #include "nsIThreadRetargetableRequest.h"
 #include "nsIAsyncVerifyRedirectCallback.h"
 #include "nsIScriptError.h"
-#include "mozilla/dom/EncodingUtils.h"
 #include "nsIContentSecurityPolicy.h"
 #include "nsContentUtils.h"
 #include "mozilla/Preferences.h"
 #include "xpcpublic.h"
 #include "nsWrapperCacheInlines.h"
 #include "mozilla/Attributes.h"
 #include "nsError.h"
+#include "mozilla/Encoding.h"
 
 namespace mozilla {
 namespace dom {
 
 using namespace workers;
 
-#define REPLACEMENT_CHAR     (char16_t)0xFFFD
-#define BOM_CHAR             (char16_t)0xFEFF
 #define SPACE_CHAR           (char16_t)0x0020
 #define CR_CHAR              (char16_t)0x000D
 #define LF_CHAR              (char16_t)0x000A
@@ -213,13 +211,7 @@ class EventSourceImpl final : public nsIObserver
    *
    * PARSE_STATE_OFF              -> PARSE_STATE_BEGIN_OF_STREAM
    *
-   * PARSE_STATE_BEGIN_OF_STREAM  -> PARSE_STATE_BOM_WAS_READ |
-   *                                 PARSE_STATE_CR_CHAR |
-   *                                 PARSE_STATE_BEGIN_OF_LINE |
-   *                                 PARSE_STATE_COMMENT |
-   *                                 PARSE_STATE_FIELD_NAME
-   *
-   * PARSE_STATE_BOM_WAS_READ     -> PARSE_STATE_CR_CHAR |
+   * PARSE_STATE_BEGIN_OF_STREAM     -> PARSE_STATE_CR_CHAR |
    *                                 PARSE_STATE_BEGIN_OF_LINE |
    *                                 PARSE_STATE_COMMENT |
    *                                 PARSE_STATE_FIELD_NAME
@@ -255,7 +247,6 @@ class EventSourceImpl final : public nsIObserver
   enum ParserStatus {
     PARSE_STATE_OFF = 0,
     PARSE_STATE_BEGIN_OF_STREAM,
-    PARSE_STATE_BOM_WAS_READ,
     PARSE_STATE_CR_CHAR,
     PARSE_STATE_COMMENT,
     PARSE_STATE_FIELD_NAME,
@@ -285,8 +276,7 @@ class EventSourceImpl final : public nsIObserver
   UniquePtr<Message> mCurrentMessage;
   nsDeque mMessagesToDispatch;
   ParserStatus mStatus;
-  nsCOMPtr<nsIUnicodeDecoder> mUnicodeDecoder;
-  nsresult mLastConvertionResult;
+  mozilla::UniquePtr<mozilla::Decoder> mUnicodeDecoder;
   nsString mLastFieldName;
   nsString mLastFieldValue;
 
@@ -351,7 +341,6 @@ EventSourceImpl::EventSourceImpl(EventSource* aEventSource)
   : mEventSource(aEventSource)
   , mReconnectionTime(0)
   , mStatus(PARSE_STATE_OFF)
-  , mLastConvertionResult(NS_OK)
   , mMutex("EventSourceImpl::mMutex")
   , mFrozen(false)
   , mGoingToDispatchAllMessages(false)
@@ -598,7 +587,7 @@ EventSourceImpl::Init(nsIPrincipal* aPrincipal,
     Preferences::GetInt("dom.server-events.default-reconnection-time",
                         DEFAULT_RECONNECTION_TIME_VALUE);
 
-  mUnicodeDecoder = EncodingUtils::DecoderForEncoding("UTF-8");
+  mUnicodeDecoder = UTF_8_ENCODING->NewDecoderWithBOMRemoval();
 
   // the constructor should throw a SYNTAX_ERROR only if it fails resolving the
   // url parameter, so we don't care about the InitChannelAndRequestEventSource
@@ -730,27 +719,27 @@ EventSourceImpl::ParseSegment(const char* aBuffer, uint32_t aLength)
   if (IsClosed()) {
     return;
   }
-  int32_t srcCount, outCount;
-  char16_t out[2];
-  const char* p = aBuffer;
-  const char* end = aBuffer + aLength;
-
-  do {
-    srcCount = aLength - (p - aBuffer);
-    outCount = 2;
-
-    mLastConvertionResult =
-      mUnicodeDecoder->Convert(p, &srcCount, out, &outCount);
-    MOZ_ASSERT(mLastConvertionResult != NS_ERROR_ILLEGAL_INPUT);
-
-    for (int32_t i = 0; i < outCount; ++i) {
-      nsresult rv = ParseCharacter(out[i]);
+  char16_t buffer[1024];
+  auto dst = MakeSpan(buffer);
+  auto src = AsBytes(MakeSpan(aBuffer, aLength));
+  // XXX EOF handling is https://bugzilla.mozilla.org/show_bug.cgi?id=1369018
+  for (;;) {
+    uint32_t result;
+    size_t read;
+    size_t written;
+    bool hadErrors;
+    Tie(result, read, written, hadErrors) =
+      mUnicodeDecoder->DecodeToUTF16(src, dst, false);
+    Unused << hadErrors;
+    for (auto c : dst.To(written)) {
+      nsresult rv = ParseCharacter(c);
       NS_ENSURE_SUCCESS_VOID(rv);
     }
-    p = p + srcCount;
-  } while (p < end &&
-           mLastConvertionResult != NS_PARTIAL_MORE_INPUT &&
-           mLastConvertionResult != NS_OK);
+    if (result == kInputEmpty) {
+      return;
+    }
+    src = src.From(read);
+  }
 }
 
 class DataAvailableRunnable final : public Runnable
@@ -1163,10 +1152,9 @@ EventSourceImpl::ResetDecoder()
 {
   AssertIsOnTargetThread();
   if (mUnicodeDecoder) {
-    mUnicodeDecoder->Reset();
+    UTF_8_ENCODING->NewDecoderWithBOMRemovalInto(*mUnicodeDecoder);
   }
   mStatus = PARSE_STATE_OFF;
-  mLastConvertionResult = NS_OK;
   ClearFields();
 }
 
@@ -1682,22 +1670,6 @@ EventSourceImpl::ParseCharacter(char16_t aChr)
       break;
 
     case PARSE_STATE_BEGIN_OF_STREAM:
-      if (aChr == BOM_CHAR) {
-        mStatus = PARSE_STATE_BOM_WAS_READ;  // ignore it
-      } else if (aChr == CR_CHAR) {
-        mStatus = PARSE_STATE_CR_CHAR;
-      } else if (aChr == LF_CHAR) {
-        mStatus = PARSE_STATE_BEGIN_OF_LINE;
-      } else if (aChr == COLON_CHAR) {
-        mStatus = PARSE_STATE_COMMENT;
-      } else {
-        mLastFieldName += aChr;
-        mStatus = PARSE_STATE_FIELD_NAME;
-      }
-
-      break;
-
-    case PARSE_STATE_BOM_WAS_READ:
       if (aChr == CR_CHAR) {
         mStatus = PARSE_STATE_CR_CHAR;
       } else if (aChr == LF_CHAR) {

dom/base/EventSource.h

@@ -24,7 +24,6 @@
 #include "nsIHttpChannel.h"
 #include "nsWeakReference.h"
 #include "nsDeque.h"
-#include "nsIUnicodeDecoder.h"
 
 class nsPIDOMWindowInner;
 

dom/base/FormData.cpp

@@ -10,14 +10,15 @@
 #include "mozilla/dom/File.h"
 #include "mozilla/dom/Directory.h"
 #include "mozilla/dom/HTMLFormElement.h"
+#include "mozilla/Encoding.h"
 
 #include "MultipartBlobImpl.h"
 
 using namespace mozilla;
 using namespace mozilla::dom;
 
 FormData::FormData(nsISupports* aOwner)
-  : HTMLFormSubmission(NS_LITERAL_CSTRING("UTF-8"), nullptr)
+  : HTMLFormSubmission(WrapNotNull(UTF_8_ENCODING), nullptr)
   , mOwner(aOwner)
 {
 }
@@ -401,7 +402,7 @@ NS_IMETHODIMP
 FormData::GetSendInfo(nsIInputStream** aBody, uint64_t* aContentLength,
                       nsACString& aContentTypeWithCharset, nsACString& aCharset)
 {
-  FSMultipartFormData fs(NS_LITERAL_CSTRING("UTF-8"), nullptr);
+  FSMultipartFormData fs(WrapNotNull(UTF_8_ENCODING), nullptr);
 
   for (uint32_t i = 0; i < mFormData.Length(); ++i) {
     if (mFormData[i].wasNullBlob) {

dom/base/WebSocket.cpp

@@ -33,7 +33,6 @@
 #include "nsError.h"
 #include "nsIScriptObjectPrincipal.h"
 #include "nsIURL.h"
-#include "nsIUnicodeEncoder.h"
 #include "nsThreadUtils.h"
 #include "nsIPromptFactory.h"
 #include "nsIWindowWatcher.h"