Skip to content
Browse files

Merge pull request #32 from asutherland/encoding-win-1252-glitch

Properly decode non-utf-8 messages, r=squib
  • Loading branch information...
2 parents 13549f5 + 546945b commit 3167b03e18595011bffb6870585ca2270677e787 @asutherland asutherland committed Sep 10, 2012
View
3 .gitmodules
@@ -25,3 +25,6 @@
[submodule "deps/bleach.js"]
path = deps/bleach.js
url = git://github.com/asutherland/bleach.js.git
+[submodule "node-deps/addressparser"]
+ path = node-deps/addressparser
+ url = git://github.com/andris9/addressparser.git
View
2 Makefile
@@ -7,7 +7,7 @@ VOLO=./scripts/volo
# Volo does its transformations in-place, so we need to copy junk across,
# transform it, then copy it to the destination dir.
-NODE_PKGS := mailparser mailcomposer mimelib simplesmtp browserify-builtins
+NODE_PKGS := addressparser mailparser mailcomposer mimelib simplesmtp browserify-builtins
SED_TRANSFORMS_mailcomposer = s/mimelib-noiconv/mimelib/g
View
2 data/lib/imap.js
@@ -227,6 +227,8 @@ ImapConnection.prototype.connect = function(loginCb) {
}
catch (ex) {
console.error('Explosion while processing data', ex);
+ if ('stack' in ex)
+ console.error('Stack:', ex.stack);
throw ex;
}
};
View
57 data/lib/js-shims/faux-encoding.js
@@ -0,0 +1,57 @@
+/**
+ * mimelib now uses an 'encoding' module to wrap its use of iconv versus
+ * iconv-lite. This is a good thing from our perspective because it allows
+ * the API to be more sane.
+ **/
+
+define(function(require, exports, module) {
+
+// from https://github.com/andris9/encoding/blob/master/index.js
+// (MIT licensed)
+/**
+ * Converts charset name if needed
+ *
+ * @param {String} name Character set
+ * @return {String} Character set name
+ */
+function checkEncoding(name){
+ name = (name || "").toString().trim().
+ replace(/^latin[\-_]?(\d+)$/i, "ISO-8859-$1").
+ replace(/^win(?:dows)?[\-_]?(\d+)$/i, "WINDOWS-$1").
+ replace(/^utf[\-_]?(\d+)$/i, "UTF-$1").
+ replace(/^ks_c_5601\-1987$/i, "CP949").
+ replace(/^us[\-_]?ascii$/i, "ASCII").
+ toUpperCase();
+ return name;
+}
+
+var ENCODER_OPTIONS = { fatal: false };
+
+exports.convert = function(str, destEnc, sourceEnc, ignoredUseLite) {
+ destEnc = checkEncoding(destEnc || 'UTF-8');
+ sourceEnc = checkEncoding(sourceEnc || 'UTF-8');
+
+ if (destEnc === sourceEnc)
+ return new Buffer(str, 'UTF-8');
+
+ // - decoding (Uint8Array => String)
+ else if (/^UTF-8/.test(destEnc)) {
+ var decoder = new TextDecoder(sourceEnc, ENCODER_OPTIONS);
+ if (typeof(str) === 'string')
+ str = new Buffer(str, 'binary');
+ // XXX strictly speaking, we should be returning a buffer...
+ return decoder.decode(str);
+ }
+ // - encoding (String => Uint8Array)
+ else {
+ var idxSlash = destEnc.indexOf('/');
+ // ignore '//TRANSLIT//IGNORE' and the like.
+ if (idxSlash !== -1 && destEnc[idxSlash+1] === '/')
+ destEnc = destEnc.substring(0, idxSlash);
+
+ var encoder = new TextEncoder(destEnc, ENCODER_OPTIONS);
+ return encoder.encode(str);
+ }
+};
+
+});
View
4 data/lib/js-shims/faux-iconv.js
@@ -18,7 +18,7 @@ exports.Iconv = function Iconv(sourceEnc, destEnc) {
// - decoding
if (/^UTF-8/.test(destEnc)) {
this.decode = true;
- this.coder = new TextDecoder('utf-8', ENCODER_OPTIONS);
+ this.coder = new TextDecoder(sourceEnc, ENCODER_OPTIONS);
}
// - encoding
else {
@@ -37,7 +37,7 @@ exports.Iconv.prototype = {
*/
convert: function(inbuf) {
if (this.decode) {
- return Buffer(this.coder.decode(inbuf));
+ return this.coder.decode(inbuf);
}
else {
return Buffer(this.coder.encode(inbuf));
View
3 data/lib/mailapi/imap/imapchew.js
@@ -360,7 +360,8 @@ exports.chewBodyParts = function chewBodyParts(rep, bodyPartContents,
for (var i = 0; i < addrs.length; i++) {
var addrPair = addrs[i];
sizeEst += OBJ_OVERHEAD_EST + 2 * STR_ATTR_OVERHEAD_EST +
- addrPair.name.length + addrPair.address.length;
+ (addrPair.name ? addrPair.name.length : 0) +
+ (addrPair.address ? addrPair.address.length : 0);
}
return addrs;
}
1 node-deps/addressparser
@@ -0,0 +1 @@
+Subproject commit 2bf021f60ef0d8074be23666b31fffeaa9167252
2 node-deps/mailcomposer
@@ -1 +1 @@
-Subproject commit b1e1c53bcb9c530ff341bbd52f2114fde9da67b6
+Subproject commit 4e70e7cb2fd00e5b1aa6a20f2251198a99c808fb
2 node-deps/mimelib
@@ -1 +1 @@
-Subproject commit 864f72ce67c80cc0f03ff8c5f1396f21f6c49008
+Subproject commit de5f8f7df64501e9ee2fe884013b93b85bf34524
View
2 scripts/gaia-email-opt.build.js
@@ -36,6 +36,7 @@
"iconv": "data/lib/js-shims/faux-iconv",
"iconv-lite": "data/libs/js-shims/faux-iconx",
+ "encoding": "data/lib/js-shims/faux-encoding",
"assert": "data/deps/browserify-builtins/assert",
"events": "data/deps/browserify-builtins/events",
@@ -44,6 +45,7 @@
// These used to be packages but we have AMD shims for their mains where
// appropriate, so we can just use paths.
+ "addressparser": "data/deps/addressparser",
"mimelib": "data/deps/mimelib",
"mailparser": "data/deps/mailparser/lib",
"simplesmtp": "data/deps/simplesmtp",
View
4 test/unit/head.js
@@ -211,6 +211,7 @@ require({
"timers": "data/lib/node-timers",
"iconv": "data/lib/js-shims/faux-iconv",
+ "encoding": "data/lib/js-shims/faux-encoding",
"assert": "data/deps/browserify-builtins/assert",
"events": "data/deps/browserify-builtins/events",
@@ -219,6 +220,7 @@ require({
// These used to be packages but we have AMD shims for their mains where
// appropriate, so we can just use paths.
+ "addressparser": "data/deps/addressparser",
"mimelib": "data/deps/mimelib",
"mailparser": "data/deps/mailparser/lib",
"simplesmtp": "data/deps/simplesmtp",
@@ -241,7 +243,7 @@ var process = window.process = {
};
// now that RequireJS has bootstrapped, we can setup navigator.
-navigator = window.navigator;
+navigator = _window_mixin.navigator;
// -- Pull relevant test environment variables out of the environment.
// The goal is to allow our unit tests to be run against varying server
View
2 test/unit/resources/b2g_shims.js
@@ -1,3 +1,3 @@
const nsIDOMTCPSocket = CC("@mozilla.org/tcp-socket;1",
"nsIDOMTCPSocket");
-window.navigator.mozTCPSocket = new nsIDOMTCPSocket();
+_window_mixin.navigator.mozTCPSocket = new nsIDOMTCPSocket();
View
9 test/unit/resources/loggest_test_framework.js
@@ -23,9 +23,16 @@ var MailAPI = null, MailBridge = null, MailUniverse = null;
var gAllAccountsSlice = null, gAllFoldersSlice = null;
var gDumpedLogs = false, gRunner;
+function dumpAsUtf8WithNewline(s) {
+ var converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]
+ .createInstance(Ci.nsIScriptableUnicodeConverter);
+ converter.charset = "UTF-8";
+ var u8s = converter.ConvertFromUnicode(s);
+ print(u8s);
+}
function dumpLogs() {
if (!gDumpedLogs) {
- gRunner.dumpLogResultsToConsole(print);
+ gRunner.dumpLogResultsToConsole(dumpAsUtf8WithNewline);
gDumpedLogs = true;
}
}
View
6 test/unit/resources/messageGenerator.js
@@ -383,6 +383,8 @@ SyntheticMessage.prototype = Object_extend(SyntheticPart.prototype, {
* e-mail address (sans wrapping greater-than/less-than).
*/
_formatMailFromNameAndAddress: function(aNameAndAddress) {
+ if (!aNameAndAddress[0])
+ return aNameAndAddress[1];
// if the name is encoded, do not put it in quotes!
return (aNameAndAddress[0][0] == "=" ?
(aNameAndAddress[0] + " ") :
@@ -470,6 +472,7 @@ SyntheticMessage.prototype = Object_extend(SyntheticPart.prototype, {
let lines = [headerKey + ": " + this._formatHeaderValues(headerValues)
for each ([headerKey, headerValues] in Iterator(this.headers))];
+
var msgString = this.bodyPart.toMessageString();
return lines.join("\r\n") + "\r\n\r\n" + msgString +
@@ -643,6 +646,9 @@ SyntheticMessage.prototype = Object_extend(SyntheticPart.prototype, {
set: function(aBodyPart) {
this._bodyPart = aBodyPart;
this.headers["Content-Type"] = this._bodyPart.contentTypeHeaderValue;
+ if (aBodyPart.hasTransferEncoding)
+ this.headers["Content-Transfer-Encoding"] =
+ aBodyPart.contentTransferEncodingHeaderValue;
},
},
});
View
36 test/unit/resources/window_shims.js
@@ -32,16 +32,6 @@ function clearTimeout(handle) {
var moduleGlobalsHack = {};
Components.utils.import("resource://test/resources/globalshack.jsm",
moduleGlobalsHack);
-var document = {
- implementation: {
- createHTMLDocument: function createHTMLDocument(str) {
- var parser = Cc["@mozilla.org/xmlextras/domparser;1"]
- .createInstance(Ci.nsIDOMParser);
- parser.init();
- return parser.parseFromString(str, 'text/html');
- }
- }
-};
/**
* A function that can be clobbered to generate events when blob-related
@@ -50,7 +40,7 @@ var document = {
var __blobLogFunc = function() {
};
-var window = {
+var _window_mixin = {
// - indexed db
indexedDB: indexedDB,
DOMException: DOMException,
@@ -129,12 +119,28 @@ var window = {
throw new Error("atob of '" + data + "' failed.");
}
},
- document: document
+ document: {
+ implementation: {
+ createHTMLDocument: function createHTMLDocument(str) {
+ var parser = Cc["@mozilla.org/xmlextras/domparser;1"]
+ .createInstance(Ci.nsIDOMParser);
+ parser.init();
+ return parser.parseFromString(str, 'text/html');
+ }
+ }
+ }
};
+
+// mix all the window stuff into the global scope for things like the string
+// encoding polyfill that really want 'this' and window to be the same.
+var window = this, self = this;
+(function(win) {
+ for (var key in _window_mixin) {
+ win[key] = _window_mixin[key];
+ }
+}(this));
// during the RequireJS bootstrap, have navigator be undefined.
-var navigator = undefined;
-// new to me, but apparently it's a thing...
-var self = window.self = window;
+navigator = undefined;
function Blob(parts, properties) {
this.parts = parts;
View
65 test/unit/test_imap_mime.js
@@ -90,8 +90,9 @@ TD.commonCase('message encodings', function(T) {
*/
TD.commonCase('MIME hierarchies', function(T) {
// -- pieces
+ var
// - bodies: text/plain
- var bpartEmptyText =
+ bpartEmptyText =
new SyntheticPartLeaf(''),
bpartStraightASCII =
new SyntheticPartLeaf('I am text! Woo!'),
@@ -106,6 +107,7 @@ TD.commonCase('MIME hierarchies', function(T) {
new SyntheticPartLeaf(
utf8UnicodeName,
{ charset: 'utf-8', format: null, encoding: '8bit' }),
+ // quoted-printable encoding utf-8
bpartQpUtf8Name =
new SyntheticPartLeaf(
qpUtf8UnicodeName,
@@ -115,6 +117,20 @@ TD.commonCase('MIME hierarchies', function(T) {
new SyntheticPartLeaf(
mwqSammySnake,
{ charset: 'utf-8', format: null, encoding: null }),
+ bstrQpWin1252 =
+ 'Ellipsis: "=85", apostrophe "=92", accented i "=ED"',
+ rawQpWin1252 =
+ 'Ellipsis: "\u2026", apostrophe "\u2019", accented i "\u00ed"',
+ bpartQpWin1252 =
+ new SyntheticPartLeaf(
+ bstrQpWin1252,
+ { charset: 'windows-1252', format: null,
+ encoding: 'quoted-printable' }),
+ bpartQpWin1252ShortenedCharset =
+ new SyntheticPartLeaf(
+ bstrQpWin1252,
+ { charset: 'win-1252', format: null,
+ encoding: 'quoted-printable' }),
// - bodies: text/enriched (ignored!)
// This exists just to test the alternatives logic.
bpartIgnoredEnriched =
@@ -201,6 +217,34 @@ TD.commonCase('MIME hierarchies', function(T) {
'This is a very long message that wants to be snippeted to a ' +
'reasonable length that is reasonable and',
},
+ {
+ name: 'text/plain utf8',
+ bodyPart: bpartUtf8Name,
+ checkBody: rawUnicodeName,
+ },
+ {
+ name: 'text/plain qp utf8',
+ bodyPart: bpartQpUtf8Name,
+ checkBody: rawUnicodeName,
+ },
+ {
+ name: 'text/plain qp windows-1252',
+ bodyPart: bpartQpWin1252,
+ checkBody: rawQpWin1252,
+ },
+ {
+ name: 'text/plain qp win-1252 (incorrectly shortened from windows-1252)',
+ bodyPart: bpartQpWin1252ShortenedCharset,
+ checkBody: rawQpWin1252,
+ },
+ // - text/plain checking things not related to bodies...
+ {
+ name: 'text/plain with sender without display name',
+ bodyPart: bpartEmptyText,
+ from: [null, 'nodisplayname@example.com'],
+ to: [[null, 'nodisplayname2@example.com']],
+ checkBody: '',
+ },
// - straight up verification we don't do mime-word decoding on bodies
{
name: 'simple text/plain with mimeword in the body',
@@ -263,6 +307,25 @@ TD.commonCase('MIME hierarchies', function(T) {
{ universe: testUniverse, restored: true }),
eCheck = T.lazyLogger('messageCheck');
+
+ var DISABLE_THRESH_USING_FUTURE = -60 * 60 * 1000;
+ testUniverse.do_adjustSyncValues({
+ // only fillSize and days are adjusted; we want to synchronize everything
+ // in one go.
+ fillSize: testMessages.length,
+ days: testMessages.length,
+ // the rest are defaults here...
+ scaleFactor: 1.6,
+ bisectThresh: 2000,
+ tooMany: 2000,
+ refreshNonInbox: DISABLE_THRESH_USING_FUTURE,
+ refreshInbox: DISABLE_THRESH_USING_FUTURE,
+ oldIsSafeForRefresh: DISABLE_THRESH_USING_FUTURE,
+ refreshOld: DISABLE_THRESH_USING_FUTURE,
+ useRangeNonInbox: DISABLE_THRESH_USING_FUTURE,
+ useRangeInbox: DISABLE_THRESH_USING_FUTURE
+ });
+
// -- create the folder, append the messages
var fullSyncFolder = testAccount.do_createTestFolder(
'test_mime_hier', function makeMessages() {

0 comments on commit 3167b03

Please sign in to comment.
Something went wrong with that request. Please try again.