Skip to content
Browse files

Bug 808716 - Lazily Fetch message bodies. r=asuth,squib,jrburke

  • Loading branch information...
1 parent 934a61c commit 7448913db30f766e59c6f18379f039d74d591346 @lightsofapollo lightsofapollo committed Mar 17, 2013
View
1 data/lib/imap.js
@@ -403,6 +403,7 @@ ImapConnection.prototype.connect = function(loginCb) {
type = type[1];
curReq._desc = desc;
curReq._msg = msg;
+ msg.size = self._state.curExpected;
curReq._fetcher.emit('message', msg);
View
30 data/lib/mailapi/activesync/folder.js
@@ -618,7 +618,7 @@ ActiveSyncFolderConn.prototype = {
attachments: [],
relatedParts: [],
references: null,
- bodyReps: null,
+ bodyReps: null
};
flagHeader = function(flag, state) {
@@ -685,13 +685,13 @@ ActiveSyncFolderConn.prototype = {
header.author = $mimelib.parseAddresses(childText)[0] || null;
break;
case em.To:
- body.to = $mimelib.parseAddresses(childText);
+ header.to = $mimelib.parseAddresses(childText);
break;
case em.Cc:
- body.cc = $mimelib.parseAddresses(childText);
+ header.cc = $mimelib.parseAddresses(childText);
break;
case em.ReplyTo:
- body.replyTo = $mimelib.parseAddresses(childText);
+ header.replyTo = $mimelib.parseAddresses(childText);
break;
case em.DateReceived:
body.date = header.date = new Date(childText).valueOf();
@@ -796,13 +796,31 @@ ActiveSyncFolderConn.prototype = {
var bodyRep = $quotechew.quoteProcessTextBody(bodyText);
header.snippet = $quotechew.generateSnippet(bodyRep,
DESIRED_SNIPPET_LENGTH);
- body.bodyReps = ['plain', bodyRep];
+ var content = bodyRep[1];
+ var len = content.length;
+
+ body.bodyReps = [{
+ type: 'plain',
+ content: bodyRep,
+ sizeEstimate: len,
+ amountDownloaded: len,
+ isDownloaded: true
+ }];
}
else if (bodyType === asbEnum.Type.HTML) {
var htmlNode = $htmlchew.sanitizeAndNormalizeHtml(bodyText);
header.snippet = $htmlchew.generateSnippet(htmlNode,
DESIRED_SNIPPET_LENGTH);
- body.bodyReps = ['html', htmlNode.innerHTML];
+ var content = htmlNode.innerHTML;
+ var len = content.length;
+
+ body.bodyReps = [{
+ type: 'html',
+ content: content,
+ sizeEstimate: len,
+ amountDownloaded: len,
+ isDownloaded: true
+ }];
}
return { header: header, body: body };
View
4 data/lib/mailapi/fake/account.js
@@ -413,7 +413,9 @@ MessageGenerator.prototype = {
'3: ...\n' +
'\nIt is a tiny screen we target, thank goodness!';
}
- bodyInfo.bodyReps = ['plain', [0x1, bodyText]];
+ bodyInfo.bodyReps = [
+ { type: 'plain', content: [0x1, bodyText] }
+ ];
if (this._mode === 'info') {
return {
View
554 data/lib/mailapi/imap/folder.js
@@ -21,6 +21,15 @@ define(
require,
exports
) {
+
+/**
+ * Lazily evaluated modules
+ */
+var $imaptextparser = null;
+var $imapsnippetparser = null;
+var $imapbodyfetcher = null;
+var $imapsync = null;
+
var allbackMaker = $allback.allbackMaker,
bsearchForInsert = $util.bsearchForInsert,
bsearchMaybeExists = $util.bsearchMaybeExists,
@@ -69,40 +78,14 @@ function compactArray(arr) {
var BASELINE_SEARCH_OPTIONS = ['!DELETED'];
/**
- * Fetch parameters to get the headers / bodystructure; exists to reuse the
- * object since every fetch is the same. Note that imap.js always gives us
- * FLAGS and INTERNALDATE so we don't need to ask for that.
- *
- * We are intentionally not using ENVELOPE because Thunderbird intentionally
- * defaults to to not using ENVELOPE. Per bienvenu in
- * https://bugzilla.mozilla.org/show_bug.cgi?id=402594#c33 "We stopped using it
- * by default because servers often had issues with it so it was more trouble
- * than it was worth."
- *
- * Of course, imap.js doesn't really support ENVELOPE outside of bodystructure
- * right now either, but that's a lesser issue. We probably don't want to trust
- * that data, however, if we don't want to trust normal ENVELOPE.
+ * Number of bytes to fetch from the server for snippets.
*/
-var INITIAL_FETCH_PARAMS = {
- request: {
- headers: ['FROM', 'TO', 'CC', 'BCC', 'SUBJECT', 'REPLY-TO', 'MESSAGE-ID',
- 'REFERENCES'],
- struct: true,
- body: false
- },
-};
+var NUMBER_OF_SNIPPET_BYTES = 256;
/**
- * Fetch parameters to just get the flags, which is no parameters because
- * imap.js always fetches them right now.
+ * Maximum bytes to request from server in a fetch request (max uint32)
*/
-var FLAG_FETCH_PARAMS = {
- request: {
- struct: false,
- headers: false,
- body: false
- },
-};
+var MAX_FETCH_BYTES = (Math.pow(2, 32) - 1);
/**
* Folder connections do the actual synchronization logic. They are associated
@@ -251,6 +234,16 @@ ImapFolderConn.prototype = {
});
},
+ syncDateRange: function() {
+ var args = Array.slice(arguments);
+ var self = this;
+
+ require(['./protocol/sync'], function(_sync) {
+ $imapsync = _sync;
+ (self.syncDateRange = self._lazySyncDateRange).apply(self, args);
+ });
+ },
+
/**
* Perform a search to find all the messages in the given date range.
* Meanwhile, load the set of messages from storage. Infer deletion of the
@@ -282,7 +275,7 @@ ImapFolderConn.prototype = {
* }
* ]
*/
- syncDateRange: function(startTS, endTS, accuracyStamp,
+ _lazySyncDateRange: function(startTS, endTS, accuracyStamp,
doneCallback, progressCallback) {
if (startTS && endTS && SINCE(startTS, endTS)) {
this._LOG.illegalSync(startTS, endTS);
@@ -364,20 +357,31 @@ console.log('BISECT CASE', serverUIDs.length, 'curDaysDelta', curDaysDelta);
if (numDeleted)
compactArray(headers);
- return self._commonSync(
- newUIDs, knownUIDs, headers,
- function(newCount, knownCount) {
+ var uidSync = new $imapsync.Sync({
+ connection: self._conn,
+ storage: self._storage,
+ newUIDs: newUIDs,
+ knownUIDs: knownUIDs,
+ knownHeaders: headers
+ });
+
+ uidSync.onprogress = progressCallback;
+
+ uidSync.oncomplete = function(newCount, knownCount) {
self._LOG.syncDateRange_end(newCount, knownCount, numDeleted,
startTS, endTS, null, null);
+
self._storage.markSyncRange(startTS, endTS, modseq,
accuracyStamp);
if (completed)
return;
+
completed = true;
doneCallback(null, null, newCount + knownCount,
skewedStartTS, skewedEndTS);
- },
- progressCallback);
+ };
+
+ return;
});
// - Adjust DB time range for server skew on INTERNALDATE
@@ -423,281 +427,233 @@ console.log('BISECT CASE', serverUIDs.length, 'curDaysDelta', curDaysDelta);
searchOptions.push(['BEFORE', endTS]);
},
+ downloadBodyReps: function() {
+ var args = Array.slice(arguments);
+ var self = this;
+
+ require(
+ ['./imapchew', './protocol/bodyfetcher', './protocol/textparser'],
+ function(
+ _imapchew,
+ _bodyfetcher,
+ _textparser
+ ) {
+
+ $imapchew =_imapchew;
+ $imapbodyfetcher = _bodyfetcher;
+ $imaptextparser = _textparser;
+
+ (self.downloadBodyReps = self._lazyDownloadBodyReps).apply(self, args);
+ });
+ },
+
/**
- * Given a list of new-to-us UIDs and known-to-us UIDs and their corresponding
- * headers, synchronize the flags for the known UIDs' headers and fetch and
- * create the header and body objects for the new UIDS.
- *
- * First we fetch the headers/bodystructures for the new UIDs all in one go;
- * all of these headers are going to end up in-memory at the same time, so
- * batching won't let us reduce the overhead right now. We process them
- * to determine the body parts we should fetch as the results come in. Once
- * we have them all, we sort them by date, endTS-to-startTS for the third
- * step and start issuing/pipelining the requests.
- *
- * Second, we issue the flag update requests for the known-to-us UIDs. This
- * is done second so it can help avoid wasting the latency of the round-trip
- * that would otherwise result between steps one and three. (Although we
- * could also mitigate that by issuing some step three requests even as
- * the step one requests are coming in; our sorting doesn't have to be
- * perfect and may already be reasonably well ordered if UIDs correlate
- * with internal date well.)
- *
- * Third, we fetch the body parts in our newest-to-startTS order, adding
- * finalized headers and bodies as we go.
+ * Initiates a request to download all body reps. If a snippet has not yet
+ * been generated this will also generate the snippet...
*/
- _commonSync: function(newUIDs, knownUIDs, knownHeaders, doneCallback,
- progressCallback) {
- require(['./imapchew'], function ($imapchew) {
- var conn = this._conn, storage = this._storage, self = this;
-console.log("_commonSync", 'newUIDs', newUIDs.length, 'knownUIDs',
- knownUIDs.length, 'knownHeaders', knownHeaders.length);
- // See the `ImapFolderConn` block comment for rationale.
- var KNOWN_HEADERS_AGGR_COST = 20,
- KNOWN_HEADERS_PER_COST = 1,
- NEW_HEADERS_AGGR_COST = 20,
- NEW_HEADERS_PER_COST = 5,
- NEW_BODIES_PER_COST = 30;
- var progressCost =
- (knownUIDs.length ? KNOWN_HEADERS_AGGR_COST : 0) +
- KNOWN_HEADERS_PER_COST * knownUIDs.length +
- (newUIDs.length ? NEW_HEADERS_AGGR_COST : 0) +
- NEW_HEADERS_PER_COST * newUIDs.length +
- NEW_BODIES_PER_COST * newUIDs.length,
- progressSoFar = 0;
-
- function updateProgress(newProgress) {
- progressSoFar += newProgress;
- if (progressCallback)
- progressCallback(0.25 + 0.75 * (progressSoFar / progressCost));
- }
+ _lazyDownloadBodyReps: function(suid, date, callback) {
+ var bodyInfo;
+ var header;
- var callbacks = allbackMaker(
- ['newMsgs', 'knownMsgs'],
- function() {
- // It is possible that async I/O will be required to add a header or a
- // body, so we need to defer declaring the synchronization done until
- // after all of the storage's deferred calls have run because the
- // header/body affecting calls will have been deferred.
- storage.runAfterDeferredCalls(
- doneCallback.bind(null, newUIDs.length, knownUIDs.length));
+ var requests = [];
+ var self = this;
+
+ var gotHeader = function gotHeader(_header) {
+ // header may have been deleted by the time we get here...
+ if (!_header) {
+ return callback();
+ }
+
+ header = _header;
+ self._storage.getMessageBody(suid, date, gotBody);
+ };
+
+ var gotBody = function gotBody(bodyInfo) {
+ if (!bodyInfo)
+ return callback();
+
+ // target for snippet generation
+ var bodyRepIdx = $imapchew.selectSnippetBodyRep(header, bodyInfo);
+
+ // build the list of requests based on downloading required.
+ bodyInfo.bodyReps.forEach(function(rep, idx) {
+
+ // attempt to be idempotent by only requesting the bytes we need if we
+ // actually need them...
+ if (rep.isDownloaded)
+ return;
+
+ var request = {
+ uid: header.srvid,
+ partInfo: rep._partInfo,
+ bodyRepIndex: idx,
+ createSnippet: idx === bodyRepIdx
+ };
+
+ // we may only need a subset of the total number of bytes.
+ if (rep.amountDownloaded) {
+ // request the remainder
+ request.bytes = [
+ rep.amountDownloaded,
+ Math.min(rep.sizeEstimate * 5, MAX_FETCH_BYTES)
+ ];
+ }
+
+ requests.push(request);
});
- // -- Fetch headers/bodystructures for new UIDs
- var newChewReps = [];
- if (newUIDs.length) {
- var newFetcher = this._conn.fetch(newUIDs, INITIAL_FETCH_PARAMS);
- newFetcher.on('message', function onNewMessage(msg) {
- msg.on('end', function onNewMsgEnd() {
-console.log(' new fetched, header processing, INTERNALDATE: ', msg.rawDate);
- newChewReps.push($imapchew.chewHeaderAndBodyStructure(msg));
-console.log(' header processed');
- });
- });
- newFetcher.on('error', function onNewFetchError(err) {
- // XXX the UID might have disappeared already? we might need to have
- // our initiating command re-do whatever it's up to. Alternatively,
- // we could drop back from a bulk fetch to a one-by-one fetch.
- console.warn('New UIDs fetch error, ideally harmless:', err);
- });
- newFetcher.on('end', function onNewFetchEnd() {
- require(['mailparser/mailparser'], function($mailparser) {
- // the fetch results will be bursty, so just update all at once
- updateProgress(NEW_HEADERS_AGGR_COST +
- NEW_HEADERS_PER_COST * newUIDs.length);
-
- // sort the messages, endTS to startTS (aka numerically descending)
- newChewReps.sort(function(a, b) {
- return b.msg.date - a.msg.date;
- });
-
- // - issue the bodypart fetches.
- // Use mailparser's body parsing capabilities, albeit not entirely in
- // the way it was intended to be used since it wants to parse full
- // messages.
- var mparser = new $mailparser.MailParser();
- function setupBodyParser(partDef) {
- mparser._state = 0x2; // body
- mparser._remainder = '';
- mparser._currentNode = null;
- mparser._currentNode = mparser._createMimeNode(null);
- // nb: mparser._multipartTree is an empty list (always)
- mparser._currentNode.meta.contentType =
- partDef.type.toLowerCase() + '/' +
- partDef.subtype.toLowerCase();
- mparser._currentNode.meta.charset =
- partDef.params && partDef.params.charset &&
- partDef.params.charset.toLowerCase();
- mparser._currentNode.meta.transferEncoding =
- partDef.encoding && partDef.encoding.toLowerCase();
- mparser._currentNode.meta.textFormat =
- partDef.params && partDef.params.format &&
- partDef.params.format.toLowerCase();
- }
- function bodyParseBuffer(buffer) {
- process.immediate = true;
- mparser.write(buffer);
- process.immediate = false;
- }
- function finishBodyParsing() {
- process.immediate = true;
- mparser._process(true);
- process.immediate = false;
- // We end up having provided an extra newline that we don't
- // want, so let's cut it off if it exists.
- var content = mparser._currentNode.content;
- if (content.charCodeAt(content.length - 1) === 10)
- content = content.substring(0, content.length - 1);
- return content;
- }
+ // we may not have any requests bail early if so.
+ if (!requests.length)
+ callback(); // no requests === success
- // XXX imap.js is currently not capable of issuing/parsing multiple
- // literal results from a single fetch result line. It's not a
- // fundamentally hard problem, but I'd rather defer messing with its
- // parse loop (and internal state tracking) until a future time when
- // I can do some other cleanup at the same time. (The subsequent
- // literals are just on their own lines with an initial space and then
- // the named literal. Ex: " BODY[1.2] {2463}".)
- //
- // So let's issue one fetch per body part and then be happy when we've
- // got them all.
- var pendingFetches = 0;
- newChewReps.forEach(function(chewRep, iChewRep) {
- var partsReceived = [];
- // If there are no parts to process, consume it now.
- if (chewRep.bodyParts.length === 0) {
- if ($imapchew.chewBodyParts(chewRep, partsReceived,
- storage.folderId,
- storage._issueNewHeaderId())) {
- storage.addMessageHeader(chewRep.header);
- storage.addMessageBody(chewRep.header, chewRep.bodyInfo);
- }
- }
+ var fetch = new $imapbodyfetcher.BodyFetcher(
+ self._conn,
+ $imaptextparser.TextParser,
+ requests
+ );
- chewRep.bodyParts.forEach(function(bodyPart) {
- var opts = {
- request: {
- struct: false,
- headers: false,
- body: bodyPart.partID
- }
- };
- pendingFetches++;
-
-console.log(' fetching body for', chewRep.msg.id, bodyPart.partID);
- var fetcher;
-try {
- fetcher = conn.fetch(chewRep.msg.id, opts);
-} catch (ex) {
- console.warn('!failure fetching body', ex);
- return;
-}
- setupBodyParser(bodyPart);
- fetcher.on('error', function(err) {
- console.warn('body fetch error', err);
- if (--pendingFetches === 0)
- callbacks.newMsgs();
- });
- fetcher.on('message', function(msg) {
- setupBodyParser(bodyPart);
- msg.on('data', bodyParseBuffer);
- msg.on('end', function() {
- updateProgress(NEW_BODIES_PER_COST);
- partsReceived.push(finishBodyParsing());
-console.log(' !fetched body part for', chewRep.msg.id, bodyPart.partID,
- partsReceived.length, chewRep.bodyParts.length);
-
- // -- Process
- if (partsReceived.length === chewRep.bodyParts.length) {
- try {
- if ($imapchew.chewBodyParts(
- chewRep, partsReceived, storage.folderId,
- storage._issueNewHeaderId())) {
- storage.addMessageHeader(chewRep.header);
- storage.addMessageBody(chewRep.header,
- chewRep.bodyInfo);
- }
- else {
- self._LOG.bodyChewError(false);
- console.error('Failed to process body!');
- }
- }
- catch (ex) {
- self._LOG.bodyChewError(ex);
- console.error('Failure processing body:', ex, '\n',
- ex.stack);
- }
- }
- // If this is the last chew rep, then use its completion
- // to report our completion.
- if (--pendingFetches === 0)
- callbacks.newMsgs();
- });
- });
- });
- });
- if (pendingFetches === 0)
- callbacks.newMsgs();
- }.bind(this));
+ self._handleBodyFetcher(fetch, header, bodyInfo, function(err) {
+ callback(err, bodyInfo);
});
- }
- else {
- callbacks.newMsgs();
- }
+ };
- // -- Fetch updated flags for known UIDs
- if (knownUIDs.length) {
- var knownFetcher = this._conn.fetch(knownUIDs, FLAG_FETCH_PARAMS);
- var numFetched = 0;
- knownFetcher.on('message', function onKnownMessage(msg) {
- // (Since we aren't requesting headers, we should be able to get
- // away without registering this next event handler and just process
- // msg right now, but let's wait on an optimization pass.)
- msg.on('end', function onKnownMsgEnd() {
- var i = numFetched++;
-console.log('FETCHED', i, 'known id', knownHeaders[i].id,
- 'known srvid', knownHeaders[i].srvid, 'actual id', msg.id);
- // RFC 3501 doesn't require that we get results in the order we
- // request them, so use indexOf if things don't line up. (In fact,
- // dovecot sorts them, so we might just want to sort ours too.)
- if (knownHeaders[i].srvid !== msg.id) {
- i = knownUIDs.indexOf(msg.id);
- // If it's telling us about a message we don't know about, run away.
- if (i === -1) {
- console.warn("Server fetch reports unexpected message:", msg.id);
- return;
- }
- }
- var header = knownHeaders[i];
- // (msg.flags comes sorted and we maintain that invariant)
- if (header.flags.toString() !== msg.flags.toString()) {
-console.warn(' FLAGS: "' + header.flags.toString() + '" VS "' +
- msg.flags.toString() + '"');
- header.flags = msg.flags;
- storage.updateMessageHeader(header.date, header.id, true, header);
- }
- else {
- storage.unchangedMessageHeader(header);
- }
- });
- });
- knownFetcher.on('error', function onKnownFetchError(err) {
- // XXX the UID might have disappeared already? we might need to have
- // our initiating command re-do whatever it's up to. Alternatively,
- // we could drop back from a bulk fetch to a one-by-one fetch.
- console.warn('Known UIDs fetch error, ideally harmless:', err);
+ self._storage.getMessageHeader(suid, date, gotHeader);
+ },
+
+ /**
+ * Download a snippet and a portion of the bodyRep to go along with it... In
+ * all cases we expect the bodyReps to be completely empty as we also will
+ * generate the snippet in the case of completely downloading a snippet.
+ */
+ _downloadSnippet: function(header, callback) {
+ var self = this;
+ this._storage.getMessageBody(header.suid, header.date, function(body) {
+ // attempt to find a rep
+ var bodyRepIndex = $imapchew.selectSnippetBodyRep(header, body);
+
+ // no suitable snippet we are done.
+ if (bodyRepIndex === -1)
+ return callback();
+
+ var rep = body.bodyReps[bodyRepIndex];
+ var requests = [{
+ uid: header.srvid,
+ bodyRepIndex: bodyRepIndex,
+ partInfo: rep._partInfo,
+ bytes: [0, NUMBER_OF_SNIPPET_BYTES],
+ createSnippet: true
+ }];
+
+ var fetch = new $imapbodyfetcher.BodyFetcher(
+ self._conn,
+ $imapsnippetparser.SnippetParser,
+ requests
+ );
+
+ self._handleBodyFetcher(
+ fetch,
+ header,
+ body,
+ callback
+ );
+ });
+ },
+
+ /**
+ * Wrapper around common bodyRep updates...
+ */
+ _handleBodyFetcher: function(fetcher, header, body, callback) {
+ var event = {
+ changeType: 'bodyReps',
+ indexes: []
+ };
+
+ var self = this;
+
+ fetcher.onparsed = function(req, resp) {
+ $imapchew.updateMessageWithFetch(header, body, req, resp);
+
+ if (req.createSnippet) {
+ self._storage.updateMessageHeader(
+ header.date,
+ header.id,
+ false,
+ header
+ );
+ }
+
+ event.indexes.push(req.bodyRepIndex);
+ };
+
+ fetcher.onerror = function(e) {
+ callback(e);
+ };
+
+ fetcher.onend = function() {
+ self._storage.updateMessageBody(
+ header,
+ body,
+ event
+ );
+
+ self._storage.runAfterDeferredCalls(callback);
+ };
+ },
+
+ /**
+ * Download snippets for a set of headers.
+ */
+ _lazyDownloadSnippets: function(headers, callback) {
+ var i = 0;
+ var len = headers.length;
+ var pending = 1;
+
+ var self = this;
+ var anyErr;
+ function next(err) {
+ if (err && !anyErr)
+ anyErr = err;
+
+ if (!--pending) {
+ self._storage.runAfterDeferredCalls(function() {
+ callback(anyErr);
});
- knownFetcher.on('end', function() {
- // the fetch results will be bursty, so just update all at once
- updateProgress(KNOWN_HEADERS_AGGR_COST +
- KNOWN_HEADERS_PER_COST * knownUIDs.length);
- callbacks.knownMsgs();
- });
+ }
}
- else {
- callbacks.knownMsgs();
+
+ for (; i < len; i++) {
+ if (!headers[i] || headers[i].snippet) {
+ continue;
+ }
+
+ pending++;
+ this._downloadSnippet(headers[i], next);
}
- }.bind(this));
+
+ // by having one pending item always this handles the case of not having any
+ // snippets needing a download and also returning in the next tick of the
+ // event loop.
+ window.setZeroTimeout(next);
+ },
+
+ downloadSnippets: function() {
+ var args = Array.slice(arguments);
+ var self = this;
+
+ require(
+ ['./imapchew', './protocol/bodyfetcher', './protocol/snippetparser'],
+ function(
+ _imapchew,
+ _bodyfetcher,
+ _snippetparser
+ ) {
+
+ $imapchew =_imapchew;
+ $imapbodyfetcher = _bodyfetcher;
+ $imapsnippetparser = _snippetparser;
+
+ (self.downloadSnippets = self._lazyDownloadSnippets).apply(self, args);
+ });
},
downloadMessageAttachments: function(uid, partInfos, callback, progress) {
View
254 data/lib/mailapi/imap/imapchew.js
@@ -14,6 +14,7 @@ define(
exports
) {
+
/**
* Process the headers and bodystructure of a message to build preliminary state
* and determine what body parts to fetch. The list of body parts will be used
@@ -46,15 +47,13 @@ define(
* attachments.
*
* @typedef[ChewRep @dict[
- * @key[msg ImapJsMsg]
- * @key[bodyParts @listof[ImapJsPart]]
+ * @key[bodyReps @listof[ImapJsPart]]
* @key[attachments @listof[AttachmentInfo]]
- * @key[header HeaderInfo]
- * @key[bodyInfo BodyInfo]
+ * @key[relatedParts @listof[RelatedPartInfo]]
* ]]
* @return[ChewRep]
*/
-exports.chewHeaderAndBodyStructure = function chewStructure(msg) {
+function chewStructure(msg) {
// imap.js builds a bodystructure tree using lists. All nodes get wrapped
// in a list so they are element zero. Children (which get wrapped in
// their own list) follow.
@@ -66,7 +65,7 @@ exports.chewHeaderAndBodyStructure = function chewStructure(msg) {
// [{alternative} [{text/plain}] [{text/html}]]
// multipart/mixed text w/attachment =>
// [{mixed} [{text/plain}] [{application/pdf}]]
- var attachments = [], bodyParts = [], unnamedPartCounter = 0,
+ var attachments = [], bodyReps = [], unnamedPartCounter = 0,
relatedParts = [];
/**
@@ -158,6 +157,23 @@ exports.chewHeaderAndBodyStructure = function chewStructure(msg) {
};
}
+ function makeTextPart(partInfo) {
+ return {
+ type: partInfo.subtype,
+ part: partInfo.partID,
+ sizeEstimate: partInfo.size,
+ amountDownloaded: 0,
+ // its important to know that sizeEstimate and amountDownloaded
+ // do _not_ determine if the bodyRep is fully downloaded the
+ // estimated amount is not reliable
+ isDownloaded: false,
+ // full internal IMAP representation
+ // it would also be entirely appropriate to move
+ // the information on the bodyRep directly?
+ _partInfo: partInfo
+ };
+ }
+
if (disposition === 'attachment') {
attachments.push(makePart(partInfo, filename));
return true;
@@ -174,7 +190,7 @@ exports.chewHeaderAndBodyStructure = function chewStructure(msg) {
case 'text':
if (partInfo.subtype === 'plain' ||
partInfo.subtype === 'html') {
- bodyParts.push(partInfo);
+ bodyReps.push(makeTextPart(partInfo));
return true;
}
break;
@@ -243,107 +259,177 @@ exports.chewHeaderAndBodyStructure = function chewStructure(msg) {
chewLeaf(msg.structure);
return {
- msg: msg,
- bodyParts: bodyParts,
+ bodyReps: bodyReps,
attachments: attachments,
relatedParts: relatedParts,
- header: null,
- bodyInfo: null,
};
};
-var DESIRED_SNIPPET_LENGTH = 100;
-
-/**
- * Call once the body parts requested by `chewHeaderAndBodyStructure` have been
- * fetched in order to finish processing of the message to produce the header
- * and body data-structures for the message.
- *
- * This method is currently synchronous because quote-chewing and HTML
- * sanitization can be performed synchronously. This may need to become
- * asynchronous if we still end up with this happening on the same thread as the
- * UI so we can time slice of something like that.
- *
- * @args[
- * @param[rep ChewRep]
- * @param[bodyPartContents @listof[String]]{
- * The fetched body parts matching the list of requested parts in
- * `rep.bodyParts`.
- * }
- * ]
- * @return[success Boolean]{
- * True if we were able to process the message and have updated `rep.header`
- * and `rep.bodyInfo` with populated objects.
- * }
- */
-exports.chewBodyParts = function chewBodyParts(rep, bodyPartContents,
- folderId, newMsgId) {
- var snippet = null, bodyReps = [];
-
- // Mailing lists can result in a text/html body part followed by a text/plain
- // body part. Can't rule out multiple HTML parts at this point either, so we
- // just process everything independently and have the UI do likewise.
- for (var i = 0; i < rep.bodyParts.length; i++) {
- var partInfo = rep.bodyParts[i],
- contents = bodyPartContents[i];
-
- // HTML parts currently can be synchronously sanitized...
- switch (partInfo.subtype) {
- case 'plain':
- var bodyRep = $quotechew.quoteProcessTextBody(contents);
- if (!snippet)
- snippet = $quotechew.generateSnippet(bodyRep,
- DESIRED_SNIPPET_LENGTH);
- bodyReps.push('plain', bodyRep);
- break;
-
- case 'html':
- var htmlNode = $htmlchew.sanitizeAndNormalizeHtml(contents);
- if (!snippet)
- snippet = $htmlchew.generateSnippet(htmlNode, DESIRED_SNIPPET_LENGTH);
- bodyReps.push('html', htmlNode.innerHTML);
- break;
- }
- }
-
+exports.chewHeaderAndBodyStructure =
+ function(msg, folderId, newMsgId) {
+ // begin by splitting up the raw imap message
+ var parts = chewStructure(msg);
+ var rep = {};
rep.header = {
// the FolderStorage issued id for this message (which differs from the
// IMAP-server-issued UID so we can do speculative offline operations like
// moves).
id: newMsgId,
- srvid: rep.msg.id,
+ srvid: msg.id,
// The sufficiently unique id is a concatenation of the UID onto the
// folder id.
suid: folderId + '/' + newMsgId,
// The message-id header value; as GUID as get for now; on gmail we can
// use their unique value, or if we could convince dovecot to tell us, etc.
- guid: rep.msg.msg.meta.messageId,
+ guid: msg.msg.meta.messageId,
// mailparser models from as an array; we do not.
- author: rep.msg.msg.from[0] || null,
- date: rep.msg.date,
- flags: rep.msg.flags,
- hasAttachments: rep.attachments.length > 0,
- subject: rep.msg.msg.subject || null,
- snippet: snippet,
+ author: msg.msg.from[0] || null,
+ to: ('to' in msg.msg) ? msg.msg.to : null,
+ cc: ('cc' in msg.msg) ? msg.msg.cc : null,
+ bcc: ('bcc' in msg.msg) ? msg.msg.bcc : null,
+
+ replyTo: ('reply-to' in msg.msg.parsedHeaders) ?
+ msg.msg.parsedHeaders['reply-to'] : null,
+
+ date: msg.date,
+ flags: msg.flags,
+ hasAttachments: parts.attachments.length > 0,
+ subject: msg.msg.subject || null,
+
+ // we lazily fetch the snippet later on
+ snippet: null
};
rep.bodyInfo = {
- date: rep.msg.date,
+ date: msg.date,
size: 0,
- to: ('to' in rep.msg.msg) ? rep.msg.msg.to : null,
- cc: ('cc' in rep.msg.msg) ? rep.msg.msg.cc : null,
- bcc: ('bcc' in rep.msg.msg) ? rep.msg.msg.bcc : null,
- replyTo: ('reply-to' in rep.msg.msg.parsedHeaders) ?
- rep.msg.msg.parsedHeaders['reply-to'] : null,
- attachments: rep.attachments,
- relatedParts: rep.relatedParts,
- references: rep.msg.msg.meta.references,
- bodyReps: bodyReps,
+ attachments: parts.attachments,
+ relatedParts: parts.relatedParts,
+ references: msg.msg.meta.references,
+ bodyReps: parts.bodyReps
};
- return true;
+ return rep;
+};
+
+var DESIRED_SNIPPET_LENGTH = 100;
+
+/**
+ * Fill a given body rep with the content from fetching
+ * part or the entire body of the message...
+ *
+ * var body = ...;
+ * var header = ...;
+ * var content = (some fetched content)..
+ *
+ * $imapchew.updateMessageWithBodyRep(
+ * header,
+ * bodyInfo,
+ * {
+ * bodyRepIndex: 0,
+ * text: '',
+ * buffer: Uint8Array|Null,
+ * bytesFetched: n,
+ * bytesRequested: n
+ * }
+ * );
+ *
+ * // what just happend?
+ * // 1. the body.bodyReps[n].content is now the value of content.
+ * //
+ * // 2. we update .downloadedAmount with the second argument
+ * // (number of bytes downloaded).
+ * //
+ * // 3. if snippet has not bee set on the header we create the snippet
+ * // and set its value.
+ *
+ */
+exports.updateMessageWithFetch =
+ function(header, body, req, res) {
+
+ var bodyRep = body.bodyReps[req.bodyRepIndex];
+
+ // check if the request was unbounded or we got back less bytes then we
+ // requested in which case the download of this bodyRep is complete.
+ if (!req.bytes || res.bytesFetched < req.bytes[1]) {
+ bodyRep.isDownloaded = true;
+
+ // clear private space for maintaining parser state.
+ delete bodyRep._partInfo;
+ }
+
+ if (!bodyRep.isDownloaded && res.buffer) {
+ bodyRep._partInfo.pendingBuffer = res.buffer;
+ }
+
+ var parsedContent;
+ var snippet;
+ switch (bodyRep.type) {
+ case 'plain':
+ parsedContent = $quotechew.quoteProcessTextBody(res.text);
+ if (req.createSnippet) {
+ header.snippet = $quotechew.generateSnippet(
+ parsedContent, DESIRED_SNIPPET_LENGTH
+ );
+ }
+ break;
+ case 'html':
+ var internalRep = $htmlchew.sanitizeAndNormalizeHtml(res.text);
+ if (req.createSnippet) {
+ header.snippet = $htmlchew.generateSnippet(
+ internalRep, DESIRED_SNIPPET_LENGTH
+ );
+ }
+ parsedContent = internalRep.innerHTML;
+ break;
+ }
+
+ bodyRep.amountDownloaded += res.bytesFetched;
+
+ // if the body rep is fully downloaded then we should set the content as text
+ // otherwise the message is likely garbled and the snippet is the best we can
+ // do.
+ if (bodyRep.isDownloaded) {
+ bodyRep.content = parsedContent;
+ }
+};
+
+/**
+ * Selects a desirable snippet body rep if the given header has no snippet.
+ */
+exports.selectSnippetBodyRep = function(header, body) {
+ if (header.snippet)
+ return -1;
+
+ var bodyReps = body.bodyReps;
+ var len = bodyReps.length;
+
+ for (var i = 0; i < len; i++) {
+ if (exports.canBodyRepFillSnippet(bodyReps[i])) {
+ return i;
+ }
+ }
+
+ return -1;
+};
+
+/**
+ * Determines if a given body rep can be converted into a snippet. Useful for
+ * determining which body rep to use when downloading partial bodies.
+ *
+ *
+ * var bodyInfo;
+ * $imapchew.canBodyRepFillSnippet(bodyInfo.bodyReps[0]) // true/false
+ *
+ */
+exports.canBodyRepFillSnippet = function(bodyRep) {
+ return (
+ bodyRep &&
+ bodyRep.type === 'plain' ||
+ bodyRep.type === 'html'
+ );
};
}); // end define
View
82 data/lib/mailapi/imap/jobs.js
@@ -269,6 +269,84 @@ ImapJobDriver.prototype = {
allJobsDone: $jobmixins.allJobsDone,
+ // snippet downloading
+
+ local_do_downloadSnippets: function(op, doneCallback) {
+ doneCallback(null);
+ },
+
+ do_downloadSnippets: function(op, doneCallback) {
+ var aggrErr;
+ this._partitionAndAccessFoldersSequentially(
+ op.messages,
+ true,
+ function perFolder(folderConn, storage, headers, namers, callWhenDone) {
+ folderConn.downloadSnippets(headers, function(err) {
+ if (err && !aggrErr) {
+ aggrErr = err;
+ }
+ callWhenDone();
+ });
+ },
+ function allDone() {
+ doneCallback(aggrErr);
+ },
+ function deadConn() {
+ aggrErr = 'aborted-retry';
+ },
+ false, // reverse?
+ 'downloadSnippets',
+ true // require headers
+ );
+ },
+
+ // body rep downloading
+
+ local_do_downloadBodyReps: function(op, doneCallback) {
+ doneCallback(null);
+ },
+
+ do_downloadBodyReps: function(op, doneCallback) {
+ var self = this;
+
+ var idxLastSlash = op.messageSuid.lastIndexOf('/'),
+ folderId = op.messageSuid.substring(0, idxLastSlash);
+
+ var folderConn, folderStorage;
+ // Once we have the connection, get the current state of the body rep.
+ var gotConn = function gotConn(_folderConn, _folderStorage) {
+ folderConn = _folderConn;
+ folderStorage = _folderStorage;
+
+ folderConn.downloadBodyReps(
+ op.messageSuid, op.messageDate, onDownloadReps
+ );
+ };
+
+ var onDownloadReps = function onDownloadReps(err, bodyInfo) {
+ if (err) {
+ console.error('Error downloading reps', err);
+ // fail we cannot download for some reason?
+ return doneCallback('unknown');
+ }
+
+ // success
+ doneCallback(null, bodyInfo, true);
+ };
+
+ var deadConn = function deadConn() {
+ doneCallback('aborted-retry');
+ };
+
+ self._accessFolderForMutation(
+ folderId,
+ true,
+ gotConn,
+ deadConn,
+ 'downloadBodyReps'
+ );
+ },
+
//////////////////////////////////////////////////////////////////////////////
// download: Download one or more attachments from a single message
@@ -603,8 +681,8 @@ ImapJobDriver.prototype = {
namer.date, newId, false,
function(header) {
// If the header isn't there because it got moved, then null
- // will be returned and it's up to the next move operation
- // to fix this up.
+ // will be returned and it's up to the next move operation to
+ // fix this up.
if (header)
header.srvid = msg.id;
else
View
128 data/lib/mailapi/imap/protocol/bodyfetcher.js
@@ -0,0 +1,128 @@
+define(
+ [
+ 'exports'
+ ],
+ function(
+ exports
+ ) {
+
+function fetchOptions(partInfo, partial) {
+ var body;
+
+ if (!partial) {
+ body = partInfo.partID;
+ } else {
+ // for some reason the imap lib uses strings to delimit the starting and
+ // ending byte range....
+ body = [
+ partInfo.partID,
+ String(partial[0]) + '-' + String(partial[1])
+ ];
+ }
+
+ return {
+ request: {
+ struct: false,
+ headers: false,
+ body: body
+ }
+ };
+}
+
+/**
+ * Convenience class and wrapper around building multiple fetching operations or
+ * the aggregation of many fetching operations into a single unit of
+ * operation...
+ *
+ *
+ * var fetcher = new $bodyfetcher.BodyFetcher(
+ * connection,
+ * BodyParser (or any other kind of parser),
+ * [
+ * { uid: X, partInfo: {}, bytes: [A, B] }
+ * ]
+ * );
+ *
+ * // in all examples item is a single element in the
+ * // array (third argument).
+ *
+ * fetcher.onerror = function(err, item) {};
+ * fetcher.ondata = function(parsed, item) {}
+ * fetcher.onend = function() {}
+ *
+ */
+function BodyFetcher(connection, parserClass, list) {
+ this.connection = connection;
+ this.parserClass = parserClass;
+ this.list = list;
+
+ this.pending = list.length;
+
+ this.onerror = null;
+ this.ondata = null;
+ this.onend = null;
+
+ list.forEach(this._fetch, this);
+}
+
+BodyFetcher.prototype = {
+ _fetch: function(request) {
+ // build the fetcher based on the request.uid
+ var fetch = this.connection.fetch(
+ request.uid,
+ fetchOptions(request.partInfo, request.bytes)
+ );
+
+ var parser = new this.parserClass(request.partInfo);
+ var self = this;
+
+ fetch.on('error', function(err) {
+ // if fetch provides an error we expect this request to be completed so we
+ // resolve here...
+ self._resolve(err, request);
+ });
+
+ fetch.on('message', function(msg) {
+ msg.on('error', function(err) {
+ // similar to the fetch error we expect this only to be called once and
+ // exclusive of the error event on the fetch itself...
+ self._resolve(err, request);
+ });
+
+ msg.on('data', function(content) {
+ parser.parse(content);
+ });
+
+ msg.on('end', function() {
+ self._resolve(null, request, parser.complete(msg));
+ });
+ });
+ },
+
+ _resolve: function() {
+ var args = Array.slice(arguments);
+ var err = args[0];
+
+ if (err) {
+ if (this.onerror) {
+ this.onerror.apply(this, args);
+ }
+ } else {
+ if (this.onparsed) {
+ // get rid of the error object
+ args.shift();
+
+ this.onparsed.apply(this, args);
+ }
+ }
+
+ if (!--this.pending && this.onend) {
+ this.onend();
+ }
+ }
+};
+
+
+exports.BodyFetcher = BodyFetcher;
+
+});
View
57 data/lib/mailapi/imap/protocol/snippetparser.js
@@ -0,0 +1,57 @@
+define(
+ [
+ './textparser',
+ 'exports'
+ ],
+ function(
+ $textparser,
+ exports
+ ) {
+
+var TextParser = $textparser.TextParser;
+
+function bufferAppend(buf1, buf2) {
+ var newBuf = new Buffer(buf1.length + buf2.length);
+ buf1.copy(newBuf, 0, 0);
+ if (Buffer.isBuffer(buf2))
+ buf2.copy(newBuf, buf1.length, 0);
+ else if (Array.isArray(buf2)) {
+ for (var i=buf1.length, len=buf2.length; i<len; i++)
+ newBuf[i] = buf2[i];
+ }
+
+ return newBuf;
+};
+
+/**
+ * Wrapper around the textparser, accumulates buffer content and returns it as
+ * part of the .complete step.
+ */
+function SnippetParser(partDef) {
+ $textparser.TextParser.apply(this, arguments);
+}
+
+SnippetParser.prototype = {
+ parse: function(buffer) {
+ if (!this._buffer) {
+ this._buffer = buffer;
+ } else {
+ this._buffer = bufferAppend(this._buffer, buffer);
+ }
+
+ // do some magic parsing
+ TextParser.prototype.parse.apply(this, arguments);
+ },
+
+ complete: function() {
+ var content =
+ TextParser.prototype.complete.apply(this, arguments);
+
+ content.buffer = this._buffer;
+ return content;
+ }
+};
+
+exports.SnippetParser = SnippetParser;
+
+});
View
269 data/lib/mailapi/imap/protocol/sync.js
@@ -0,0 +1,269 @@
+define(
+ [
+ 'mailparser/mailparser',
+ '../imapchew',
+ 'exports'
+ ],
+ function(
+ $mailparser,
+ $imapchew,
+ exports
+ ) {
+
+/**
+ * Fetch parameters to get the headers / bodystructure; exists to reuse the
+ * object since every fetch is the same. Note that imap.js always gives us
+ * FLAGS and INTERNALDATE so we don't need to ask for that.
+ *
+ * We are intentionally not using ENVELOPE because Thunderbird intentionally
+ * defaults to to not using ENVELOPE. Per bienvenu in
+ * https://bugzilla.mozilla.org/show_bug.cgi?id=402594#c33 "We stopped using it
+ * by default because servers often had issues with it so it was more trouble
+ * than it was worth."
+ *
+ * Of course, imap.js doesn't really support ENVELOPE outside of bodystructure
+ * right now either, but that's a lesser issue. We probably don't want to trust
+ * that data, however, if we don't want to trust normal ENVELOPE.
+ */
+var INITIAL_FETCH_PARAMS = {
+ request: {
+ headers: ['FROM', 'TO', 'CC', 'BCC', 'SUBJECT', 'REPLY-TO', 'MESSAGE-ID',
+ 'REFERENCES'],
+ struct: true,
+ body: false
+ }
+};
+
+/**
+ * Fetch parameters to just get the flags, which is no parameters because
+ * imap.js always fetches them right now.
+ */
+var FLAG_FETCH_PARAMS = {
+ request: {
+ struct: false,
+ headers: false,
+ body: false
+ }
+};
+
+
+// Number of bytes to fetch for snippet.
+var SNIPPET_BYTES = 256;
+
+// See the `ImapFolderConn` block comment for rationale.
+var KNOWN_HEADERS_AGGR_COST = 20,
+ KNOWN_HEADERS_PER_COST = 1,
+ NEW_HEADERS_AGGR_COST = 20,
+ NEW_HEADERS_PER_COST = 5,
+ NEW_BODIES_PER_COST = 30;
+
+/**
+ * Given a list of new-to-us UIDs and known-to-us UIDs and their corresponding
+ * headers, synchronize the flags for the known UIDs' headers and fetch and
+ * create the header and body objects for the new UIDS.
+ *
+ * First we fetch the headers/bodystructures for the new UIDs all in one go;
+ * all of these headers are going to end up in-memory at the same time, so
+ * batching won't let us reduce the overhead right now. We process them
+ * to determine the body parts we should fetch as the results come in. Once
+ * we have them all, we sort them by date, endTS-to-startTS for the third
+ * step and start issuing/pipelining the requests.
+ *
+ * Second, we issue the flag update requests for the known-to-us UIDs. This
+ * is done second so it can help avoid wasting the latency of the round-trip
+ * that would otherwise result between steps one and three. (Although we
+ * could also mitigate that by issuing some step three requests even as
+ * the step one requests are coming in; our sorting doesn't have to be
+ * perfect and may already be reasonably well ordered if UIDs correlate
+ * with internal date well.)
+ *
+ * Third, we fetch the body parts in our newest-to-startTS order, adding
+ * finalized headers and bodies as we go.
+ *
+ * == Usage
+ *
+ * var sync = new ImapUidSync({
+ * initialProgress: 0.25,
+ * connection: (ImapConnection),
+ *
+ * storage: (FolderStorage),
+ *
+ * knownHeaders: [],
+ *
+ * knownUIDs: [],
+ * newUIDs: []
+ * });
+ *
+ */
+function Sync(options) {
+ // storage and connections
+ this.storage = options.storage;
+ this.connection = options.connection;
+
+ this.knownHeaders = options.knownHeaders || [];
+
+ this.knownUIDs = options.knownUIDs || [];
+ this.newUIDs = options.newUIDs || [];
+
+ this._progress = options.initialProgress || 0.25;
+
+ this._progressCost =
+ (this.knownUIDs.length ? KNOWN_HEADERS_AGGR_COST : 0) +
+ KNOWN_HEADERS_PER_COST * this.knownUIDs.length +
+ (this.newUIDs.length ? NEW_HEADERS_AGGR_COST : 0) +
+ NEW_HEADERS_PER_COST * this.newUIDs.length;
+
+ // events
+ this.onprogress = null;
+ this.oncomplete = null;
+
+ this._beginSync();
+}
+
+Sync.prototype = {
+ _updateProgress: function(newProgress) {
+ this._progress += newProgress;
+
+ if (this.onprogress) {
+ this.onprogress(
+ 0.25 + 0.75 * (this._progress / this._progressCost)
+ );
+ }
+ },
+
+ _beginSync: function() {
+ // pending operations
+ var pending = 1;
+ var self = this;
+
+ function next() {
+ if (!--pending) {
+ self.storage.runAfterDeferredCalls(function() {
+ if (!self.oncomplete)
+ return;
+
+ self.oncomplete(
+ self.newUIDs.length,
+ self.knownUIDs.length
+ );
+ });
+ }
+ }
+
+ if (this.newUIDs.length) {
+ pending++;
+ this._handleNewUids(next);
+ }
+
+ if (this.knownUIDs.length) {
+ pending++;
+ this._handleKnownUids(next);
+ }
+
+ window.setZeroTimeout(next);
+ },
+
+ _handleNewUids: function(callback) {
+ var newFetcher = this.connection.fetch(
+ this.newUIDs, INITIAL_FETCH_PARAMS
+ );
+
+ var pendingSnippets = [];
+ var self = this;
+
+ newFetcher.on('message', function onNewMessage(msg) {
+ msg.on('end', function onNewMsgEnd() {
+ console.log(' new fetched, header processing, INTERNALDATE: ', msg.rawDate);
+
+ // at this point we can build the first batch of header/body
+ // information.
+ var chewRep = $imapchew.chewHeaderAndBodyStructure(
+ msg,
+ self.storage.folderId,
+ self.storage._issueNewHeaderId()
+ );
+
+ pendingSnippets.push(chewRep);
+
+ // flush our body/header information ? should we do some sorting,
+ // etc.. here or just let the UI update ASAP?
+ self.storage.addMessageHeader(chewRep.header);
+ self.storage.addMessageBody(chewRep.header, chewRep.bodyInfo);
+ });
+ });
+
+ newFetcher.on('error', function onNewFetchError(err) {
+ // XXX the UID might have disappeared already? we might need to have
+ // our initiating command re-do whatever it's up to. Alternatively,
+ // we could drop back from a bulk fetch to a one-by-one fetch.
+ console.warn('New UIDs fetch error, ideally harmless:', err);
+ });
+
+ newFetcher.on('end', callback);
+ },
+
+ _handleKnownUids: function(callback) {
+ var self = this;
+ var knownFetcher = this.connection.fetch(
+ self.knownUIDs, FLAG_FETCH_PARAMS
+ );
+
+ var numFetched = 0;
+ knownFetcher.on('message', function onKnownMessage(msg) {
+ // (Since we aren't requesting headers, we should be able to get
+ // away without registering this next event handler and just process
+ // msg right now, but let's wait on an optimization pass.)
+ msg.on('end', function onKnownMsgEnd() {
+ var i = numFetched++;
+
+console.log('FETCHED', i, 'known id', self.knownHeaders[i].id,
+
+ 'known srvid', self.knownHeaders[i].srvid, 'actual id', msg.id);
+ // RFC 3501 doesn't require that we get results in the order we
+ // request them, so use indexOf if things don't line up. (In fact,
+ // dovecot sorts them, so we might just want to sort ours too.)
+ if (self.knownHeaders[i].srvid !== msg.id) {
+ i = self.knownUIDs.indexOf(msg.id);
+ // If it's telling us about a message we don't know about, run away.
+ if (i === -1) {
+ console.warn("Server fetch reports unexpected message:", msg.id);
+ return;
+ }
+ }
+ var header = self.knownHeaders[i];
+ // (msg.flags comes sorted and we maintain that invariant)
+ if (header.flags.toString() !== msg.flags.toString()) {
+
+console.warn(' FLAGS: "' + header.flags.toString() + '" VS "' +
+
+ msg.flags.toString() + '"');
+ header.flags = msg.flags;
+ self.storage.updateMessageHeader(header.date, header.id, true, header);
+ }
+ else {
+ self.storage.unchangedMessageHeader(header);
+ }
+ });
+ });
+
+ knownFetcher.on('error', function onKnownFetchError(err) {
+ // XXX the UID might have disappeared already? we might need to have
+ // our initiating command re-do whatever it's up to. Alternatively,
+ // we could drop back from a bulk fetch to a one-by-one fetch.
+ console.warn('Known UIDs fetch error, ideally harmless:', err);
+ });
+
+ knownFetcher.on('end', function() {
+ // the fetch results will be bursty, so just update all at once
+ self._updateProgress(KNOWN_HEADERS_AGGR_COST +
+ KNOWN_HEADERS_PER_COST * self.knownUIDs.length);
+
+ callback();
+ });
+ }
+
+};
+
+exports.Sync = Sync;
+
+});
View
82 data/lib/mailapi/imap/protocol/textparser.js
@@ -0,0 +1,82 @@
+define(
+ [
+ 'mailparser/mailparser',
+ 'exports'
+ ],
+ function(
+ $mailparser,
+ exports
+ ) {
+
+
+/**
+ * Simple wrapper around mailparser hacks allows us to reuse data from the
+ * BODYSTRUCT request that contained the mime type, etc....
+ *
+ * var parser = $textparser.TextParser(
+ * bodyInfo.bodyReps[n]
+ * );
+ *
+ * // msg is some stream thing from fetcher
+ *
+ * msg.on('data', parser.parse.bind(parser));
+ * msg.on('end', function() {
+ * var content = parser.complete();
+ * });
+ *
+ */
+function TextParser(partDef) {
+ var mparser = this._mparser = new $mailparser.MailParser();
+
+ mparser._state = 0x2; // body
+ mparser._remainder = '';
+ mparser._currentNode = null;
+ mparser._currentNode = mparser._createMimeNode(null);
+ // nb: mparser._multipartTree is an empty list (always)
+ mparser._currentNode.meta.contentType =
+ partDef.type.toLowerCase() + '/' +
+ partDef.subtype.toLowerCase();
+
+ mparser._currentNode.meta.charset =
+ partDef.params && partDef.params.charset &&
+ partDef.params.charset.toLowerCase();
+
+ mparser._currentNode.meta.transferEncoding =
+ partDef.encoding && partDef.encoding.toLowerCase();
+
+ mparser._currentNode.meta.textFormat =
+ partDef.params && partDef.params.format &&
+ partDef.params.format.toLowerCase();
+
+ if (partDef.pendingBuffer) {
+ this.parse(partDef.pendingBuffer);
+ }
+}
+
+TextParser.prototype = {
+ parse: function(buffer) {
+ process.immediate = true;
+ this._mparser.write(buffer);
+ process.immediate = false;
+ },
+
+ complete: function(msg) {
+ process.immediate = true;
+ this._mparser._process(true);
+ process.immediate = false;
+ // We end up having provided an extra newline that we don't want, so let's
+ // cut it off if it exists.
+ var content = this._mparser._currentNode.content;
+ if (content.charCodeAt(content.length - 1) === 10)
+ content = content.substring(0, content.length - 1);
+
+ return {
+ bytesFetched: msg.size,
+ text: content
+ };
+ }
+};
+
+exports.TextParser = TextParser;
+
+});
View
14 data/lib/mailapi/jobmixins.js
@@ -302,9 +302,11 @@ exports.do_download = function(op, callback) {
if (!pendingStorageWrites)
done();
};
+
function done() {
- folderStorage.updateMessageBody(op.messageSuid, op.messageDate, bodyInfo);
- callback(downloadErr, bodyInfo, true);
+ folderStorage.updateMessageBody(header, bodyInfo, function() {
+ callback(downloadErr, bodyInfo, true);
+ });
};
self._accessFolderForMutation(folderId, true, gotConn, deadConn,
@@ -405,6 +407,9 @@ exports.allJobsDone = function() {
* @param[label String]{
* The label to use to name the usage of the folder connection.
* }
+ * @param[requireHeaders Boolean]{
+ * True if connection & headers are needed.
+ * }
* ]
*/
exports._partitionAndAccessFoldersSequentially = function(
@@ -414,7 +419,8 @@ exports._partitionAndAccessFoldersSequentially = function(
callWhenDone,
callOnConnLoss,
reverse,
- label) {
+ label,
+ requireHeaders) {
var partitions = $util.partitionMessagesByFolderId(allMessageNamers);
var folderConn, storage, self = this,
folderId = null, folderMessageNamers = null, serverIds = null,
@@ -453,7 +459,7 @@ exports._partitionAndAccessFoldersSequentially = function(
folderConn = _folderConn;
storage = _storage;
// - Get headers or resolve current server id from name map
- if (needConn) {
+ if (needConn && !requireHeaders) {
var neededHeaders = [],
suidToServerId = self._state.suidToServerId;
serverIds = [];
View
164 data/lib/mailapi/mailapi.js
@@ -266,6 +266,12 @@ function MailHeader(slice, wireRep) {
this.author = wireRep.author;
this.date = new Date(wireRep.date);
+
+ this.to = wireRep.to;
+ this.cc = wireRep.cc;
+ this.bcc = wireRep.bcc;
+ this.replyTo = wireRep.replyTo;
+
this.__update(wireRep);
this.hasAttachments = wireRep.hasAttachments;
@@ -291,6 +297,9 @@ MailHeader.prototype = {
},
__update: function(wireRep) {
+ if (wireRep.snippet)
+ this.snippet = wireRep.snippet;
+
this.isRead = wireRep.flags.indexOf('\\Seen') !== -1;
this.isStarred = wireRep.flags.indexOf('\\Flagged') !== -1;
this.isRepliedTo = wireRep.flags.indexOf('\\Answered') !== -1;
@@ -347,8 +356,12 @@ MailHeader.prototype = {
* Request the `MailBody` instance for this message, passing it to the
* provided callback function once retrieved.
*/
- getBody: function(callback) {
- this._slice._api._getBodyForMessage(this, callback);
+ getBody: function(options, callback) {
+ if (typeof(options) === 'function') {
+ callback = options;
+ options = null;
+ }
+ this._slice._api._getBodyForMessage(this, options, callback);
},
/**
@@ -443,15 +456,12 @@ MailMatchedHeader.prototype = {
* management to worry about. However, you should keep the `MailHeader` alive
* and worry about its lifetime since the message can get deleted, etc.
*/
-function MailBody(api, suid, wireRep) {
+function MailBody(api, suid, wireRep, handle) {
this._api = api;
this.id = suid;
this._date = wireRep.date;
+ this._handle = handle;
- this.to = wireRep.to;
- this.cc = wireRep.cc;
- this.bcc = wireRep.bcc;
- this.replyTo = wireRep.replyTo;
this.attachments = null;
if (wireRep.attachments) {
this.attachments = [];
@@ -463,6 +473,9 @@ function MailBody(api, suid, wireRep) {
this._relatedParts = wireRep.relatedParts;
this.bodyReps = wireRep.bodyReps;
this._cleanup = null;
+
+ this.onchange = null;
+ this.ondead = null;
}
MailBody.prototype = {
toString: function() {
@@ -486,6 +499,21 @@ MailBody.prototype = {
},
/**
+ * true if all the bodyReps are downloaded.
+ */
+ get bodyRepsDownloaded() {
+ var i = 0;
+ var len = this.bodyReps.length;
+
+ for (; i < len; i++) {
+ if (!this.bodyReps[i].isDownloaded) {
+ return false;
+ }
+ }
+ return true;
+ },
+
+ /**
* true if all of the images are already downloaded.
*/
get embeddedImagesDownloaded() {
@@ -603,7 +631,16 @@ MailBody.prototype = {
this._cleanup();
this._cleanup = null;
}
- },
+
+ // Remember to cleanup event listeners except ondead!
+ this.onchange = null;
+
+ this._api.__bridgeSend({
+ type: 'killBody',
+ id: this.id,
+ handle: this._handle
+ });
+ }
};
/**
@@ -922,6 +959,9 @@ FoldersViewSlice.prototype.getFirstFolderWithName = function(name, items) {
function HeadersViewSlice(api, handle) {
BridgedViewSlice.call(this, api, 'headers', handle);
+
+ this._snippetRequestId = 1;
+ this._snippetRequests = {};
}
HeadersViewSlice.prototype = Object.create(BridgedViewSlice.prototype);
/**
@@ -939,6 +979,51 @@ HeadersViewSlice.prototype.refresh = function() {
});
};
+HeadersViewSlice.prototype._notifyRequestSnippetsComplete = function(reqId) {
+ var callback = this._snippetRequests[reqId];
+ if (reqId && callback) {
+ callback(true);
+ delete this._snippetRequests[reqId];
+ }
+};
+
+/**
+ * Request snippets for range of headers in the slice.
+ *
+ * // start/end inclusive
+ * slice.maybeRequestSnippets(5, 10);
+ *
+ * The results will be sent through the standard slice/header events.
+ */
+HeadersViewSlice.prototype.maybeRequestSnippets = function(idxStart, idxEnd, callback) {
+ var messages = [];
+
+ idxEnd = Math.min(idxEnd, this.items.length - 1);
+
+ for (; idxStart <= idxEnd; idxStart++) {
+ if (this.items[idxStart] && !this.items[idxStart].snippet) {
+ messages.push({
+ suid: this.items[idxStart].id,
+ // backend does not care about Date objects
+ date: this.items[idxStart].date.valueOf()
+ });
+ }
+ }
+
+ if (!messages.length)
+ return callback && window.setZeroTimeout(callback, false);
+
+ var reqId = this._snippetRequestId++;
+ this._snippetRequests[reqId] = callback;
+
+ this._api.__bridgeSend({
+ type: 'requestSnippets',
+ handle: this._handle,
+ requestId: reqId,
+ messages: messages
+ });
+};
+
/**
* Handle for a current/ongoing message composition process. The UI reads state
@@ -1200,6 +1285,7 @@ function MailAPI() {
this._slices = {};
this._pendingRequests = {};
+ this._liveBodies = {};
/**
* @dict[
@@ -1454,18 +1540,26 @@ MailAPI.prototype = {
slice.ondead = null;
},
- _getBodyForMessage: function(header, callback) {
+ _getBodyForMessage: function(header, options, callback) {
+
+ var downloadBodyReps = false;
+
+ if (options && options.downloadBodyReps) {
+ downloadBodyReps = options.downloadBodyReps;
+ }
+
var handle = this._nextHandle++;
this._pendingRequests[handle] = {
type: 'getBody',
suid: header.id,
- callback: callback,
+ callback: callback
};
this.__bridgeSend({
type: 'getBody',
handle: handle,
suid: header.id,
date: header.date.valueOf(),
+ downloadBodyReps: downloadBodyReps
});
},
@@ -1477,10 +1571,58 @@ MailAPI.prototype = {
}
delete this._pendingRequests[msg.handle];
- var body = msg.bodyInfo ? new MailBody(this, req.suid, msg.bodyInfo) : null;
+ var body = msg.bodyInfo ?
+ new MailBody(this, req.suid, msg.bodyInfo, msg.handle) :
+ null;
+
+ if (body) {
+ this._liveBodies[msg.handle] = body;
+ }
+
req.callback.call(null, body);
},
+ _recv_requestSnippetsComplete: function(msg) {
+ var slice = this._slices[msg.handle];
+ slice._notifyRequestSnippetsComplete(msg.requestId);
+ },
+
+ _recv_bodyModified: function(msg) {
+ var body = this._liveBodies[msg.handle];
+
+ if (!body) {
+ unexpectedBridgeDataError('body modified for dead handle', msg.handle);
+ // possible but very unlikely race condition where body is modified while
+ // we are removing the reference to the observer...
+ return;
+ }
+
+ if (body.onchange) {
+ // there may be many kinds of updates we want to support but we only
+ // support updating the bodyReps reference currently.
+ switch (msg.detail.changeType) {
+ case 'bodyReps':
+ body.bodyReps = msg.bodyInfo.bodyReps;
+ break;
+ }
+
+ body.onchange(
+ msg.detail,
+ msg.bodyInfo
+ );
+ }
+ },
+
+ _recv_bodyDead: function(msg) {
+ var body = this._liveBodies[msg.handle];
+
+ if (body && body.ondead) {
+ body.ondead();
+ }
+
+ delete this._liveBodies[msg.handle];
+ },
+
_downloadAttachments: function(body, relPartIndices, attachmentIndices,
callWhenDone, callOnProgress) {
var handle = this._nextHandle++;
View
363 data/lib/mailapi/mailbridge.js
@@ -101,6 +101,22 @@ function MailBridge(universe) {
headers: [],
matchedHeaders: [],
};
+
+ /**
+ * Observed bodies in the format of:
+ *
+ * @dictof[
+ * @key[suid]
+ * @value[@dictof[
+ * @key[handleId]
+ * @value[@oneof[Function null]]
+ * ]]
+ * ]
+ *
+ * Similar in concept to folder slices but specific to bodies.
+ */
+ this._observedBodies = {};
+
// outstanding persistent objects that aren't slices. covers: composition
this._pendingRequests = {};
//
@@ -132,6 +148,17 @@ MailBridge.prototype = {
this.universe.modifyConfig(msg.mods);
},
+ /**
+ * Public api to verify if body has observers.
+ *
+ *
+ * MailBridge.bodyHasObservers(header.id) // => true/false.
+ *
+ */
+ bodyHasObservers: function(suid) {
+ return !!this._observedBodies[suid];
+ },
+
notifyConfig: function(config) {
this.__sendMessage({
type: 'config',
@@ -355,6 +382,17 @@ MailBridge.prototype = {
proxy.sendSplice(0, 0, wireReps, true, false);
},
+ _cmd_requestSnippets: function(msg) {
+ var self = this;
+ this.universe.downloadSnippets(msg.messages, function() {
+ self.__sendMessage({
+ type: 'requestSnippetsComplete',
+ handle: msg.handle,
+ requestId: msg.requestId
+ });
+ });
+ },
+
notifyFolderAdded: function(account, folderMeta) {
var newMarker = makeFolderSortString(account, folderMeta);
@@ -396,6 +434,40 @@ MailBridge.prototype = {
}
},
+ /**
+ * Sends a notification of a change in the body.
+ *
+ * bridge.notifyBodyModified(
+ * suid,
+ * 'bodyRep',
+ * { index: 0 }
+ * newBodyInfo
+ * );
+ *
+ *
+ */
+ notifyBodyModified: function(suid, detail, body) {
+ var handles = this._observedBodies[suid];
+ var defaultHandler = this.__sendMessage;
+
+ if (handles) {
+ for (var handle in handles) {
+ // the suid may have an existing handler which captures the output of
+ // the notification instead of dispatching here... This allows us to
+ // aggregate pending notifications while fetching the bodies so updates
+ // never come before the actual body.
+ var emit = handles[handle] || defaultHandler;
+
+ emit.call(this, {
+ type: 'bodyModified',
+ handle: handle,
+ bodyInfo: body,
+ detail: detail
+ });
+ }
+ }
+ },
+
_cmd_viewFolders: function mb__cmd_viewFolders(msg) {
var proxy = this._slices[msg.handle] =
new SliceBridgeProxy(this, 'folders', msg.handle);
@@ -523,12 +595,71 @@ MailBridge.prototype = {
var self = this;
// map the message id to the folder storage
var folderStorage = this.universe.getFolderStorageForMessageSuid(msg.suid);
+
+ // when requesting the body we also create a observer to notify the client
+ // of events... We never want to send the updates before fetching the body
+ // so we buffer them here with a temporary handler.
+ var pendingUpdates = [];
+
+ var catchPending = function catchPending(msg) {
+ pendingUpdates.push(msg);
+ };
+
+ if (!this._observedBodies[msg.suid])
+ this._observedBodies[msg.suid] = {};
+
+ this._observedBodies[msg.suid][msg.handle] = catchPending;
+
folderStorage.getMessageBody(msg.suid, msg.date, function(bodyInfo) {
self.__sendMessage({
type: 'gotBody',
handle: msg.handle,
- bodyInfo: bodyInfo,
+ bodyInfo: bodyInfo
});
+
+ // if all body reps where requested we verify that all are present
+ // otherwise we begin the request for more body reps.
+ if (
+ msg.downloadBodyReps &&
+ !folderStorage.messageBodyRepsDownloaded(bodyInfo)
+ ) {
+
+ self.universe.downloadMessageBodyReps(
+ msg.suid,
+ msg.date,
+ function() { /* we don't care it will send update events */ }
+ );
+ }
+
+ // dispatch pending updates...
+ pendingUpdates.forEach(self.__sendMessage, self);
+ pendingUpdates = null;
+
+ // revert to default handler. Note! this is intentionally
+ // set to null and not deleted if deleted the observer is removed.
+ self._observedBodies[msg.suid][msg.handle] = null;
+ });
+ },
+
+ _cmd_killBody: function(msg) {
+ var handles = this._observedBodies[msg.id];
+ if (handles) {
+ delete handles[msg.handle];
+
+ var purgeHandles = true;
+ for (var key in handles) {
+ purgeHandles = false;
+ break;
+ }
+
+ if (purgeHandles) {
+ delete this._observedBodies[msg.id];
+ }
+ }
+
+ this.__sendMessage({
+ type: 'bodyDead',
+ handle: msg.handle
});
},
@@ -595,6 +726,7 @@ MailBridge.prototype = {
//////////////////////////////////////////////////////////////////////////////
// Composition
+
_cmd_beginCompose: function mb__cmd_beginCompose(msg) {
require(['./composer'], function ($composer) {
var req = this._pendingRequests[msg.handle] = {
@@ -613,120 +745,131 @@ MailBridge.prototype = {
identity = account.identities[0];
- if (msg.mode === 'reply' ||
- msg.mode === 'forward') {
- var folderStorage = this.universe.getFolderStorageForMessageSuid(
- msg.refSuid);
- var self = this;
- folderStorage.getMessageBody(
- msg.refSuid, msg.refDate,
- function(bodyInfo) {
- if (msg.mode === 'reply') {
- var rTo, rCc, rBcc;
- // clobber the sender's e-mail with the reply-to
- var effectiveAuthor = {
- name: msg.refAuthor.name,
- address: (bodyInfo.replyTo && bodyInfo.replyTo.address) ||
- msg.refAuthor.address,
- };
- switch (msg.submode) {
- case 'list':
- // XXX we can't do this without headers we're not retrieving,
- // fall through for now.
- case null:
- case 'sender':
- rTo = [effectiveAuthor];
- rCc = rBcc = [];
- break;
- case 'all':
- // No need to change the lists if the author is already on the
- // reply lists.
- //
- // nb: Our logic here is fairly simple; Thunderbird's
- // nsMsgCompose.cpp does a lot of checking that we should
- // audit, although much of it could just be related to its
- // much more extensive identity support.
- if (checkIfAddressListContainsAddress(bodyInfo.to,
- effectiveAuthor) ||
- checkIfAddressListContainsAddress(bodyInfo.cc,
- effectiveAuthor)) {
- rTo = bodyInfo.to;