Imap Copy #321

Closed
myoung69 opened this Issue Oct 23, 2013 · 13 comments

3 participants

@myoung69

Hello,

Is there any chance of someone having a example of how to do a all mailboxes to all mailboxes copy from one server to another?

@mscdex
Owner

Assuming both servers use the same hierarchy delimiters (if applicable), it might look something like this (untested):

// imapsrc and imapdest are connected and 'ready'

var async = require('async');

function toBoxList(boxes, prefix, list) {
  prefix || (prefix = '');
  var box, start = false;
  if (list === undefined) {
    start = true;
    list = [];
  }
  for (var name in boxes) {
    box = boxes[name];
    if (!box.children && box.attribs.indexOf('\\Noselect') === -1)
      list.push(prefix + name);
    else if (box.children)
      toBoxList(box.children, prefix + name + box.delimiter, list);
  }
  if (start)
    return list;
}

function makeBoxes(callback) {
  imapsrc.getBoxes(function(err, boxes) {
    if (err) return callback(err);
    boxes = toBoxList(boxes);
    async.eachSeries(boxes, function(name, cb) {
      imapsrc.openBox(name, function(err) {
        if (err) return cb(err);
        imapdest.addBox(name, function(err) { cb(); });
      });
    }, function(err) {
      if (err) return callback(err);
      callback(undefined, boxes);
    });
  });
}

function copyMessages(boxes, callback) {
  async.eachSeries(boxes, function(name, cb) {
    imapsrc.openBox(name, function(err) {
      if (err) return cb(err);
      imapdest.openBox(name, function(err) {
        if (err) return cb(err);
        imapsrc.search([ 'ALL' ], function(err, uids) {
          if (err) return cb(err);
          async.eachSeries(uids, function(uid, cb) {
            var body = '', date;
            imapsrc.fetch(uid, { bodies: '' })
            .on('message', function(msg, seqno) {
              msg.on('body', function(stream, info) {
                stream.on('data', function(chunk) {
                  body += chunk.toString('utf8');
                });
              });
              msg.on('attributes', function(attrs) {
                date = attrs.date;
              });
            })
            .on('end', function() {
              if (body.length) {
                var opts = (date ? { date: date } : {});
                imapdest.append(body, opts, cb);
              } else
                cb();
            });
          }, cb);
        });
      });
    });
  }, callback);
}

// let's do this thing!
makeBoxes(function(err, boxes) {
  if (err) throw err;
  copyMessages(boxes, function(err) {
    if (err) throw err;
  });
});
@myoung69

First off thanks mscdex for the excellent starting example for me.

If you could give me advice on a few items i am trying to solve now it would be very much appreciated.

item one would be how to make it honor the seen flags.

item two the script crashes if you don't make the parent folders first to folders that have children.

item three is there any way to speed it up a little. It is still the fastest option I have found but I have 1000's of accounts to migrate. What is your opinion on maybe using async.each instead of async.eachSeries I know i might hit connection limits on servers outside of my control.
would using node clustering be worth the headache of implementing?

What would be the best way to diagnose / debug this when it hangs on a message before completing all the mailbox copies?

@mscdex
Owner
  1. It should be as simple as settings flags to attrs.flags in the opts object when append()'ing, similar to how date is currently being set.
  2. Yeah, I didn't test the code after I wrote it, but it was meant to be just enough to give you a general idea.
  3. node-imap forces queueing internally to avoid some gotchas inherent in the protocol itself, so the eachSeries() is there mostly by habit. As far as increasing performance goes, you can definitely use multiple connections to make things faster if you're currently just doing one account at a time. As you said, just be aware of server connection limits.

Regarding hanging, can you explain a bit more about what you're seeing? You can get debug output by setting debug: console.log in the constructor object.

@myoung69

Ok i added a message counter and every time it just stops responding around message 1135 or about 35MB of mail left to copy.

Ok I enabled the debug on destination server constructor and the following it what is showing in the console when it hangs.

AAAAAAAAD///8AAP//AAAAAP////8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r\nAAAAAAAAAAAAAP///01TRFQEAAAAAAAAAGx2ZnJhbWlkAMGbAGcPAABNU0RUBAAAAAAAAAB2\r\nZGZyYW1pZADBmwBnDwAATVNEVAQAAAAAAAAAbHZmcmFtaWQAwZsAaA8AAE1TRFQEAAAAAAAA\r\nAHZkZnJhbWlkAMGbAGgPAABNU0RUxAAAAAAAAABleHBldmVudAAAAABoDwAA3gEAAAAAAAAA\r\nAAAArAKQAZgTkwnzCIsFAAcICwUAAAAYAAAAAACQAWL////zCAAA4AEAAJEJAACQAQAAoAAA\r\nAAIAAAAFAAAAOJUAADiVAAABAAAAAAAAAAkAAAAAAAAA/////wAAAAAAAAAAAAAAAAAAAAAA\r\nAAAAAAAAAAAAAAAAAAAAAP///wAA//8AAAAA///9/wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r\nAAAAAAAAAAAAAAAAAAAAAAAA////TVNEVAQAAAAAAAAAbHZmcmFtaWQAwZsAaQ8AAE1TRFQE\r\nAAAAAAAAAHZkZnJhbWlkAMGbAGkPAABNU0RUBAAAAAAAAABsdmZyYW1pZADBmwBqDwAATVNE\r\nVAQAAAAAAAAAdmRmcmFtaWQAwZsAag8AAE1TRFTEAAAAAAAAAGV4cGV2ZW50AAAAAGoPAADb\r\nAQAAAAAAAAAAAACsApABmBOVCfUIiwUABwoLBQAAABgAAAAAAJABZP////UIAADeAQAAkwkA\r\nAJABAACgAAAAAgAAAAUAAABQlQAAUJUAAAEAAAAAAAAACQAAAAAAAAD7////AAAAAAAAAAAA\r\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AAD//wAAAAD//v//AAAAAAAAAAAAAAAAAAAA\r\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///9NU0RUBAAAAAAAAABsdmZyYW1pZADBmwBr\r\nDwAATVNEVAQAAAAAAAAAdmRmcmFtaWQAwZsAaw8AAE1TRFQEAAAAAAAAAGx2ZnJhbWlkAMGb\r\nAGwPAABNU0RUBAAAAAAAAAB2ZGZyYW1pZADBmwBsDwAATVNEVIQAAAAAAAAAdmRpbmZvYnMA\r\nAAAAREhEVAEAAABwAAAATVNEVAQAAAAAAAAAdmNmcm1jbnQAAAAAAAAAAE1TRFQEAAAAAAAA\r\nAGR1cmF0aW9uAAAAAAAAAABNU0RUBAAAAAAAAAB2ZHJlc2x2bAAAAAAFAAAATVNEVAQAAAAA\r\nAAAAdmRjbXBsdmwAAAAABAAAAJwSAABURkRUTVNEVAQAAAAAAAAAY2Nkd2lkdGgAAAAAUA4A\r\nAE1TRFQEAAAAAAAAAGNjZGhlaWd0AAAAAMAKAADPKiEAVEZEVAAAAbk=\r\n\r\n--part1_c9b.15402be1.33e11f04_boundary--\r\n\r\n'
<= 'A1206 OK [APPENDUID 2823061117 7] APPEND file copied'
=> 'IDLE IDLE'
=> DONE
<= '* 5 EXISTS'
<= '+ IDLE accepted, awaiting DONE command.'
<= '* BYE Session terminated by server because idle timeout was exceeded'
[connection] Ended
Connection ended
[connection] Closed

any ideas on the cause of what it appears as not receiving the DONE command?

@mscdex
Owner

Can you try the node-imap master branch?

@myoung69

Ok i am using the node-imap master branch now.

I have also found the message that was hanging the copy process forever it is a email with around 15MB of picture attachments.

Is there a easy way to timeout the copy of that message and move to the next one?

@mscdex
Owner

Wait, so is it still hanging with the master branch or no?

@myoung69
@mscdex
Owner

The debug output shows the same thing (at the end) as before? Namely...

<= 'A1206 OK [APPENDUID 2823061117 7] APPEND file copied'
=> 'IDLE IDLE'
=> DONE
<= '* 5 EXISTS'
<= '+ IDLE accepted, awaiting DONE command.'
<= '* BYE Session terminated by server because idle timeout was exceeded'
[connection] Ended
Connection ended
[connection] Closed
@myoung69
@myoung69

Ok using master it appears to work for that email if i copy the message from a windows smartermail server to a windows mailenable server.

But it hangs if i try to copy same message for a Gmail account to a windows mailenable server.

below is the console dubug output when it hangs.

///AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD//99NU0RUBAAA\r\nAAAAAABsdmZyYW1pZADBmwBhDwAATVNEVAQAAAAAAAAAdmRmcmFtaWQAwZsAYQ8AAE1TRFQE\r\nAAAAAAAAAGx2ZnJhbWlkAMGbAGIPAABNU0RUBAAAAAAAAAB2ZGZyYW1pZADBmwBiDwAATVNE\r\nVAQAAAAAAAAAbHZmcmFtaWQAwZsAYw8AAE1TRFQEAAAAAAAAAHZkZnJhbWlkAMGbAGMPAABN\r\nU0RUBAAAAAAAAABsdmZyYW1pZADBmwBkDwAATVNEVAQAAAAAAAAAdmRmcmFtaWQAwZsAZA8A\r\nAE1TRFTEAAAAAAAAAGV4cGV2ZW50AAAAAGMPAADgAQAAAAAAAAAAAACsApABmBORCfEIiwUA\r\nBwYLBQAAABgAAAAAAJABXf////EIAADdAQAAlAkAAJABAACgAAAAAwAAAAUAAAAQlQAAEJUA\r\nAAEAAAAAAAAACQAAAAAAAAD////+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r\n////AAD//wAAAAC/////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r\nAAD///9NU0RUBAAAAAAAAABsdmZyYW1pZADBmwBlDwAATVNEVAQAAAAAAAAAdmRmcmFtaWQA\r\nwZsAZQ8AAE1TRFQEAAAAAAAAAGx2ZnJhbWlkAMGbAGYPAABNU0RUBAAAAAAAAAB2ZGZyYW1p\r\nZADBmwBmDwAATVNEVMQAAAAAAAAAZXhwZXZlbnQAAAAAZA8AAN4BAAAAAAAAAAAAAKwCkAGY\r\nE5MJ8wiLBQAHCAsFAAAAGAAAAAAAkAFk////8wgAAN8BAACSCQAAkAEAAKAAAAACAAAABQAA\r\nACSVAAAklQAAAQAAAAAAAAAJAAAAAAAAAPf///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r\nAAAAAAAAAAD///8AAP//AAAAAP////8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r\nAAAAAAAAAAAAAP///01TRFQEAAAAAAAAAGx2ZnJhbWlkAMGbAGcPAABNU0RUBAAAAAAAAAB2\r\nZGZyYW1pZADBmwBnDwAATVNEVAQAAAAAAAAAbHZmcmFtaWQAwZsAaA8AAE1TRFQEAAAAAAAA\r\nAHZkZnJhbWlkAMGbAGgPAABNU0RUxAAAAAAAAABleHBldmVudAAAAABoDwAA3gEAAAAAAAAA\r\nAAAArAKQAZgTkwnzCIsFAAcICwUAAAAYAAAAAACQAWL////zCAAA4AEAAJEJAACQAQAAoAAA\r\nAAIAAAAFAAAAOJUAADiVAAABAAAAAAAAAAkAAAAAAAAA/////wAAAAAAAAAAAAAAAAAAAAAA\r\nAAAAAAAAAAAAAAAAAAAAAP///wAA//8AAAAA///9/wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r\nAAAAAAAAAAAAAAAAAAAAAAAA////TVNEVAQAAAAAAAAAbHZmcmFtaWQAwZsAaQ8AAE1TRFQE\r\nAAAAAAAAAHZkZnJhbWlkAMGbAGkPAABNU0RUBAAAAAAAAABsdmZyYW1pZADBmwBqDwAATVNE\r\nVAQAAAAAAAAAdmRmcmFtaWQAwZsAag8AAE1TRFTEAAAAAAAAAGV4cGV2ZW50AAAAAGoPAADb\r\nAQAAAAAAAAAAAACsApABmBOVCfUIiwUABwoLBQAAABgAAAAAAJABZP////UIAADeAQAAkwkA\r\nAJABAACgAAAAAgAAAAUAAABQlQAAUJUAAAEAAAAAAAAACQAAAAAAAAD7////AAAAAAAAAAAA\r\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////AAD//wAAAAD//v//AAAAAAAAAAAAAAAAAAAA\r\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///9NU0RUBAAAAAAAAABsdmZyYW1pZADBmwBr\r\nDwAATVNEVAQAAAAAAAAAdmRmcmFtaWQAwZsAaw8AAE1TRFQEAAAAAAAAAGx2ZnJhbWlkAMGb\r\nAGwPAABNU0RUBAAAAAAAAAB2ZGZyYW1pZADBmwBsDwAATVNEVIQAAAAAAAAAdmRpbmZvYnMA\r\nAAAAREhEVAEAAABwAAAATVNEVAQAAAAAAAAAdmNmcm1jbnQAAAAAAAAAAE1TRFQEAAAAAAAA\r\nAGR1cmF0aW9uAAAAAAAAAABNU0RUBAAAAAAAAAB2ZHJlc2x2bAAAAAAFAAAATVNEVAQAAAAA\r\nAAAAdmRjbXBsdmwAAAAABAAAAJwSAABURkRUTVNEVAQAAAAAAAAAY2Nkd2lkdGgAAAAAUA4A\r\nAE1TRFQEAAAAAAAAAGNjZGhlaWd0AAAAAMAKAADPKiEAVEZEVAAAAbk=\r\n\r\n--part1_c9b.15402be1.33e11f04_boundary--'
<= 'A3245 OK [APPENDUID 1303586320 1] APPEND file copied'
=> 'IDLE IDLE'
=> DONE
<= '* 1 EXISTS'
<= '+ IDLE accepted, awaiting DONE command.'

@SamDecrock

That looks like binary data. Maybe it's because you're converting the content of the body to utf8:

body += chunk.toString('utf8');

You can just leave it in binary form, like this:

// at the start
var chunks = [];
// on data:
chunks.push(chunk);
// on end:
var body = Buffer.concat(chunks);

Here's a piece of code copying the INBOX:
https://gist.github.com/SamDecrock/dc936a17ffa86bfeb8a1

+ I'm not using async ;-)

@mscdex
Owner

@myoung69's issue with all of the base64 data showing up in the debug log should be fixed as of 8376f21.

@mscdex mscdex closed this Apr 3, 2015
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment