Contacts is working

* the contacts synclet runs against the synced data from the headers synclet and does not other external calls. * Fixed a small bug in the testSynclet script.
leggetter · Sep 28, 2012 · 3466c31 · 3466c31
1 parent 03e258e
commit 3466c31
Show file tree

Hide file tree

Showing 4 changed files with 117 additions and 55 deletions.
diff --git a/lib/services/gmail/contacts.js b/lib/services/gmail/contacts.js
@@ -1,46 +1,99 @@
 var ijod = require("ijod");
 var encoder = require("encoding");
 var mimelib = require("mimelib");
+var async = require("async");
 
 var IMAP_PAGE_SIZE = 1000;
 
 exports.sync = function(pi, cbDone) {
-  var curFriends = {};
-  ijod.getRange("envelope:" + pi.auth.pid + "/headers", {since:1, limit:500}, function(entry) {
-    var labels = entry.data && entry.data["x-gm-labels"];
-    if (labels && labels.indexOf("\\\\Sent") > 0) {
-      var contacts = entry.data.headers.to;
-      console.log(contacts);
-      if (!contacts) return;
+  var allContacts = {};
+  var curSince = (pi.config && pi.config.contactsSince) || 0;
+  var gotEntry = true;
+  async.whilst(function() {
+    return gotEntry && Object.keys(allContacts).length < 200;
+  }, function(cbStep) {
+    curContacts = [];
+    // Go through all of the envelopes and pull out contats
+    //console.log("Starting with the chunk since %s", curSince);
+    gotEntry = false;
+    ijod.getRange("envelope:" + pi.auth.pid + "/headers", {since:curSince, reverse:true, limit:500}, function(entry) {
+      gotEntry = true;
+      if (!entry) return;
+      // For each entry we quickly make sure it's a clean email and name then move on
+      curSince = entry.at + 1;
+      var labels = entry.data && entry.data["x-gm-labels"];
+      // We only do people we've sent to because that's a real contact and less spam filtering
+      if (labels && labels.indexOf("\\\\Sent") > 0) {
+        var contacts = entry.data.headers.to;
+        //console.log(contacts);
+        if (!contacts) return;
 
-      for (var idx = 0; idx < contacts.length; ++idx) {
-        var contactInfo = contacts[idx];
-        var ltIndex = contactInfo.indexOf("<");
-        var contactName = contactInfo.substr(0, ltIndex - 1);
-        if (contactName.substr(0, 2) == "=?" && contactName.substr(contactName.length - 2, 2) == "?=") {
-          var encodingEnd = contactName.indexOf("?", 3);
-          var encoding = contactName.substr(2, encodingEnd - 2);
-          var encodingType = contactName[encodingEnd + 1];
-          var encodedText = contactName.substr(encodingEnd + 3, contactName.length - encodingEnd - 5);
-          var decodedText = encodedText.replace(/_/g, " ");
-          decodedText = decodedText.replace(/=[A-Fa-f0-9]{2}/g, function(match) {
-            return String.fromCharCode(parseInt(match.substr(1, 2), 16));
-          });
-          console.log("Decoded to: %s", decodedText);
-          contactName = encoder.convert(decodedText, "UTF-8", encoding).toString("utf8");
-          console.log("decoded %s using coding %s of type %s to %s", encodedText, encodingType, encoding, contactName);
-        }
-        var contactEmail = contactInfo.substr(ltIndex + 1, contactInfo.indexOf(">") - ltIndex - 1);
+        for (var idx = 0; idx < contacts.length; ++idx) {
+          var contactInfo = contacts[idx];
+          var ltIndex = contactInfo.indexOf("<");
+          var contactName = contactInfo.substr(0, ltIndex - 1);
+          // This checks for RFC 2047 encoded names and decodes them
+          if (contactName.substr(0, 2) == "=?" && contactName.substr(contactName.length - 2, 2) == "?=") {
+            var encodingEnd = contactName.indexOf("?", 3);
+            var encoding = contactName.substr(2, encodingEnd - 2);
+            var encodingType = contactName[encodingEnd + 1];
+            var encodedText = contactName.substr(encodingEnd + 3, contactName.length - encodingEnd - 5);
+            var decodedText = encodedText.replace(/_/g, " ");
+            decodedText = decodedText.replace(/=[A-Fa-f0-9]{2}/g, function(match) {
+              return String.fromCharCode(parseInt(match.substr(1, 2), 16));
+            });
+            //console.log("Decoded to: %s", decodedText);
+            contactName = encoder.convert(decodedText, "UTF-8", encoding).toString("utf8");
+            //console.log("decoded %s using coding %s of type %s to %s", encodedText, encodingType, encoding, contactName);
+          }
+          var contactEmail = contactInfo.substr(ltIndex + 1, contactInfo.indexOf(">") - ltIndex - 1);
 
-        if (!curFriends[contactEmail]) curFriends[contactEmail] = [];
-        if (curFriends[contactEmail].indexOf(contactName) < 0) curFriends[contactEmail].push(contactName);
-
-        console.log("Got an entry for: %s [%s]", contactName, contactEmail);
+          curContacts.push([contactEmail, contactName, entry.at]);
+          //console.log("Got an entry for: %s [%s]", contactName, contactEmail);
+        }
       }
-    }
+    }, function(error) {
+      // This is the end of the ijod range, let's do our processing now
+      if (error) return cbStep(error);
+
+      // Run through the matches and merge with any existing data or create entries
+      async.forEachSeries(curContacts, function(contact, cbContactStep) {
+        var contactEmail = contact[0];
+        var contactName = contact[1];
+        var contactAt = contact[2];
+        function updateContact(contact) {
+          if (contact.names.indexOf(contactName) < 0) contact.names.push(contactName);
+          contact.interactions++;
+          contact.at = contactAt;
+          return cbContactStep(null);
+        }
+        // XXX: Need to write a finally pattern!
+        if (!allContacts[contactEmail]) {
+          ijod.getOne("contact:" + pi.auth.pid + "/contacts", function(error, entry) {
+            if (error || !entry) {
+              allContacts[contactEmail] = {interactions:0, names:[], email:contactEmail, id:encodeURIComponent(contactEmail)};
+            } else {
+              allContacts[contactsEmail] = entry.data;
+            }
+            updateContact(allContacts[contactEmail]);
+          });
+        } else {
+          updateContact(allContacts[contactEmail]);
+        }
+      }, function(err) {
+        //console.log(err);
+        //console.log("Did a chunk, currently: %j", allContacts);
+        cbStep(err);
+      });
+    });
   }, function(error) {
-    console.log("Friends: %j", curFriends);
-    cbDone(error, {});
+    // Setup our new config
+    pi.config.contactsSince = curSince;
+    var contacts = {};
+    console.log("Got %d contacts", Object.keys(allContacts).length);
+    contacts["contact:" + pi.auth.pid + "/contacts"] = Object.keys(allContacts).map(function(key) { return allContacts[key]; });
+    // Transform into a contacts array
+    cbDone(error, {auth:pi.auth, config:pi.config, data:contacts});
   });
 }
 
diff --git a/lib/services/gmail/map.js b/lib/services/gmail/map.js
@@ -1,5 +1,6 @@
 exports.defaults = {
   self: 'profile',
-  headers: 'envelope'
+  headers: 'envelope',
+  contacts: "contact"
 }
 
diff --git a/lib/services/gmail/synclets.json b/lib/services/gmail/synclets.json
@@ -8,6 +8,7 @@
   ],
     "synclets":[
         {"name": "self",    "frequency": 3600, "class":"core"},
-        {"name": "headers",    "frequency": 3600, "class":"core"}
+        {"name": "headers",    "frequency": 3600, "class":"core"},
+        {"name": "contacts", "frequency":3600, "class":"core"}
     ]
 }
diff --git a/testSynclet.js b/testSynclet.js
@@ -13,39 +13,46 @@ var idr = require("idr");
 var logger = require("logger").logger("testSynclet");
 var profileManager = require("profileManager");
 var path = require("path");
+var ijod = require("ijod");
 
 var profile = process.argv[2];
 var synclet = process.argv[3];
 
 var service = profile.split("@")[1];
 
+
 logger.info("Running %s/%s for %s", service, synclet, profile);
 
 function exitWithError() {
   logger.error.apply(logger, arguments);
   process.exit(1);
 }
 
-dal.query("SELECT service FROM Profiles WHERE id=?", [profile], function(error, rows) {
-  if (error) exitWithError("Error finding the profile %s: %s", profile, error);
-  if (rows.length != 1 || rows[0].service != service) exitWithError("Did not find a valid profile for %s", service);
-
-  profileManager.allGet(profile, function(error, pi) {
-    if (error) exitWithError("Error getting the profile information for %s: %s", profile, error);
-    if (!pi.auth) exitWithError("No auth information was found for the profile %s, you must auth before you can run the synclet.", profile);
-
-    //logger.debug("%j", pi);
-    try {
-      var mod = require(path.join(__dirname, "lib", "services", service, synclet) + ".js");
-      if (!mod) exitWithError("Could not find the synclet for %s/%s", service, synclet);
-      mod.sync(pi, function(error, data) {
-        if (error) exitWithError("%s/%s returned the error: %s", service, synclet, E);
-        logger.info("%s/%s returned: %j", service, synclet, data);
-        process.exit(0);
-      });
-    } catch(E) {
-      exitWithError("Got an exception running %s/%s: %s", service, synclet, E);
-    }
+function runService() {
+  dal.query("SELECT service FROM Profiles WHERE id=?", [profile], function(error, rows) {
+    if (error) exitWithError("Error finding the profile %s: %s", profile, error);
+    if (rows.length != 1 || rows[0].service != service) exitWithError("Did not find a valid profile for %s", service);
+
+    profileManager.allGet(profile, function(error, pi) {
+      if (error) exitWithError("Error getting the profile information for %s: %s", profile, error);
+      if (!pi.auth) exitWithError("No auth information was found for the profile %s, you must auth before you can run the synclet.", profile);
+
+      //logger.debug("%j", pi);
+      try {
+        var mod = require(path.join(__dirname, "lib", "services", service, synclet) + ".js");
+        if (!mod) exitWithError("Could not find the synclet for %s/%s", service, synclet);
+        mod.sync(pi, function(error, data) {
+          if (error) exitWithError("%s/%s returned the error: %s", service, synclet, error);
+          logger.info("%s/%s returned: %j", service, synclet, data);
+          process.exit(0);
+        });
+      } catch(E) {
+        exitWithError("Got an exception running %s/%s: %s", service, synclet, E);
+      }
+    });
   });
-});
+}
 
+ijod.initDB(function(error) {
+  runService();
+});