instead of read FB directly, read the processed mongodb

cakefile

@@ -1,6 +1,7 @@
 express           = require 'express'
 ejs               = require 'ejs'
 ISODate           = require 'isodate'
+{_}               = require 'underscore'
 
 {FacebookService} = require './lib/facebook_service'
 {MongoService}    = require './lib/mongo_service'
@@ -14,29 +15,43 @@ task 'import', 'fetch facebook update and insert into mongo', (options) ->
   fb        = new FacebookService accessToken
   mongo     = new MongoService mongo_url
 
-  fb.getFeed groupId, null, (error, feeds) => 
+  # fetch all feeds
+  all_feeds = []
+
+  # create a fetch callback: if more pages available, fetch them
+  onFeedFetched = (error, feeds) => 
     if error
       console.log error
       console.log "Error contacting Facebook: #{error.message}"
-
     else
       data    = feeds.data
       paging  = feeds.paging
 
-      mongo.save data, (error, posts) =>
-        if error
-          console.log("error saving post: ", error)
-        else
-          console.log("#{posts.length} saved")
-
-        mongo.close()
-
-      # Parse last request time
-      # last_feed = data[data.length-1]
-      # last_feed_create_datestr = "#{last_feed.created_time[0..21]}:#{last_feed.created_time[22..24]}"
-      # last_feed_create_date    = ISODate(last_feed_create_datestr)
-      # console.log last_feed_create_date.valueOf() / 1000
+      all_feeds.push(data)
+      all_feeds = _.flatten(all_feeds)
 
+      # fetch more if needed
+      if data.length > 0 && paging?.next
+        last_feed = data[data.length-1]
+        last_feed_create_datestr = "#{last_feed.created_time[0..21]}:#{last_feed.created_time[22..24]}"
+        last_feed_create_date    = ISODate(last_feed_create_datestr)
+        timestamp = last_feed_create_date.valueOf() / 1000
+
+        console.log("fetch message before: #{last_feed_create_date}")
+        fb.getFeed groupId, {until: timestamp}, onFeedFetched
+
+      else
+        mongo.save all_feeds, (error, records) =>
+          if error
+            console.log("Failed saving to mongo", error)
+          else
+            console.log("#{records.length} saved.")
+
+          mongo.close()
+
+  # fetch first page
+  console.log("fetch message ...")
+  fb.getFeed groupId, null, onFeedFetched
 
 task 'export', "export mongo database", (options) ->
   mongo     = new MongoService mongo_url

lib/mongo_service.coffee

@@ -1,18 +1,16 @@
-Mongo     = require('mongoskin')
+Mongo     = require 'mongoskin'
 ISODate   = require 'isodate'
 
 class MongoService
   constructor: (url) ->
     @db = Mongo.db("#{url}?auto_reconnect")
-
-  getCollection: ->
-    @db.collection('posts')
+    @db.collection('posts').ensureIndex([['facebook_id', 1]], true, -> )
 
   close: ->
     @db.close()
 
   findAll: (callback) ->
-    @getCollection().open (error, collection) ->
+    @db.collection('posts').open (error, collection) ->
       if error
         callback(error)
       else
@@ -22,32 +20,46 @@ class MongoService
           else 
             callback(null, posts)
 
+  # save posts to database, update records as needed
   save: (posts, callback) ->
     # if posts is not an array, make it an array
     if typeof(posts.length) == "undefined"
       posts = [posts]
 
     # preprocess the posts
+    counter = 0
     for post in posts
+      counter += 1
+      console.log("(#{counter}) preprocessing message: #{post.id}")
+
+      post.facebook_id = post.id
       post._id = post.id
+      delete post.id
 
       post.created_at = ISODate("#{post.created_time[0..21]}:#{post.created_time[22..24]}")
-      post.updated_at = ISODate("#{post.updated_time[0..21]}:#{post.updated_time[22..24]}")
-
+      delete post.created_time if post.created_time
+
+      post.updated_at = ISODate("#{post.updated_time[0..21]}:#{post.updated_time[22..24]}") if post.updated_time
+      delete post.updated_time if post.updated_time
+
       if post.comments == undefined 
         post.comments = []
 
       for comment in post.comments
-        comment.created_at = ISODate("#{comment.created_time[0..21]}:#{comment.created_time[22..24]}")
-        comment.updated_at = ISODate("#{comment.updated_time[0..21]}:#{comment.updated_time[22..24]}")
-    
+        comment.created_at = ISODate("#{comment.created_time[0..21]}:#{comment.created_time[22..24]}") if comment.created_time
+        comment.updated_at = ISODate("#{comment.updated_time[0..21]}:#{comment.updated_time[22..24]}") if comment.updated_time
+
     # save posts
-    @getCollection().open (error, collection) ->
-      collection.insert posts, {}, (error, results) ->
-        if error 
-          callback(error)
-        else 
-          callback(null, results)
+    @db.collection('posts').open (error, collection) =>
+      postCount = posts.length
+      counter   = 0
+      for idx, post of posts        
+        collection.save post, {safe: true, upsert: true}, (error, result) ->
+          counter++
+          if error
+            callback(error)
+          else if counter == postCount
+            callback(null, posts)
 
 root = exports ? window
 root.MongoService = MongoService

package.json

@@ -6,7 +6,8 @@
     "coffee-script": ">= 1.1.2",
     "facebook-graph": ">= 0.0.6",
     "ejs": ">= 0.4.3",
-    "isodate": ">= 0.0.2"
-    "mongoskin": ">= 0.1.3"
+    "isodate": ">= 0.0.2",
+    "mongoskin": ">= 0.1.3",
+    "underscore": "1.1.7"
   }
 }

views/index.ejs

@@ -5,7 +5,7 @@
   <div class="span9 columns">
     <article class="post">
       <div class="body">
-        <h2><%- feed.caption %> <small><%- feed.from.name %></small> <small class="easydate"><%- feed.created_time %></small></h2>    
+        <h2><%- feed.caption %> <small><%- feed.from.name %></small> <small class="easydate"><%- feed.created_at %></small></h2>    
         <pre><%- feed.message %></pre>
         <small class='comment-count'><%= feed.comments.count %> comments</small>
       </div>

views/layout.ejs

@@ -23,18 +23,7 @@
 
   <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.6.2/jquery.min.js" type="text/javascript" charset="utf-8"></script>
   <script src="/js/jquert-eastdate-0.2.4.min.js" type="text/javascript" charset="utf-8"></script>
-  <script>jQuery(function($) { 
-    $(".easydate").each(function(idx, elem){
-      var dateStr = $(elem).text()
-      var year    = dateStr.substring(0, 4)
-      var month   = dateStr.substring(5, 7)
-      var day     = dateStr.substring(8, 10)
-      var hour    = dateStr.substring(11, 13)
-      var min     = dateStr.substring(14, 16)
-      var sec     = dateStr.substring(17, 19)
-      var date    = new Date(year, month-1, day, hour-8, min, sec)
-      $(elem).text(date.toString())
-    }).easydate([]);
+  <script>jQuery(function($) {
     $(".easydate").easydate([]); 
   });
   </script>

web.coffee

@@ -1,26 +1,25 @@
 express           = require 'express'
 ejs               = require 'ejs'
-{FacebookService} = require './lib/facebook_service'
+{MongoService}    = require './lib/mongo_service'
 
 # Configuration
-groupId       = process.env.FB_GROUP_ID ? "133426573417117"
-accessToken   = process.env.FB_GRAPH_TOKEN
 port          = process.env.PORT || 3000
+mongo_url     = process.env.MONGOHQ_URL
 
 # Setup services
-fbService     = new FacebookService accessToken
+mongo         = new MongoService mongo_url
 app           = express.createServer express.logger()
 app.use express.static("#{__dirname}/public")
 
 # Handle Requests
 app.get '/', (req, res) ->
-  feeds   = fbService.getFeed groupId, null, (error, feeds) => 
+  feeds   = mongo.findAll (error, feeds) => 
     if error
       console.log error
-      res.send "Error contacting Facebook: #{error.message}"
+      res.send "Error contacting mongo: #{error.message}"
+
     else
-      data    = feeds.data
-      paging  = feeds.paging
+      data    = feeds
       res.render 'index.ejs', {data}
 
 # Start Listening