for updating existing user

ktmud · Mar 3, 2014 · 524ff74 · 524ff74
1 parent 5f702c7
commit 524ff74
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 34 deletions.
diff --git a/tasks/compute/index.js b/tasks/compute/index.js
@@ -209,7 +209,7 @@ function run_toplist(user, total) {
   toplist_timer = setTimeout(function() {
     // generate toplist one by one
     require('../toplist').run(total)
-  }, central.DEBUG ? 2000 : 60 * 1000); // 30 minutes of free
+  }, central.DEBUG ? 10000 : 60 * 1000); // 30 minutes of free
 }
 
 central.DOUBAN_APPS.forEach(function(ns) {

diff --git a/tools/update.js b/tools/update.js
@@ -3,67 +3,82 @@
 /**
  * Update all users in background.
  */
-var log = require('debug')('dbj:tool:update');
+var log = require('debug')('dbj:tool:update')
 
-var User = require('../models/user');
-var tasks = require('../tasks');
+var User = require('../models/user')
+var cached = require('../lib/cached')
+var tasks = require('../tasks')
 
-tasks.setKeyPrefix('dbj-cron-update-');
+tasks.setKeyPrefix('dbj-cron-update-')
 
-var oneday = 60 * 60 * 24 * 1000;
-var oneweek = oneday * 7;
+var oneday = 60 * 60 * 24 * 1000
+var oneweek = oneday * 7
 
 function updateAll(query) {
-  var now = new Date();
+  var now = new Date()
+  var canExit = false
+  var blacklist = []
 
-  if (tatks.getQueueLength()) {
-    log('There are unfinished task. Exit.');
-    return;
+  if (tasks.getQueueLength()) {
+    log('There are unfinished task. Exit.')
+    return
   }
 
+  cached.get('user_blacklist', function(err, res) {
+    if (res && res.length) {
+      blacklist = res
+    }
+  })
+
   User.stream(query, { limit: null }, function(stream) {
+    function resume() {
+      if (stream.paused) {
+        stream.resume()
+      }
+      if (canExit) {
+        log('About to exit.')
+        process.exit()
+      }
+    }
     stream.on('data', function(doc) {
-      var u = new User(doc);
-
-      stream.pause();
-
+      if (~blacklist.indexOf(doc._id)) {
+        return resume()
+      }
+      var u = new User(doc)
+      stream.pause()
       u.pull(function() {
         tasks.interest.collect_book({
           user: u,
           force: true,
           fresh: false,
           success: function() {
-            log('Callback sussess for user %s [%s].', u.uid, u.name);
-            if (stream.paused) {
-              stream.resume();
-            }
+            log('Callback sussess for user %s [%s].', u.uid, u.name)
+            resume()
           },
           error: function() {
-            log('Callback error for user %s [%s].', u.uid, u.name);
-            if (stream.paused) {
-              stream.resume();
-            }
+            log('Callback error for user %s [%s].', u.uid, u.name)
+            resume()
           }
-        });
-      });
-      log('Queue user %s [%s]', u.uid, u.name);
-    });
+        })
+      })
+      log('Queue user %s [%s]', u.uid, u.name)
+    })
     stream.on('close', function() {
-      log('=== Stream closed. ===');
-      setTimeout(process.exit, 120000);
-    });
-  });
+      log('=== Stream closed. ===')
+      canExit = true
+    })
+  })
 }
 
-setTimeout(updateAll, 2000, {
+setTimeout(updateAll, 1000, {
   last_synced_status: {
     $ne: 'ing'
   },
   book_n: {
-    $gt: 100
+    $gt: 500
   },
   // 一周之内更新过的用户就不再更新
   last_synced: {
     $lt: new Date(new Date() - oneweek)
   },
-});
+})