Permalink
Browse files

saving tweeters to mongo

  • Loading branch information...
1 parent 52d55ff commit 1047afbf40b57da15030b659829ea2b9d1ed6be4 @peterbe committed Oct 13, 2011
Showing with 216 additions and 22 deletions.
  1. +1 −0 bin/_run_tests.py
  2. +60 −6 handlers.py
  3. +33 −6 models.py
  4. +1 −1 static/js/lookups.js
  5. +3 −2 templates/test.html
  6. +2 −2 tests/base.py
  7. +84 −5 tests/test_handlers.py
  8. +32 −0 tests/test_models.py
View
@@ -11,6 +11,7 @@
TEST_MODULES = [
'tests.test_handlers',
+ 'tests.test_models',
]
View
@@ -1,18 +1,18 @@
+import re
+import datetime
import os
import logging
from pprint import pprint, pformat
import tornado.auth
import tornado.web
import tornado.gen
-#from tornado import gen
from tornado.web import HTTPError
from tornado_utils.routes import route
from tornado.escape import json_decode, json_encode
from pymongo.objectid import InvalidId, ObjectId
import utils
-#import settings
-from models import User
+from models import User, Tweeter
class BaseHandler(tornado.web.RequestHandler):
@@ -41,6 +41,53 @@ def redis(self):
def db(self):
return self.application.db
+ def save_tweeter_user(self, user):
+ user_id = user['id']
+ tweeter = self.db.Tweeter.find_one({'user_id': user_id})
+ _save = False
+ if not tweeter:
+ tweeter = self.db.Tweeter()
+ tweeter['user_id'] = user_id
+ _save = True
+
+ if tweeter['name'] != user['name']:
+ tweeter['name'] = user['name']
+ _save = True
+
+ if tweeter['username'] != user['screen_name']:
+ tweeter['username'] = user['screen_name']
+ _save = True
+
+ if tweeter['followers'] != user['followers_count']:
+ tweeter['followers'] = user['followers_count']
+ _save = True
+
+ if tweeter['following'] != user['friends_count']:
+ tweeter['following'] = user['friends_count']
+ _save = True
+
+ def parse_status_date(dstr):
+ dstr = re.sub('\+\d{1,4}', '', dstr)
+ return datetime.datetime.strptime(
+ dstr,
+ '%a %b %d %H:%M:%S %Y'
+ )
+ last_tweet_date = None
+ if 'status' in user:
+ last_tweet_date = user['status']['created_at']
+ last_tweet_date = parse_status_date(last_tweet_date)
+ if tweeter['last_tweet_date'] != last_tweet_date:
+ tweeter['last_tweet_date'] = last_tweet_date
+ _save = True
+
+ ratio_before = tweeter['ratio']
+ ratio = tweeter.set_ratio()
+ if ratio != ratio_before:
+ _save = True
+
+ if _save:
+ tweeter.save()
+
@route('/')
class HomeHandler(BaseHandler):
@@ -75,6 +122,7 @@ def increment_lookup_count(self, username, usernames, jsonp=False):
self.redis.incr(key)
key = 'lookups:username:%s' % username
+ assert username
self.redis.incr(key)
key = 'lookups:usernames'
@@ -248,7 +296,9 @@ def _on_auth(self, user_struct):
options['page_title'] = "Twitter authentication failed"
self.render('twitter_auth_failed.html', **options)
return
- username = user_struct.get('username')
+
+ username = user_struct.get('username',
+ user_struct.get('screen_name'))
access_token = user_struct['access_token']
assert access_token
user = self.db.User.find_one({'username': username})
@@ -263,6 +313,8 @@ def _on_auth(self, user_struct):
self.set_secure_cookie("user",
str(user['_id']),
expires_days=30, path='/')
+
+ self.save_tweeter_user(user_struct)
self.redirect('/')
@@ -349,6 +401,8 @@ def _fetch_info(self, options, username=None):
"/users/show",
screen_name=username,
access_token=access_token)
+ if result:
+ self.save_tweeter_user(result)
else:
result = json_decode(value)
key = None
@@ -455,8 +509,6 @@ def get(self):
@route('/everyone', name='everyone')
class EveryoneIFollowHandler(BaseHandler, tornado.auth.TwitterMixin):
- #@tornado.web.asynchronous
- #@tornado.gen.engine
def get(self):
current_user = self.get_current_user()
if not current_user:
@@ -526,6 +578,7 @@ def get(self):
for user in users:
username = user['screen_name']
key = 'screen_name:%s' % user['id']
+ self.save_tweeter_user(user)
self.redis.setex(key, username, 7 * 24 * 60 * 60)
screen_names.append(username)
@@ -536,6 +589,7 @@ def get(self):
self.finish()
+
@route('/lookups', name='lookups')
class LookupsHandler(BaseHandler):
View
@@ -2,8 +2,22 @@
from mongolite import Connection, Document
connection = Connection()
+class BaseDocument(Document):
+ skeleton = {
+ 'modify_date': datetime.datetime
+ }
+
+ default_values = {'modify_date': datetime.datetime.utcnow}
+
+ def save(self, *args, **kwargs):
+ if '_id' in self and kwargs.get('update_modify_date', True):
+ m = datetime.datetime.utcnow()
+ self['modify_date'] = m
+ super(BaseDocument, self).save(*args, **kwargs)
+
+
@connection.register
-class User(Document):
+class User(BaseDocument):
__collection__ = 'users'
skeleton = {
'username': unicode,
@@ -14,9 +28,22 @@ class User(Document):
'user_id': int,
}
- default_values = {'modify_date':datetime.datetime.utcnow}
- def save(self, *args, **kwargs):
- if '_id' in self and kwargs.get('update_modify_date', True):
- self.modify_date = datetime.datetime.utcnow()
- super(User, self).save(*args, **kwargs)
+@connection.register
+class Tweeter(BaseDocument):
+ __collection__ = 'tweeters'
+ skeleton = {
+ 'user_id': int,
+ 'username': unicode,
+ 'name': unicode,
+ 'followers': int,
+ 'following': int,
+ 'ratio': float,
+ 'last_tweet_date': datetime.datetime,
+ }
+ optional = {
+ 'ratio_rank': int,
+ }
+
+ def set_ratio(self):
+ self['ratio'] = 1.0 * self['followers'] / max(self['following'], 1)
View
@@ -89,7 +89,7 @@ function update() {
if (before !== '' + num) {
// there's a change!
$(key).fadeTo(200, 0.1, function() {
- $(this).text(num).fadeTo(300, 1.0);
+ $(this).text(num).fadeIn(300);
});
}
}
View
@@ -8,6 +8,7 @@
<div style="float:left">
<textarea name="usernames" cols="20" rows="20">katyperry
justinbieber
+peterbe
</textarea><input type="submit" value="Test">
</div>
</form>
@@ -42,8 +43,8 @@
c = $('ul#followsyou');
else
c = $('ul#followsyounot');
- $('<a>', {href: 'https://twitter.com/' + x})
- .appendTo($('<li>', {text: x}).appendTo(c));
+ $('<a>', {href: '/following/' + x, text: x})
+ .appendTo($('<li>').appendTo(c));
});
});
return false;
View
@@ -13,7 +13,7 @@
import app
from tornado_utils.http_test_client import TestClient, HTTPClientMixin
-class _DatabaseTestCaseMixin(object):
+class DatabaseTestCaseMixin(object):
_once = False
def setup_connection(self):
@@ -30,7 +30,7 @@ def _emptyCollections(self):
if x not in ('system.indexes',)]
-class BaseAsyncTestCase(AsyncHTTPTestCase, _DatabaseTestCaseMixin):
+class BaseAsyncTestCase(AsyncHTTPTestCase, DatabaseTestCaseMixin):
def setUp(self):
super(BaseAsyncTestCase, self).setUp()
Oops, something went wrong.

0 comments on commit 1047afb

Please sign in to comment.