Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

saving tweeters to mongo

  • Loading branch information...
commit 1047afbf40b57da15030b659829ea2b9d1ed6be4 1 parent 52d55ff
Peter Bengtsson authored
1  bin/_run_tests.py
@@ -11,6 +11,7 @@
11 11
12 12 TEST_MODULES = [
13 13 'tests.test_handlers',
  14 + 'tests.test_models',
14 15 ]
15 16
16 17
66 handlers.py
... ... @@ -1,18 +1,18 @@
  1 +import re
  2 +import datetime
1 3 import os
2 4 import logging
3 5 from pprint import pprint, pformat
4 6 import tornado.auth
5 7 import tornado.web
6 8 import tornado.gen
7   -#from tornado import gen
8 9 from tornado.web import HTTPError
9 10 from tornado_utils.routes import route
10 11 from tornado.escape import json_decode, json_encode
11 12 from pymongo.objectid import InvalidId, ObjectId
12 13 import utils
13   -#import settings
14 14
15   -from models import User
  15 +from models import User, Tweeter
16 16
17 17
18 18 class BaseHandler(tornado.web.RequestHandler):
@@ -41,6 +41,53 @@ def redis(self):
41 41 def db(self):
42 42 return self.application.db
43 43
  44 + def save_tweeter_user(self, user):
  45 + user_id = user['id']
  46 + tweeter = self.db.Tweeter.find_one({'user_id': user_id})
  47 + _save = False
  48 + if not tweeter:
  49 + tweeter = self.db.Tweeter()
  50 + tweeter['user_id'] = user_id
  51 + _save = True
  52 +
  53 + if tweeter['name'] != user['name']:
  54 + tweeter['name'] = user['name']
  55 + _save = True
  56 +
  57 + if tweeter['username'] != user['screen_name']:
  58 + tweeter['username'] = user['screen_name']
  59 + _save = True
  60 +
  61 + if tweeter['followers'] != user['followers_count']:
  62 + tweeter['followers'] = user['followers_count']
  63 + _save = True
  64 +
  65 + if tweeter['following'] != user['friends_count']:
  66 + tweeter['following'] = user['friends_count']
  67 + _save = True
  68 +
  69 + def parse_status_date(dstr):
  70 + dstr = re.sub('\+\d{1,4}', '', dstr)
  71 + return datetime.datetime.strptime(
  72 + dstr,
  73 + '%a %b %d %H:%M:%S %Y'
  74 + )
  75 + last_tweet_date = None
  76 + if 'status' in user:
  77 + last_tweet_date = user['status']['created_at']
  78 + last_tweet_date = parse_status_date(last_tweet_date)
  79 + if tweeter['last_tweet_date'] != last_tweet_date:
  80 + tweeter['last_tweet_date'] = last_tweet_date
  81 + _save = True
  82 +
  83 + ratio_before = tweeter['ratio']
  84 + ratio = tweeter.set_ratio()
  85 + if ratio != ratio_before:
  86 + _save = True
  87 +
  88 + if _save:
  89 + tweeter.save()
  90 +
44 91
45 92 @route('/')
46 93 class HomeHandler(BaseHandler):
@@ -75,6 +122,7 @@ def increment_lookup_count(self, username, usernames, jsonp=False):
75 122 self.redis.incr(key)
76 123
77 124 key = 'lookups:username:%s' % username
  125 + assert username
78 126 self.redis.incr(key)
79 127
80 128 key = 'lookups:usernames'
@@ -248,7 +296,9 @@ def _on_auth(self, user_struct):
248 296 options['page_title'] = "Twitter authentication failed"
249 297 self.render('twitter_auth_failed.html', **options)
250 298 return
251   - username = user_struct.get('username')
  299 +
  300 + username = user_struct.get('username',
  301 + user_struct.get('screen_name'))
252 302 access_token = user_struct['access_token']
253 303 assert access_token
254 304 user = self.db.User.find_one({'username': username})
@@ -263,6 +313,8 @@ def _on_auth(self, user_struct):
263 313 self.set_secure_cookie("user",
264 314 str(user['_id']),
265 315 expires_days=30, path='/')
  316 +
  317 + self.save_tweeter_user(user_struct)
266 318 self.redirect('/')
267 319
268 320
@@ -349,6 +401,8 @@ def _fetch_info(self, options, username=None):
349 401 "/users/show",
350 402 screen_name=username,
351 403 access_token=access_token)
  404 + if result:
  405 + self.save_tweeter_user(result)
352 406 else:
353 407 result = json_decode(value)
354 408 key = None
@@ -455,8 +509,6 @@ def get(self):
455 509 @route('/everyone', name='everyone')
456 510 class EveryoneIFollowHandler(BaseHandler, tornado.auth.TwitterMixin):
457 511
458   - #@tornado.web.asynchronous
459   - #@tornado.gen.engine
460 512 def get(self):
461 513 current_user = self.get_current_user()
462 514 if not current_user:
@@ -526,6 +578,7 @@ def get(self):
526 578 for user in users:
527 579 username = user['screen_name']
528 580 key = 'screen_name:%s' % user['id']
  581 + self.save_tweeter_user(user)
529 582 self.redis.setex(key, username, 7 * 24 * 60 * 60)
530 583 screen_names.append(username)
531 584
@@ -536,6 +589,7 @@ def get(self):
536 589 self.finish()
537 590
538 591
  592 +
539 593 @route('/lookups', name='lookups')
540 594 class LookupsHandler(BaseHandler):
541 595
39 models.py
@@ -2,8 +2,22 @@
2 2 from mongolite import Connection, Document
3 3 connection = Connection()
4 4
  5 +class BaseDocument(Document):
  6 + skeleton = {
  7 + 'modify_date': datetime.datetime
  8 + }
  9 +
  10 + default_values = {'modify_date': datetime.datetime.utcnow}
  11 +
  12 + def save(self, *args, **kwargs):
  13 + if '_id' in self and kwargs.get('update_modify_date', True):
  14 + m = datetime.datetime.utcnow()
  15 + self['modify_date'] = m
  16 + super(BaseDocument, self).save(*args, **kwargs)
  17 +
  18 +
5 19 @connection.register
6   -class User(Document):
  20 +class User(BaseDocument):
7 21 __collection__ = 'users'
8 22 skeleton = {
9 23 'username': unicode,
@@ -14,9 +28,22 @@ class User(Document):
14 28 'user_id': int,
15 29 }
16 30
17   - default_values = {'modify_date':datetime.datetime.utcnow}
18 31
19   - def save(self, *args, **kwargs):
20   - if '_id' in self and kwargs.get('update_modify_date', True):
21   - self.modify_date = datetime.datetime.utcnow()
22   - super(User, self).save(*args, **kwargs)
  32 +@connection.register
  33 +class Tweeter(BaseDocument):
  34 + __collection__ = 'tweeters'
  35 + skeleton = {
  36 + 'user_id': int,
  37 + 'username': unicode,
  38 + 'name': unicode,
  39 + 'followers': int,
  40 + 'following': int,
  41 + 'ratio': float,
  42 + 'last_tweet_date': datetime.datetime,
  43 + }
  44 + optional = {
  45 + 'ratio_rank': int,
  46 + }
  47 +
  48 + def set_ratio(self):
  49 + self['ratio'] = 1.0 * self['followers'] / max(self['following'], 1)
2  static/js/lookups.js
@@ -89,7 +89,7 @@ function update() {
89 89 if (before !== '' + num) {
90 90 // there's a change!
91 91 $(key).fadeTo(200, 0.1, function() {
92   - $(this).text(num).fadeTo(300, 1.0);
  92 + $(this).text(num).fadeIn(300);
93 93 });
94 94 }
95 95 }
5 templates/test.html
@@ -8,6 +8,7 @@
8 8 <div style="float:left">
9 9 <textarea name="usernames" cols="20" rows="20">katyperry
10 10 justinbieber
  11 +peterbe
11 12 </textarea><input type="submit" value="Test">
12 13 </div>
13 14 </form>
@@ -42,8 +43,8 @@
42 43 c = $('ul#followsyou');
43 44 else
44 45 c = $('ul#followsyounot');
45   - $('<a>', {href: 'https://twitter.com/' + x})
46   - .appendTo($('<li>', {text: x}).appendTo(c));
  46 + $('<a>', {href: '/following/' + x, text: x})
  47 + .appendTo($('<li>').appendTo(c));
47 48 });
48 49 });
49 50 return false;
4 tests/base.py
@@ -13,7 +13,7 @@
13 13 import app
14 14 from tornado_utils.http_test_client import TestClient, HTTPClientMixin
15 15
16   -class _DatabaseTestCaseMixin(object):
  16 +class DatabaseTestCaseMixin(object):
17 17 _once = False
18 18
19 19 def setup_connection(self):
@@ -30,7 +30,7 @@ def _emptyCollections(self):
30 30 if x not in ('system.indexes',)]
31 31
32 32
33   -class BaseAsyncTestCase(AsyncHTTPTestCase, _DatabaseTestCaseMixin):
  33 +class BaseAsyncTestCase(AsyncHTTPTestCase, DatabaseTestCaseMixin):
34 34
35 35 def setUp(self):
36 36 super(BaseAsyncTestCase, self).setUp()
89 tests/test_handlers.py
... ... @@ -1,8 +1,10 @@
  1 +import datetime
1 2 import os
2 3 import json
3 4 from urllib import urlencode
4 5 from .base import BaseHTTPTestCase
5   -from handlers import TwitterAuthHandler, FollowsHandler, FollowingHandler
  6 +from handlers import (TwitterAuthHandler, FollowsHandler, FollowingHandler,
  7 + EveryoneIFollowJSONHandler)
6 8
7 9 class HandlersTestCase(BaseHTTPTestCase):
8 10
@@ -59,12 +61,16 @@ def test_twitter_login_twitter_failing(self):
59 61
60 62 def _login(self, username=u'peterbe', name=u'Peter Bengtsson',
61 63 email=None):
  64 + assert username
62 65 struct = {
63 66 'name': name,
64   - 'username': username,
  67 + 'screen_name': username,
65 68 'email': email,
66 69 'access_token': {'key': '0123456789',
67   - 'secret': 'xxx'}
  70 + 'secret': 'xxx'},
  71 + 'id': 9876543210,
  72 + 'followers_count': 333,
  73 + 'friends_count': 222,
68 74 }
69 75 TwitterAuthHandler.get_authenticated_user = \
70 76 make_twitter_get_authenticated_user_callback(struct)
@@ -282,6 +288,7 @@ def test_following(self):
282 288 u'friends_count': 1300,
283 289 u'name': u'Barak',
284 290 u'screen_name': u'obama',
  291 + 'id': 9876543210,
285 292 },
286 293 "/users/show?screen_name=peterbe": {
287 294 u'followers_count': 417,
@@ -289,6 +296,7 @@ def test_following(self):
289 296 u'friends_count': 330,
290 297 u'name': u'Peter Bengtsson',
291 298 u'screen_name': u'peterbe',
  299 + 'id': 123456789,
292 300 }
293 301 })
294 302
@@ -311,6 +319,7 @@ def test_following(self):
311 319 u'friends_count': 1301,
312 320 u'name': u'Barak',
313 321 u'screen_name': u'obama',
  322 + 'id': 9876543210,
314 323 },
315 324 "/users/show?screen_name=peterbe": {
316 325 u'followers_count': 417,
@@ -318,6 +327,7 @@ def test_following(self):
318 327 u'friends_count': 331,
319 328 u'name': u'Peter Bengtsson',
320 329 u'screen_name': u'peterbe',
  330 + 'id': 123456789,
321 331 }
322 332 })
323 333
@@ -339,6 +349,7 @@ def test_following(self):
339 349 u'friends_count': 1000,
340 350 u'name': u'West',
341 351 u'screen_name': u'chris',
  352 + 'id': 112233445566,
342 353 },
343 354 })
344 355 response = self.client.get(url)
@@ -366,6 +377,7 @@ def test_following_temporary_glitch_on_info(self):
366 377 u'friends_count': 1300,
367 378 u'name': u'Barak',
368 379 u'screen_name': u'obama',
  380 + u'id': 1233456365
369 381 },
370 382 "/users/show?screen_name=peterbe": None
371 383 })
@@ -427,6 +439,7 @@ def test_following_someone_following_0(self):
427 439 u'friends_count': 0,
428 440 u'name': u'Barak',
429 441 u'screen_name': u'obama',
  442 + 'id': 987654321,
430 443 },
431 444 "/users/show?screen_name=peterbe": {
432 445 u'followers_count': 417,
@@ -434,6 +447,7 @@ def test_following_someone_following_0(self):
434 447 u'friends_count': 330,
435 448 u'name': u'Peter Bengtsson',
436 449 u'screen_name': u'peterbe',
  450 + 'id': 123456789
437 451 }
438 452 })
439 453
@@ -468,6 +482,66 @@ def test_screenshots(self):
468 482 self.assertTrue('alt="%s"' % title in response.body)
469 483 self.assertTrue('title="%s"' % title in response.body)
470 484
  485 + def test_everyone_json(self):
  486 + url = self.reverse_url('everyone_json')
  487 + response = self.client.get(url)
  488 + self.assertEqual(response.code, 403)
  489 + self._login()
  490 +
  491 + EveryoneIFollowJSONHandler.twitter_request = \
  492 + make_mock_twitter_request({
  493 + "/friends/ids": [123456789, 987654321],
  494 + "/users/lookup": [
  495 + {'id': 987654321,
  496 + 'followers_count': 41700,
  497 + 'following': False,
  498 + 'friends_count': 0,
  499 + 'name': u'Barak',
  500 + 'screen_name': u'obama',
  501 + 'status': {
  502 + 'created_at': u'Wed Oct 12 20:12:27 +0000 2011',
  503 + }
  504 + },
  505 + {'id': 123456789,
  506 + u'followers_count': 417,
  507 + u'following': False,
  508 + u'friends_count': 330,
  509 + u'name': u'Peter Bengtsson',
  510 + u'screen_name': u'peter',
  511 + 'created_at': u'Thu Jan 04 17:13:11 +0000 2007',
  512 + }
  513 + ]
  514 + })
  515 +
  516 + response = self.client.get(url)
  517 + struct = json.loads(response.body)
  518 + self.assertEqual(struct, ['obama', 'peter'])
  519 +
  520 + # this should have created some Tweeters
  521 + self.assertEqual(self.db.Tweeter
  522 + .find({'username': {'$ne':'peterbe'}})
  523 + .count(), 2)
  524 + obama = self.db.Tweeter.find_one({'username': 'obama'})
  525 + self.assertEqual(obama['user_id'], 987654321)
  526 + self.assertEqual(obama['ratio'], 41700.0)
  527 + self.assertEqual(obama['following'], 0)
  528 + self.assertEqual(obama['followers'], 41700)
  529 + self.assertEqual(obama['name'], 'Barak')
  530 + fmt = '%Y-%m-%d-%H-%M-%S'
  531 + self.assertEqual(
  532 + obama['last_tweet_date'].strftime(fmt),
  533 + datetime.datetime(2011, 10, 12, 20, 12, 27).strftime(fmt)
  534 + )
  535 +
  536 + peter = self.db.Tweeter.find_one({'username': 'peter'})
  537 + self.assertEqual(peter['user_id'], 123456789)
  538 + self.assertEqual(peter['ratio'], 417.0/330)
  539 + self.assertEqual(peter['following'], 330)
  540 + self.assertEqual(peter['followers'], 417)
  541 + self.assertEqual(peter['name'], 'Peter Bengtsson')
  542 + self.assertEqual(peter['last_tweet_date'], None)
  543 +
  544 +
471 545 def make_twitter_get_authenticated_user_callback(struct):
472 546 def twitter_get_authenticated_user(self, callback, **kw):
473 547 callback(struct)
@@ -478,7 +552,11 @@ def twitter_get_authenticated_user(self, callback, **kw):
478 552 'name': u'Peter Bengtsson',
479 553 'username': u'peterbe',
480 554 'email': None,
481   - 'access_token': {'key': '0123456789', 'secret': 'xxx'}
  555 + 'access_token': {'key': '0123456789', 'secret': 'xxx'},
  556 + 'id': 123456789,
  557 + 'screen_name': u'peterbe',
  558 + 'followers_count': 300,
  559 + 'friends_count': 255,
482 560 })
483 561
484 562 def twitter_authenticate_redirect(self):
@@ -551,7 +629,8 @@ def twitter_request(self, path, callback, **kw):
551 629 u'screen_name': u'fox2mike'}]
552 630
553 631 """
554   - kw.pop('access_token')
  632 + if 'access_token' in kw:
  633 + kw.pop('access_token')
555 634 long_path = path + '?' + urlencode(kw)
556 635 if isinstance(result, dict):
557 636 send_back = result.get(long_path, result.get(path, result))
32 tests/test_models.py
... ... @@ -0,0 +1,32 @@
  1 +from unittest import TestCase
  2 +from models import User, connection
  3 +from .base import DatabaseTestCaseMixin
  4 +
  5 +class ModelsTestCase(TestCase, DatabaseTestCaseMixin):
  6 +
  7 + def setUp(self):
  8 + self.db = connection.test
  9 + super(ModelsTestCase, self).setUp()
  10 + self.setup_connection()
  11 +
  12 + def tearDown(self):
  13 + self.teardown_connection()
  14 +
  15 + def test_create_user(self):
  16 + assert not self.db.User.find().count()
  17 + user = self.db.User()
  18 + user.username = u'bob'
  19 + user.access_token = {'key': 'xxx', 'secret': 'yyy'}
  20 + user.save()
  21 +
  22 + b = user['modify_date']
  23 + self.assertTrue(b)
  24 + from time import sleep
  25 + # mongodb is a bit weird when it comes to actual saving time
  26 + # so introduce a realistic phsyical waiting time.
  27 + sleep(0.001)
  28 + user.username = u'bobby'
  29 + user.save()
  30 +
  31 + a = user['modify_date']
  32 + self.assertTrue(a.microsecond > b.microsecond)

0 comments on commit 1047afb

Please sign in to comment.
Something went wrong with that request. Please try again.