Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Implemented Celery MQ (using Redis) on user details lookups.

  • Loading branch information...
commit 1d872f05e1961c5d2cd9ee93836a056d436780fe 1 parent f3077e2
Peter Bengtsson authored November 02, 2011
10  README.md
Source Rendered
@@ -11,4 +11,12 @@ Running tests
11 11
 
12 12
 Run this:
13 13
 
14  
-        python bin/_run_tests.py --logging=error
  14
+        $ python bin/_run_tests.py --logging=error
  15
+
  16
+
  17
+Running celeryd
  18
+---------------
  19
+
  20
+Run celeryd like this:
  21
+
  22
+        $ celeryd --loglevel=INFO
8  bin/run_shell.py
... ...
@@ -1,7 +1,13 @@
1 1
 #!/usr/bin/env python
2 2
 
3 3
 import code, re
4  
-import here
  4
+try:
  5
+    import here
  6
+except ImportError:
  7
+    import sys
  8
+    import os.path as op
  9
+    sys.path.insert(0, op.abspath(op.join(op.dirname(__file__), '..')))
  10
+    import here
5 11
 
6 12
 if __name__ == '__main__':
7 13
 
15  celeryconfig.py
... ...
@@ -0,0 +1,15 @@
  1
+import here
  2
+# http://docs.celeryproject.org/en/latest/tutorials/otherqueues.html#redis
  3
+BROKER_TRANSPORT = "redis"
  4
+
  5
+import settings
  6
+BROKER_HOST = settings.REDIS_HOST
  7
+BROKER_PORT = settings.REDIS_PORT
  8
+BROKER_VHOST = "0"         # Maps to database number.
  9
+
  10
+CELERY_IGNORE_RESULT = True
  11
+
  12
+CELERY_IMPORTS = ("tasks", )
  13
+
  14
+import os
  15
+CELERY_ALWAYS_EAGER = bool(os.environ.get('ALWAYS_EAGER', False))
134  handlers.py
@@ -14,7 +14,7 @@
14 14
 from tornado.escape import json_decode, json_encode
15 15
 from pymongo.objectid import InvalidId, ObjectId
16 16
 import utils
17  
-
  17
+import tasks
18 18
 from models import User, Tweeter
19 19
 
20 20
 
@@ -61,50 +61,10 @@ def save_following(self, source_username, dest_username, result):
61 61
     def save_tweeter_user(self, user):
62 62
         user_id = user['id']
63 63
         tweeter = self.db.Tweeter.find_one({'user_id': user_id})
64  
-        _save = False
65 64
         if not tweeter:
66 65
             tweeter = self.db.Tweeter()
67 66
             tweeter['user_id'] = user_id
68  
-            _save = True
69  
-
70  
-        if tweeter['name'] != user['name']:
71  
-            tweeter['name'] = user['name']
72  
-            _save = True
73  
-
74  
-        if tweeter['username'] != user['screen_name']:
75  
-            tweeter['username'] = user['screen_name']
76  
-            _save = True
77  
-
78  
-        if tweeter['followers'] != user['followers_count']:
79  
-            tweeter['followers'] = user['followers_count']
80  
-            _save = True
81  
-
82  
-        if tweeter['following'] != user['friends_count']:
83  
-            tweeter['following'] = user['friends_count']
84  
-            _save = True
85  
-
86  
-        def parse_status_date(dstr):
87  
-            dstr = re.sub('\+\d{1,4}', '', dstr)
88  
-            return datetime.datetime.strptime(
89  
-              dstr,
90  
-              '%a %b %d %H:%M:%S %Y'
91  
-            )
92  
-        last_tweet_date = None
93  
-        if 'status' in user:
94  
-            last_tweet_date = user['status']['created_at']
95  
-            last_tweet_date = parse_status_date(last_tweet_date)
96  
-            if tweeter['last_tweet_date'] != last_tweet_date:
97  
-                tweeter['last_tweet_date'] = last_tweet_date
98  
-                _save = True
99  
-
100  
-        ratio_before = tweeter['ratio']
101  
-        ratio = tweeter.set_ratio()
102  
-        if ratio != ratio_before:
103  
-            _save = True
104  
-
105  
-        if _save:
106  
-            tweeter.save()
107  
-
  67
+        Tweeter.update_tweeter(tweeter, user)
108 68
         return tweeter
109 69
 
110 70
     def assert_tweeter_user(self, user):
@@ -520,53 +480,51 @@ def _fetch_info(self, options, username=None):
520 480
         if username is None:
521 481
             username = options['username']
522 482
 
523  
-        key = 'info:%s' % username
524  
-        value = self.redis.get(key)
  483
+        def age(d):
  484
+            return (datetime.datetime.utcnow() - d).seconds
525 485
 
526  
-        if value is None:
527  
-            user = self.db.User.find_one({'username': options['this_username']})
528  
-            access_token = user['access_token']
  486
+        tweeter = self.db.Tweeter.find_one({'username': username})
  487
+        current_user = self.get_current_user()
  488
+        if not tweeter:
  489
+            access_token = current_user['access_token']
529 490
             result = yield tornado.gen.Task(self.twitter_request,
530 491
                                             "/users/show",
531 492
                                             screen_name=username,
532 493
                                             access_token=access_token)
533  
-            if result:
534  
-                self.save_tweeter_user(result)
535  
-        else:
536  
-            result = json_decode(value)
537  
-            self.assert_tweeter_user(result)
538  
-            key = None
539 494
 
540  
-        if result is None:
  495
+            tweeter = self.save_tweeter_user(result)
  496
+        elif age(tweeter['modify_date']) > 3600:
  497
+            tasks.refresh_user_info.delay(
  498
+              username, current_user['access_token'])
  499
+
  500
+        if not tweeter:
541 501
             options['error'] = "Unable to look up info for %s" % username
542 502
             self._render(options)
543 503
             return
544  
-        if isinstance(result, basestring):
545  
-            result = json_decode(result)
546  
-        if key:
547  
-            self.redis.setex(key, json_encode(result), 60 * 60)
  504
+
548 505
         if 'info' not in options:
549  
-            options['info'] = {options['username']: result}
  506
+            options['info'] = {options['username']: tweeter}
550 507
             self._fetch_info(options, username=options['this_username'])
551 508
         else:
552  
-            options['info'][options['this_username']] = result
  509
+            options['info'][options['this_username']] = tweeter
553 510
             self._render(options)
554 511
 
555 512
     def _render(self, options):
556  
-        if 'error' not in options:
557  
-            if options['follows']:
558  
-                page_title = '%s follows me'
559  
-            else:
560  
-                page_title = '%s is too cool for me'
561  
-            self._set_ratio(options, 'username')
562  
-            self._set_ratio(options, 'this_username')
563  
-            options['page_title'] = page_title % options['username']
564  
-            options['perm_url'] = self.get_following_perm_url(
565  
-              options['username'], options['this_username'])
566  
-            self.render('following.html', **options)
567  
-        else:
  513
+        if 'error' in options:
568 514
             options['page_title'] = 'Error :('
569 515
             self.render('following_error.html', **options)
  516
+            return
  517
+
  518
+        if options['follows']:
  519
+            page_title = '%s follows me'
  520
+        else:
  521
+            page_title = '%s is too cool for me'
  522
+        options['page_title'] = page_title % options['username']
  523
+        options['perm_url'] = self.get_following_perm_url(
  524
+          options['username'],
  525
+          options['this_username']
  526
+        )
  527
+        self.render('following.html', **options)
570 528
 
571 529
     def _set_ratio(self, options, key):
572 530
         value = options[key]
@@ -680,11 +638,16 @@ class FollowingComparedtoHandler(FollowingHandler):
680 638
     @tornado.gen.engine
681 639
     def get(self, username, compared_to):
682 640
         options = {'compared_to': compared_to}
683  
-        tweeter = self.db.Tweeter.find_by_username(self.db, username)
684  
-        compared_tweeter = self.db.Tweeter.find_by_username(self.db, compared_to)
  641
+        tweeter = self.db.Tweeter.find_one({'username': username})
  642
+        compared_tweeter = self.db.Tweeter.find_one({'username': compared_to})
  643
+
  644
+        def age(d):
  645
+            return (datetime.datetime.utcnow() - d).seconds
  646
+
685 647
 
686 648
         current_user = self.get_current_user()
687 649
         if current_user:
  650
+
688 651
             # if we don't have tweeter info on any of them, fetch it
689 652
             if not tweeter:
690 653
                 # fetch it
@@ -693,12 +656,19 @@ def get(self, username, compared_to):
693 656
                                         screen_name=username,
694 657
                                         access_token=current_user['access_token'])
695 658
                 tweeter = self.save_tweeter_user(result)
  659
+            elif age(tweeter['modify_date']) > 3600:
  660
+                tasks.refresh_user_info.delay(
  661
+                  username, current_user['access_token'])
  662
+
696 663
             if not compared_tweeter:
697 664
                 result = yield tornado.gen.Task(self.twitter_request,
698 665
                                         "/users/show",
699 666
                                         screen_name=compared_to,
700 667
                                         access_token=current_user['access_token'])
701 668
                 compared_tweeter = self.save_tweeter_user(result)
  669
+            elif age(compared_tweeter['modify_date']) > 3600:
  670
+                tasks.refresh_user_info.delay(
  671
+                  compared_to, current_user['access_token'])
702 672
 
703 673
         elif not tweeter or not compared_tweeter:
704 674
             options = {
@@ -717,8 +687,8 @@ def get(self, username, compared_to):
717 687
         value = self.redis.get(key)
718 688
         if value is None:
719 689
             following = (self.db.Following
720  
-                       .find_one({'user': tweeter['_id'],
721  
-                                  'follows': compared_tweeter['_id']}))
  690
+                         .find_one({'user': tweeter['_id'],
  691
+                                    'follows': compared_tweeter['_id']}))
722 692
             if following:
723 693
                 options['follows'] = following['following']
724 694
             else:
@@ -735,19 +705,11 @@ def get(self, username, compared_to):
735 705
                                      (username, compared_to))
736 706
 
737 707
         options['info'] = {
738  
-          username: {
739  
-            'followers_count': tweeter['followers'],
740  
-            'friends_count': tweeter['following'],
741  
-          },
742  
-          compared_to: {
743  
-            'followers_count': compared_tweeter['followers'],
744  
-            'friends_count': compared_tweeter['following'],
745  
-          }
  708
+          username: tweeter,
  709
+          compared_to: compared_tweeter
746 710
         }
747 711
         options['username'] = username
748 712
         options['this_username'] = compared_to
749  
-        self._set_ratio(options, 'username')
750  
-        self._set_ratio(options, 'this_username')
751 713
         options['compared_to'] = compared_to
752 714
         options['perm_url'] = self.get_following_perm_url(
753 715
           options['username'], options['this_username'])
35  models.py
@@ -2,6 +2,9 @@
2 2
 import datetime
3 3
 from pymongo.objectid import ObjectId
4 4
 from mongolite import Connection, Document
  5
+
  6
+
  7
+
5 8
 connection = Connection()
6 9
 
7 10
 class BaseDocument(Document):
@@ -57,6 +60,38 @@ def find_by_username(db, username):
57 60
             tweeter = db.Tweeter.find_one({'username': re.compile(re.escape(username), re.I)})
58 61
         return tweeter
59 62
 
  63
+    @staticmethod
  64
+    def update_tweeter(tweeter, user):
  65
+        if tweeter['name'] != user['name']:
  66
+            tweeter['name'] = user['name']
  67
+
  68
+        if tweeter['username'] != user['screen_name']:
  69
+            tweeter['username'] = user['screen_name']
  70
+
  71
+        if tweeter['followers'] != user['followers_count']:
  72
+            tweeter['followers'] = user['followers_count']
  73
+
  74
+        if tweeter['following'] != user['friends_count']:
  75
+            tweeter['following'] = user['friends_count']
  76
+
  77
+        def parse_status_date(dstr):
  78
+            dstr = re.sub('\+\d{1,4}', '', dstr)
  79
+            return datetime.datetime.strptime(
  80
+              dstr,
  81
+              '%a %b %d %H:%M:%S %Y'
  82
+            )
  83
+        last_tweet_date = None
  84
+        if 'status' in user:
  85
+            last_tweet_date = user['status']['created_at']
  86
+            last_tweet_date = parse_status_date(last_tweet_date)
  87
+            if tweeter['last_tweet_date'] != last_tweet_date:
  88
+                tweeter['last_tweet_date'] = last_tweet_date
  89
+
  90
+        ratio_before = tweeter['ratio']
  91
+        tweeter.set_ratio()
  92
+        tweeter.save()
  93
+
  94
+
60 95
 
61 96
 @connection.register
62 97
 class Following(BaseDocument):
2  requirements.txt
@@ -2,3 +2,5 @@ redis
2 2
 tornado
3 3
 mongolite
4 4
 mock
  5
+tornado-utils
  6
+Celery
57  tasks.py
... ...
@@ -0,0 +1,57 @@
  1
+import logging
  2
+import tornado.escape
  3
+import tornado.auth
  4
+import tornado.ioloop
  5
+from celery.task import task
  6
+from celery import conf
  7
+import settings
  8
+from models import Tweeter, connection
  9
+
  10
+
  11
+
  12
+@task
  13
+def refresh_user_info(*args, **kwargs):
  14
+    try:
  15
+        _refresh_user_info(*args, **kwargs)
  16
+    except:
  17
+        logging.error("_refresh_user_info() failed", exc_info=True)
  18
+        if conf.ALWAYS_EAGER:
  19
+            raise
  20
+
  21
+def _refresh_user_info(username, access_token):
  22
+    #from time import sleep; sleep(5)
  23
+    uu = UserUpdate()
  24
+    def cb(r, *args, **kwargs):
  25
+        try:
  26
+            uu.callback(username, r)
  27
+        finally:
  28
+            if not conf.ALWAYS_EAGER:
  29
+                tornado.ioloop.IOLoop.instance().stop()
  30
+    uu.twitter_request("/users/show", cb, access_token=access_token,
  31
+                       screen_name=username)
  32
+    if not conf.ALWAYS_EAGER:
  33
+        tornado.ioloop.IOLoop.instance().start()
  34
+
  35
+
  36
+class UserUpdate(tornado.auth.TwitterMixin):
  37
+    def __init__(self):
  38
+        self.settings = dict(
  39
+            twitter_consumer_key=settings.TWITTER_CONSUMER_KEY,
  40
+            twitter_consumer_secret=settings.TWITTER_CONSUMER_SECRET,
  41
+        )
  42
+
  43
+    @property
  44
+    def db(self):
  45
+        return connection[settings.DATABASE_NAME]
  46
+
  47
+    def require_setting(self, key, error):
  48
+        assert key in self.settings, "%s (%s)" % (error, key)
  49
+
  50
+    def async_callback(self, func, callback):
  51
+        return callback
  52
+
  53
+    def callback(self, username, response):
  54
+        result = tornado.escape.json_decode(response.body)
  55
+        tweeter = self.db.Tweeter.find_one({'user_id': result['id']})
  56
+        assert tweeter['username'].lower() == username.lower()
  57
+        Tweeter.update_tweeter(tweeter, result)
16  templates/following.html
@@ -71,11 +71,11 @@
71 71
     data.addColumn('number', 'following');
72 72
     data.addRows(2);
73 73
     data.setValue(0, 0, USERNAME);
74  
-    data.setValue(0, 1, {{ info[username]['followers_count'] }});
75  
-    data.setValue(0, 2, {{ info[username]['friends_count'] }});
  74
+    data.setValue(0, 1, {{ info[username]['followers'] }});
  75
+    data.setValue(0, 2, {{ info[username]['following'] }});
76 76
     data.setValue(1, 0, COMPARED_TO ? COMPARED_TO : 'you');
77  
-    data.setValue(1, 1, {{ info[this_username]['followers_count'] }});
78  
-    data.setValue(1, 2, {{ info[this_username]['friends_count'] }});
  77
+    data.setValue(1, 1, {{ info[this_username]['followers'] }});
  78
+    data.setValue(1, 2, {{ info[this_username]['following'] }});
79 79
      var chart = new google.visualization.ColumnChart(document.getElementById('chart_div'));
80 80
     chart.draw(data, {width: 600, height: 400,
81 81
                       title: 'Coolness in terms of following and followers',
@@ -124,13 +124,13 @@
124 124
   </tr>
125 125
   <tr>
126 126
     <td class="label">followers:</td>
127  
-    <td>{{ info[username]['followers_count'] }}</td>
128  
-    <td>{{ info[this_username]['followers_count'] }}</td>
  127
+    <td>{{ info[username]['followers'] }}</td>
  128
+    <td>{{ info[this_username]['followers'] }}</td>
129 129
   </tr>
130 130
   <tr>
131 131
     <td class="label">following:</td>
132  
-    <td>{{ info[username]['friends_count'] }}</td>
133  
-    <td>{{ info[this_username]['friends_count'] }}</td>
  132
+    <td>{{ info[username]['following'] }}</td>
  133
+    <td>{{ info[this_username]['following'] }}</td>
134 134
   </tr>
135 135
   <tr>
136 136
     <td class="label">ratio:</td>
7  tests/base.py
@@ -8,11 +8,16 @@
8 8
 import hashlib
9 9
 import unittest
10 10
 
  11
+
11 12
 from tornado.testing import LogTrapTestCase, AsyncHTTPTestCase
  13
+os.environ['ALWAYS_EAGER'] = 'true'
  14
+import celery
  15
+import settings
12 16
 
13 17
 import app
14 18
 from tornado_utils.http_test_client import TestClient, HTTPClientMixin
15 19
 
  20
+
16 21
 class DatabaseTestCaseMixin(object):
17 22
     _once = False
18 23
 
@@ -51,6 +56,8 @@ def setUp(self):
51 56
           'tornado_utils.send_mail.backends.locmem.EmailBackend'
52 57
         self._app.settings['email_exceptions'] = False
53 58
         self.client = TestClient(self)
  59
+        celery.conf.ALWAYS_EAGER = True
  60
+        settings.DATABASE_NAME = 'test'
54 61
 
55 62
     def tearDown(self):
56 63
         super(BaseHTTPTestCase, self).tearDown()
88  tests/test_handlers.py
@@ -2,6 +2,7 @@
2 2
 import os
3 3
 import json
4 4
 from urllib import urlencode
  5
+import tornado.escape
5 6
 from .base import BaseHTTPTestCase
6 7
 from handlers import (TwitterAuthHandler, FollowsHandler, FollowingHandler,
7 8
                       EveryoneIFollowJSONHandler)
@@ -809,7 +810,6 @@ def test_suggest_tweet(self):
809 810
 
810 811
         struct = json.loads(response.body)
811 812
         self.assertTrue(len(struct['text']) <= 140)
812  
-        #self.assertTrue(struct['text'].endswith('#toocool'))
813 813
         self.assertTrue('@Mr_Billy_Nomates' in struct['text'])
814 814
 
815 815
         peterbe = self.db.Tweeter.find_one({'username': 'peterbe'})
@@ -841,6 +841,92 @@ def test_default_page_not_found(self):
841 841
         self.assertEqual(response.code, 404)
842 842
         self.assertTrue('restart your computer' in response.body)
843 843
 
  844
+    def test_following_compared_refresh(self):
  845
+        url = self.reverse_url('following_compared', 'obama', 'kimk')
  846
+        self._login()
  847
+
  848
+        FollowingHandler.twitter_request = \
  849
+          make_mock_twitter_request({
  850
+            "/friendships/show": {u'relationship': {
  851
+                                    u'target': {u'followed_by': False,
  852
+                                    u'following': False,
  853
+                                    u'screen_name': u'obama'}}},
  854
+            "/users/show?screen_name=obama": {u'followers_count': 41700,
  855
+                            u'following': False,
  856
+                            u'friends_count': 1300,
  857
+                            u'name': u'Barak Obama',
  858
+                            u'screen_name': u'obama',
  859
+                            'id': 9876543210,
  860
+                            },
  861
+            "/users/show?screen_name=kimk": {
  862
+                            u'followers_count': 40117,
  863
+                            u'following': False,
  864
+                            u'friends_count': 200,
  865
+                            u'name': u'Kim Kardashian',
  866
+                            u'screen_name': u'kimk',
  867
+                            'id': 123456789,
  868
+                            }
  869
+            })
  870
+
  871
+        response = self.client.get(url)
  872
+        self.assertEqual(response.code, 200)
  873
+
  874
+        obama = self.db.Tweeter.find_one({'username': 'obama'})
  875
+        self.assertEqual(obama['ratio'], 41700.0 / 1300)
  876
+        self.assertTrue('%.1f' % (41700.0 / 1300) in response.body)
  877
+
  878
+        kimk = self.db.Tweeter.find_one({'username': 'kimk'})
  879
+        self.assertEqual(kimk['ratio'], 40117.0 / 200)
  880
+        self.assertTrue('%.1f' % (40117.0 / 200) in response.body)
  881
+
  882
+        # change the stats
  883
+        import tasks
  884
+        def mock_twitter_request(self, url, callback, access_token, screen_name):
  885
+            results = {
  886
+              'obama': {
  887
+                u'followers_count': 40700,
  888
+                u'following': False,
  889
+                u'friends_count': 1333,
  890
+                u'name': u'Barak Obama',
  891
+                u'screen_name': u'obama',
  892
+                'id': 9876543210,
  893
+              },
  894
+              'kimk': {
  895
+                u'followers_count': 41117,
  896
+                u'following': False,
  897
+                u'friends_count': 222,
  898
+                u'name': u'Kim Kardashian',
  899
+                u'screen_name': u'kimk',
  900
+                'id': 123456789,
  901
+                }
  902
+            }
  903
+            class R(object):
  904
+                def __init__(self, result):
  905
+                    self.body = tornado.escape.json_encode(result)
  906
+            callback(R(results[screen_name]))
  907
+
  908
+        tasks.UserUpdate.twitter_request = mock_twitter_request
  909
+
  910
+        # now, pretend time passes
  911
+        obama['modify_date'] -= datetime.timedelta(seconds=60 * 60 + 1)
  912
+        obama.save(update_modify_date=False)
  913
+        kimk['modify_date'] -= datetime.timedelta(seconds=60 * 60 + 1)
  914
+        kimk.save(update_modify_date=False)
  915
+
  916
+        # second time it's going to use the saved data
  917
+        response = self.client.get(url)
  918
+        self.assertEqual(response.code, 200)
  919
+        # the old numbers will still be there
  920
+        self.assertTrue('%.1f' % (41700.0 / 1300) in response.body)
  921
+        self.assertTrue('%.1f' % (40117.0 / 200) in response.body)
  922
+
  923
+        # but the actual numbers will be updated!
  924
+        obama = self.db.Tweeter.find_one({'username': 'obama'})
  925
+        self.assertEqual(obama['ratio'], 40700.0 / 1333)  # new
  926
+
  927
+        kimk = self.db.Tweeter.find_one({'username': 'kimk'})
  928
+        self.assertEqual(kimk['ratio'], 41117.0 / 222)  # new
  929
+
844 930
 def make_twitter_get_authenticated_user_callback(struct):
845 931
     def twitter_get_authenticated_user(self, callback, **kw):
846 932
         callback(struct)

0 notes on commit 1d872f0

Please sign in to comment.
Something went wrong with that request. Please try again.