[bug 880194] Use Twython for AoA tweet collection

This does a couple things: - Adds Twython to the vendor libs - Adds Twython's requirements to the vendor libs (oauthlib & request_oauthlib) - Switches the AoA cron to use Twython instead of Tweepy
mozilla · Jun 10, 2013 · 02dc36d · 02dc36d
1 parent 1a33fcd
commit 02dc36d
Show file tree

Hide file tree

Showing 9 changed files with 77 additions and 47 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -142,3 +142,12 @@
 [submodule "kitsune/sumo/static/js/libs/ace"]
 	path = kitsune/sumo/static/js/libs/ace
 	url = git://github.com/ajaxorg/ace-builds.git
+[submodule "vendor/src/twython"]
+	path = vendor/src/twython
+	url = https://github.com/ryanmcgrath/twython.git
+[submodule "vendor/src/requests-oauthlib"]
+	path = vendor/src/requests-oauthlib
+	url = https://github.com/requests/requests-oauthlib.git
+[submodule "vendor/src/oauthlib"]
+	path = vendor/src/oauthlib
+	url = https://github.com/idan/oauthlib.git
diff --git a/kitsune/customercare/cron.py b/kitsune/customercare/cron.py
@@ -9,10 +9,9 @@
 from django.db.utils import IntegrityError
 
 import cronjobs
-import tweepy
 from multidb.pinning import pin_this_thread
 from statsd import statsd
-from tweepy.parsers import RawParser
+from twython import Twython
 
 from kitsune.customercare.models import Tweet, Reply
 from kitsune.sumo.redis_utils import redis_client, RedisError
@@ -34,18 +33,14 @@
 def collect_tweets():
     """Collect new tweets about Firefox."""
     with statsd.timer('customercare.tweets.time_elapsed'):
-        auth = tweepy.OAuthHandler(settings.TWITTER_CONSUMER_KEY,
-                                   settings.TWITTER_CONSUMER_SECRET,
-                                   secure=True)
-
-        auth.set_access_token(settings.TWITTER_ACCESS_TOKEN,
-                              settings.TWITTER_ACCESS_TOKEN_SECRET)
-
-        api = tweepy.API(auth, parser=RawParser())
+        t = Twython(settings.TWITTER_CONSUMER_KEY,
+                    settings.TWITTER_CONSUMER_SECRET,
+                    settings.TWITTER_ACCESS_TOKEN,
+                    settings.TWITTER_ACCESS_TOKEN_SECRET)
 
         search_options = {
             'q': 'firefox OR #fxinput OR @firefoxbrasil OR #firefoxos',
-            'rpp': settings.CC_TWEETS_PERPAGE,  # Items per page.
+            'count': settings.CC_TWEETS_PERPAGE,  # Items per page.
             'result_type': 'recent',  # Retrieve tweets by date.
         }
 
@@ -54,25 +49,21 @@ def collect_tweets():
         try:
             latest_tweet = Tweet.latest()
         except Tweet.DoesNotExist:
-            log.debug('No existing tweets. Retrieving %d tweets from search.' % (
-                settings.CC_TWEETS_PERPAGE))
+            log.debug('No existing tweets. Retrieving %d tweets from search.' %
+                      settings.CC_TWEETS_PERPAGE)
         else:
             search_options['since_id'] = latest_tweet.tweet_id
             log.info('Retrieving tweets with id >= %s' % latest_tweet.tweet_id)
 
         # Retrieve Tweets
-        try:
-            raw_data = json.loads(str(api.search(**search_options)))
-        except tweepy.TweepError, e:
-            log.warning('Twitter request failed: %s' % e)
-            return
+        results = t.search(**search_options)
 
-        if not ('results' in raw_data and raw_data['results']):
+        if len(results['statuses']) == 0:
             # Twitter returned 0 results.
             return
 
         # Drop tweets into DB
-        for item in raw_data['results']:
+        for item in results['statuses']:
             # Apply filters to tweet before saving
             # Allow links in #fxinput tweets
             statsd.incr('customercare.tweet.collected')
@@ -162,7 +153,7 @@ def _filter_tweet(item, allow_links=False):
         return None
 
     # Exclude filtered users
-    if item['from_user'] in settings.CC_IGNORE_USERS:
+    if item['user']['screen_name'] in settings.CC_IGNORE_USERS:
         statsd.incr('customercare.tweet.rejected.user')
         return None
 
@@ -206,8 +197,8 @@ def get_customercare_stats():
         if user not in contributor_stats:
             contributor_stats[user] = {
                 'twitter_username': user,
-                'avatar': raw['profile_image_url'],
-                'avatar_https': raw['profile_image_url_https'],
+                'avatar': raw['user']['profile_image_url'],
+                'avatar_https': raw['user']['profile_image_url_https'],
                 'all': 0, '1m': 0, '1w': 0, '1d': 0,
             }
         contributor = contributor_stats[reply.twitter_username]

diff --git a/kitsune/customercare/tests/__init__.py b/kitsune/customercare/tests/__init__.py
@@ -18,17 +18,31 @@ def tweet(**kwargs):
     """
     global next_tweet_id
     # TODO: Escape quotes and such
-    defaults = {'locale': 'en', 'raw_json':
-        '{"iso_language_code": "en", "text": "%s", '
-        '"created_at": "%s", '
-        '"profile_image_url": '
-        '"http://a1.twimg.com/profile_images/1117809237/cool_cat_normal.jpg", '
-        '"source": "&lt;a href=&quot;http://www.tweetdeck.com&quot; '
-            'rel=&quot;nofollow&quot;&gt;TweetDeck&lt;/a&gt;", '
-            '"from_user": "__jimcasey__", "from_user_id": 142651388, '
-            '"to_user_id": null, "geo": null, "id": 25309168521, '
-            '"metadata": {"result_type": "recent"}}' %
-            (kwargs.pop('text', 'Hey #Firefox'), tweet_created)}
+    defaults = {
+        'locale': 'en',
+        'raw_json': json.dumps({
+            'iso_language_code': 'en',
+            'text': kwargs.pop('text', 'Hey #Firefox'),
+            'created_at': tweet_created,
+            'source': '&lt;a href=&quot;http://www.tweetdeck.com&quot; '
+                      'rel=&quot;nofollow&quot;&gt;TweetDeck&lt;/a&gt;',
+            'user': {
+                'screen_name': '__jimcasey__',
+                'profile_image_url': 'http://a1.twimg.com/profile_images/'
+                                     '1117809237/cool_cat_normal.jpg',
+                'profile_image_url_https': 'http://si0.twimg.com/'
+                                           'profile_images/1117809237/'
+                                           'cool_cat_normal.jpg',
+            },
+            'to_user_id': None,
+            'geo': None,
+            'id': 25309168521,
+            'metadata': {
+                'results_type': 'recent',
+            }
+        })
+    }
+
     defaults.update(kwargs)
     if 'tweet_id' not in kwargs:
         defaults['tweet_id'] = next_tweet_id
@@ -53,14 +67,16 @@ def reply(**kwargs):
             'iso_language_code': 'en',
             'text': kwargs.pop('text', 'Hey #Firefox'),
             'created_at': 'Thu, 23 Sep 2010 13:58:06 +0000',
-            'profile_image_url':
-                'http://a1.twimg.com/profile_images/1117809237/cool_cat_normal.jpg',
-            'profile_image_url_https':
-                'http://si0.twimg.com/profile_images/1117809237/cool_cat_normal.jpg',
             'source': '&lt;a href=&quot;http://www.tweetdeck.com&quot; '
                 'rel=&quot;nofollow&quot;&gt;TweetDeck&lt;/a&gt;',
-            'from_user': '__jimcasey__',
-            'from_user_id': 142651388,
+            'user': {
+                'screen_name': '__jimcasey__',
+                'profile_image_url': 'http://a1.twimg.com/profile_images/'
+                                     '1117809237/cool_cat_normal.jpg',
+                'profile_image_url_https': 'http://si0.twimg.com/'
+                                           'profile_images/1117809237/'
+                                           'cool_cat_normal.jpg',
+            },
             'to_user_id': None,
             'geo': None,
             'id': 25309168521,

diff --git a/kitsune/customercare/tests/test_cron.py b/kitsune/customercare/tests/test_cron.py
@@ -25,14 +25,15 @@ class TwitterCronTestCase(TestCase):
             "http://a3.twimg.com/profile_images/688562959/"
             "jspeis_gmail.com_852af0c8__1__normal.jpg"),
         "created_at": "Mon, 25 Oct 2010 18:12:20 +0000",
-        "from_user": "jspeis",
+        "user": {
+            "screen_name": "jspeis",
+        },
         "metadata": {
             "result_type": "recent",
         },
         "to_user_id": None,
         "text": "giving the Firefox 4 beta a whirl",
         "id": 28713868836,
-        "from_user_id": 2385258,
         "geo": None,
         "iso_language_code": "en",
         "source": "&lt;a href=&quot;http://twitter.com/&quot;&gt;web&lt;/a&gt;"
@@ -93,11 +94,11 @@ def test_links(self):
 
     def test_fx4status(self):
         """Ensure fx4status tweets are filtered out."""
-        self.tweet['from_user'] = 'fx4status'
+        self.tweet['user']['screen_name'] = 'fx4status'
         assert _filter_tweet(self.tweet) is None
 
     def test_username_and_tweet_contain_firefox(self):
-        self.tweet['from_user'] = 'ilovefirefox4ever'
+        self.tweet['user']['screen_name'] = 'ilovefirefox4ever'
         self.tweet['text'] = 'My Firefox crashes :-( Any advice?'
         assert _filter_tweet(self.tweet) is not None
 

diff --git a/kitsune/customercare/views.py b/kitsune/customercare/views.py
@@ -45,13 +45,20 @@ def _tweet_for_template(tweet, https=False):
     else:
         replies = None
 
+    if 'from_user' in data: #For tweets collected using v1 API
+        user_data = data
+        from_user = data['from_user']
+    else:
+        user_data = data['user']
+        from_user = user_data['screen_name']
+
     if https:
-        img = bleach.clean(data['profile_image_url_https'])
+        img = bleach.clean(user_data['profile_image_url_https'])
     else:
-        img = bleach.clean(data['profile_image_url'])
+        img = bleach.clean(user_data['profile_image_url'])
 
     return {'profile_img': img,
-            'user': bleach.clean(data['from_user']),
+            'user': from_user,
             'text': bleach.clean(data['text']),
             'id': tweet.pk,
             'date': date,

diff --git a/vendor/kitsune.pth b/vendor/kitsune.pth
@@ -86,3 +86,6 @@ src/django-statsd
 src/pyelasticsearch
 src/requests
 src/premailer
+src/oauthlib
+src/requests-oauthlib
+src/twython
diff --git a/vendor/src/oauthlib b/vendor/src/oauthlib
diff --git a/vendor/src/requests-oauthlib b/vendor/src/requests-oauthlib
diff --git a/vendor/src/twython b/vendor/src/twython