diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8dada3e --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README b/README new file mode 100644 index 0000000..8c77d91 --- /dev/null +++ b/README @@ -0,0 +1,38 @@ +The PHEME conversation collection script allows the user to collect the set of tweets replying to a specific tweet, forming a conversation or a thread. The user needs to specify a single source tweet, providing its URL or tweet ID, and the tool collects the replies and stories them in the 'data/tweet-id' directory. + +NOTE: Since retrieval of replying tweets has been discontinued in Twitter API v1.1 (it used to be available through the 'related_results/show' endpoint in v1.0), this script scrapes the replies from the HTML of the source tweet. + +----- +SETUP +----- + +The script is developed using PHP (it was initially intended to run as a web service) and Python (for the back-end to access the Twitter API using the Tweepy library). It is ready to run on a system with PHP installed (i.e., the 'php5-cli' package on UNIX systems, or the equivalent PHP CLI for Windows). However, it does need to define a few settings in order to be able to access Twitter.com and the API. There are basically two steps: + +1. Create a cookie file to enable web access to replies: + +This can be done using Mozilla Firefox and the add-on Cookies Export/Import, which can be installed from: +https://addons.mozilla.org/en-US/firefox/addon/cookies-exportimport/ + +It is recommended to delete all existing cookies on Firefox before proceeding. Then, you should log in on Twitter.com with some user account. After logging in, click on 'Tool -> Export cookies', and save the file with the name 'cookies.txt'. This file should be copied into the main directory of this script. + +2. Add Twitter API credentials to twitter.ini: + +This is the Twitter API configuration file. You will see four lines there where you can specify the Twitter API credentials. + +------------- +HOW TO RUN IT +------------- + +Once you have the ID of the tweet that you want to get the conversation for, you can run the following command: + +php get.thread.php tweet-id + +Where tweet-id is the ID of the tweet that you are interested in. If successful, the script will output the number of replies that have been collected, and a new directory will be created in the 'data' folder. + +--------- +REFERENCE +--------- + +This conversation collection script was used for the following paper: + +Arkaitz Zubiaga, Geraldine Wong Sak Hoi, Maria Liakata, Rob Procter, Peter Tolmie. Analysing How People Orient to and Spread Rumours in Social Media by Looking at Conversational Threads. arXiv. 2015. diff --git a/get.thread.php b/get.thread.php new file mode 100755 index 0000000..959ec2d --- /dev/null +++ b/get.thread.php @@ -0,0 +1,151 @@ +", $content); + $content = str_replace("\\/", "/", $content); + $content = str_replace("\\\"", "\"", $content); + + if (preg_match_all("| $reptweet) { + $reptweettokens = explode("/", $reptweet); + $repusername = $reptweettokens[1]; + $reptweetid = $reptweettokens[count($reptweettokens) - 1]; + + if (!in_array($reptweetid, $replyingids)) { + array_push($replyingids, $reptweetid); + get_replying_ids($reptweetid, $repusername); + } + } + } + + $maxposition = ""; + if (preg_match("|data-min-position=\"([^\"]*)\"|U", $content, $mp) || preg_match("|\"min_position\":\"([^\"]*)\"|U", $content, $mp)) { + $maxposition = $mp[1]; + } + } while ($maxposition != ""); +} + +function add_to_structure ($tweetid, $inreplyto) { + global $structure; + + foreach ($structure as $id => $substructure) { + if ($id == $inreplyto) { + $structure[$id] = $tweetid; + } + else { + add_to_structure($tweetid, $inreplyto, $structure[$id]); + } + } +} + +function collect_replying_tweets ($tweetid, $username) { + global $argv, $replyingids; + $replycount = 0; + + @mkdir("data/" . $tweetid . "/reactions/"); + @chmod("data/" . $tweetid . "/reactions/", 0777); + get_replying_ids($tweetid, $username); + + $idsstr = ""; + $idcount = 0; + $allcount = 0; + foreach ($replyingids as $replyingid) { + $allcount++; + $idsstr .= $replyingid . ","; + $idcount++; + if ($idcount == 100 || $allcount == count($replyingids)) { + $tweets = @shell_exec("python retrieve.tweet.list.py " . substr($idsstr, 0, strlen($idsstr) - 1)); + $tweets = explode("\n", $tweets); + foreach ($tweets as $tweet) { + $tweetobj = @json_decode($tweet); + if (isset($tweetobj->id_str)) { + file_put_contents("data/" . $tweetid . "/reactions/" . $tweetobj->id_str . ".json", $tweet); + $replycount++; + } + } + + $idsstr = ""; + $idcount = 0; + } + } + + if (isset($argv[1])) { + echo $tweetid . " - source tweet and " . $replycount . " replies collected.\n"; + } +} + +function create_structure($tweetid) { + global $structure; + + $parents = array(); + $dir = dir("data/" . $tweetid . "/reactions/"); + while (($file = $dir->read()) !== false) { + if ($file != "." && $file != "..") { + $tweet = json_decode(file_get_contents("data/" . $tweetid . "/reactions/" . $file)); + + $inreplyto = $tweet->in_reply_to_status_id_str; + $id = $tweet->id; + + if (!isset($parents[$inreplyto])) { + $parents[$inreplyto] = array(); + } + array_push($parents[$inreplyto], $id); + } + } + + foreach ($structure as $sid => $substructure) { + if (isset($parents[$sid])) { + foreach ($parents[$sid] as $cid) { + $structure[$sid][$cid] = array(); + } + } + } + + file_put_contents("data/" . $tweetid . "/structure.json", json_encode($structure)); + chmod("data/" . $tweetid . "/structure.json", 0777); +} + +if (!isset($argv[1])) { + exit(0); +} +$tweetid = $argv[1]; + +if (strstr($tweetid, "/")) { + $tweetid = explode("/", $tweetid); + $tweetid = $tweetid[count($tweetid) - 1]; +} + +$replyingids = array(); +$structure = array($tweetid => array()); + +$sourcetweet = @shell_exec("python retrieve.tweet.py " . $tweetid); +$sourcetweetobj = json_decode($sourcetweet); +if (isset($sourcetweetobj->id_str)) { + $username = $sourcetweetobj->user->screen_name; + + @mkdir("data/" . $tweetid); + @chmod("data/" . $tweetid, 0766); + @mkdir("data/" . $tweetid . "/source-tweets/"); + @chmod("data/" . $tweetid . "/source-tweets/", 0766); + file_put_contents("data/" . $tweetid . "/source-tweets/" . $tweetid . ".json", $sourcetweet); + + collect_replying_tweets($tweetid, $username); + + create_structure($tweetid); +} +?> diff --git a/index.php b/index.php new file mode 100755 index 0000000..8b73a2c --- /dev/null +++ b/index.php @@ -0,0 +1,17 @@ + + + + + PHEME - Collection of Twitter Threads + + +

Collection of Twitter Threads/Conversations

+

This web application enables to collect the whole conversation in the form of replies, given the source tweet as the input. The output is rendered in an HTML file that visualises the conversation in a forum-like visualisation where replies are indented.

+ +

To start, please input the URL of a tweet for which you would like to see the whole conversation:

+ +
+ +
+ + diff --git a/retrieve.tweet.list.py b/retrieve.tweet.list.py new file mode 100755 index 0000000..1222f68 --- /dev/null +++ b/retrieve.tweet.list.py @@ -0,0 +1,29 @@ +import json +import tweepy +import sys +import pprint +import os +import ConfigParser +import time + +tweetid = sys.argv[1] + +config = ConfigParser.ConfigParser() +config.read('twitter.ini') + +consumer_key = config.get('Twitter', 'consumer_key') +consumer_secret = config.get('Twitter', 'consumer_secret') +access_key = config.get('Twitter', 'access_key') +access_secret = config.get('Twitter', 'access_secret') + +auth = tweepy.OAuthHandler(consumer_key, consumer_secret) +auth.set_access_token(access_key, access_secret) +api = tweepy.API(auth) + +try: + tweets = api._statuses_lookup(id=tweetid) + for tweet in tweets: + print json.dumps(tweet.json) + print tweet +except: + sys.exit() diff --git a/retrieve.tweet.py b/retrieve.tweet.py new file mode 100755 index 0000000..cc7b428 --- /dev/null +++ b/retrieve.tweet.py @@ -0,0 +1,27 @@ +import json +import tweepy +import sys +import pprint +import os +import ConfigParser +import time + +tweetid = sys.argv[1] + +config = ConfigParser.ConfigParser() +config.read('twitter.ini') + +consumer_key = config.get('Twitter', 'consumer_key') +consumer_secret = config.get('Twitter', 'consumer_secret') +access_key = config.get('Twitter', 'access_key') +access_secret = config.get('Twitter', 'access_secret') + +auth = tweepy.OAuthHandler(consumer_key, consumer_secret) +auth.set_access_token(access_key, access_secret) +api = tweepy.API(auth) + +try: + tweet = api.get_status(tweetid) + print json.dumps(tweet.json) +except: + sys.exit() diff --git a/tweepy/LICENSE b/tweepy/LICENSE new file mode 100644 index 0000000..545a75c --- /dev/null +++ b/tweepy/LICENSE @@ -0,0 +1,20 @@ +MIT License +Copyright (c) 2013-2014 Joshua Roesslein + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/tweepy/__init__.py b/tweepy/__init__.py new file mode 100755 index 0000000..05dbfc3 --- /dev/null +++ b/tweepy/__init__.py @@ -0,0 +1,27 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +""" +Tweepy Twitter API library +""" +__version__ = '2.2' +__author__ = 'Joshua Roesslein' +__license__ = 'MIT' + +from tweepy.models import Status, User, DirectMessage, Friendship, SavedSearch, SearchResults, ModelFactory, Category +from tweepy.error import TweepError +from tweepy.api import API +from tweepy.cache import Cache, MemoryCache, FileCache +from tweepy.auth import OAuthHandler +from tweepy.streaming import Stream, StreamListener +from tweepy.cursor import Cursor + +# Global, unauthenticated instance of API +api = API() + +def debug(enable=True, level=1): + + import httplib + httplib.HTTPConnection.debuglevel = level + diff --git a/tweepy/__init__.pyc b/tweepy/__init__.pyc new file mode 100644 index 0000000..14cff65 Binary files /dev/null and b/tweepy/__init__.pyc differ diff --git a/tweepy/api.py b/tweepy/api.py new file mode 100755 index 0000000..b6d3cae --- /dev/null +++ b/tweepy/api.py @@ -0,0 +1,751 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +import os +import mimetypes + +from tweepy.binder import bind_api +from tweepy.error import TweepError +from tweepy.parsers import ModelParser +from tweepy.utils import list_to_csv + + +class API(object): + """Twitter API""" + + def __init__(self, auth_handler=None, + host='api.twitter.com', search_host='search.twitter.com', + cache=None, secure=True, api_root='/1.1', search_root='', + retry_count=0, retry_delay=0, retry_errors=None, timeout=60, + parser=None, compression=False): + self.auth = auth_handler + self.host = host + self.search_host = search_host + self.api_root = api_root + self.search_root = search_root + self.cache = cache + self.secure = secure + self.compression = compression + self.retry_count = retry_count + self.retry_delay = retry_delay + self.retry_errors = retry_errors + self.timeout = timeout + self.parser = parser or ModelParser() + + """ statuses/home_timeline """ + home_timeline = bind_api( + path = '/statuses/home_timeline.json', + payload_type = 'status', payload_list = True, + allowed_param = ['since_id', 'max_id', 'count'], + require_auth = True + ) + + """ statuses/user_timeline """ + user_timeline = bind_api( + path = '/statuses/user_timeline.json', + payload_type = 'status', payload_list = True, + allowed_param = ['id', 'user_id', 'screen_name', 'since_id', + 'max_id', 'count', 'include_rts'] + ) + + """ statuses/mentions """ + mentions_timeline = bind_api( + path = '/statuses/mentions_timeline.json', + payload_type = 'status', payload_list = True, + allowed_param = ['since_id', 'max_id', 'count'], + require_auth = True + ) + + """/related_results/show/:id.format""" + related_results = bind_api( + path = '/related_results/show/{id}.json', + payload_type = 'relation', payload_list = True, + allowed_param = ['id'], + require_auth = False + ) + + """ statuses/retweets_of_me """ + retweets_of_me = bind_api( + path = '/statuses/retweets_of_me.json', + payload_type = 'status', payload_list = True, + allowed_param = ['since_id', 'max_id', 'count'], + require_auth = True + ) + + """ statuses/lookup """ + def statuses_lookup(self, id, include_entities=None, trim_user=None, map=None): + return self._statuses_lookup(list_to_csv(id), include_entities, trim_user, map) + + _statuses_lookup = bind_api( + path = '/statuses/lookup.json', + payload_type = 'status', payload_list = True, + allowed_param = ['id', 'include_entities', 'trim_user', 'map'], + require_auth = True + ) + + """ statuses/show """ + get_status = bind_api( + path = '/statuses/show.json', + payload_type = 'status', + allowed_param = ['id'] + ) + + """ statuses/update """ + update_status = bind_api( + path = '/statuses/update.json', + method = 'POST', + payload_type = 'status', + allowed_param = ['status', 'in_reply_to_status_id', 'lat', 'long', 'source', 'place_id'], + require_auth = True + ) + + """ statuses/update_with_media """ + def update_with_media(self, filename, *args, **kwargs): + headers, post_data = API._pack_image(filename, 3072, form_field='media[]') + kwargs.update({'headers': headers, 'post_data': post_data}) + + return bind_api( + path='/statuses/update_with_media.json', + method = 'POST', + payload_type='status', + allowed_param = [ + 'status', 'possibly_sensitive', 'in_reply_to_status_id', 'lat', 'long', + 'place_id', 'display_coordinates' + ], + require_auth=True + )(self, *args, **kwargs) + + """ statuses/destroy """ + destroy_status = bind_api( + path = '/statuses/destroy/{id}.json', + method = 'POST', + payload_type = 'status', + allowed_param = ['id'], + require_auth = True + ) + + """ statuses/retweet """ + retweet = bind_api( + path = '/statuses/retweet/{id}.json', + method = 'POST', + payload_type = 'status', + allowed_param = ['id'], + require_auth = True + ) + + """ statuses/retweets """ + retweets = bind_api( + path = '/statuses/retweets/{id}.json', + payload_type = 'status', payload_list = True, + allowed_param = ['id', 'count'], + require_auth = True + ) + + retweeters = bind_api( + path = '/statuses/retweeters/ids.json', + payload_type = 'ids', + allowed_param = ['id', 'cursor', 'stringify_ids'] + ) + + """ users/show """ + get_user = bind_api( + path = '/users/show.json', + payload_type = 'user', + allowed_param = ['id', 'user_id', 'screen_name'] + ) + + ''' statuses/oembed ''' + get_oembed = bind_api( + path = '/statuses/oembed.json', + payload_type = 'json', + allowed_param = ['id', 'url', 'maxwidth', 'hide_media', 'omit_script', 'align', 'related', 'lang'] + ) + + """ Perform bulk look up of users from user ID or screenname """ + def lookup_users(self, user_ids=None, screen_names=None): + return self._lookup_users(list_to_csv(user_ids), list_to_csv(screen_names)) + + _lookup_users = bind_api( + path = '/users/lookup.json', + payload_type = 'user', payload_list = True, + allowed_param = ['user_id', 'screen_name'], + ) + + """ Get the authenticated user """ + def me(self): + return self.get_user(screen_name=self.auth.get_username()) + + """ users/search """ + search_users = bind_api( + path = '/users/search.json', + payload_type = 'user', payload_list = True, + require_auth = True, + allowed_param = ['q', 'per_page', 'page'] + ) + + """ users/suggestions/:slug """ + suggested_users = bind_api( + path = '/users/suggestions/{slug}.json', + payload_type = 'user', payload_list = True, + require_auth = True, + allowed_param = ['slug', 'lang'] + ) + + """ users/suggestions """ + suggested_categories = bind_api( + path = '/users/suggestions.json', + payload_type = 'category', payload_list = True, + allowed_param = ['lang'], + require_auth = True + ) + + """ users/suggestions/:slug/members """ + suggested_users_tweets = bind_api( + path = '/users/suggestions/{slug}/members.json', + payload_type = 'status', payload_list = True, + allowed_param = ['slug'], + require_auth = True + ) + + """ direct_messages """ + direct_messages = bind_api( + path = '/direct_messages.json', + payload_type = 'direct_message', payload_list = True, + allowed_param = ['since_id', 'max_id', 'count'], + require_auth = True + ) + + """ direct_messages/show """ + get_direct_message = bind_api( + path = '/direct_messages/show/{id}.json', + payload_type = 'direct_message', + allowed_param = ['id'], + require_auth = True + ) + + """ direct_messages/sent """ + sent_direct_messages = bind_api( + path = '/direct_messages/sent.json', + payload_type = 'direct_message', payload_list = True, + allowed_param = ['since_id', 'max_id', 'count', 'page'], + require_auth = True + ) + + """ direct_messages/new """ + send_direct_message = bind_api( + path = '/direct_messages/new.json', + method = 'POST', + payload_type = 'direct_message', + allowed_param = ['user', 'screen_name', 'user_id', 'text'], + require_auth = True + ) + + """ direct_messages/destroy """ + destroy_direct_message = bind_api( + path = '/direct_messages/destroy.json', + method = 'DELETE', + payload_type = 'direct_message', + allowed_param = ['id'], + require_auth = True + ) + + """ friendships/create """ + create_friendship = bind_api( + path = '/friendships/create.json', + method = 'POST', + payload_type = 'user', + allowed_param = ['id', 'user_id', 'screen_name', 'follow'], + require_auth = True + ) + + """ friendships/destroy """ + destroy_friendship = bind_api( + path = '/friendships/destroy.json', + method = 'DELETE', + payload_type = 'user', + allowed_param = ['id', 'user_id', 'screen_name'], + require_auth = True + ) + + """ friendships/show """ + show_friendship = bind_api( + path = '/friendships/show.json', + payload_type = 'friendship', + allowed_param = ['source_id', 'source_screen_name', + 'target_id', 'target_screen_name'] + ) + + """ Perform bulk look up of friendships from user ID or screenname """ + def lookup_friendships(self, user_ids=None, screen_names=None): + return self._lookup_friendships(list_to_csv(user_ids), list_to_csv(screen_names)) + + _lookup_friendships = bind_api( + path = '/friendships/lookup.json', + payload_type = 'relationship', payload_list = True, + allowed_param = ['user_id', 'screen_name'], + require_auth = True + ) + + + """ friends/ids """ + friends_ids = bind_api( + path = '/friends/ids.json', + payload_type = 'ids', + allowed_param = ['id', 'user_id', 'screen_name', 'cursor'] + ) + + """ friends/list """ + friends = bind_api( + path = '/friends/list.json', + payload_type = 'user', payload_list = True, + allowed_param = ['id', 'user_id', 'screen_name', 'cursor'] + ) + + """ friendships/incoming """ + friendships_incoming = bind_api( + path = '/friendships/incoming.json', + payload_type = 'ids', + allowed_param = ['cursor'] + ) + + """ friendships/outgoing""" + friendships_outgoing = bind_api( + path = '/friendships/outgoing.json', + payload_type = 'ids', + allowed_param = ['cursor'] + ) + + """ followers/ids """ + followers_ids = bind_api( + path = '/followers/ids.json', + payload_type = 'ids', + allowed_param = ['id', 'user_id', 'screen_name', 'cursor'] + ) + + """ followers/list """ + followers = bind_api( + path = '/followers/list.json', + payload_type = 'user', payload_list = True, + allowed_param = ['id', 'user_id', 'screen_name', 'cursor', 'count', + 'skip_status', 'include_user_entities'] + ) + + """ account/verify_credentials """ + def verify_credentials(self, **kargs): + try: + return bind_api( + path = '/account/verify_credentials.json', + payload_type = 'user', + require_auth = True, + allowed_param = ['include_entities', 'skip_status'], + )(self, **kargs) + except TweepError, e: + if e.response and e.response.status == 401: + return False + raise + + """ account/rate_limit_status """ + rate_limit_status = bind_api( + path = '/application/rate_limit_status.json', + payload_type = 'json', + allowed_param = ['resources'], + use_cache = False + ) + + """ account/update_delivery_device """ + set_delivery_device = bind_api( + path = '/account/update_delivery_device.json', + method = 'POST', + allowed_param = ['device'], + payload_type = 'user', + require_auth = True + ) + + """ account/update_profile_colors """ + update_profile_colors = bind_api( + path = '/account/update_profile_colors.json', + method = 'POST', + payload_type = 'user', + allowed_param = ['profile_background_color', 'profile_text_color', + 'profile_link_color', 'profile_sidebar_fill_color', + 'profile_sidebar_border_color'], + require_auth = True + ) + + """ account/update_profile_image """ + def update_profile_image(self, filename): + headers, post_data = API._pack_image(filename, 700) + return bind_api( + path = '/account/update_profile_image.json', + method = 'POST', + payload_type = 'user', + require_auth = True + )(self, post_data=post_data, headers=headers) + + """ account/update_profile_background_image """ + def update_profile_background_image(self, filename, *args, **kargs): + headers, post_data = API._pack_image(filename, 800) + bind_api( + path = '/account/update_profile_background_image.json', + method = 'POST', + payload_type = 'user', + allowed_param = ['tile'], + require_auth = True + )(self, post_data=post_data, headers=headers) + + """ account/update_profile_banner """ + def update_profile_banner(self, filename, *args, **kargs): + headers, post_data = API._pack_image(filename, 700, form_field="banner") + bind_api( + path = '/account/update_profile_banner.json', + method = 'POST', + allowed_param = ['width', 'height', 'offset_left', 'offset_right'], + require_auth = True + )(self, post_data=post_data, headers=headers) + + + """ account/update_profile """ + update_profile = bind_api( + path = '/account/update_profile.json', + method = 'POST', + payload_type = 'user', + allowed_param = ['name', 'url', 'location', 'description'], + require_auth = True + ) + + """ favorites """ + favorites = bind_api( + path = '/favorites/list.json', + payload_type = 'status', payload_list = True, + allowed_param = ['screen_name', 'user_id', 'max_id', 'count', 'since_id', 'max_id'] + ) + + """ favorites/create """ + create_favorite = bind_api( + path = '/favorites/create.json', + method = 'POST', + payload_type = 'status', + allowed_param = ['id'], + require_auth = True + ) + + """ favorites/destroy """ + destroy_favorite = bind_api( + path = '/favorites/destroy.json', + method = 'POST', + payload_type = 'status', + allowed_param = ['id'], + require_auth = True + ) + + """ blocks/create """ + create_block = bind_api( + path = '/blocks/create.json', + method = 'POST', + payload_type = 'user', + allowed_param = ['id', 'user_id', 'screen_name'], + require_auth = True + ) + + """ blocks/destroy """ + destroy_block = bind_api( + path = '/blocks/destroy.json', + method = 'DELETE', + payload_type = 'user', + allowed_param = ['id', 'user_id', 'screen_name'], + require_auth = True + ) + + """ blocks/blocking """ + blocks = bind_api( + path = '/blocks/list.json', + payload_type = 'user', payload_list = True, + allowed_param = ['cursor'], + require_auth = True + ) + + """ blocks/blocking/ids """ + blocks_ids = bind_api( + path = '/blocks/ids.json', + payload_type = 'json', + require_auth = True + ) + + """ report_spam """ + report_spam = bind_api( + path = '/users/report_spam.json', + method = 'POST', + payload_type = 'user', + allowed_param = ['user_id', 'screen_name'], + require_auth = True + ) + + """ saved_searches """ + saved_searches = bind_api( + path = '/saved_searches/list.json', + payload_type = 'saved_search', payload_list = True, + require_auth = True + ) + + """ saved_searches/show """ + get_saved_search = bind_api( + path = '/saved_searches/show/{id}.json', + payload_type = 'saved_search', + allowed_param = ['id'], + require_auth = True + ) + + """ saved_searches/create """ + create_saved_search = bind_api( + path = '/saved_searches/create.json', + method = 'POST', + payload_type = 'saved_search', + allowed_param = ['query'], + require_auth = True + ) + + """ saved_searches/destroy """ + destroy_saved_search = bind_api( + path = '/saved_searches/destroy/{id}.json', + method = 'POST', + payload_type = 'saved_search', + allowed_param = ['id'], + require_auth = True + ) + + create_list = bind_api( + path = '/lists/create.json', + method = 'POST', + payload_type = 'list', + allowed_param = ['name', 'mode', 'description'], + require_auth = True + ) + + destroy_list = bind_api( + path = '/lists/destroy.json', + method = 'POST', + payload_type = 'list', + allowed_param = ['owner_screen_name', 'owner_id', 'list_id', 'slug'], + require_auth = True + ) + + update_list = bind_api( + path = '/lists/update.json', + method = 'POST', + payload_type = 'list', + allowed_param = ['list_id', 'slug', 'name', 'mode', 'description', 'owner_screen_name', 'owner_id'], + require_auth = True + ) + + lists_all = bind_api( + path = '/lists/list.json', + payload_type = 'list', payload_list = True, + allowed_param = ['screen_name', 'user_id'], + require_auth = True + ) + + lists_memberships = bind_api( + path = '/lists/memberships.json', + payload_type = 'list', payload_list = True, + allowed_param = ['screen_name', 'user_id', 'filter_to_owned_lists', 'cursor'], + require_auth = True + ) + + lists_subscriptions = bind_api( + path = '/lists/subscriptions.json', + payload_type = 'list', payload_list = True, + allowed_param = ['screen_name', 'user_id', 'cursor'], + require_auth = True + ) + + list_timeline = bind_api( + path = '/lists/statuses.json', + payload_type = 'status', payload_list = True, + allowed_param = ['owner_screen_name', 'slug', 'owner_id', 'list_id', 'since_id', 'max_id', 'count', 'include_rts'] + ) + + get_list = bind_api( + path = '/lists/show.json', + payload_type = 'list', + allowed_param = ['owner_screen_name', 'owner_id', 'slug', 'list_id'] + ) + + add_list_member = bind_api( + path = '/lists/members/create.json', + method = 'POST', + payload_type = 'list', + allowed_param = ['screen_name', 'user_id', 'owner_screen_name', 'owner_id', 'slug', 'list_id'], + require_auth = True + ) + + remove_list_member = bind_api( + path = '/lists/members/destroy.json', + method = 'POST', + payload_type = 'list', + allowed_param = ['screen_name', 'user_id', 'owner_screen_name', 'owner_id', 'slug', 'list_id'], + require_auth = True + ) + + list_members = bind_api( + path = '/lists/members.json', + payload_type = 'user', payload_list = True, + allowed_param = ['owner_screen_name', 'slug', 'list_id', 'owner_id', 'cursor'] + ) + + show_list_member = bind_api( + path = '/lists/members/show.json', + payload_type = 'user', + allowed_param = ['list_id', 'slug', 'user_id', 'screen_name', 'owner_screen_name', 'owner_id'] + ) + + subscribe_list = bind_api( + path = '/lists/subscribers/create.json', + method = 'POST', + payload_type = 'list', + allowed_param = ['owner_screen_name', 'slug', 'owner_id', 'list_id'], + require_auth = True + ) + + unsubscribe_list = bind_api( + path = '/lists/subscribers/destroy.json', + method = 'POST', + payload_type = 'list', + allowed_param = ['owner_screen_name', 'slug', 'owner_id', 'list_id'], + require_auth = True + ) + + list_subscribers = bind_api( + path = '/lists/subscribers.json', + payload_type = 'user', payload_list = True, + allowed_param = ['owner_screen_name', 'slug', 'owner_id', 'list_id', 'cursor'] + ) + + show_list_subscriber = bind_api( + path = '/lists/subscribers/show.json', + payload_type = 'user', + allowed_param = ['owner_screen_name', 'slug', 'screen_name', 'owner_id', 'list_id', 'user_id'] + ) + + """ trends/available """ + trends_available = bind_api( + path = '/trends/available.json', + payload_type = 'json' + ) + + trends_place = bind_api( + path = '/trends/place.json', + payload_type = 'json', + allowed_param = ['id', 'exclude'] + ) + + trends_closest = bind_api( + path = '/trends/closest.json', + payload_type = 'json', + allowed_param = ['lat', 'long'] + ) + + """ search """ + search = bind_api( + path = '/search/tweets.json', + payload_type = 'search_results', + allowed_param = ['q', 'lang', 'locale', 'since_id', 'geocode', 'max_id', 'since', 'until', 'result_type', 'count', 'include_entities', 'from', 'to', 'source'] + ) + + """ trends/daily """ + trends_daily = bind_api( + path = '/trends/daily.json', + payload_type = 'json', + allowed_param = ['date', 'exclude'] + ) + + """ trends/weekly """ + trends_weekly = bind_api( + path = '/trends/weekly.json', + payload_type = 'json', + allowed_param = ['date', 'exclude'] + ) + + """ geo/reverse_geocode """ + reverse_geocode = bind_api( + path = '/geo/reverse_geocode.json', + payload_type = 'place', payload_list = True, + allowed_param = ['lat', 'long', 'accuracy', 'granularity', 'max_results'] + ) + + """ geo/id """ + geo_id = bind_api( + path = '/geo/id/{id}.json', + payload_type = 'place', + allowed_param = ['id'] + ) + + """ geo/search """ + geo_search = bind_api( + path = '/geo/search.json', + payload_type = 'place', payload_list = True, + allowed_param = ['lat', 'long', 'query', 'ip', 'granularity', 'accuracy', 'max_results', 'contained_within'] + ) + + """ geo/similar_places """ + geo_similar_places = bind_api( + path = '/geo/similar_places.json', + payload_type = 'place', payload_list = True, + allowed_param = ['lat', 'long', 'name', 'contained_within'] + ) + + """ help/languages.json """ + supported_languages = bind_api( + path = '/help/languages.json', + payload_type = 'json', + require_auth = True + ) + + """ help/configuration """ + configuration = bind_api( + path = '/help/configuration.json', + payload_type = 'json', + require_auth = True + ) + + """ Internal use only """ + @staticmethod + def _pack_image(filename, max_size, form_field="image"): + """Pack image from file into multipart-formdata post body""" + # image must be less than 700kb in size + try: + if os.path.getsize(filename) > (max_size * 1024): + raise TweepError('File is too big, must be less than 700kb.') + except os.error: + raise TweepError('Unable to access file') + + # image must be gif, jpeg, or png + file_type = mimetypes.guess_type(filename) + if file_type is None: + raise TweepError('Could not determine file type') + file_type = file_type[0] + if file_type not in ['image/gif', 'image/jpeg', 'image/png']: + raise TweepError('Invalid file type for image: %s' % file_type) + + # build the mulitpart-formdata body + fp = open(filename, 'rb') + BOUNDARY = 'Tw3ePy' + body = [] + body.append('--' + BOUNDARY) + body.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (form_field, filename)) + body.append('Content-Type: %s' % file_type) + body.append('') + body.append(fp.read()) + body.append('--' + BOUNDARY + '--') + body.append('') + fp.close() + body = '\r\n'.join(body) + + # build headers + headers = { + 'Content-Type': 'multipart/form-data; boundary=Tw3ePy', + 'Content-Length': str(len(body)) + } + + return headers, body + diff --git a/tweepy/api.pyc b/tweepy/api.pyc new file mode 100644 index 0000000..6030059 Binary files /dev/null and b/tweepy/api.pyc differ diff --git a/tweepy/auth.py b/tweepy/auth.py new file mode 100755 index 0000000..29df5b2 --- /dev/null +++ b/tweepy/auth.py @@ -0,0 +1,156 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +from urllib2 import Request, urlopen +import base64 + +from tweepy import oauth +from tweepy.error import TweepError +from tweepy.api import API + + +class AuthHandler(object): + + def apply_auth(self, url, method, headers, parameters): + """Apply authentication headers to request""" + raise NotImplementedError + + def get_username(self): + """Return the username of the authenticated user""" + raise NotImplementedError + + +class OAuthHandler(AuthHandler): + """OAuth authentication handler""" + + OAUTH_HOST = 'api.twitter.com' + OAUTH_ROOT = '/oauth/' + + def __init__(self, consumer_key, consumer_secret, callback=None, secure=True): + if type(consumer_key) == unicode: + consumer_key = bytes(consumer_key) + + if type(consumer_secret) == unicode: + consumer_secret = bytes(consumer_secret) + + self._consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) + self._sigmethod = oauth.OAuthSignatureMethod_HMAC_SHA1() + self.request_token = None + self.access_token = None + self.callback = callback + self.username = None + self.secure = secure + + def _get_oauth_url(self, endpoint, secure=True): + if self.secure or secure: + prefix = 'https://' + else: + prefix = 'http://' + + return prefix + self.OAUTH_HOST + self.OAUTH_ROOT + endpoint + + def apply_auth(self, url, method, headers, parameters): + request = oauth.OAuthRequest.from_consumer_and_token( + self._consumer, http_url=url, http_method=method, + token=self.access_token, parameters=parameters + ) + request.sign_request(self._sigmethod, self._consumer, self.access_token) + headers.update(request.to_header()) + + def _get_request_token(self): + try: + url = self._get_oauth_url('request_token') + request = oauth.OAuthRequest.from_consumer_and_token( + self._consumer, http_url=url, callback=self.callback + ) + request.sign_request(self._sigmethod, self._consumer, None) + resp = urlopen(Request(url, headers=request.to_header())) + return oauth.OAuthToken.from_string(resp.read()) + except Exception, e: + raise TweepError(e) + + def set_request_token(self, key, secret): + self.request_token = oauth.OAuthToken(key, secret) + + def set_access_token(self, key, secret): + self.access_token = oauth.OAuthToken(key, secret) + + def get_authorization_url(self, signin_with_twitter=False): + """Get the authorization URL to redirect the user""" + try: + # get the request token + self.request_token = self._get_request_token() + + # build auth request and return as url + if signin_with_twitter: + url = self._get_oauth_url('authenticate') + else: + url = self._get_oauth_url('authorize') + request = oauth.OAuthRequest.from_token_and_callback( + token=self.request_token, http_url=url + ) + + return request.to_url() + except Exception, e: + raise TweepError(e) + + def get_access_token(self, verifier=None): + """ + After user has authorized the request token, get access token + with user supplied verifier. + """ + try: + url = self._get_oauth_url('access_token') + + # build request + request = oauth.OAuthRequest.from_consumer_and_token( + self._consumer, + token=self.request_token, http_url=url, + verifier=str(verifier) + ) + request.sign_request(self._sigmethod, self._consumer, self.request_token) + + # send request + resp = urlopen(Request(url, headers=request.to_header())) + self.access_token = oauth.OAuthToken.from_string(resp.read()) + return self.access_token + except Exception, e: + raise TweepError(e) + + def get_xauth_access_token(self, username, password): + """ + Get an access token from an username and password combination. + In order to get this working you need to create an app at + http://twitter.com/apps, after that send a mail to api@twitter.com + and request activation of xAuth for it. + """ + try: + url = self._get_oauth_url('access_token', secure=True) # must use HTTPS + request = oauth.OAuthRequest.from_consumer_and_token( + oauth_consumer=self._consumer, + http_method='POST', http_url=url, + parameters = { + 'x_auth_mode': 'client_auth', + 'x_auth_username': username, + 'x_auth_password': password + } + ) + request.sign_request(self._sigmethod, self._consumer, None) + + resp = urlopen(Request(url, data=request.to_postdata())) + self.access_token = oauth.OAuthToken.from_string(resp.read()) + return self.access_token + except Exception, e: + raise TweepError(e) + + def get_username(self): + if self.username is None: + api = API(self) + user = api.verify_credentials() + if user: + self.username = user.screen_name + else: + raise TweepError("Unable to get username, invalid oauth token!") + return self.username + diff --git a/tweepy/auth.pyc b/tweepy/auth.pyc new file mode 100644 index 0000000..ce77a5f Binary files /dev/null and b/tweepy/auth.pyc differ diff --git a/tweepy/binder.py b/tweepy/binder.py new file mode 100755 index 0000000..29eacae --- /dev/null +++ b/tweepy/binder.py @@ -0,0 +1,213 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +import httplib +import urllib +import time +import re +from StringIO import StringIO +import gzip + +from tweepy.error import TweepError +from tweepy.utils import convert_to_utf8_str +from tweepy.models import Model + +re_path_template = re.compile('{\w+}') + + +def bind_api(**config): + + class APIMethod(object): + + path = config['path'] + payload_type = config.get('payload_type', None) + payload_list = config.get('payload_list', False) + allowed_param = config.get('allowed_param', []) + method = config.get('method', 'GET') + require_auth = config.get('require_auth', False) + search_api = config.get('search_api', False) + use_cache = config.get('use_cache', True) + + def __init__(self, api, args, kargs): + # If authentication is required and no credentials + # are provided, throw an error. + if self.require_auth and not api.auth: + raise TweepError('Authentication required!') + + self.api = api + self.post_data = kargs.pop('post_data', None) + self.retry_count = kargs.pop('retry_count', api.retry_count) + self.retry_delay = kargs.pop('retry_delay', api.retry_delay) + self.retry_errors = kargs.pop('retry_errors', api.retry_errors) + self.headers = kargs.pop('headers', {}) + self.build_parameters(args, kargs) + + # Pick correct URL root to use + if self.search_api: + self.api_root = api.search_root + else: + self.api_root = api.api_root + + # Perform any path variable substitution + self.build_path() + + if api.secure: + self.scheme = 'https://' + else: + self.scheme = 'http://' + + if self.search_api: + self.host = api.search_host + else: + self.host = api.host + + # Manually set Host header to fix an issue in python 2.5 + # or older where Host is set including the 443 port. + # This causes Twitter to issue 301 redirect. + # See Issue https://github.com/tweepy/tweepy/issues/12 + self.headers['Host'] = self.host + + def build_parameters(self, args, kargs): + self.parameters = {} + for idx, arg in enumerate(args): + if arg is None: + continue + + try: + self.parameters[self.allowed_param[idx]] = convert_to_utf8_str(arg) + except IndexError: + raise TweepError('Too many parameters supplied!') + + for k, arg in kargs.items(): + if arg is None: + continue + if k in self.parameters: + raise TweepError('Multiple values for parameter %s supplied!' % k) + + self.parameters[k] = convert_to_utf8_str(arg) + + def build_path(self): + for variable in re_path_template.findall(self.path): + name = variable.strip('{}') + + if name == 'user' and 'user' not in self.parameters and self.api.auth: + # No 'user' parameter provided, fetch it from Auth instead. + value = self.api.auth.get_username() + else: + try: + value = urllib.quote(self.parameters[name]) + except KeyError: + raise TweepError('No parameter value found for path variable: %s' % name) + del self.parameters[name] + + self.path = self.path.replace(variable, value) + + def execute(self): + self.api.cached_result = False + + # Build the request URL + url = self.api_root + self.path + if len(self.parameters): + url = '%s?%s' % (url, urllib.urlencode(self.parameters)) + + # Query the cache if one is available + # and this request uses a GET method. + if self.use_cache and self.api.cache and self.method == 'GET': + cache_result = self.api.cache.get(url) + # if cache result found and not expired, return it + if cache_result: + # must restore api reference + if isinstance(cache_result, list): + for result in cache_result: + if isinstance(result, Model): + result._api = self.api + else: + if isinstance(cache_result, Model): + cache_result._api = self.api + self.api.cached_result = True + return cache_result + + # Continue attempting request until successful + # or maximum number of retries is reached. + retries_performed = 0 + while retries_performed < self.retry_count + 1: + # Open connection + if self.api.secure: + conn = httplib.HTTPSConnection(self.host, timeout=self.api.timeout) + else: + conn = httplib.HTTPConnection(self.host, timeout=self.api.timeout) + + # Apply authentication + if self.api.auth: + self.api.auth.apply_auth( + self.scheme + self.host + url, + self.method, self.headers, self.parameters + ) + + # Request compression if configured + if self.api.compression: + self.headers['Accept-encoding'] = 'gzip' + + # Execute request + try: + conn.request(self.method, url, headers=self.headers, body=self.post_data) + resp = conn.getresponse() + except Exception, e: + raise TweepError('Failed to send request: %s' % e) + + # Exit request loop if non-retry error code + if self.retry_errors: + if resp.status not in self.retry_errors: break + else: + if resp.status == 200: break + + # Sleep before retrying request again + time.sleep(self.retry_delay) + retries_performed += 1 + + # If an error was returned, throw an exception + self.api.last_response = resp + if resp.status and not 200 <= resp.status < 300: + try: + error_msg = self.api.parser.parse_error(resp.read()) + except Exception: + error_msg = "Twitter error response: status code = %s" % resp.status + raise TweepError(error_msg, resp) + + # Parse the response payload + body = resp.read() + if resp.getheader('Content-Encoding', '') == 'gzip': + try: + zipper = gzip.GzipFile(fileobj=StringIO(body)) + body = zipper.read() + except Exception, e: + raise TweepError('Failed to decompress data: %s' % e) + result = self.api.parser.parse(self, body) + + conn.close() + + # Store result into cache if one is available. + if self.use_cache and self.api.cache and self.method == 'GET' and result: + self.api.cache.store(url, result) + + return result + + + def _call(api, *args, **kargs): + + method = APIMethod(api, args, kargs) + return method.execute() + + + # Set pagination mode + if 'cursor' in APIMethod.allowed_param: + _call.pagination_mode = 'cursor' + elif 'max_id' in APIMethod.allowed_param and \ + 'since_id' in APIMethod.allowed_param: + _call.pagination_mode = 'id' + elif 'page' in APIMethod.allowed_param: + _call.pagination_mode = 'page' + + return _call + diff --git a/tweepy/binder.pyc b/tweepy/binder.pyc new file mode 100644 index 0000000..1b759eb Binary files /dev/null and b/tweepy/binder.pyc differ diff --git a/tweepy/cache.py b/tweepy/cache.py new file mode 100755 index 0000000..25564a3 --- /dev/null +++ b/tweepy/cache.py @@ -0,0 +1,424 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +import time +import datetime +import threading +import os + +try: + import cPickle as pickle +except ImportError: + import pickle + +try: + import hashlib +except ImportError: + # python 2.4 + import md5 as hashlib + +try: + import fcntl +except ImportError: + # Probably on a windows system + # TODO: use win32file + pass + + +class Cache(object): + """Cache interface""" + + def __init__(self, timeout=60): + """Initialize the cache + timeout: number of seconds to keep a cached entry + """ + self.timeout = timeout + + def store(self, key, value): + """Add new record to cache + key: entry key + value: data of entry + """ + raise NotImplementedError + + def get(self, key, timeout=None): + """Get cached entry if exists and not expired + key: which entry to get + timeout: override timeout with this value [optional] + """ + raise NotImplementedError + + def count(self): + """Get count of entries currently stored in cache""" + raise NotImplementedError + + def cleanup(self): + """Delete any expired entries in cache.""" + raise NotImplementedError + + def flush(self): + """Delete all cached entries""" + raise NotImplementedError + + +class MemoryCache(Cache): + """In-memory cache""" + + def __init__(self, timeout=60): + Cache.__init__(self, timeout) + self._entries = {} + self.lock = threading.Lock() + + def __getstate__(self): + # pickle + return {'entries': self._entries, 'timeout': self.timeout} + + def __setstate__(self, state): + # unpickle + self.lock = threading.Lock() + self._entries = state['entries'] + self.timeout = state['timeout'] + + def _is_expired(self, entry, timeout): + return timeout > 0 and (time.time() - entry[0]) >= timeout + + def store(self, key, value): + self.lock.acquire() + self._entries[key] = (time.time(), value) + self.lock.release() + + def get(self, key, timeout=None): + self.lock.acquire() + try: + # check to see if we have this key + entry = self._entries.get(key) + if not entry: + # no hit, return nothing + return None + + # use provided timeout in arguments if provided + # otherwise use the one provided during init. + if timeout is None: + timeout = self.timeout + + # make sure entry is not expired + if self._is_expired(entry, timeout): + # entry expired, delete and return nothing + del self._entries[key] + return None + + # entry found and not expired, return it + return entry[1] + finally: + self.lock.release() + + def count(self): + return len(self._entries) + + def cleanup(self): + self.lock.acquire() + try: + for k, v in self._entries.items(): + if self._is_expired(v, self.timeout): + del self._entries[k] + finally: + self.lock.release() + + def flush(self): + self.lock.acquire() + self._entries.clear() + self.lock.release() + + +class FileCache(Cache): + """File-based cache""" + + # locks used to make cache thread-safe + cache_locks = {} + + def __init__(self, cache_dir, timeout=60): + Cache.__init__(self, timeout) + if os.path.exists(cache_dir) is False: + os.mkdir(cache_dir) + self.cache_dir = cache_dir + if cache_dir in FileCache.cache_locks: + self.lock = FileCache.cache_locks[cache_dir] + else: + self.lock = threading.Lock() + FileCache.cache_locks[cache_dir] = self.lock + + if os.name == 'posix': + self._lock_file = self._lock_file_posix + self._unlock_file = self._unlock_file_posix + elif os.name == 'nt': + self._lock_file = self._lock_file_win32 + self._unlock_file = self._unlock_file_win32 + else: + print 'Warning! FileCache locking not supported on this system!' + self._lock_file = self._lock_file_dummy + self._unlock_file = self._unlock_file_dummy + + def _get_path(self, key): + md5 = hashlib.md5() + md5.update(key) + return os.path.join(self.cache_dir, md5.hexdigest()) + + def _lock_file_dummy(self, path, exclusive=True): + return None + + def _unlock_file_dummy(self, lock): + return + + def _lock_file_posix(self, path, exclusive=True): + lock_path = path + '.lock' + if exclusive is True: + f_lock = open(lock_path, 'w') + fcntl.lockf(f_lock, fcntl.LOCK_EX) + else: + f_lock = open(lock_path, 'r') + fcntl.lockf(f_lock, fcntl.LOCK_SH) + if os.path.exists(lock_path) is False: + f_lock.close() + return None + return f_lock + + def _unlock_file_posix(self, lock): + lock.close() + + def _lock_file_win32(self, path, exclusive=True): + # TODO: implement + return None + + def _unlock_file_win32(self, lock): + # TODO: implement + return + + def _delete_file(self, path): + os.remove(path) + if os.path.exists(path + '.lock'): + os.remove(path + '.lock') + + def store(self, key, value): + path = self._get_path(key) + self.lock.acquire() + try: + # acquire lock and open file + f_lock = self._lock_file(path) + datafile = open(path, 'wb') + + # write data + pickle.dump((time.time(), value), datafile) + + # close and unlock file + datafile.close() + self._unlock_file(f_lock) + finally: + self.lock.release() + + def get(self, key, timeout=None): + return self._get(self._get_path(key), timeout) + + def _get(self, path, timeout): + if os.path.exists(path) is False: + # no record + return None + self.lock.acquire() + try: + # acquire lock and open + f_lock = self._lock_file(path, False) + datafile = open(path, 'rb') + + # read pickled object + created_time, value = pickle.load(datafile) + datafile.close() + + # check if value is expired + if timeout is None: + timeout = self.timeout + if timeout > 0 and (time.time() - created_time) >= timeout: + # expired! delete from cache + value = None + self._delete_file(path) + + # unlock and return result + self._unlock_file(f_lock) + return value + finally: + self.lock.release() + + def count(self): + c = 0 + for entry in os.listdir(self.cache_dir): + if entry.endswith('.lock'): + continue + c += 1 + return c + + def cleanup(self): + for entry in os.listdir(self.cache_dir): + if entry.endswith('.lock'): + continue + self._get(os.path.join(self.cache_dir, entry), None) + + def flush(self): + for entry in os.listdir(self.cache_dir): + if entry.endswith('.lock'): + continue + self._delete_file(os.path.join(self.cache_dir, entry)) + +class MemCacheCache(Cache): + """Cache interface""" + + def __init__(self, client, timeout=60): + """Initialize the cache + client: The memcache client + timeout: number of seconds to keep a cached entry + """ + self.client = client + self.timeout = timeout + + def store(self, key, value): + """Add new record to cache + key: entry key + value: data of entry + """ + self.client.set(key, value, time=self.timeout) + + def get(self, key, timeout=None): + """Get cached entry if exists and not expired + key: which entry to get + timeout: override timeout with this value [optional]. DOES NOT WORK HERE + """ + return self.client.get(key) + + def count(self): + """Get count of entries currently stored in cache. RETURN 0""" + raise NotImplementedError + + def cleanup(self): + """Delete any expired entries in cache. NO-OP""" + raise NotImplementedError + + def flush(self): + """Delete all cached entries. NO-OP""" + raise NotImplementedError + +class RedisCache(Cache): + '''Cache running in a redis server''' + + def __init__(self, client, timeout=60, keys_container = 'tweepy:keys', pre_identifier = 'tweepy:'): + Cache.__init__(self, timeout) + self.client = client + self.keys_container = keys_container + self.pre_identifier = pre_identifier + + def _is_expired(self, entry, timeout): + # Returns true if the entry has expired + return timeout > 0 and (time.time() - entry[0]) >= timeout + + def store(self, key, value): + '''Store the key, value pair in our redis server''' + # Prepend tweepy to our key, this makes it easier to identify tweepy keys in our redis server + key = self.pre_identifier + key + # Get a pipe (to execute several redis commands in one step) + pipe = self.client.pipeline() + # Set our values in a redis hash (similar to python dict) + pipe.set(key, pickle.dumps((time.time(), value))) + # Set the expiration + pipe.expire(key, self.timeout) + # Add the key to a set containing all the keys + pipe.sadd(self.keys_container, key) + # Execute the instructions in the redis server + pipe.execute() + + def get(self, key, timeout=None): + '''Given a key, returns an element from the redis table''' + key = self.pre_identifier + key + # Check to see if we have this key + unpickled_entry = self.client.get(key) + if not unpickled_entry: + # No hit, return nothing + return None + + entry = pickle.loads(unpickled_entry) + # Use provided timeout in arguments if provided + # otherwise use the one provided during init. + if timeout is None: + timeout = self.timeout + + # Make sure entry is not expired + if self._is_expired(entry, timeout): + # entry expired, delete and return nothing + self.delete_entry(key) + return None + # entry found and not expired, return it + return entry[1] + + def count(self): + '''Note: This is not very efficient, since it retreives all the keys from the redis + server to know how many keys we have''' + return len(self.client.smembers(self.keys_container)) + + def delete_entry(self, key): + '''Delete an object from the redis table''' + pipe = self.client.pipeline() + pipe.srem(self.keys_container, key) + pipe.delete(key) + pipe.execute() + + def cleanup(self): + '''Cleanup all the expired keys''' + keys = self.client.smembers(self.keys_container) + for key in keys: + entry = self.client.get(key) + if entry: + entry = pickle.loads(entry) + if self._is_expired(entry, self.timeout): + self.delete_entry(key) + + def flush(self): + '''Delete all entries from the cache''' + keys = self.client.smembers(self.keys_container) + for key in keys: + self.delete_entry(key) + + +class MongodbCache(Cache): + """A simple pickle-based MongoDB cache sytem.""" + + def __init__(self, db, timeout=3600, collection='tweepy_cache'): + """Should receive a "database" cursor from pymongo.""" + Cache.__init__(self, timeout) + self.timeout = timeout + self.col = db[collection] + self.col.create_index('created', expireAfterSeconds=timeout) + + def store(self, key, value): + from bson.binary import Binary + + now = datetime.datetime.utcnow() + blob = Binary(pickle.dumps(value)) + + self.col.insert({'created': now, '_id': key, 'value': blob}) + + def get(self, key, timeout=None): + if timeout: + raise NotImplementedError + obj = self.col.find_one({'_id': key}) + if obj: + return pickle.loads(obj['value']) + + def count(self): + return self.col.find({}).count() + + def delete_entry(self, key): + return self.col.remove({'_id': key}) + + def cleanup(self): + """MongoDB will automatically clear expired keys.""" + pass + + def flush(self): + self.col.drop() + self.col.create_index('created', expireAfterSeconds=self.timeout) diff --git a/tweepy/cache.pyc b/tweepy/cache.pyc new file mode 100644 index 0000000..1681ad1 Binary files /dev/null and b/tweepy/cache.pyc differ diff --git a/tweepy/cursor.py b/tweepy/cursor.py new file mode 100755 index 0000000..4c06f17 --- /dev/null +++ b/tweepy/cursor.py @@ -0,0 +1,171 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +from tweepy.error import TweepError + +class Cursor(object): + """Pagination helper class""" + + def __init__(self, method, *args, **kargs): + if hasattr(method, 'pagination_mode'): + if method.pagination_mode == 'cursor': + self.iterator = CursorIterator(method, args, kargs) + elif method.pagination_mode == 'id': + self.iterator = IdIterator(method, args, kargs) + elif method.pagination_mode == 'page': + self.iterator = PageIterator(method, args, kargs) + else: + raise TweepError('Invalid pagination mode.') + else: + raise TweepError('This method does not perform pagination') + + def pages(self, limit=0): + """Return iterator for pages""" + if limit > 0: + self.iterator.limit = limit + return self.iterator + + def items(self, limit=0): + """Return iterator for items in each page""" + i = ItemIterator(self.iterator) + i.limit = limit + return i + +class BaseIterator(object): + + def __init__(self, method, args, kargs): + self.method = method + self.args = args + self.kargs = kargs + self.limit = 0 + + def next(self): + raise NotImplementedError + + def prev(self): + raise NotImplementedError + + def __iter__(self): + return self + +class CursorIterator(BaseIterator): + + def __init__(self, method, args, kargs): + BaseIterator.__init__(self, method, args, kargs) + start_cursor = kargs.pop('cursor', None) + self.next_cursor = start_cursor or -1 + self.prev_cursor = start_cursor or 0 + self.count = 0 + + def next(self): + if self.next_cursor == 0 or (self.limit and self.count == self.limit): + raise StopIteration + data, cursors = self.method( + cursor=self.next_cursor, *self.args, **self.kargs + ) + self.prev_cursor, self.next_cursor = cursors + if len(data) == 0: + raise StopIteration + self.count += 1 + return data + + def prev(self): + if self.prev_cursor == 0: + raise TweepError('Can not page back more, at first page') + data, self.next_cursor, self.prev_cursor = self.method( + cursor=self.prev_cursor, *self.args, **self.kargs + ) + self.count -= 1 + return data + +class IdIterator(BaseIterator): + + def __init__(self, method, args, kargs): + BaseIterator.__init__(self, method, args, kargs) + self.max_id = kargs.get('max_id') + self.since_id = kargs.get('since_id') + self.count = 0 + + def next(self): + """Fetch a set of items with IDs less than current set.""" + if self.limit and self.limit == self.count: + raise StopIteration + + # max_id is inclusive so decrement by one + # to avoid requesting duplicate items. + max_id = self.since_id - 1 if self.max_id else None + data = self.method(max_id = max_id, *self.args, **self.kargs) + if len(data) == 0: + raise StopIteration + self.max_id = data.max_id + self.since_id = data.since_id + self.count += 1 + return data + + def prev(self): + """Fetch a set of items with IDs greater than current set.""" + if self.limit and self.limit == self.count: + raise StopIteration + + since_id = self.max_id + data = self.method(since_id = since_id, *self.args, **self.kargs) + if len(data) == 0: + raise StopIteration + self.max_id = data.max_id + self.since_id = data.since_id + self.count += 1 + return data + +class PageIterator(BaseIterator): + + def __init__(self, method, args, kargs): + BaseIterator.__init__(self, method, args, kargs) + self.current_page = 0 + + def next(self): + self.current_page += 1 + items = self.method(page=self.current_page, *self.args, **self.kargs) + if len(items) == 0 or (self.limit > 0 and self.current_page > self.limit): + raise StopIteration + return items + + def prev(self): + if (self.current_page == 1): + raise TweepError('Can not page back more, at first page') + self.current_page -= 1 + return self.method(page=self.current_page, *self.args, **self.kargs) + +class ItemIterator(BaseIterator): + + def __init__(self, page_iterator): + self.page_iterator = page_iterator + self.limit = 0 + self.current_page = None + self.page_index = -1 + self.count = 0 + + def next(self): + if self.limit > 0 and self.count == self.limit: + raise StopIteration + if self.current_page is None or self.page_index == len(self.current_page) - 1: + # Reached end of current page, get the next page... + self.current_page = self.page_iterator.next() + self.page_index = -1 + self.page_index += 1 + self.count += 1 + return self.current_page[self.page_index] + + def prev(self): + if self.current_page is None: + raise TweepError('Can not go back more, at first page') + if self.page_index == 0: + # At the beginning of the current page, move to next... + self.current_page = self.page_iterator.prev() + self.page_index = len(self.current_page) + if self.page_index == 0: + raise TweepError('No more items') + self.page_index -= 1 + self.count -= 1 + return self.current_page[self.page_index] + diff --git a/tweepy/cursor.pyc b/tweepy/cursor.pyc new file mode 100644 index 0000000..9fe446f Binary files /dev/null and b/tweepy/cursor.pyc differ diff --git a/tweepy/error.py b/tweepy/error.py new file mode 100755 index 0000000..753e2fe --- /dev/null +++ b/tweepy/error.py @@ -0,0 +1,15 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +class TweepError(Exception): + """Tweepy exception""" + + def __init__(self, reason, response=None): + self.reason = unicode(reason) + self.response = response + Exception.__init__(self, reason) + + def __str__(self): + return self.reason + diff --git a/tweepy/error.pyc b/tweepy/error.pyc new file mode 100644 index 0000000..c85db90 Binary files /dev/null and b/tweepy/error.pyc differ diff --git a/tweepy/models.py b/tweepy/models.py new file mode 100755 index 0000000..9432463 --- /dev/null +++ b/tweepy/models.py @@ -0,0 +1,433 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +from tweepy.error import TweepError +from tweepy.utils import parse_datetime, parse_html_value, parse_a_href + + +class ResultSet(list): + """A list like object that holds results from a Twitter API query.""" + def __init__(self, max_id=None, since_id=None): + super(ResultSet, self).__init__() + self._max_id = max_id + self._since_id = since_id + + @property + def max_id(self): + if self._max_id: + return self._max_id + ids = self.ids() + return max(ids) if ids else None + + @property + def since_id(self): + if self._since_id: + return self._since_id + ids = self.ids() + return min(ids) if ids else None + + def ids(self): + return [item.id for item in self if hasattr(item, 'id')] + +class Model(object): + + def __init__(self, api=None): + self._api = api + + def __getstate__(self): + # pickle + pickle = dict(self.__dict__) + try: + del pickle['_api'] # do not pickle the API reference + except KeyError: + pass + return pickle + + @classmethod + def parse(cls, api, json): + """Parse a JSON object into a model instance.""" + raise NotImplementedError + + @classmethod + def parse_list(cls, api, json_list): + """Parse a list of JSON objects into a result set of model instances.""" + results = ResultSet() + for obj in json_list: + if obj: + results.append(cls.parse(api, obj)) + return results + + +class Status(Model): + + @classmethod + def parse(cls, api, json): + status = cls(api) + for k, v in json.items(): + if k == 'user': + user_model = getattr(api.parser.model_factory, 'user') if api else User + user = user_model.parse(api, v) + setattr(status, 'author', user) + setattr(status, 'user', user) # DEPRECIATED + elif k == 'created_at': + setattr(status, k, parse_datetime(v)) + elif k == 'source': + if '<' in v: + setattr(status, k, parse_html_value(v)) + setattr(status, 'source_url', parse_a_href(v)) + else: + setattr(status, k, v) + setattr(status, 'source_url', None) + elif k == 'retweeted_status': + setattr(status, k, Status.parse(api, v)) + elif k == 'place': + if v is not None: + setattr(status, k, Place.parse(api, v)) + else: + setattr(status, k, None) + else: + setattr(status, k, v) + setattr(status, 'json', json) + return status + + def destroy(self): + return self._api.destroy_status(self.id) + + def retweet(self): + return self._api.retweet(self.id) + + def retweets(self): + return self._api.retweets(self.id) + + def favorite(self): + return self._api.create_favorite(self.id) + + +class User(Model): + + @classmethod + def parse(cls, api, json): + user = cls(api) + for k, v in json.items(): + if k == 'created_at': + setattr(user, k, parse_datetime(v)) + elif k == 'status': + setattr(user, k, Status.parse(api, v)) + elif k == 'following': + # twitter sets this to null if it is false + if v is True: + setattr(user, k, True) + else: + setattr(user, k, False) + else: + setattr(user, k, v) + return user + + @classmethod + def parse_list(cls, api, json_list): + if isinstance(json_list, list): + item_list = json_list + else: + item_list = json_list['users'] + + results = ResultSet() + for obj in item_list: + results.append(cls.parse(api, obj)) + return results + + def timeline(self, **kargs): + return self._api.user_timeline(user_id=self.id, **kargs) + + def friends(self, **kargs): + return self._api.friends(user_id=self.id, **kargs) + + def followers(self, **kargs): + return self._api.followers(user_id=self.id, **kargs) + + def follow(self): + self._api.create_friendship(user_id=self.id) + self.following = True + + def unfollow(self): + self._api.destroy_friendship(user_id=self.id) + self.following = False + + def lists_memberships(self, *args, **kargs): + return self._api.lists_memberships(user=self.screen_name, *args, **kargs) + + def lists_subscriptions(self, *args, **kargs): + return self._api.lists_subscriptions(user=self.screen_name, *args, **kargs) + + def lists(self, *args, **kargs): + return self._api.lists_all(user=self.screen_name, *args, **kargs) + + def followers_ids(self, *args, **kargs): + return self._api.followers_ids(user_id=self.id, *args, **kargs) + + +class DirectMessage(Model): + + @classmethod + def parse(cls, api, json): + dm = cls(api) + for k, v in json.items(): + if k == 'sender' or k == 'recipient': + setattr(dm, k, User.parse(api, v)) + elif k == 'created_at': + setattr(dm, k, parse_datetime(v)) + else: + setattr(dm, k, v) + return dm + + def destroy(self): + return self._api.destroy_direct_message(self.id) + + +class Friendship(Model): + + @classmethod + def parse(cls, api, json): + relationship = json['relationship'] + + # parse source + source = cls(api) + for k, v in relationship['source'].items(): + setattr(source, k, v) + + # parse target + target = cls(api) + for k, v in relationship['target'].items(): + setattr(target, k, v) + + return source, target + + +class Category(Model): + + @classmethod + def parse(cls, api, json): + category = cls(api) + for k, v in json.items(): + setattr(category, k, v) + return category + + +class SavedSearch(Model): + + @classmethod + def parse(cls, api, json): + ss = cls(api) + for k, v in json.items(): + if k == 'created_at': + setattr(ss, k, parse_datetime(v)) + else: + setattr(ss, k, v) + return ss + + def destroy(self): + return self._api.destroy_saved_search(self.id) + + +class SearchResults(ResultSet): + + @classmethod + def parse(cls, api, json): + metadata = json['search_metadata'] + results = SearchResults(metadata.get('max_id'), metadata.get('since_id')) + results.refresh_url = metadata.get('refresh_url') + results.completed_in = metadata.get('completed_in') + results.query = metadata.get('query') + results.count = metadata.get('count') + results.next_results = metadata.get('next_results') + + for status in json['statuses']: + results.append(Status.parse(api, status)) + return results + + +class List(Model): + + @classmethod + def parse(cls, api, json): + lst = List(api) + for k,v in json.items(): + if k == 'user': + setattr(lst, k, User.parse(api, v)) + elif k == 'created_at': + setattr(lst, k, parse_datetime(v)) + else: + setattr(lst, k, v) + return lst + + @classmethod + def parse_list(cls, api, json_list, result_set=None): + results = ResultSet() + if isinstance(json_list, dict): + json_list = json_list['lists'] + for obj in json_list: + results.append(cls.parse(api, obj)) + return results + + def update(self, **kargs): + return self._api.update_list(self.slug, **kargs) + + def destroy(self): + return self._api.destroy_list(self.slug) + + def timeline(self, **kargs): + return self._api.list_timeline(self.user.screen_name, self.slug, **kargs) + + def add_member(self, id): + return self._api.add_list_member(self.slug, id) + + def remove_member(self, id): + return self._api.remove_list_member(self.slug, id) + + def members(self, **kargs): + return self._api.list_members(self.user.screen_name, self.slug, **kargs) + + def is_member(self, id): + return self._api.is_list_member(self.user.screen_name, self.slug, id) + + def subscribe(self): + return self._api.subscribe_list(self.user.screen_name, self.slug) + + def unsubscribe(self): + return self._api.unsubscribe_list(self.user.screen_name, self.slug) + + def subscribers(self, **kargs): + return self._api.list_subscribers(self.user.screen_name, self.slug, **kargs) + + def is_subscribed(self, id): + return self._api.is_subscribed_list(self.user.screen_name, self.slug, id) + +class Relation(Model): + @classmethod + def parse(cls, api, json): + result = cls(api) + for k,v in json.items(): + if k == 'value' and json['kind'] in ['Tweet', 'LookedupStatus']: + setattr(result, k, Status.parse(api, v)) + elif k == 'results': + setattr(result, k, Relation.parse_list(api, v)) + else: + setattr(result, k, v) + return result + +class Relationship(Model): + @classmethod + def parse(cls, api, json): + result = cls(api) + for k,v in json.items(): + if k == 'connections': + setattr(result, 'is_following', 'following' in v) + setattr(result, 'is_followed_by', 'followed_by' in v) + else: + setattr(result, k, v) + return result + +class JSONModel(Model): + + @classmethod + def parse(cls, api, json): + return json + + +class IDModel(Model): + + @classmethod + def parse(cls, api, json): + if isinstance(json, list): + return json + else: + return json['ids'] + + +class BoundingBox(Model): + + @classmethod + def parse(cls, api, json): + result = cls(api) + if json is not None: + for k, v in json.items(): + setattr(result, k, v) + return result + + def origin(self): + """ + Return longitude, latitude of southwest (bottom, left) corner of + bounding box, as a tuple. + + This assumes that bounding box is always a rectangle, which + appears to be the case at present. + """ + return tuple(self.coordinates[0][0]) + + def corner(self): + """ + Return longitude, latitude of northeast (top, right) corner of + bounding box, as a tuple. + + This assumes that bounding box is always a rectangle, which + appears to be the case at present. + """ + return tuple(self.coordinates[0][2]) + + +class Place(Model): + + @classmethod + def parse(cls, api, json): + place = cls(api) + for k, v in json.items(): + if k == 'bounding_box': + # bounding_box value may be null (None.) + # Example: "United States" (id=96683cc9126741d1) + if v is not None: + t = BoundingBox.parse(api, v) + else: + t = v + setattr(place, k, t) + elif k == 'contained_within': + # contained_within is a list of Places. + setattr(place, k, Place.parse_list(api, v)) + else: + setattr(place, k, v) + return place + + @classmethod + def parse_list(cls, api, json_list): + if isinstance(json_list, list): + item_list = json_list + else: + item_list = json_list['result']['places'] + + results = ResultSet() + for obj in item_list: + results.append(cls.parse(api, obj)) + return results + +class ModelFactory(object): + """ + Used by parsers for creating instances + of models. You may subclass this factory + to add your own extended models. + """ + + status = Status + user = User + direct_message = DirectMessage + friendship = Friendship + saved_search = SavedSearch + search_results = SearchResults + category = Category + list = List + relation = Relation + relationship = Relationship + + json = JSONModel + ids = IDModel + place = Place + bounding_box = BoundingBox + diff --git a/tweepy/models.pyc b/tweepy/models.pyc new file mode 100644 index 0000000..f90612d Binary files /dev/null and b/tweepy/models.pyc differ diff --git a/tweepy/oauth.py b/tweepy/oauth.py new file mode 100755 index 0000000..286de18 --- /dev/null +++ b/tweepy/oauth.py @@ -0,0 +1,655 @@ +""" +The MIT License + +Copyright (c) 2007 Leah Culver + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import cgi +import urllib +import time +import random +import urlparse +import hmac +import binascii + + +VERSION = '1.0' # Hi Blaine! +HTTP_METHOD = 'GET' +SIGNATURE_METHOD = 'PLAINTEXT' + + +class OAuthError(RuntimeError): + """Generic exception class.""" + def __init__(self, message='OAuth error occured.'): + self.message = message + +def build_authenticate_header(realm=''): + """Optional WWW-Authenticate header (401 error)""" + return {'WWW-Authenticate': 'OAuth realm="%s"' % realm} + +def escape(s): + """Escape a URL including any /.""" + return urllib.quote(s, safe='~') + +def _utf8_str(s): + """Convert unicode to utf-8.""" + if isinstance(s, unicode): + return s.encode("utf-8") + else: + return str(s) + +def generate_timestamp(): + """Get seconds since epoch (UTC).""" + return int(time.time()) + +def generate_nonce(length=8): + """Generate pseudorandom number.""" + return ''.join([str(random.randint(0, 9)) for i in range(length)]) + +def generate_verifier(length=8): + """Generate pseudorandom number.""" + return ''.join([str(random.randint(0, 9)) for i in range(length)]) + + +class OAuthConsumer(object): + """Consumer of OAuth authentication. + + OAuthConsumer is a data type that represents the identity of the Consumer + via its shared secret with the Service Provider. + + """ + key = None + secret = None + + def __init__(self, key, secret): + self.key = key + self.secret = secret + + +class OAuthToken(object): + """OAuthToken is a data type that represents an End User via either an access + or request token. + + key -- the token + secret -- the token secret + + """ + key = None + secret = None + callback = None + callback_confirmed = None + verifier = None + + def __init__(self, key, secret): + self.key = key + self.secret = secret + + def set_callback(self, callback): + self.callback = callback + self.callback_confirmed = 'true' + + def set_verifier(self, verifier=None): + if verifier is not None: + self.verifier = verifier + else: + self.verifier = generate_verifier() + + def get_callback_url(self): + if self.callback and self.verifier: + # Append the oauth_verifier. + parts = urlparse.urlparse(self.callback) + scheme, netloc, path, params, query, fragment = parts[:6] + if query: + query = '%s&oauth_verifier=%s' % (query, self.verifier) + else: + query = 'oauth_verifier=%s' % self.verifier + return urlparse.urlunparse((scheme, netloc, path, params, + query, fragment)) + return self.callback + + def to_string(self): + data = { + 'oauth_token': self.key, + 'oauth_token_secret': self.secret, + } + if self.callback_confirmed is not None: + data['oauth_callback_confirmed'] = self.callback_confirmed + return urllib.urlencode(data) + + def from_string(s): + """ Returns a token from something like: + oauth_token_secret=xxx&oauth_token=xxx + """ + params = cgi.parse_qs(s, keep_blank_values=False) + key = params['oauth_token'][0] + secret = params['oauth_token_secret'][0] + token = OAuthToken(key, secret) + try: + token.callback_confirmed = params['oauth_callback_confirmed'][0] + except KeyError: + pass # 1.0, no callback confirmed. + return token + from_string = staticmethod(from_string) + + def __str__(self): + return self.to_string() + + +class OAuthRequest(object): + """OAuthRequest represents the request and can be serialized. + + OAuth parameters: + - oauth_consumer_key + - oauth_token + - oauth_signature_method + - oauth_signature + - oauth_timestamp + - oauth_nonce + - oauth_version + - oauth_verifier + ... any additional parameters, as defined by the Service Provider. + """ + parameters = None # OAuth parameters. + http_method = HTTP_METHOD + http_url = None + version = VERSION + + def __init__(self, http_method=HTTP_METHOD, http_url=None, parameters=None): + self.http_method = http_method + self.http_url = http_url + self.parameters = parameters or {} + + def set_parameter(self, parameter, value): + self.parameters[parameter] = value + + def get_parameter(self, parameter): + try: + return self.parameters[parameter] + except: + raise OAuthError('Parameter not found: %s' % parameter) + + def _get_timestamp_nonce(self): + return self.get_parameter('oauth_timestamp'), self.get_parameter( + 'oauth_nonce') + + def get_nonoauth_parameters(self): + """Get any non-OAuth parameters.""" + parameters = {} + for k, v in self.parameters.iteritems(): + # Ignore oauth parameters. + if k.find('oauth_') < 0: + parameters[k] = v + return parameters + + def to_header(self, realm=''): + """Serialize as a header for an HTTPAuth request.""" + auth_header = 'OAuth realm="%s"' % realm + # Add the oauth parameters. + if self.parameters: + for k, v in self.parameters.iteritems(): + if k[:6] == 'oauth_': + auth_header += ', %s="%s"' % (k, escape(str(v))) + return {'Authorization': auth_header} + + def to_postdata(self): + """Serialize as post data for a POST request.""" + return '&'.join(['%s=%s' % (escape(str(k)), escape(str(v))) \ + for k, v in self.parameters.iteritems()]) + + def to_url(self): + """Serialize as a URL for a GET request.""" + return '%s?%s' % (self.get_normalized_http_url(), self.to_postdata()) + + def get_normalized_parameters(self): + """Return a string that contains the parameters that must be signed.""" + params = self.parameters + try: + # Exclude the signature if it exists. + del params['oauth_signature'] + except: + pass + # Escape key values before sorting. + key_values = [(escape(_utf8_str(k)), escape(_utf8_str(v))) \ + for k,v in params.items()] + # Sort lexicographically, first after key, then after value. + key_values.sort() + # Combine key value pairs into a string. + return '&'.join(['%s=%s' % (k, v) for k, v in key_values]) + + def get_normalized_http_method(self): + """Uppercases the http method.""" + return self.http_method.upper() + + def get_normalized_http_url(self): + """Parses the URL and rebuilds it to be scheme://host/path.""" + parts = urlparse.urlparse(self.http_url) + scheme, netloc, path = parts[:3] + # Exclude default port numbers. + if scheme == 'http' and netloc[-3:] == ':80': + netloc = netloc[:-3] + elif scheme == 'https' and netloc[-4:] == ':443': + netloc = netloc[:-4] + return '%s://%s%s' % (scheme, netloc, path) + + def sign_request(self, signature_method, consumer, token): + """Set the signature parameter to the result of build_signature.""" + # Set the signature method. + self.set_parameter('oauth_signature_method', + signature_method.get_name()) + # Set the signature. + self.set_parameter('oauth_signature', + self.build_signature(signature_method, consumer, token)) + + def build_signature(self, signature_method, consumer, token): + """Calls the build signature method within the signature method.""" + return signature_method.build_signature(self, consumer, token) + + def from_request(http_method, http_url, headers=None, parameters=None, + query_string=None): + """Combines multiple parameter sources.""" + if parameters is None: + parameters = {} + + # Headers + if headers and 'Authorization' in headers: + auth_header = headers['Authorization'] + # Check that the authorization header is OAuth. + if auth_header[:6] == 'OAuth ': + auth_header = auth_header[6:] + try: + # Get the parameters from the header. + header_params = OAuthRequest._split_header(auth_header) + parameters.update(header_params) + except: + raise OAuthError('Unable to parse OAuth parameters from ' + 'Authorization header.') + + # GET or POST query string. + if query_string: + query_params = OAuthRequest._split_url_string(query_string) + parameters.update(query_params) + + # URL parameters. + param_str = urlparse.urlparse(http_url)[4] # query + url_params = OAuthRequest._split_url_string(param_str) + parameters.update(url_params) + + if parameters: + return OAuthRequest(http_method, http_url, parameters) + + return None + from_request = staticmethod(from_request) + + def from_consumer_and_token(oauth_consumer, token=None, + callback=None, verifier=None, http_method=HTTP_METHOD, + http_url=None, parameters=None): + if not parameters: + parameters = {} + + defaults = { + 'oauth_consumer_key': oauth_consumer.key, + 'oauth_timestamp': generate_timestamp(), + 'oauth_nonce': generate_nonce(), + 'oauth_version': OAuthRequest.version, + } + + defaults.update(parameters) + parameters = defaults + + if token: + parameters['oauth_token'] = token.key + if token.callback: + parameters['oauth_callback'] = token.callback + # 1.0a support for verifier. + if verifier: + parameters['oauth_verifier'] = verifier + elif callback: + # 1.0a support for callback in the request token request. + parameters['oauth_callback'] = callback + + return OAuthRequest(http_method, http_url, parameters) + from_consumer_and_token = staticmethod(from_consumer_and_token) + + def from_token_and_callback(token, callback=None, http_method=HTTP_METHOD, + http_url=None, parameters=None): + if not parameters: + parameters = {} + + parameters['oauth_token'] = token.key + + if callback: + parameters['oauth_callback'] = callback + + return OAuthRequest(http_method, http_url, parameters) + from_token_and_callback = staticmethod(from_token_and_callback) + + def _split_header(header): + """Turn Authorization: header into parameters.""" + params = {} + parts = header.split(',') + for param in parts: + # Ignore realm parameter. + if param.find('realm') > -1: + continue + # Remove whitespace. + param = param.strip() + # Split key-value. + param_parts = param.split('=', 1) + # Remove quotes and unescape the value. + params[param_parts[0]] = urllib.unquote(param_parts[1].strip('\"')) + return params + _split_header = staticmethod(_split_header) + + def _split_url_string(param_str): + """Turn URL string into parameters.""" + parameters = cgi.parse_qs(param_str, keep_blank_values=False) + for k, v in parameters.iteritems(): + parameters[k] = urllib.unquote(v[0]) + return parameters + _split_url_string = staticmethod(_split_url_string) + +class OAuthServer(object): + """A worker to check the validity of a request against a data store.""" + timestamp_threshold = 300 # In seconds, five minutes. + version = VERSION + signature_methods = None + data_store = None + + def __init__(self, data_store=None, signature_methods=None): + self.data_store = data_store + self.signature_methods = signature_methods or {} + + def set_data_store(self, data_store): + self.data_store = data_store + + def get_data_store(self): + return self.data_store + + def add_signature_method(self, signature_method): + self.signature_methods[signature_method.get_name()] = signature_method + return self.signature_methods + + def fetch_request_token(self, oauth_request): + """Processes a request_token request and returns the + request token on success. + """ + try: + # Get the request token for authorization. + token = self._get_token(oauth_request, 'request') + except OAuthError: + # No token required for the initial token request. + version = self._get_version(oauth_request) + consumer = self._get_consumer(oauth_request) + try: + callback = self.get_callback(oauth_request) + except OAuthError: + callback = None # 1.0, no callback specified. + self._check_signature(oauth_request, consumer, None) + # Fetch a new token. + token = self.data_store.fetch_request_token(consumer, callback) + return token + + def fetch_access_token(self, oauth_request): + """Processes an access_token request and returns the + access token on success. + """ + version = self._get_version(oauth_request) + consumer = self._get_consumer(oauth_request) + try: + verifier = self._get_verifier(oauth_request) + except OAuthError: + verifier = None + # Get the request token. + token = self._get_token(oauth_request, 'request') + self._check_signature(oauth_request, consumer, token) + new_token = self.data_store.fetch_access_token(consumer, token, verifier) + return new_token + + def verify_request(self, oauth_request): + """Verifies an api call and checks all the parameters.""" + # -> consumer and token + version = self._get_version(oauth_request) + consumer = self._get_consumer(oauth_request) + # Get the access token. + token = self._get_token(oauth_request, 'access') + self._check_signature(oauth_request, consumer, token) + parameters = oauth_request.get_nonoauth_parameters() + return consumer, token, parameters + + def authorize_token(self, token, user): + """Authorize a request token.""" + return self.data_store.authorize_request_token(token, user) + + def get_callback(self, oauth_request): + """Get the callback URL.""" + return oauth_request.get_parameter('oauth_callback') + + def build_authenticate_header(self, realm=''): + """Optional support for the authenticate header.""" + return {'WWW-Authenticate': 'OAuth realm="%s"' % realm} + + def _get_version(self, oauth_request): + """Verify the correct version request for this server.""" + try: + version = oauth_request.get_parameter('oauth_version') + except: + version = VERSION + if version and version != self.version: + raise OAuthError('OAuth version %s not supported.' % str(version)) + return version + + def _get_signature_method(self, oauth_request): + """Figure out the signature with some defaults.""" + try: + signature_method = oauth_request.get_parameter( + 'oauth_signature_method') + except: + signature_method = SIGNATURE_METHOD + try: + # Get the signature method object. + signature_method = self.signature_methods[signature_method] + except: + signature_method_names = ', '.join(self.signature_methods.keys()) + raise OAuthError('Signature method %s not supported try one of the ' + 'following: %s' % (signature_method, signature_method_names)) + + return signature_method + + def _get_consumer(self, oauth_request): + consumer_key = oauth_request.get_parameter('oauth_consumer_key') + consumer = self.data_store.lookup_consumer(consumer_key) + if not consumer: + raise OAuthError('Invalid consumer.') + return consumer + + def _get_token(self, oauth_request, token_type='access'): + """Try to find the token for the provided request token key.""" + token_field = oauth_request.get_parameter('oauth_token') + token = self.data_store.lookup_token(token_type, token_field) + if not token: + raise OAuthError('Invalid %s token: %s' % (token_type, token_field)) + return token + + def _get_verifier(self, oauth_request): + return oauth_request.get_parameter('oauth_verifier') + + def _check_signature(self, oauth_request, consumer, token): + timestamp, nonce = oauth_request._get_timestamp_nonce() + self._check_timestamp(timestamp) + self._check_nonce(consumer, token, nonce) + signature_method = self._get_signature_method(oauth_request) + try: + signature = oauth_request.get_parameter('oauth_signature') + except: + raise OAuthError('Missing signature.') + # Validate the signature. + valid_sig = signature_method.check_signature(oauth_request, consumer, + token, signature) + if not valid_sig: + key, base = signature_method.build_signature_base_string( + oauth_request, consumer, token) + raise OAuthError('Invalid signature. Expected signature base ' + 'string: %s' % base) + built = signature_method.build_signature(oauth_request, consumer, token) + + def _check_timestamp(self, timestamp): + """Verify that timestamp is recentish.""" + timestamp = int(timestamp) + now = int(time.time()) + lapsed = abs(now - timestamp) + if lapsed > self.timestamp_threshold: + raise OAuthError('Expired timestamp: given %d and now %s has a ' + 'greater difference than threshold %d' % + (timestamp, now, self.timestamp_threshold)) + + def _check_nonce(self, consumer, token, nonce): + """Verify that the nonce is uniqueish.""" + nonce = self.data_store.lookup_nonce(consumer, token, nonce) + if nonce: + raise OAuthError('Nonce already used: %s' % str(nonce)) + + +class OAuthClient(object): + """OAuthClient is a worker to attempt to execute a request.""" + consumer = None + token = None + + def __init__(self, oauth_consumer, oauth_token): + self.consumer = oauth_consumer + self.token = oauth_token + + def get_consumer(self): + return self.consumer + + def get_token(self): + return self.token + + def fetch_request_token(self, oauth_request): + """-> OAuthToken.""" + raise NotImplementedError + + def fetch_access_token(self, oauth_request): + """-> OAuthToken.""" + raise NotImplementedError + + def access_resource(self, oauth_request): + """-> Some protected resource.""" + raise NotImplementedError + + +class OAuthDataStore(object): + """A database abstraction used to lookup consumers and tokens.""" + + def lookup_consumer(self, key): + """-> OAuthConsumer.""" + raise NotImplementedError + + def lookup_token(self, oauth_consumer, token_type, token_token): + """-> OAuthToken.""" + raise NotImplementedError + + def lookup_nonce(self, oauth_consumer, oauth_token, nonce): + """-> OAuthToken.""" + raise NotImplementedError + + def fetch_request_token(self, oauth_consumer, oauth_callback): + """-> OAuthToken.""" + raise NotImplementedError + + def fetch_access_token(self, oauth_consumer, oauth_token, oauth_verifier): + """-> OAuthToken.""" + raise NotImplementedError + + def authorize_request_token(self, oauth_token, user): + """-> OAuthToken.""" + raise NotImplementedError + + +class OAuthSignatureMethod(object): + """A strategy class that implements a signature method.""" + def get_name(self): + """-> str.""" + raise NotImplementedError + + def build_signature_base_string(self, oauth_request, oauth_consumer, oauth_token): + """-> str key, str raw.""" + raise NotImplementedError + + def build_signature(self, oauth_request, oauth_consumer, oauth_token): + """-> str.""" + raise NotImplementedError + + def check_signature(self, oauth_request, consumer, token, signature): + built = self.build_signature(oauth_request, consumer, token) + return built == signature + + +class OAuthSignatureMethod_HMAC_SHA1(OAuthSignatureMethod): + + def get_name(self): + return 'HMAC-SHA1' + + def build_signature_base_string(self, oauth_request, consumer, token): + sig = ( + escape(oauth_request.get_normalized_http_method()), + escape(oauth_request.get_normalized_http_url()), + escape(oauth_request.get_normalized_parameters()), + ) + + key = '%s&' % escape(consumer.secret) + if token: + key += escape(token.secret) + raw = '&'.join(sig) + return key, raw + + def build_signature(self, oauth_request, consumer, token): + """Builds the base signature string.""" + key, raw = self.build_signature_base_string(oauth_request, consumer, + token) + + # HMAC object. + try: + import hashlib # 2.5 + hashed = hmac.new(key, raw, hashlib.sha1) + except: + import sha # Deprecated + hashed = hmac.new(key, raw, sha) + + # Calculate the digest base 64. + return binascii.b2a_base64(hashed.digest())[:-1] + + +class OAuthSignatureMethod_PLAINTEXT(OAuthSignatureMethod): + + def get_name(self): + return 'PLAINTEXT' + + def build_signature_base_string(self, oauth_request, consumer, token): + """Concatenates the consumer key and secret.""" + sig = '%s&' % escape(consumer.secret) + if token: + sig = sig + escape(token.secret) + return sig, sig + + def build_signature(self, oauth_request, consumer, token): + key, raw = self.build_signature_base_string(oauth_request, consumer, + token) + return key \ No newline at end of file diff --git a/tweepy/oauth.pyc b/tweepy/oauth.pyc new file mode 100644 index 0000000..8d718e9 Binary files /dev/null and b/tweepy/oauth.pyc differ diff --git a/tweepy/parsers.py b/tweepy/parsers.py new file mode 100755 index 0000000..55a5ba8 --- /dev/null +++ b/tweepy/parsers.py @@ -0,0 +1,97 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +from tweepy.models import ModelFactory +from tweepy.utils import import_simplejson +from tweepy.error import TweepError + + +class Parser(object): + + def parse(self, method, payload): + """ + Parse the response payload and return the result. + Returns a tuple that contains the result data and the cursors + (or None if not present). + """ + raise NotImplementedError + + def parse_error(self, payload): + """ + Parse the error message from payload. + If unable to parse the message, throw an exception + and default error message will be used. + """ + raise NotImplementedError + + +class RawParser(Parser): + + def __init__(self): + pass + + def parse(self, method, payload): + return payload + + def parse_error(self, payload): + return payload + + +class JSONParser(Parser): + + payload_format = 'json' + + def __init__(self): + self.json_lib = import_simplejson() + + def parse(self, method, payload): + try: + json = self.json_lib.loads(payload) + except Exception, e: + raise TweepError('Failed to parse JSON payload: %s' % e) + + needsCursors = method.parameters.has_key('cursor') + if needsCursors and isinstance(json, dict) and 'previous_cursor' in json and 'next_cursor' in json: + cursors = json['previous_cursor'], json['next_cursor'] + return json, cursors + else: + return json + + def parse_error(self, payload): + error = self.json_lib.loads(payload) + if error.has_key('error'): + return error['error'] + else: + return error['errors'] + + +class ModelParser(JSONParser): + + def __init__(self, model_factory=None): + JSONParser.__init__(self) + self.model_factory = model_factory or ModelFactory + + def parse(self, method, payload): + try: + if method.payload_type is None: return + model = getattr(self.model_factory, method.payload_type) + except AttributeError: + raise TweepError('No model for this payload type: %s' % method.payload_type) + + json = JSONParser.parse(self, method, payload) + if isinstance(json, tuple): + json, cursors = json + else: + cursors = None + + if method.payload_list: + result = model.parse_list(method.api, json) + else: + result = model.parse(method.api, json) + + if cursors: + return result, cursors + else: + return result + diff --git a/tweepy/parsers.pyc b/tweepy/parsers.pyc new file mode 100644 index 0000000..b74926d Binary files /dev/null and b/tweepy/parsers.pyc differ diff --git a/tweepy/streaming.py b/tweepy/streaming.py new file mode 100755 index 0000000..b17b4f9 --- /dev/null +++ b/tweepy/streaming.py @@ -0,0 +1,319 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +import logging +import httplib +from socket import timeout +from threading import Thread +from time import sleep +import ssl + +from tweepy.models import Status +from tweepy.api import API +from tweepy.error import TweepError + +from tweepy.utils import import_simplejson, urlencode_noplus +json = import_simplejson() + +STREAM_VERSION = '1.1' + + +class StreamListener(object): + + def __init__(self, api=None): + self.api = api or API() + + def on_connect(self): + """Called once connected to streaming server. + + This will be invoked once a successful response + is received from the server. Allows the listener + to perform some work prior to entering the read loop. + """ + pass + + def on_data(self, raw_data): + """Called when raw data is received from connection. + + Override this method if you wish to manually handle + the stream data. Return False to stop stream and close connection. + """ + data = json.loads(raw_data) + + if 'in_reply_to_status_id' in data: + status = Status.parse(self.api, data) + if self.on_status(status) is False: + return False + elif 'delete' in data: + delete = data['delete']['status'] + if self.on_delete(delete['id'], delete['user_id']) is False: + return False + elif 'event' in data: + status = Status.parse(self.api, data) + if self.on_event(status) is False: + return False + elif 'direct_message' in data: + status = Status.parse(self.api, data) + if self.on_direct_message(status) is False: + return False + elif 'limit' in data: + if self.on_limit(data['limit']['track']) is False: + return False + elif 'disconnect' in data: + if self.on_disconnect(data['disconnect']) is False: + return False + else: + logging.error("Unknown message type: " + str(raw_data)) + + def on_status(self, status): + """Called when a new status arrives""" + return + + def on_exception(self, exception): + """Called when an unhandled exception occurs.""" + return + + def on_delete(self, status_id, user_id): + """Called when a delete notice arrives for a status""" + return + + def on_event(self, status): + """Called when a new event arrives""" + return + + def on_direct_message(self, status): + """Called when a new direct message arrives""" + return + + def on_limit(self, track): + """Called when a limitation notice arrvies""" + return + + def on_error(self, status_code): + """Called when a non-200 status code is returned""" + return False + + def on_timeout(self): + """Called when stream connection times out""" + return + + def on_disconnect(self, notice): + """Called when twitter sends a disconnect notice + + Disconnect codes are listed here: + https://dev.twitter.com/docs/streaming-apis/messages#Disconnect_messages_disconnect + """ + return + + +class Stream(object): + + host = 'stream.twitter.com' + + def __init__(self, auth, listener, **options): + self.auth = auth + self.listener = listener + self.running = False + self.timeout = options.get("timeout", 300.0) + self.retry_count = options.get("retry_count") + # values according to https://dev.twitter.com/docs/streaming-apis/connecting#Reconnecting + self.retry_time_start = options.get("retry_time", 5.0) + self.retry_420_start = options.get("retry_420", 60.0) + self.retry_time_cap = options.get("retry_time_cap", 320.0) + self.snooze_time_step = options.get("snooze_time", 0.25) + self.snooze_time_cap = options.get("snooze_time_cap", 16) + self.buffer_size = options.get("buffer_size", 1500) + if options.get("secure", True): + self.scheme = "https" + else: + self.scheme = "http" + + self.api = API() + self.headers = options.get("headers") or {} + self.parameters = None + self.body = None + self.retry_time = self.retry_time_start + self.snooze_time = self.snooze_time_step + + def _run(self): + # Authenticate + url = "%s://%s%s" % (self.scheme, self.host, self.url) + + # Connect and process the stream + error_counter = 0 + conn = None + exception = None + while self.running: + if self.retry_count is not None and error_counter > self.retry_count: + # quit if error count greater than retry count + break + try: + if self.scheme == "http": + conn = httplib.HTTPConnection(self.host, timeout=self.timeout) + else: + conn = httplib.HTTPSConnection(self.host, timeout=self.timeout) + self.auth.apply_auth(url, 'POST', self.headers, self.parameters) + conn.connect() + conn.request('POST', self.url, self.body, headers=self.headers) + resp = conn.getresponse() + if resp.status != 200: + if self.listener.on_error(resp.status) is False: + break + error_counter += 1 + if resp.status == 420: + self.retry_time = max(self.retry_420_start, self.retry_time) + sleep(self.retry_time) + self.retry_time = min(self.retry_time * 2, self.retry_time_cap) + else: + error_counter = 0 + self.retry_time = self.retry_time_start + self.snooze_time = self.snooze_time_step + self.listener.on_connect() + self._read_loop(resp) + except (timeout, ssl.SSLError), exc: + # If it's not time out treat it like any other exception + if isinstance(exc, ssl.SSLError) and not (exc.args and 'timed out' in str(exc.args[0])): + exception = exc + break + + if self.listener.on_timeout() == False: + break + if self.running is False: + break + conn.close() + sleep(self.snooze_time) + self.snooze_time = min(self.snooze_time + self.snooze_time_step, + self.snooze_time_cap) + except Exception, exception: + # any other exception is fatal, so kill loop + break + + # cleanup + self.running = False + if conn: + conn.close() + + if exception: + # call a handler first so that the exception can be logged. + self.listener.on_exception(exception) + raise + + def _data(self, data): + if self.listener.on_data(data) is False: + self.running = False + + def _read_loop(self, resp): + + while self.running and not resp.isclosed(): + + # Note: keep-alive newlines might be inserted before each length value. + # read until we get a digit... + c = '\n' + while c == '\n' and self.running and not resp.isclosed(): + c = resp.read(1) + delimited_string = c + + # read rest of delimiter length.. + d = '' + while d != '\n' and self.running and not resp.isclosed(): + d = resp.read(1) + delimited_string += d + + # read the next twitter status object + if delimited_string.strip().isdigit(): + next_status_obj = resp.read( int(delimited_string) ) + self._data(next_status_obj) + + if resp.isclosed(): + self.on_closed(resp) + + def _start(self, async): + self.running = True + if async: + Thread(target=self._run).start() + else: + self._run() + + def on_closed(self, resp): + """ Called when the response has been closed by Twitter """ + pass + + def userstream(self, stall_warnings=False, _with=None, replies=None, + track=None, locations=None, async=False, encoding='utf8'): + self.parameters = {'delimited': 'length'} + if self.running: + raise TweepError('Stream object already connected!') + self.url = '/%s/user.json?delimited=length' % STREAM_VERSION + self.host='userstream.twitter.com' + if stall_warnings: + self.parameters['stall_warnings'] = stall_warnings + if _with: + self.parameters['with'] = _with + if replies: + self.parameters['replies'] = replies + if locations and len(locations) > 0: + assert len(locations) % 4 == 0 + self.parameters['locations'] = ','.join(['%.2f' % l for l in locations]) + if track: + encoded_track = [s.encode(encoding) for s in track] + self.parameters['track'] = ','.join(encoded_track) + self.body = urlencode_noplus(self.parameters) + self._start(async) + + def firehose(self, count=None, async=False): + self.parameters = {'delimited': 'length'} + if self.running: + raise TweepError('Stream object already connected!') + self.url = '/%s/statuses/firehose.json?delimited=length' % STREAM_VERSION + if count: + self.url += '&count=%s' % count + self._start(async) + + def retweet(self, async=False): + self.parameters = {'delimited': 'length'} + if self.running: + raise TweepError('Stream object already connected!') + self.url = '/%s/statuses/retweet.json?delimited=length' % STREAM_VERSION + self._start(async) + + def sample(self, count=None, async=False): + self.parameters = {'delimited': 'length'} + if self.running: + raise TweepError('Stream object already connected!') + self.url = '/%s/statuses/sample.json?delimited=length' % STREAM_VERSION + if count: + self.url += '&count=%s' % count + self._start(async) + + def filter(self, follow=None, track=None, async=False, locations=None, + count=None, stall_warnings=False, languages=None, encoding='utf8'): + self.parameters = {} + self.headers['Content-type'] = "application/x-www-form-urlencoded" + if self.running: + raise TweepError('Stream object already connected!') + self.url = '/%s/statuses/filter.json?delimited=length' % STREAM_VERSION + if follow: + encoded_follow = [s.encode(encoding) for s in follow] + self.parameters['follow'] = ','.join(encoded_follow) + if track: + encoded_track = [s.encode(encoding) for s in track] + self.parameters['track'] = ','.join(encoded_track) + if locations and len(locations) > 0: + assert len(locations) % 4 == 0 + self.parameters['locations'] = ','.join(['%.2f' % l for l in locations]) + if count: + self.parameters['count'] = count + if stall_warnings: + self.parameters['stall_warnings'] = stall_warnings + if languages: + self.parameters['language'] = ','.join(map(str, languages)) + self.body = urlencode_noplus(self.parameters) + self.parameters['delimited'] = 'length' + self._start(async) + + def disconnect(self): + if self.running is False: + return + self.running = False + diff --git a/tweepy/streaming.pyc b/tweepy/streaming.pyc new file mode 100644 index 0000000..2ab45ac Binary files /dev/null and b/tweepy/streaming.pyc differ diff --git a/tweepy/utils.py b/tweepy/utils.py new file mode 100755 index 0000000..7c2d498 --- /dev/null +++ b/tweepy/utils.py @@ -0,0 +1,60 @@ +# Tweepy +# Copyright 2010 Joshua Roesslein +# See LICENSE for details. + +from datetime import datetime +import time +import htmlentitydefs +import re +import locale +from urllib import quote +from email.utils import parsedate + + +def parse_datetime(string): + return datetime(*(parsedate(string)[:6])) + + +def parse_html_value(html): + + return html[html.find('>')+1:html.rfind('<')] + + +def parse_a_href(atag): + + start = atag.find('"') + 1 + end = atag.find('"', start) + return atag[start:end] + + +def convert_to_utf8_str(arg): + # written by Michael Norton (http://docondev.blogspot.com/) + if isinstance(arg, unicode): + arg = arg.encode('utf-8') + elif not isinstance(arg, str): + arg = str(arg) + return arg + + + +def import_simplejson(): + try: + import simplejson as json + except ImportError: + try: + import json # Python 2.6+ + except ImportError: + try: + from django.utils import simplejson as json # Google App Engine + except ImportError: + raise ImportError, "Can't load a json library" + + return json + +def list_to_csv(item_list): + if item_list: + return ','.join([str(i) for i in item_list]) + +def urlencode_noplus(query): + return '&'.join(['%s=%s' % (quote(str(k), ''), quote(str(v), '')) \ + for k, v in query.iteritems()]) diff --git a/tweepy/utils.pyc b/tweepy/utils.pyc new file mode 100644 index 0000000..ed34bc1 Binary files /dev/null and b/tweepy/utils.pyc differ diff --git a/twitter.ini b/twitter.ini new file mode 100644 index 0000000..1f14349 --- /dev/null +++ b/twitter.ini @@ -0,0 +1,5 @@ +[Twitter] +consumer_key = +consumer_secret = +access_key = +access_secret =