From 3c28ce8764d91c9b635093e7503f72003bcc5ded Mon Sep 17 00:00:00 2001 From: essamjoubori Date: Fri, 11 Mar 2016 17:17:54 -0500 Subject: [PATCH] Write twitter_process tests --- gaia/inputs.py | 115 ++++++++++++------------ tests/cases/test_processors.py | 19 +++- tests/data/twitter_feed.json | 1 + tests/data/twitter_process_results.json | 1 + 4 files changed, 79 insertions(+), 57 deletions(-) create mode 100644 tests/data/twitter_feed.json create mode 100644 tests/data/twitter_process_results.json diff --git a/gaia/inputs.py b/gaia/inputs.py index 12fbcb0..8f95fd1 100644 --- a/gaia/inputs.py +++ b/gaia/inputs.py @@ -177,65 +177,16 @@ class TwitterIO(FileIO): Convert twitter data into geojson """ - def read(self, uri=None, format=None): - if not format: - format = self.default_output - super(TwitterIO, self).read() - if self.data is None: - self.data = open(self.uri).read() - self.data = json.loads(self.data)['data_inputs'] - try: - read_input = raw_input - except NameError: - read_input = input - - - twitter = OAuth1Service( - consumer_key= self.data['consumer_key'], - consumer_secret= self.data['consumer_secret'], - request_token_url= self.data['request_token_url'], - access_token_url= self.data['access_token_url'], - authorize_url= self.data['authorize_url'], - base_url= self.data['base_url'] - ) - - request_token, request_token_secret = twitter.get_request_token() - - authorize_url = twitter.get_authorize_url(request_token) - - print '' - print 'I will try to start a browser to visit the following Twitter page' - print 'if a browser will not start, copy the URL to your browser' - print 'and retrieve the pincode to be used' - print 'in the next step to obtaining an Authentication Token:' - print '' - print authorize_url - print '' - - webbrowser.open(authorize_url) - pincode = raw_input('Enter PIN from browser: ') - - session = twitter.get_auth_session(request_token, - request_token_secret, - method='POST', - data={'oauth_verifier': pincode}) - - params = {'include_rts': self.data['include_retweets'], # Include retweets - 'count': self.data['count']} - - r = session.get('statuses/home_timeline.json', params=params, verify=True) - - - # Convert twitter data into geojson - # Create Feature if one tweet was found, otherwise create FeatureCollection + def convertToGeojson(self, data): geojson = {} - if len(r.json()) > 1: + if len(data) > 1: geojson = { "type": "FeatureCollection", "features": [] } - for i, tweet in enumerate(r.json(), 1): + data = json.loads(data) + for i, tweet in enumerate(data, 1): resolver = carmen.get_resolver() resolver.load_locations() location = resolver.resolve_tweet(tweet) @@ -270,7 +221,7 @@ def read(self, uri=None, format=None): "properties": {} } - for i, tweet in enumerate(r.json(), 1): + for i, tweet in enumerate(data, 1): resolver = carmen.get_resolver() resolver.load_locations() location = resolver.resolve_tweet(tweet) @@ -297,10 +248,62 @@ class geoEmptyClass: return self.data else: self.data = GeoDataFrame.from_features(geojson["features"]) - print self.data + + return self.data.to_json() + + def read(self, uri=None, format=None): + if not format: + format = self.default_output + super(TwitterIO, self).read() + if self.data is None: + self.data = open(self.uri).read() + self.data = json.loads(self.data)['data_inputs'] + try: + read_input = raw_input + except NameError: + read_input = input + + + twitter = OAuth1Service( + consumer_key= self.data['consumer_key'], + consumer_secret= self.data['consumer_secret'], + request_token_url= self.data['request_token_url'], + access_token_url= self.data['access_token_url'], + authorize_url= self.data['authorize_url'], + base_url= self.data['base_url'] + ) + + request_token, request_token_secret = twitter.get_request_token() + + authorize_url = twitter.get_authorize_url(request_token) + + print '' + print 'I will try to start a browser to visit the following Twitter page' + print 'if a browser will not start, copy the URL to your browser' + print 'and retrieve the pincode to be used' + print 'in the next step to obtaining an Authentication Token:' + print '' + print authorize_url + print '' + + webbrowser.open(authorize_url) + pincode = raw_input('Enter PIN from browser: ') + + session = twitter.get_auth_session(request_token, + request_token_secret, + method='POST', + data={'oauth_verifier': pincode}) + + params = {'include_rts': self.data['include_retweets'], # Include retweets + 'count': self.data['count']} + + r = session.get('statuses/home_timeline.json', params=params, verify=True) + + # Convert twitter data into geojson + # Create Feature if one tweet was found, otherwise create FeatureCollection + self.convertToGeojson(r.json()) if format == formats.JSON: result = self.data.to_json() - print result return result else: return self.data diff --git a/tests/cases/test_processors.py b/tests/cases/test_processors.py index b4eea88..a091b35 100644 --- a/tests/cases/test_processors.py +++ b/tests/cases/test_processors.py @@ -7,7 +7,8 @@ from gaia import formats import gaia.geo.processes_vector as pv import gaia.geo.processes_raster as pr -from gaia.inputs import RasterFileIO, VectorFileIO, FeatureIO +import gaia.geo.processes_twitter as tw +from gaia.inputs import RasterFileIO, VectorFileIO, FeatureIO, TwitterIO testfile_path = os.path.join(os.path.dirname( os.path.realpath(__file__)), '../data') @@ -396,3 +397,19 @@ def test_weight(self): finally: if process: process.purge() + + def test_twitter_process(self): + """ + Test TwitterProcess for twitter data + """ + twitterData = open(os.path.join(testfile_path, 'twitter_feed.json')).read() + twitterIO = TwitterIO() + + geojson = twitterIO.convertToGeojson(twitterData) + with open(os.path.join( + testfile_path, + 'twitter_process_results.json')) as exp: + expected_json = json.load(exp) + actual_json = json.loads(geojson) + self.assertEquals(len(expected_json['features']), + len(actual_json['features'])) diff --git a/tests/data/twitter_feed.json b/tests/data/twitter_feed.json new file mode 100644 index 0000000..bbb072a --- /dev/null +++ b/tests/data/twitter_feed.json @@ -0,0 +1 @@ +[{"contributors": null, "truncated": false, "text": "Steve McClaren fired as Newcastle manager: NEWCASTLE, England \u2014 Steve McClaren was fired as Newcastle ... https://t.co/48VlfcwvPR #sports", "is_quote_status": false, "in_reply_to_status_id": null, "id": 708272595016126464, "favorite_count": 0, "source": "twitterfeed", "retweeted": false, "coordinates": null, "entities": {"symbols": [], "user_mentions": [], "hashtags": [{"indices": [130, 137], "text": "sports"}], "urls": [{"url": "https://t.co/48VlfcwvPR", "indices": [106, 129], "expanded_url": "http://bit.ly/225w9sD", "display_url": "bit.ly/225w9sD"}]}, "in_reply_to_screen_name": null, "id_str": "708272595016126464", "retweet_count": 0, "in_reply_to_user_id": null, "favorited": false, "user": {"follow_request_sent": false, "has_extended_profile": false, "profile_use_background_image": true, "default_profile_image": false, "id": 193921355, "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/439163436958625792/TfIlNIoj.jpeg", "verified": false, "profile_text_color": "333333", "profile_image_url_https": "https://pbs.twimg.com/profile_images/1133345532/boston_normal.jpg", "profile_sidebar_fill_color": "DDEEF6", "entities": {"description": {"urls": []}}, "followers_count": 4315, "profile_sidebar_border_color": "FFFFFF", "id_str": "193921355", "profile_background_color": "C0DEED", "listed_count": 90, "is_translation_enabled": false, "utc_offset": -18000, "statuses_count": 64198, "description": "email: bostongadget@gmail.com Local news, events, offers and updates about Boston city!\r\nThe Walking City!", "friends_count": 111, "location": "Boston, MA", "profile_link_color": "0084B4", "profile_image_url": "http://pbs.twimg.com/profile_images/1133345532/boston_normal.jpg", "following": true, "geo_enabled": false, "profile_banner_url": "https://pbs.twimg.com/profile_banners/193921355/1393539428", "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/439163436958625792/TfIlNIoj.jpeg", "screen_name": "Boston", "lang": "en", "profile_background_tile": false, "favourites_count": 1, "name": "Boston", "notifications": false, "url": null, "created_at": "Thu Sep 23 00:12:59 +0000 2010", "contributors_enabled": false, "time_zone": "Eastern Time (US & Canada)", "protected": false, "default_profile": false, "is_translator": false}, "geo": null, "in_reply_to_user_id_str": null, "possibly_sensitive": false, "possibly_sensitive_appealable": false, "lang": "en", "created_at": "Fri Mar 11 12:45:20 +0000 2016", "in_reply_to_status_id_str": null, "place": null}] diff --git a/tests/data/twitter_process_results.json b/tests/data/twitter_process_results.json new file mode 100644 index 0000000..caa96bc --- /dev/null +++ b/tests/data/twitter_process_results.json @@ -0,0 +1 @@ +{"type": "FeatureCollection", "features": [{"geometry": {"type": "Point", "coordinates": [42.3604823, -71.0595677]}, "type": "Feature", "id": "0", "properties": {"contributors": null, "truncated": false, "text": "Steve McClaren fired as Newcastle manager: NEWCASTLE, England \u2014 Steve McClaren was fired as Newcastle ... https://t.co/48VlfcwvPR #sports", "is_quote_status": false, "in_reply_to_status_id": null, "id": 708272595016126464, "favorite_count": 0, "source": "twitterfeed", "retweeted": false, "coordinates": null, "entities": {"symbols": [], "user_mentions": [], "hashtags": [{"indices": [130, 137], "text": "sports"}], "urls": [{"url": "https://t.co/48VlfcwvPR", "indices": [106, 129], "expanded_url": "http://bit.ly/225w9sD", "display_url": "bit.ly/225w9sD"}]}, "in_reply_to_screen_name": null, "in_reply_to_user_id": null, "retweet_count": 0, "id_str": "708272595016126464", "favorited": false, "user": {"follow_request_sent": false, "has_extended_profile": false, "profile_use_background_image": true, "default_profile_image": false, "id": 193921355, "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/439163436958625792/TfIlNIoj.jpeg", "verified": false, "profile_text_color": "333333", "profile_image_url_https": "https://pbs.twimg.com/profile_images/1133345532/boston_normal.jpg", "profile_sidebar_fill_color": "DDEEF6", "entities": {"description": {"urls": []}}, "followers_count": 4315, "profile_sidebar_border_color": "FFFFFF", "id_str": "193921355", "profile_background_color": "C0DEED", "listed_count": 90, "is_translation_enabled": false, "utc_offset": -18000, "statuses_count": 64198, "description": "email: bostongadget@gmail.com Local news, events, offers and updates about Boston city!\r\nThe Walking City!", "friends_count": 111, "location": "Boston, MA", "profile_link_color": "0084B4", "profile_image_url": "http://pbs.twimg.com/profile_images/1133345532/boston_normal.jpg", "following": true, "geo_enabled": false, "profile_banner_url": "https://pbs.twimg.com/profile_banners/193921355/1393539428", "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/439163436958625792/TfIlNIoj.jpeg", "screen_name": "Boston", "lang": "en", "profile_background_tile": false, "favourites_count": 1, "name": "Boston", "notifications": false, "url": null, "created_at": "Thu Sep 23 00:12:59 +0000 2010", "contributors_enabled": false, "time_zone": "Eastern Time (US & Canada)", "protected": false, "default_profile": false, "is_translator": false}, "geo": null, "in_reply_to_user_id_str": null, "possibly_sensitive": false, "possibly_sensitive_appealable": false, "lang": "en", "created_at": "Fri Mar 11 12:45:20 +0000 2016", "in_reply_to_status_id_str": null, "place": null}}]} \ No newline at end of file