Write twitter_process tests

gaia/inputs.py

@@ -177,65 +177,16 @@ class TwitterIO(FileIO):
     Convert twitter data into geojson
     """
 
-    def read(self, uri=None, format=None):
-        if not format:
-            format = self.default_output
-        super(TwitterIO, self).read()
-        if self.data is None:
-            self.data = open(self.uri).read()
-            self.data = json.loads(self.data)['data_inputs']
-            try:
-                read_input = raw_input
-            except NameError:
-                read_input = input
-
-
-            twitter = OAuth1Service(
-                        consumer_key= self.data['consumer_key'],
-                        consumer_secret= self.data['consumer_secret'],
-                        request_token_url= self.data['request_token_url'],
-                        access_token_url= self.data['access_token_url'],
-                        authorize_url= self.data['authorize_url'],
-                        base_url= self.data['base_url']
-            )
-
-            request_token, request_token_secret = twitter.get_request_token()
-
-            authorize_url = twitter.get_authorize_url(request_token)
-
-            print ''
-            print 'I will try to start a browser to visit the following Twitter page'
-            print 'if a browser will not start, copy the URL to your browser'
-            print 'and retrieve the pincode to be used'
-            print 'in the next step to obtaining an Authentication Token:'
-            print ''
-            print authorize_url
-            print ''
-
-            webbrowser.open(authorize_url)
-            pincode = raw_input('Enter PIN from browser: ')
-
-            session = twitter.get_auth_session(request_token,
-                                       request_token_secret,
-                                       method='POST',
-                                       data={'oauth_verifier': pincode})
-
-            params = {'include_rts': self.data['include_retweets'],  # Include retweets
-                'count': self.data['count']}
-
-            r = session.get('statuses/home_timeline.json', params=params, verify=True)
-
-
-            # Convert twitter data into geojson
-            # Create Feature if one tweet was found, otherwise create FeatureCollection
+    def convertToGeojson(self, data):
             geojson = {}
-            if len(r.json()) > 1:
+            if len(data) > 1:
                 geojson = {
                                     "type": "FeatureCollection",
                                     "features": []
 
                 }
-                for i, tweet in enumerate(r.json(), 1):
+                data = json.loads(data)
+                for i, tweet in enumerate(data, 1):
                     resolver = carmen.get_resolver()
                     resolver.load_locations()
                     location = resolver.resolve_tweet(tweet)
@@ -270,7 +221,7 @@ def read(self, uri=None, format=None):
                                     "properties": {}
                 }
 
-                for i, tweet in enumerate(r.json(), 1):
+                for i, tweet in enumerate(data, 1):
                     resolver = carmen.get_resolver()
                     resolver.load_locations()
                     location = resolver.resolve_tweet(tweet)
@@ -297,10 +248,62 @@ class geoEmptyClass:
                     return self.data
             else:
                 self.data = GeoDataFrame.from_features(geojson["features"])
-        print self.data
+
+            return self.data.to_json()
+
+    def read(self, uri=None, format=None):
+        if not format:
+            format = self.default_output
+        super(TwitterIO, self).read()
+        if self.data is None:
+            self.data = open(self.uri).read()
+            self.data = json.loads(self.data)['data_inputs']
+            try:
+                read_input = raw_input
+            except NameError:
+                read_input = input
+
+
+            twitter = OAuth1Service(
+                        consumer_key= self.data['consumer_key'],
+                        consumer_secret= self.data['consumer_secret'],
+                        request_token_url= self.data['request_token_url'],
+                        access_token_url= self.data['access_token_url'],
+                        authorize_url= self.data['authorize_url'],
+                        base_url= self.data['base_url']
+            )
+
+            request_token, request_token_secret = twitter.get_request_token()
+
+            authorize_url = twitter.get_authorize_url(request_token)
+
+            print ''
+            print 'I will try to start a browser to visit the following Twitter page'
+            print 'if a browser will not start, copy the URL to your browser'
+            print 'and retrieve the pincode to be used'
+            print 'in the next step to obtaining an Authentication Token:'
+            print ''
+            print authorize_url
+            print ''
+
+            webbrowser.open(authorize_url)
+            pincode = raw_input('Enter PIN from browser: ')
+
+            session = twitter.get_auth_session(request_token,
+                                       request_token_secret,
+                                       method='POST',
+                                       data={'oauth_verifier': pincode})
+
+            params = {'include_rts': self.data['include_retweets'],  # Include retweets
+                'count': self.data['count']}
+
+            r = session.get('statuses/home_timeline.json', params=params, verify=True)
+
+            # Convert twitter data into geojson
+            # Create Feature if one tweet was found, otherwise create FeatureCollection
+            self.convertToGeojson(r.json())
         if format == formats.JSON:
             result = self.data.to_json()
-            print result
             return result
         else:
             return self.data

tests/cases/test_processors.py

@@ -7,7 +7,8 @@
 from gaia import formats
 import gaia.geo.processes_vector as pv
 import gaia.geo.processes_raster as pr
-from gaia.inputs import RasterFileIO, VectorFileIO, FeatureIO
+import gaia.geo.processes_twitter as tw
+from gaia.inputs import RasterFileIO, VectorFileIO, FeatureIO, TwitterIO
 
 testfile_path = os.path.join(os.path.dirname(
     os.path.realpath(__file__)), '../data')
@@ -396,3 +397,19 @@ def test_weight(self):
         finally:
             if process:
                 process.purge()
+
+    def test_twitter_process(self):
+        """
+        Test TwitterProcess for twitter data
+        """
+        twitterData = open(os.path.join(testfile_path, 'twitter_feed.json')).read()
+        twitterIO = TwitterIO()
+
+        geojson = twitterIO.convertToGeojson(twitterData)
+        with open(os.path.join(
+                testfile_path,
+                'twitter_process_results.json')) as exp:
+            expected_json = json.load(exp)
+        actual_json = json.loads(geojson)
+        self.assertEquals(len(expected_json['features']),
+                          len(actual_json['features']))

tests/data/twitter_feed.json

@@ -0,0 +1 @@
+[{"contributors": null, "truncated": false, "text": "Steve McClaren fired as Newcastle manager: NEWCASTLE, England \u2014 Steve McClaren was fired as Newcastle ... https://t.co/48VlfcwvPR #sports", "is_quote_status": false, "in_reply_to_status_id": null, "id": 708272595016126464, "favorite_count": 0, "source": "<a href=\"http://twitterfeed.com\" rel=\"nofollow\">twitterfeed</a>", "retweeted": false, "coordinates": null, "entities": {"symbols": [], "user_mentions": [], "hashtags": [{"indices": [130, 137], "text": "sports"}], "urls": [{"url": "https://t.co/48VlfcwvPR", "indices": [106, 129], "expanded_url": "http://bit.ly/225w9sD", "display_url": "bit.ly/225w9sD"}]}, "in_reply_to_screen_name": null, "id_str": "708272595016126464", "retweet_count": 0, "in_reply_to_user_id": null, "favorited": false, "user": {"follow_request_sent": false, "has_extended_profile": false, "profile_use_background_image": true, "default_profile_image": false, "id": 193921355, "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/439163436958625792/TfIlNIoj.jpeg", "verified": false, "profile_text_color": "333333", "profile_image_url_https": "https://pbs.twimg.com/profile_images/1133345532/boston_normal.jpg", "profile_sidebar_fill_color": "DDEEF6", "entities": {"description": {"urls": []}}, "followers_count": 4315, "profile_sidebar_border_color": "FFFFFF", "id_str": "193921355", "profile_background_color": "C0DEED", "listed_count": 90, "is_translation_enabled": false, "utc_offset": -18000, "statuses_count": 64198, "description": "email: bostongadget@gmail.com Local news, events, offers and updates about Boston city!\r\nThe Walking City!", "friends_count": 111, "location": "Boston, MA", "profile_link_color": "0084B4", "profile_image_url": "http://pbs.twimg.com/profile_images/1133345532/boston_normal.jpg", "following": true, "geo_enabled": false, "profile_banner_url": "https://pbs.twimg.com/profile_banners/193921355/1393539428", "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/439163436958625792/TfIlNIoj.jpeg", "screen_name": "Boston", "lang": "en", "profile_background_tile": false, "favourites_count": 1, "name": "Boston", "notifications": false, "url": null, "created_at": "Thu Sep 23 00:12:59 +0000 2010", "contributors_enabled": false, "time_zone": "Eastern Time (US & Canada)", "protected": false, "default_profile": false, "is_translator": false}, "geo": null, "in_reply_to_user_id_str": null, "possibly_sensitive": false, "possibly_sensitive_appealable": false, "lang": "en", "created_at": "Fri Mar 11 12:45:20 +0000 2016", "in_reply_to_status_id_str": null, "place": null}]

tests/data/twitter_process_results.json

@@ -0,0 +1 @@
+{"type": "FeatureCollection", "features": [{"geometry": {"type": "Point", "coordinates": [42.3604823, -71.0595677]}, "type": "Feature", "id": "0", "properties": {"contributors": null, "truncated": false, "text": "Steve McClaren fired as Newcastle manager: NEWCASTLE, England \u2014 Steve McClaren was fired as Newcastle ... https://t.co/48VlfcwvPR #sports", "is_quote_status": false, "in_reply_to_status_id": null, "id": 708272595016126464, "favorite_count": 0, "source": "<a href=\"http://twitterfeed.com\" rel=\"nofollow\">twitterfeed</a>", "retweeted": false, "coordinates": null, "entities": {"symbols": [], "user_mentions": [], "hashtags": [{"indices": [130, 137], "text": "sports"}], "urls": [{"url": "https://t.co/48VlfcwvPR", "indices": [106, 129], "expanded_url": "http://bit.ly/225w9sD", "display_url": "bit.ly/225w9sD"}]}, "in_reply_to_screen_name": null, "in_reply_to_user_id": null, "retweet_count": 0, "id_str": "708272595016126464", "favorited": false, "user": {"follow_request_sent": false, "has_extended_profile": false, "profile_use_background_image": true, "default_profile_image": false, "id": 193921355, "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/439163436958625792/TfIlNIoj.jpeg", "verified": false, "profile_text_color": "333333", "profile_image_url_https": "https://pbs.twimg.com/profile_images/1133345532/boston_normal.jpg", "profile_sidebar_fill_color": "DDEEF6", "entities": {"description": {"urls": []}}, "followers_count": 4315, "profile_sidebar_border_color": "FFFFFF", "id_str": "193921355", "profile_background_color": "C0DEED", "listed_count": 90, "is_translation_enabled": false, "utc_offset": -18000, "statuses_count": 64198, "description": "email: bostongadget@gmail.com Local news, events, offers and updates about Boston city!\r\nThe Walking City!", "friends_count": 111, "location": "Boston, MA", "profile_link_color": "0084B4", "profile_image_url": "http://pbs.twimg.com/profile_images/1133345532/boston_normal.jpg", "following": true, "geo_enabled": false, "profile_banner_url": "https://pbs.twimg.com/profile_banners/193921355/1393539428", "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/439163436958625792/TfIlNIoj.jpeg", "screen_name": "Boston", "lang": "en", "profile_background_tile": false, "favourites_count": 1, "name": "Boston", "notifications": false, "url": null, "created_at": "Thu Sep 23 00:12:59 +0000 2010", "contributors_enabled": false, "time_zone": "Eastern Time (US & Canada)", "protected": false, "default_profile": false, "is_translator": false}, "geo": null, "in_reply_to_user_id_str": null, "possibly_sensitive": false, "possibly_sensitive_appealable": false, "lang": "en", "created_at": "Fri Mar 11 12:45:20 +0000 2016", "in_reply_to_status_id_str": null, "place": null}}]}