Add twitter_process

docs/gaia.rst

@@ -68,6 +68,13 @@ gaia.processes_vector module
     :undoc-members:
     :show-inheritance:
 
+gaia.processes_twitter module
+---------------------
+
+.. automodule:: gaia.processes_twitter
+    :members:
+    :undoc-members:
+    :show-inheritance:
 
 Module contents
 ---------------

gaia/conf/gaia.local.cfg

@@ -0,0 +1,7 @@
+[gaia]
+fileio_paths: ../tests/data/
+output_path: ../tests/data/output
+tmp_dir: /tmp
+default_epsg: 3857
+celery_backend: amqp
+celery_broker: amqp://guest@localhost:5672//

gaia/datasources.py

@@ -5,3 +5,4 @@
 POSTGIS = 'pg'
 PROCESS = 'process'
 FEATURES = 'features'
+Twitter = 'twitter'

gaia/geo/processes_twitter.py

@@ -0,0 +1,24 @@
+from gaia.inputs import TwitterIO, VectorFileIO
+from gaia.geo.processes_base import GaiaProcess
+import gaia.formats as formats
+
+class TwitterProcess(GaiaProcess):
+    """
+    Takes a twitter output and geocode it
+    """
+    required_inputs = ('twitter_config', formats.JSON,)
+    default_output = formats.JSON
+
+    def __init__(self, **kwargs):
+        super(TwitterProcess, self).__init__(**kwargs)
+        if not self.output:
+            self.output = VectorFileIO(name='result', uri=self.get_outpath())
+
+    def compute(self):
+        super(TwitterProcess, self).compute()
+
+        first_df = self.inputs[0]
+        print type(first_df)
+        self.output.data = first_df.read()
+        print self.output.data
+        self.output.write()

gaia/inputs.py

@@ -5,6 +5,9 @@
 import pysal
 import gdal
 import geopandas
+import webbrowser
+import carmen
+
 
 try:
     import osr
@@ -14,6 +17,11 @@
 from gaia.core import GaiaException, config
 from gaia.filters import filter_pandas
 from gaia.geo.gdal_functions import gdal_reproject
+from rauth import OAuth1Service
+from geopy.geocoders import Nominatim
+from geopandas import GeoDataFrame
+geolocator = Nominatim()
+
 
 
 class MissingParameterError(GaiaException):
@@ -164,6 +172,139 @@ def delete(self):
         if os.path.exists(self.uri):
             shutil.rmtree(os.path.dirname(self.uri))
 
+class TwitterIO(FileIO):
+    """
+    Convert twitter data into geojson
+    """
+
+    def read(self, uri=None, format=None):
+        if not format:
+            format = self.default_output
+        super(TwitterIO, self).read()
+        if self.data is None:
+            self.data = open(self.uri).read()
+            self.data = json.loads(self.data)['data_inputs']
+            try:
+                read_input = raw_input
+            except NameError:
+                read_input = input
+
+
+            twitter = OAuth1Service(
+                        consumer_key= self.data['consumer_key'],
+                        consumer_secret= self.data['consumer_secret'],
+                        request_token_url= self.data['request_token_url'],
+                        access_token_url= self.data['access_token_url'],
+                        authorize_url= self.data['authorize_url'],
+                        base_url= self.data['base_url']
+            )
+
+            request_token, request_token_secret = twitter.get_request_token()
+
+            authorize_url = twitter.get_authorize_url(request_token)
+
+            print ''
+            print 'I will try to start a browser to visit the following Twitter page'
+            print 'if a browser will not start, copy the URL to your browser'
+            print 'and retrieve the pincode to be used'
+            print 'in the next step to obtaining an Authentication Token:'
+            print ''
+            print authorize_url
+            print ''
+
+            webbrowser.open(authorize_url)
+            pincode = raw_input('Enter PIN from browser: ')
+
+            session = twitter.get_auth_session(request_token,
+                                       request_token_secret,
+                                       method='POST',
+                                       data={'oauth_verifier': pincode})
+
+            params = {'include_rts': self.data['include_retweets'],  # Include retweets
+                'count': self.data['count']}
+
+            r = session.get('statuses/home_timeline.json', params=params, verify=True)
+
+
+            # Convert twitter data into geojson
+            # Create Feature if one tweet was found, otherwise create FeatureCollection
+            geojson = {}
+            if len(r.json()) > 1:
+                geojson = {
+                                    "type": "FeatureCollection",
+                                    "features": []
+
+                }
+                for i, tweet in enumerate(r.json(), 1):
+                    resolver = carmen.get_resolver()
+                    resolver.load_locations()
+                    location = resolver.resolve_tweet(tweet)
+                    if location != None:
+                        for x in location:
+                            if (x != False):
+                                location_string = x.country + ',' + x.state + ',' + x.county + ',' + x.city
+                                coord = geolocator.geocode(location_string)
+
+                                feature = {
+                                                        "type": "Feature",
+                                                        "geometry": {
+                                                            "type": "Point",
+                                                            "coordinates": [coord.latitude, coord.longitude]
+                                                        },
+                                                        "properties": {
+
+                                                        }
+                                }
+                                # Iterate over the tweet and create properties
+                                for property in tweet:
+                                    feature["properties"][property] = tweet[property]
+                        geojson['features'].append(feature)
+
+            else:
+                geojson = {
+                                    "type": "Feature",
+                                    "geometry": {
+                                        "type": "Point",
+                                        "coordinates": []
+                                    },
+                                    "properties": {}
+                }
+
+                for i, tweet in enumerate(r.json(), 1):
+                    resolver = carmen.get_resolver()
+                    resolver.load_locations()
+                    location = resolver.resolve_tweet(tweet)
+                    if location != None:
+                        for x in location:
+                            if (x != False):
+                                location_string = x.country + ',' + x.state + ',' + x.county + ',' + x.city
+                                coord = geolocator.geocode(location_string)
+                                geojson["geometry"]["coordinates"] = [coord.latitude, coord.longitude]
+                                # Iterate over the tweet and create properties
+                                for property in tweet:
+                                    geojson["properties"][property] = tweet[property]
+
+            class geoEmptyClass:
+                pass
+
+            if geojson["type"] == "Feature":
+                results = geoEmptyClass()
+                results.__geo_interface__ = geojson
+                self.data = GeoDataFrame.from_features([results])
+                if format == formats.JSON:
+                    return self.data.to_json()
+                else:
+                    return self.data
+            else:
+                self.data = GeoDataFrame.from_features(geojson["features"])
+        print self.data
+        if format == formats.JSON:
+            result = self.data.to_json()
+            print result
+            return result
+        else:
+            return self.data
+
 
 class VectorFileIO(FileIO):
     """Read and write vector file data (such as GeoJSON)"""

gaia/parser.py

@@ -8,13 +8,17 @@
 import gaia.inputs
 import gaia.geo.processes_raster
 import gaia.geo.processes_vector
+import gaia.geo.processes_twitter
 from gaia.core import GaiaException
 
+
 _process_r = [(x[0].replace('Process', ''), x[1]) for x in inspect.getmembers(
     gaia.geo.processes_raster, inspect.isclass) if x[0].endswith('Process')]
 _process_v = ([(x[0].replace('Process', ''), x[1]) for x in inspect.getmembers(
     gaia.geo.processes_vector, inspect.isclass) if x[0].endswith('Process')])
-_processes = dict(_process_r + _process_v)
+_process_t = ([(x[0].replace('Process', ''), x[1]) for x in inspect.getmembers(
+    gaia.geo.processes_twitter, inspect.isclass) if x[0].endswith('Process')])
+_processes = dict(_process_r + _process_v + _process_t)
 
 
 class GaiaRequestParser(object):
@@ -79,6 +83,8 @@ def create_io(process, data):
         io = gaia.inputs.ProcessIO(process=parser.process)
     elif data['source'] == 'features':
         io = gaia.inputs.FeatureIO(**data)
+    elif data['source'] == 'twitter':
+        io = gaia.inputs.TwitterIO(**data)
     else:
         raise NotImplementedError()
     return io

tests/data/globalairtemp.tif.aux.xml

@@ -0,0 +1,10 @@
+<PAMDataset>
+  <PAMRasterBand band="1">
+    <Metadata>
+      <MDI key="STATISTICS_MAXIMUM">31594</MDI>
+      <MDI key="STATISTICS_MEAN">27684.131635802</MDI>
+      <MDI key="STATISTICS_MINIMUM">22419</MDI>
+      <MDI key="STATISTICS_STDDEV">2183.1054283199</MDI>
+    </Metadata>
+  </PAMRasterBand>
+</PAMDataset>

tests/data/globalprecip.tif.aux.xml

@@ -0,0 +1,10 @@
+<PAMDataset>
+  <PAMRasterBand band="1">
+    <Metadata>
+      <MDI key="STATISTICS_MAXIMUM">7689</MDI>
+      <MDI key="STATISTICS_MEAN">705.20518958254</MDI>
+      <MDI key="STATISTICS_MINIMUM">0</MDI>
+      <MDI key="STATISTICS_STDDEV">680.79516145832</MDI>
+    </Metadata>
+  </PAMRasterBand>
+</PAMDataset>

tests/data/output/a86a3306-d8fe-42bc-aadb-06a1833d6f02/a86a3306-d8fe-42bc-aadb-06a1833d6f02.json

@@ -0,0 +1 @@
+{"type": "FeatureCollection", "features": [{"geometry": {"type": "Point", "coordinates": [42.3604823, -71.0595677]}, "type": "Feature", "id": "0", "properties": {"contributors": null, "truncated": false, "text": "Steve McClaren fired as Newcastle manager: NEWCASTLE, England \u2014 Steve McClaren was fired as Newcastle ... https://t.co/48VlfcwvPR #sports", "is_quote_status": false, "in_reply_to_status_id": null, "id": 708272595016126464, "favorite_count": 0, "source": "<a href=\"http://twitterfeed.com\" rel=\"nofollow\">twitterfeed</a>", "retweeted": false, "coordinates": null, "entities": {"symbols": [], "user_mentions": [], "hashtags": [{"indices": [130, 137], "text": "sports"}], "urls": [{"url": "https://t.co/48VlfcwvPR", "indices": [106, 129], "expanded_url": "http://bit.ly/225w9sD", "display_url": "bit.ly/225w9sD"}]}, "in_reply_to_screen_name": null, "in_reply_to_user_id": null, "retweet_count": 0, "id_str": "708272595016126464", "favorited": false, "user": {"follow_request_sent": false, "has_extended_profile": false, "profile_use_background_image": true, "default_profile_image": false, "id": 193921355, "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/439163436958625792/TfIlNIoj.jpeg", "verified": false, "profile_text_color": "333333", "profile_image_url_https": "https://pbs.twimg.com/profile_images/1133345532/boston_normal.jpg", "profile_sidebar_fill_color": "DDEEF6", "entities": {"description": {"urls": []}}, "followers_count": 4315, "profile_sidebar_border_color": "FFFFFF", "id_str": "193921355", "profile_background_color": "C0DEED", "listed_count": 90, "is_translation_enabled": false, "utc_offset": -18000, "statuses_count": 64198, "description": "email: bostongadget@gmail.com Local news, events, offers and updates about Boston city!\r\nThe Walking City!", "friends_count": 111, "location": "Boston, MA", "profile_link_color": "0084B4", "profile_image_url": "http://pbs.twimg.com/profile_images/1133345532/boston_normal.jpg", "following": true, "geo_enabled": false, "profile_banner_url": "https://pbs.twimg.com/profile_banners/193921355/1393539428", "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/439163436958625792/TfIlNIoj.jpeg", "screen_name": "Boston", "lang": "en", "profile_background_tile": false, "favourites_count": 1, "name": "Boston", "notifications": false, "url": null, "created_at": "Thu Sep 23 00:12:59 +0000 2010", "contributors_enabled": false, "time_zone": "Eastern Time (US & Canada)", "protected": false, "default_profile": false, "is_translator": false}, "geo": null, "in_reply_to_user_id_str": null, "possibly_sensitive": false, "possibly_sensitive_appealable": false, "lang": "en", "created_at": "Fri Mar 11 12:45:20 +0000 2016", "in_reply_to_status_id_str": null, "place": null}}]}