Fork of code from ScraperWiki at https://classic.scraperwiki.com/scra…

…pers/aras_election_data_78/

.gitignore

@@ -0,0 +1,2 @@
+# Ignore output of scraper
+data.sqlite

scraper.py

@@ -0,0 +1,32 @@
+###################################################################################
+# Twitter API scraper - designed to be forked and used for more interesting things
+###################################################################################
+
+import scraperwiki
+import simplejson
+import urllib2
+
+
+QUERY = 'hochwasser'
+GEOINFO = '49.486816,8.442256,500km'
+RESULTS_PER_PAGE = '1000'
+LANGUAGE = 'de'
+NUM_PAGES = 15 
+
+for page in range(1, NUM_PAGES+1):
+    base_url = 'http://search.twitter.com/search.json?q=%s&geocode=%s&rpp=%s&lang=%s&page=%s' \
+         % (urllib2.quote(QUERY), urllib2.quote(GEOINFO), RESULTS_PER_PAGE, LANGUAGE, page)
+    try:
+        results_json = simplejson.loads(scraperwiki.scrape(base_url))
+        for result in results_json['results']:
+            data = {}
+            data['id'] = result['id']
+            data['text'] = result['text']
+            data['from_user'] = result['from_user']
+            data['geo'] = result['geo']
+            print data['from_user'], data['text']
+            scraperwiki.sqlite.save(["id"], data) 
+    except:
+        print 'Oh dear, failed to scrape %s' % base_url
+
+