Merge branch 'master' of github.com:chriskelvinlee/trivial_pursuit

chriskelvinlee · Dec 9, 2011 · f8d65c2 · f8d65c2
2 parents 2ba56a2 + 30357bc
commit f8d65c2
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/trivialpursuitfunctions.py b/trivialpursuitfunctions.py
@@ -84,8 +84,13 @@ def getAnswerKeywords(answers):
 
 def getTokens(urls):
     combinedtokens = []
+
+    user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7'
+    headers={'User-Agent':user_agent,} 
+
     for url in urls:
-        html = urlopen(url).read()
+        req = urllib2.Request(url,None,headers)
+        html = urlopen(req).read()
         raw = nltk.clean_html(html)
         combinedtokens += nltk.word_tokenize(raw)
     # may need to adjust 2 value here depending on answer choices