Merge pull request #5 from danieleewww/patch-2

Update nytimes_pull.py
hmason · Dec 11, 2014 · 33af094 · 33af094
2 parents dbc59f1 + b195d45
commit 33af094
Showing 1 changed file with 14 additions and 15 deletions.
diff --git a/intro_web_data/nytimes_pull.py b/intro_web_data/nytimes_pull.py
@@ -1,12 +1,3 @@
-#!/usr/bin/env python
-# encoding: utf-8
-"""
-nytimes_pull.py
-
-Created by Hilary Mason on 2011-02-17.
-Copyright (c) 2011 Hilary Mason. All rights reserved.
-"""
-
 import urllib
 import json
 
@@ -16,16 +7,24 @@ def main(api_key, category, label):
     for i in range(0,5):
         # print "http://api.nytimes.com/svc/search/v2/articlesearch.json?fq=news_desk:('%s')&api-key=%s&page=%s" % (category, api_key, i)
         h = urllib.urlopen("http://api.nytimes.com/svc/search/v2/articlesearch.json?fq=news_desk:(\"%s\")&api-key=%s&page=%s" % (category, api_key, i))
-        data = json.loads(h.read())
+        print h
-        for result in data['results']:
+        try:
-            content.append(result['body'])
+            result = json.loads(h.read())
-
+            content.append(result)
+        except ValueError:
+            print "Malformed JSON: " + data
+            continue #In the rare cases that JSON refuses to parse
+
     f = open(label, 'w')
     for line in content:
-        f.write('%s\n' % line.encode('utf-8'))
+        try:
+            f.write('%s\n' % line)
+        except UnicodeEncodeError:
+            pass
 
     f.close()
 
 if __name__ == '__main__':
     main("f7b4a1749764aec0364b215c354e3a0f:18:25759498", "Arts","arts")
-    main("f7b4a1749764aec0364b215c354e3a0f:18:25759498", "Sports","sports")  
+    main("f7b4a1749764aec0364b215c354e3a0f:18:25759498", "Sports","sports")
+