Merge pull request #432 from bollwyvl/unicode-body

Force UTF-8 on notebooks from truncated gists

nbviewer/handlers.py

@@ -668,7 +668,7 @@ def get(self, user, gist_id, filename=''):
             if file['truncated']:
                 app_log.debug("Gist %s/%s truncated, fetching %s", gist_id, filename, file['raw_url'])
                 response = yield self.fetch(file['raw_url'])
-                content = response_text(response)
+                content = response_text(response, encoding='utf-8')
             else:
                 content = file['content']
 

nbviewer/tests/test_gist.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2013 The IPython Development Team
 #
@@ -31,28 +32,37 @@ def test_gist_list(self):
         self.assertEqual(r.status_code, 200)
         html = r.text
         self.assertIn('<th>Name</th>', html)
-    
+
     def test_multifile_gist(self):
         url = self.url('7518294', 'Untitled0.ipynb')
         r = requests.get(url)
         self.assertEqual(r.status_code, 200)
         html = r.text
         self.assertIn('Download Notebook', html)
-    
+
     def test_anonymous_gist(self):
         url = self.url('gist/4465051')
         r = requests.get(url)
         self.assertEqual(r.status_code, 200)
         html = r.text
         self.assertIn('Download Notebook', html)
-    
+
     def test_gist_unicode(self):
         url = self.url('gist/amueller/3974344')
         r = requests.get(url)
         self.assertEqual(r.status_code, 200)
         html = r.text
         self.assertIn('<th>Name</th>', html)
 
+    def test_gist_unicode_content(self):
+        url = self.url('gist/ocefpaf/cf023a8db7097bd9fe92')
+        r = requests.get(url)
+        self.assertEqual(r.status_code, 200)
+        html = r.text
+        self.assertNotIn('param&#195;&#169;trica', html)
+        self.assertIn('param&#233;trica', html)
+
+
 
 class FormatHTMLGistTestCase(GistTestCase, FormatHTMLMixin):
     pass

nbviewer/utils.py

@@ -126,9 +126,13 @@ def get_encoding_from_headers(headers):
     if 'text' in content_type:
         return 'ISO-8859-1'
 
-def response_text(response):
+def response_text(response, encoding=None):
     """mimic requests.text property, but for plain HTTPResponse"""
-    encoding = get_encoding_from_headers(response.headers) or 'utf-8'
+    encoding = (
+        encoding or
+        get_encoding_from_headers(response.headers) or
+        'utf-8'
+    )
     return response.body.decode(encoding, 'replace')
 
 # parse_header_links from requests.util