more encoding fixes

georgewhewell · Jan 19, 2015 · 59a3319 · 59a3319
1 parent 7fa1cf7
commit 59a3319
Show file tree

Hide file tree

Showing 4 changed files with 67 additions and 15 deletions.
diff --git a/sheets/templatetags/sheets.py b/sheets/templatetags/sheets.py
@@ -14,6 +14,7 @@
 
 def get_sheet(key):
     try:
+        print(gdocs_format.format(key=key))
         response = requests.get(gdocs_format.format(key=key))
         response.raise_for_status()
         return response
@@ -40,4 +41,6 @@ def csv_tag(key):
     if response_data is None:
         return None
 
-    return read_csv(response_data.text)
+    reader = read_csv(response_data.content)
+
+    return [[cell.decode('utf8') for cell in row] for row in reader]
diff --git a/tests/sample_output.html b/tests/sample_output.html
@@ -1 +1 @@
-<table><tr><td>first_name</td><td>last_name</td><td>company_name</td><td>address</td><td>city</td><td>county</td><td>postal</td><td>phone1</td><td>phone2</td><td>email</td><td>web</td></tr><tr><td>Aleshia</td><td>Tomkiewicz</td><td>Alan D Rosenburg Cpa Pc</td><td>14 Taylor St</td><td>St. Stephens Ward</td><td>Kent</td><td>CT2 7PP</td><td>01835-703597</td><td>01944-369967</td><td>atomkiewicz@hotmail.com</td><td>http://www.alandrosenburgcpapc.co.uk</td></tr><tr><td>Evan</td><td>Zigomalas</td><td>Cap Gemini America</td><td>5 Binney St</td><td>Abbey Ward</td><td>Buckinghamshire</td><td>HP11 2AX</td><td>01937-864715</td><td>01714-737668</td><td>evan.zigomalas@gmail.com</td><td>http://www.capgeminiamerica.co.uk</td></tr><tr><td>France</td><td>Andrade</td><td>Elliott, John W Esq</td><td>8 Moor Place</td><td>East Southbourne and Tuckton W</td><td>Bournemouth</td><td>BH6 3BE</td><td>01347-368222</td><td>01935-821636</td><td>france.andrade@hotmail.com</td><td>http://www.elliottjohnwesq.co.uk</td></tr><tr><td>Ulysses</td><td>Mcwalters</td><td>Mcmahan, Ben L</td><td>505 Exeter Rd</td><td>Hawerby cum Beesby</td><td>Lincolnshire</td><td>DN36 5RP</td><td>01912-771311</td><td>01302-601380</td><td>ulysses@hotmail.com</td><td>http://www.mcmahanbenl.co.uk</td></tr><tr><td>Tyisha</td><td>Veness</td><td>Champagne Room</td><td>5396 Forth Street</td><td>Greets Green and Lyng Ward</td><td>West Midlands</td><td>B70 9DT</td><td>01547-429341</td><td>01290-367248</td><td>tyisha.veness@hotmail.com</td><td>http://www.champagneroom.co.uk</td></tr><tr><td>Eric</td><td>Rampy</td><td>Thompson, Michael C Esq</td><td>9472 Lind St</td><td>Desborough</td><td>Northamptonshire</td><td>NN14 2GH</td><td>01969-886290</td><td>01545-817375</td><td>erampy@rampy.co.uk</td><td>http://www.thompsonmichaelcesq.co.uk</td></tr><tr><td>Marg</td><td>Grasmick</td><td>Wrangle Hill Auto Auct &amp; Slvg</td><td>7457 Cowl St #70</td><td>Bargate Ward</td><td>Southampton</td><td>SO14 3TY</td><td>01865-582516</td><td>01362-620532</td><td>marg@hotmail.com</td><td>http://www.wranglehillautoauctslvg.co.uk</td></tr><tr><td>Laquita</td><td>Hisaw</td><td>In Communications Inc</td><td>20 Gloucester Pl #96</td><td>Chirton Ward</td><td>Tyne &amp; Wear</td><td>NE29 7AD</td><td>01746-394243</td><td>01590-982428</td><td>laquita@yahoo.com</td><td>http://www.incommunicationsinc.co.uk</td></tr><tr><td>Lura</td><td>Manzella</td><td>Bizerba Usa Inc</td><td>929 Augustine St</td><td>Staple Hill Ward</td><td>South Gloucestershire</td><td>BS16 4LL</td><td>01907-538509</td><td>01340-713951</td><td>lura@hotmail.com</td><td>http://www.bizerbausainc.co.uk</td></tr></table>
+<html><head><meta charset="utf-8"></head><table><tr><td>Origin (English)</td><td>Name (English)</td><td>Origin (Native)</td><td>Name (Native)</td></tr><tr><td>Australia</td><td>Nicole Kidman</td><td>Australia</td><td>Nicole Kidman</td></tr><tr><td>Austria</td><td>Johann Strauss</td><td>Österreich</td><td>Johann Strauß</td></tr><tr><td>Belgium (Flemish)</td><td>Rene Magritte</td><td>België</td><td>René Magritte</td></tr><tr><td>Belgium (French)</td><td>Rene Magritte</td><td>Belgique</td><td>René Magritte</td></tr><tr><td>Belgium (German)</td><td>Rene Magritte</td><td>Belgien</td><td>René Magritte</td></tr><tr><td>Bhutan</td><td>Gonpo Dorji</td><td>འབྲུག་ཡུལ།</td><td>མགོན་པོ་རྡོ་རྗེ།</td></tr><tr><td>Canada</td><td>Celine Dion</td><td>Canada</td><td>Céline Dion</td></tr><tr><td>Canada - Nunavut (Inuktitut)</td><td>Susan Aglukark</td><td>ᓄᓇᕗᒻᒥᐅᑦ</td><td>ᓱᓴᓐ ᐊᒡᓗᒃᑲᖅ</td></tr><tr><td>Democratic People&#39;s Rep. of Korea</td><td>LEE Sol-Hee</td><td>조선 민주주의 인민 공화국</td><td>이설희</td></tr><tr><td>Denmark</td><td>Soren Hauch-Fausboll</td><td>Danmark</td><td>Søren Hauch-Fausbøll</td></tr><tr><td>Denmark</td><td>Soren Kierkegaard</td><td>Danmark</td><td>Søren Kierkegård</td></tr><tr><td>Egypt</td><td>Abdel Halim Hafez</td><td>ﻣﺼﺮ</td><td>ﻋﺑﺪﺍﻠﺣﻟﻳﻢ ﺤﺎﻓﻅ</td></tr><tr><td>Egypt</td><td>Om Kolthoum</td><td>ﻣﺼﺮ</td><td>ﺃﻡ ﻛﻟﺛﻭﻡ</td></tr><tr><td>Eritrea</td><td>Berhane Zeray</td><td>ብርሃነ ዘርኣይ</td><td>ኤርትራ</td></tr><tr><td>Ethiopia</td><td>Haile Gebreselassie</td><td>ኃይሌ ገብረሥላሴ</td><td>ኢትዮጵያ</td></tr><tr><td>France</td><td>Gerard Depardieu</td><td>France</td><td>Gérard Depardieu</td></tr><tr><td>France</td><td>Jean Reno</td><td>France</td><td>Jean Réno</td></tr><tr><td>France</td><td>Camille Saint-Saens</td><td>France</td><td>Camille Saint-Saëns</td></tr><tr><td>France</td><td>Mylene Demongeot</td><td>France</td><td>Mylène Demongeot</td></tr><tr><td>France</td><td>Francois Truffaut</td><td>France</td><td>François Truffaut</td></tr><tr><td>France (Braille)</td><td>Louis Braille</td><td>⠋⠗⠁⠝⠉⠑</td><td>⠇⠕⠥⠊⠎⠀&lt;BR&gt;⠃⠗⠁⠊⠇⠇⠑</td></tr><tr><td>Georgia</td><td>Eduard Shevardnadze</td><td>საქართველო</td><td>ედუარდ შევარდნაძე</td></tr><tr><td>Germany</td><td>Rudi Voeller</td><td>Deutschland</td><td>Rudi Völler</td></tr><tr><td>Germany</td><td>Walter Schultheiss</td><td>Deutschland</td><td>Walter Schultheiß</td></tr><tr><td>Greece</td><td>Giorgos Dalaras</td><td>Ελλάς</td><td>Γιώργος Νταλάρας</td></tr><tr><td>Iceland</td><td>Bjork Gudmundsdottir</td><td>Ísland</td><td>Björk Guðmundsdóttir</td></tr><tr><td>India (Hindi)</td><td>Madhuri Dixit</td><td>भारत</td><td>माधुरी दिछित</td></tr><tr><td>Ireland</td><td>Sinead O&#39;Connor</td><td>Éire</td><td>Sinéad O&#39;Connor</td></tr><tr><td>Israel</td><td>Yehoram Gaon</td><td>ישראל</td><td>יהורם גאון</td></tr><tr><td>Italy</td><td>Fabrizio DeAndre</td><td>Italia</td><td>Fabrizio De André</td></tr><tr><td>Japan</td><td>KUBOTA Toshinobu</td><td>日本</td><td>久保田    利伸</td></tr><tr><td>Japan</td><td>HAYASHIBARA Megumi</td><td>日本</td><td>林原 めぐみ</td></tr><tr><td>Japan</td><td>Mori Ogai</td><td>日本</td><td>森鷗外</td></tr><tr><td>Japan</td><td>Tex Texin</td><td>日本</td><td>テクス テクサン</td></tr><tr><td>Norway</td><td>Tor Age Bringsvaerd</td><td>Noreg</td><td>Tor Åge Bringsværd</td></tr><tr><td>Pakistan (Urdu)</td><td>Nusrat Fatah Ali Khan</td><td>پاکستان</td><td>نصرت فتح علی خان</td></tr><tr><td>People&#39;s Rep. of China</td><td>ZHANG Ziyi</td><td>中国</td><td>章子怡</td></tr><tr><td>People&#39;s Rep. of China</td><td>WONG Faye</td><td>中国</td><td>王菲</td></tr><tr><td>Poland</td><td>Lech Walesa</td><td>Polska</td><td>Lech Wałęsa</td></tr><tr><td>Puerto Rico</td><td>Olga Tanon</td><td>Puerto Rico</td><td>Olga Tañón</td></tr><tr><td>Rep. of China</td><td>Hsu Chi</td><td>臺灣</td><td>舒淇</td></tr><tr><td>Rep. of China</td><td>Ang Lee</td><td>臺灣</td><td>李安</td></tr><tr><td>Rep. of Korea</td><td>AHN Sung-Gi</td><td>한민국</td><td>안성기</td></tr><tr><td>Rep. of Korea</td><td>SHIM Eun-Ha</td><td>한민국</td><td>심은하</td></tr><tr><td>Russia</td><td>Mikhail Gorbachev</td><td>Россия</td><td>Михаил Горбачёв</td></tr><tr><td>Russia</td><td>Boris Grebenshchikov</td><td>Россия</td><td>Борис Гребенщиков</td></tr><tr><td>Slovenia</td><td>Frane &quot;Jezek&quot; Milcinski</td><td>Slovenija</td><td>Frane Milčinski - Ježek</td></tr><tr><td>Syracuse (Sicily)</td><td>Archimedes</td><td>Συρακούσα</td><td>Ἀρχιμήδης</td></tr><tr><td>Thailand</td><td>Thongchai McIntai</td><td>ประเทศไทย</td><td>ธงไชย แม็คอินไตย์</td></tr><tr><td>U.S.A.</td><td>Brad Pitt</td><td>U.S.A.</td><td>Brad Pitt</td></tr><tr><td>Yugoslavia (Cyrillic)</td><td>Djordje Balasevic</td><td>Југославија</td><td>Ђорђе Балашевић</td></tr><tr><td>Yugoslavia (Latin)</td><td>Djordje Balasevic</td><td>Jugoslavija</td><td>Đorđe Balašević</td></tr></table></html>
diff --git a/tests/sample_response.csv b/tests/sample_response.csv
@@ -1,10 +1,53 @@
-first_name,last_name,company_name,address,city,county,postal,phone1,phone2,email,web
-Aleshia,Tomkiewicz,Alan D Rosenburg Cpa Pc,14 Taylor St,St. Stephens Ward,Kent,CT2 7PP,01835-703597,01944-369967,atomkiewicz@hotmail.com,http://www.alandrosenburgcpapc.co.uk
-Evan,Zigomalas,Cap Gemini America,5 Binney St,Abbey Ward,Buckinghamshire,HP11 2AX,01937-864715,01714-737668,evan.zigomalas@gmail.com,http://www.capgeminiamerica.co.uk
-France,Andrade,"Elliott, John W Esq",8 Moor Place,East Southbourne and Tuckton W,Bournemouth,BH6 3BE,01347-368222,01935-821636,france.andrade@hotmail.com,http://www.elliottjohnwesq.co.uk
-Ulysses,Mcwalters,"Mcmahan, Ben L",505 Exeter Rd,Hawerby cum Beesby,Lincolnshire,DN36 5RP,01912-771311,01302-601380,ulysses@hotmail.com,http://www.mcmahanbenl.co.uk
-Tyisha,Veness,Champagne Room,5396 Forth Street,Greets Green and Lyng Ward,West Midlands,B70 9DT,01547-429341,01290-367248,tyisha.veness@hotmail.com,http://www.champagneroom.co.uk
-Eric,Rampy,"Thompson, Michael C Esq",9472 Lind St,Desborough,Northamptonshire,NN14 2GH,01969-886290,01545-817375,erampy@rampy.co.uk,http://www.thompsonmichaelcesq.co.uk
-Marg,Grasmick,Wrangle Hill Auto Auct & Slvg,7457 Cowl St #70,Bargate Ward,Southampton,SO14 3TY,01865-582516,01362-620532,marg@hotmail.com,http://www.wranglehillautoauctslvg.co.uk
-Laquita,Hisaw,In Communications Inc,20 Gloucester Pl #96,Chirton Ward,Tyne & Wear,NE29 7AD,01746-394243,01590-982428,laquita@yahoo.com,http://www.incommunicationsinc.co.uk
-Lura,Manzella,Bizerba Usa Inc,929 Augustine St,Staple Hill Ward,South Gloucestershire,BS16 4LL,01907-538509,01340-713951,lura@hotmail.com,http://www.bizerbausainc.co.uk
+Origin (English),Name (English),Origin (Native),Name (Native)
+Australia,Nicole Kidman,Australia,Nicole Kidman
+Austria,Johann Strauss,Österreich,Johann Strauß
+Belgium (Flemish),Rene Magritte,België,René Magritte
+Belgium (French),Rene Magritte,Belgique,René Magritte
+Belgium (German),Rene Magritte,Belgien,René Magritte
+Bhutan,Gonpo Dorji,འབྲུག་ཡུལ།,མགོན་པོ་རྡོ་རྗེ།
+Canada,Celine Dion,Canada,Céline Dion
+Canada - Nunavut (Inuktitut),Susan Aglukark,ᓄᓇᕗᒻᒥᐅᑦ,ᓱᓴᓐ ᐊᒡᓗᒃᑲᖅ
+Democratic People's Rep. of Korea,LEE Sol-Hee,조선 민주주의 인민 공화국,이설희
+Denmark,Soren Hauch-Fausboll,Danmark,Søren Hauch-Fausbøll
+Denmark,Soren Kierkegaard,Danmark,Søren Kierkegård
+Egypt,Abdel Halim Hafez,ﻣﺼﺮ,ﻋﺑﺪﺍﻠﺣﻟﻳﻢ ﺤﺎﻓﻅ
+Egypt,Om Kolthoum,ﻣﺼﺮ,ﺃﻡ ﻛﻟﺛﻭﻡ
+Eritrea,Berhane Zeray,ብርሃነ ዘርኣይ,ኤርትራ
+Ethiopia,Haile Gebreselassie,ኃይሌ ገብረሥላሴ,ኢትዮጵያ
+France,Gerard Depardieu,France,Gérard Depardieu
+France,Jean Reno,France,Jean Réno
+France,Camille Saint-Saens,France,Camille Saint-Saëns
+France,Mylene Demongeot,France,Mylène Demongeot
+France,Francois Truffaut,France,François Truffaut
+France (Braille),Louis Braille,⠋⠗⠁⠝⠉⠑,⠇⠕⠥⠊⠎⠀<BR>⠃⠗⠁⠊⠇⠇⠑
+Georgia,Eduard Shevardnadze,საქართველო,ედუარდ შევარდნაძე
+Germany,Rudi Voeller,Deutschland,Rudi Völler
+Germany,Walter Schultheiss,Deutschland,Walter Schultheiß
+Greece,Giorgos Dalaras,Ελλάς,Γιώργος Νταλάρας
+Iceland,Bjork Gudmundsdottir,Ísland,Björk Guðmundsdóttir
+India (Hindi),Madhuri Dixit,भारत,माधुरी दिछित
+Ireland,Sinead O'Connor,Éire,Sinéad O'Connor
+Israel,Yehoram Gaon,ישראל,יהורם גאון
+Italy,Fabrizio DeAndre,Italia,Fabrizio De André
+Japan,KUBOTA Toshinobu,日本,久保田    利伸
+Japan,HAYASHIBARA Megumi,日本,林原 めぐみ
+Japan,Mori Ogai,日本,森鷗外
+Japan,Tex Texin,日本,テクス テクサン
+Norway,Tor Age Bringsvaerd,Noreg,Tor Åge Bringsværd
+Pakistan (Urdu),Nusrat Fatah Ali Khan,پاکستان,نصرت فتح علی خان
+People's Rep. of China,ZHANG Ziyi,中国,章子怡
+People's Rep. of China,WONG Faye,中国,王菲
+Poland,Lech Walesa,Polska,Lech Wałęsa
+Puerto Rico,Olga Tanon,Puerto Rico,Olga Tañón
+Rep. of China,Hsu Chi,臺灣,舒淇
+Rep. of China,Ang Lee,臺灣,李安
+Rep. of Korea,AHN Sung-Gi,한민국,안성기
+Rep. of Korea,SHIM Eun-Ha,한민국,심은하
+Russia,Mikhail Gorbachev,Россия,Михаил Горбачёв
+Russia,Boris Grebenshchikov,Россия,Борис Гребенщиков
+Slovenia,"Frane ""Jezek"" Milcinski",Slovenija,Frane Milčinski - Ježek
+Syracuse (Sicily),Archimedes,Συρακούσα,Ἀρχιμήδης
+Thailand,Thongchai McIntai,ประเทศไทย,ธงไชย แม็คอินไตย์
+U.S.A.,Brad Pitt,U.S.A.,Brad Pitt
+Yugoslavia (Cyrillic),Djordje Balasevic,Југославија,Ђорђе Балашевић
+Yugoslavia (Latin),Djordje Balasevic,Jugoslavija,Đorđe Balašević
diff --git a/tests/test_templatetags.py b/tests/test_templatetags.py
@@ -17,6 +17,10 @@
 
 sample_template = \
     """{% spaceless %}
+    <html>
+    <head>
+    <meta charset="utf-8">
+    </head>
     {% load sheets %}
     {% csv key as uk500 %}
     <table>
@@ -28,8 +32,9 @@
         </tr>
     {% endfor %}
     </table>
+    </html>
     {% endspaceless %}"""
-sample_key = '1bJNR7SLqpzWJNvstNcFR4gtS-M7Bmn0D1X2lGTJPvGM'
+sample_key = '1uPsdcGUnUsf3d2xGHRGUUb7_k5IQPtBvfQY61u8Z8wE'
 sample_response = os.path.join(
     os.path.dirname(__file__), 'sample_response.csv')
 sample_output = os.path.join(
@@ -61,19 +66,20 @@ def test_404(self):
         t = template.Template(sample_template)
         self.assertEqual(
             t.render(template.Context({'key': 'test'})),
-            '<table></table>')
+            '<html><head><meta charset="utf-8"></head><table></table></html>')
         self.assertEqual(1, len(responses.calls))
 
     @responses.activate
     def test_sample_mocked(self):
         responses.add(
             responses.GET, gdocs_format.format(key=sample_key),
-            body=open(sample_response, 'rt', encoding='utf-8').read(),
+            body=open(sample_response).read(),
             match_querystring=True, status=200)
         t = template.Template(sample_template)
         output = t.render(template.Context({'key': sample_key}))
         self.assertEqual(1, len(responses.calls))
         self.assertEqual(
             responses.calls[0].request.url,
             gdocs_format.format(key=sample_key))
+
         self.assertEqual(output, open(sample_output).read())