Merge pull request #135 from nyaruka/utf-8-sig

open candidate csv files in mode that deals with BoM
nyaruka · Mar 23, 2020 · bc1bfbe · bc1bfbe
2 parents bfbdee9 + 03d35ab
commit bc1bfbe
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 2 deletions.
diff --git a/smartmin/models.py b/smartmin/models.py
@@ -104,7 +104,7 @@ def get_import_file_headers(cls, csv_file):
                     break
             reader.close()
 
-            reader = open(filename.name, "rU")
+            reader = open(filename.name, "rU", encoding='utf-8-sig')
 
             def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
                 csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs)
@@ -300,7 +300,7 @@ def import_raw_csv(cls, filename, user, import_params, log=None, import_results=
                 break
         reader.close()
 
-        reader = open(filename.name, "rU")
+        reader = open(filename.name, "rU", encoding='utf-8-sig')
 
         def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
             csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs)

diff --git a/test_runner/blog/test_files/bom_import.csv b/test_runner/blog/test_files/bom_import.csv
@@ -0,0 +1,2 @@
+URN:tel,Name,Field:email-address
+12065551212,Fred,fred@gmail.com
diff --git a/test_runner/blog/tests.py b/test_runner/blog/tests.py
@@ -463,6 +463,9 @@ def test_get_import_file_headers(self):
         with open('test_runner/blog/test_files/posts.csv', 'rb') as open_file:
             self.assertEqual(Post.get_import_file_headers(open_file), ['title', 'body', 'order', 'tags'])
 
+        with open('test_runner/blog/test_files/bom_import.csv', 'rb') as open_file:
+            self.assertEqual(Post.get_import_file_headers(open_file), ['urn:tel', 'name', 'field:email-address'])
+
     def test_csv_import(self):
         with self.settings(CELERY_ALWAYS_EAGER=True, CELERY_RESULT_BACKEND='cache', CELERY_CACHE_BACKEND='memory'):
             import_url = reverse('blog.post_csv_import')