From 3ed8e7b3f35ab352da75dc6ab8488232cee085fd Mon Sep 17 00:00:00 2001 From: Nic Pottier Date: Mon, 23 Mar 2020 13:41:18 -0700 Subject: [PATCH 1/5] open candidate csv files in mode that deals with BoM --- smartmin/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smartmin/models.py b/smartmin/models.py index 1051776..2a8e448 100644 --- a/smartmin/models.py +++ b/smartmin/models.py @@ -104,7 +104,7 @@ def get_import_file_headers(cls, csv_file): break reader.close() - reader = open(filename.name, "rU") + reader = open(filename.name, "rU", encoding='utf-8-sig') def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs): csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs) From 2cdc63677a4f6198088676d26eb1f6ede83bbeeb Mon Sep 17 00:00:00 2001 From: Nic Pottier Date: Mon, 23 Mar 2020 13:43:37 -0700 Subject: [PATCH 2/5] also update where we read the csv file in --- smartmin/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smartmin/models.py b/smartmin/models.py index 2a8e448..576942d 100644 --- a/smartmin/models.py +++ b/smartmin/models.py @@ -300,7 +300,7 @@ def import_raw_csv(cls, filename, user, import_params, log=None, import_results= break reader.close() - reader = open(filename.name, "rU") + reader = open(filename.name, "rU", encoding='utf-8-sig') def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs): csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs) From 4e62c28a92372a7a3e8e690f5e449b349d31ebce Mon Sep 17 00:00:00 2001 From: Nic Pottier Date: Mon, 23 Mar 2020 13:49:21 -0700 Subject: [PATCH 3/5] add test --- test_runner/blog/tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test_runner/blog/tests.py b/test_runner/blog/tests.py index d34228a..3bda6ea 100644 --- a/test_runner/blog/tests.py +++ b/test_runner/blog/tests.py @@ -463,6 +463,9 @@ def test_get_import_file_headers(self): with open('test_runner/blog/test_files/posts.csv', 'rb') as open_file: self.assertEqual(Post.get_import_file_headers(open_file), ['title', 'body', 'order', 'tags']) + with open('test_runner/blog/test_files/bom_import.csv', 'rb') as open_file: + self.assertEqual(Post.get_import_file_headers(open_file), ['URN:tel', 'Name', 'Field:email-address']) + def test_csv_import(self): with self.settings(CELERY_ALWAYS_EAGER=True, CELERY_RESULT_BACKEND='cache', CELERY_CACHE_BACKEND='memory'): import_url = reverse('blog.post_csv_import') From 80d9488ef3c476b7f69bc746f10f543cb829ea4b Mon Sep 17 00:00:00 2001 From: Nic Pottier Date: Mon, 23 Mar 2020 14:07:52 -0700 Subject: [PATCH 4/5] missing file --- test_runner/blog/test_files/bom_import.csv | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 test_runner/blog/test_files/bom_import.csv diff --git a/test_runner/blog/test_files/bom_import.csv b/test_runner/blog/test_files/bom_import.csv new file mode 100644 index 0000000..6edb414 --- /dev/null +++ b/test_runner/blog/test_files/bom_import.csv @@ -0,0 +1,2 @@ +URN:tel,Name,Field:email-address +12065551212,Fred,fred@gmail.com \ No newline at end of file From 03d35ab00821b28cc0eb06543ff0cabce6cfc851 Mon Sep 17 00:00:00 2001 From: Nic Pottier Date: Mon, 23 Mar 2020 14:21:49 -0700 Subject: [PATCH 5/5] fix test --- test_runner/blog/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner/blog/tests.py b/test_runner/blog/tests.py index 3bda6ea..c3f2f51 100644 --- a/test_runner/blog/tests.py +++ b/test_runner/blog/tests.py @@ -464,7 +464,7 @@ def test_get_import_file_headers(self): self.assertEqual(Post.get_import_file_headers(open_file), ['title', 'body', 'order', 'tags']) with open('test_runner/blog/test_files/bom_import.csv', 'rb') as open_file: - self.assertEqual(Post.get_import_file_headers(open_file), ['URN:tel', 'Name', 'Field:email-address']) + self.assertEqual(Post.get_import_file_headers(open_file), ['urn:tel', 'name', 'field:email-address']) def test_csv_import(self): with self.settings(CELERY_ALWAYS_EAGER=True, CELERY_RESULT_BACKEND='cache', CELERY_CACHE_BACKEND='memory'):