Merge pull request #59 from mdn/unicode_csv_1208238

bug 1208238 - Handle unicode in CSV

+r groovecoder

mdn/html.py

@@ -234,6 +234,7 @@ def __str__(self):
     (\s|                # Any whitespace, or
      (<\s*br\s*/?>)|    # A variant of <br>, or
      \xa0|              # Unicode non-breaking space, or
+     \ufeff|            # Unicode BOM
      (\&nbsp;)          # HTML nbsp character
     )+                  # One or more in a row
     ''')

mdn/tests/test_html.py

@@ -33,6 +33,10 @@ def test_to_text(self):
         text = HTMLText(raw='\nSome Text\n')
         self.assertEqual('Some Text', text.to_text())
 
+    def test_bom_removed(self):
+        text = HTMLText(raw='BOM -> \ufeff')
+        self.assertEqual('BOM ->', text.to_text())
+
 
 class TestHTMLBaseTag(TestCase):
     def test_str(self):

mdn/tests/test_views.py

@@ -340,6 +340,19 @@ def test_get(self):
         ]
         self.assert_csv_response(url, expected)
 
+    def test_get_unicode(self):
+        text = "Pas avant les éléments en-ligne"
+        self.issue.params = {"text": text}
+        self.issue.save()
+
+        url = reverse('issues_detail_csv', kwargs={'slug': 'inline-text'})
+        full_url = 'http://testserver/importer/{}'.format(self.fp.pk)
+        expected = [
+            'MDN Slug,Import URL,Source Start,Source End,text',
+            'docs/Web/CSS/float,{},10,20,{}'.format(full_url, text),
+        ]
+        self.assert_csv_response(url, expected)
+
 
 class TestIssuesSummaryCSV(CSVTestCase):
     def test_get(self):

mdn/views.py

@@ -2,7 +2,6 @@
 from collections import Counter
 from json import loads
 from math import floor
-import csv
 
 from django import forms
 from django.contrib.auth.decorators import user_passes_test
@@ -15,6 +14,7 @@
 from django.views.generic.base import TemplateResponseMixin, View
 from django.views.generic.detail import BaseDetailView
 from django.views.generic.edit import CreateView, FormMixin, UpdateView
+import unicodecsv as csv
 
 from .models import FeaturePage, Issue, ISSUES, SEVERITIES, validate_mdn_url
 from .tasks import start_crawl, parse_page
@@ -378,20 +378,24 @@ def get_context_data(self, **kwargs):
         return ctx
 
 
-def issues_summary_csv(request):
-    raw_counts = Issue.objects.values('slug').annotate(total=Count('slug'))
-    counts = [(raw['total'], raw['slug']) for raw in raw_counts]
-    counts.sort(reverse=True)
-
+def csv_response(filename, headers, rows):
+    """Return a CSV-for-download response."""
     response = HttpResponse(content_type='text/csv')
     response['Content-Disposition'] = (
-        'attachment; filename="import_issue_counts.csv"')
+        'attachment; filename="{}"'.format(filename))
     writer = csv.writer(response)
-    writer.writerow(['Count', 'Issue'])
-    writer.writerows(counts)
+    writer.writerow(headers)
+    writer.writerows(rows)
     return response
 
 
+def issues_summary_csv(request):
+    raw_counts = Issue.objects.values('slug').annotate(total=Count('slug'))
+    counts = [(raw['total'], raw['slug']) for raw in raw_counts]
+    counts.sort(reverse=True)
+    return csv_response("import_issue_counts.csv", ['Count', 'Issue'], counts)
+
+
 def issues_detail_csv(request, slug):
     issues = Issue.objects.filter(slug=slug).select_related('page__url')
     raw_headers = set()
@@ -408,15 +412,10 @@ def issues_detail_csv(request, slug):
     for line, params in zip(lines, raw_params):
         line.extend([params.get(header, "") for header in headers])
 
-    response = HttpResponse(content_type='text/csv')
     filename = 'import_issues_for_{}.csv'.format(slug)
-    response['Content-Disposition'] = (
-        'attachment; filename="{}"'.format(filename))
-    writer = csv.writer(response)
-    writer.writerow(
+    csv_headers = (
         ['MDN Slug', 'Import URL', 'Source Start', 'Source End'] + headers)
-    writer.writerows(lines)
-    return response
+    return csv_response(filename, csv_headers, lines)
 
 
 feature_page_create = user_passes_test(can_create)(

requirements.txt

@@ -84,3 +84,6 @@ requests==2.7.0
 oauthlib==1.0.3
 requests-oauthlib==0.5.0
 django-allauth==0.23.0
+
+# Unicode-aware CSVs
+unicodecsv==0.14.1