get rid of UnicodeWriter and update exports for invoices, articles, c…

…hapters and events

The UnicodeWriter can significantly increase the size of the exported files

tendenci/apps/articles/utils.py

@@ -1,13 +1,13 @@
 import time as ttime
 from datetime import datetime, date, time
+import csv
 
 from django.contrib.auth.models import User
 from django.core.files.storage import default_storage
 from django.urls import reverse
 from django.template.loader import render_to_string
 
 from tendenci.apps.articles.models import Article
-from tendenci.apps.base.utils import UnicodeWriter
 from tendenci.apps.emails.models import Email
 from tendenci.apps.site_settings.utils import get_setting
 from tendenci.apps.base.utils import escape_csv
@@ -43,14 +43,13 @@ def process_export(identifier, user_id):
     identifier = identifier or int(ttime.time())
     file_name_temp = 'export/articles/%s_temp.csv' % (identifier)
 
-    with default_storage.open(file_name_temp, 'wb') as csvfile:
-        csv_writer = UnicodeWriter(csvfile, encoding='utf-8')
-        csv_writer.writerow(field_list)
+    with default_storage.open(file_name_temp, 'w') as csvfile:
+        csv_writer = csv.DictWriter(csvfile, fieldnames=field_list)
 
         articles = Article.objects.filter(status_detail='active')
 
         for article in articles:
-            items_list = []
+            row_dict = {}
             for field_name in field_list:
                 item = getattr(article, field_name)
 
@@ -62,8 +61,8 @@ def process_export(identifier, user_id):
                     item = item.strftime('%H:%M:%S')
                 else:
                     item = escape_csv(item)
-                items_list.append(item)
-            csv_writer.writerow(items_list)
+                row_dict[field_name] = item
+            csv_writer.writerow(row_dict)
 
     # rename the file name
     file_name = 'export/articles/%s.csv' % identifier

tendenci/apps/base/utils.py

@@ -876,53 +876,6 @@ def __next__(self):
         return next(self.reader).encode("utf-8")
 
 
-class UnicodeReader(object):
-    """
-    A CSV reader which will iterate over lines in the CSV file "f",
-    which is encoded in the given encoding.
-    """
-
-    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
-        f = UTF8Recoder(f, encoding)
-        self.reader = csv.reader(f, dialect=dialect, **kwds)
-
-    def __next__(self):
-        row = next(self.reader)
-        return [str(s, "utf-8") for s in row]
-
-    def __iter__(self):
-        return self
-
-
-class UnicodeWriter:
-    """
-    A CSV writer which will write rows to CSV file "f",
-    which is encoded in the given encoding.
-    """
-
-    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
-        # Redirect output to a queue
-        self.queue = StringIO()
-        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
-        self.stream = f
-        self.encoder = codecs.getincrementalencoder(encoding)()
-
-    def writerow(self, row):
-        self.writer.writerow(row)
-        # Fetch output from the queue ...
-        data = self.queue.getvalue()
-        # ... and encode it into the target encoding
-        data = self.encoder.encode(data)
-        # write to the target stream
-        self.stream.write(data)
-        # empty queue
-        self.queue.truncate(0)
-
-    def writerows(self, rows):
-        for row in rows:
-            self.writerow(row)
-
-
 def get_salesforce_access():
 
     required_settings = (hasattr(settings, 'SALESFORCE_USERNAME'),

tendenci/apps/chapters/models.py

@@ -47,7 +47,6 @@
 from tendenci.apps.base.utils import validate_email
 from tendenci.apps.notifications import models as notification
 from tendenci.apps.base.models import BaseImport, BaseImportData
-from tendenci.apps.base.utils import UnicodeWriter
 from tendenci.apps.base.utils import correct_filename, get_us_state_name
 from tendenci.apps.event_logs.models import EventLog
 from tendenci.libs.utils import python_executable
@@ -2077,26 +2076,28 @@ def get_header_and_first_row(self):
         return header_row, first_row
 
     def generate_recap(self):
+        import csv
         if not self.recap_file and self.header_line:
             file_name = 'chapter_memberships_import_%d_recap.csv' % self.id
             file_path = '%s/%s' % (os.path.split(self.upload_file.name)[0],
                                    file_name)
-            f = default_storage.open(file_path, 'wb')
-            recap_writer = UnicodeWriter(f, encoding='utf-8')
             header_row = self.header_line.split(',')
             if 'status' in header_row:
                 header_row.remove('status')
             if 'status_detail' in header_row:
                 header_row.remove('status_detail')
             header_row.extend(['action', 'error'])
-            recap_writer.writerow(header_row)
-            data_list = ChapterMembershipImportData.objects.filter(
-                mimport=self).order_by('row_num')
-            for idata in data_list:
-                data_dict = idata.row_data
-                row = [data_dict[k] for k in header_row if k in data_dict]
-                row.extend([idata.action_taken, idata.error])
-                recap_writer.writerow(row)
+            with default_storage.open(file_path, 'w') as f:
+                recap_writer = csv.DictWriter(f, fieldnames=header_row)
+
+                data_list = ChapterMembershipImportData.objects.filter(
+                    mimport=self).order_by('row_num')
+                for idata in data_list:
+                    data_dict = idata.row_data
+                    data_dict['action'] = idata.action_taken
+                    data_dict['error'] = idata.error
+
+                    recap_writer.writerow(data_dict)
 
             f.close()
             self.recap_file.name = file_path

tendenci/apps/directories/utils.py

@@ -3,6 +3,7 @@
 from io import BytesIO
 from PIL import Image
 import time as ttime
+import csv
 
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
@@ -15,7 +16,6 @@
 
 from tendenci.apps.directories.models import Directory, DirectoryPricing
 from tendenci.apps.invoices.models import Invoice
-from tendenci.apps.base.utils import UnicodeWriter
 from tendenci.apps.emails.models import Email
 from tendenci.apps.payments.models import Payment
 from tendenci.apps.site_settings.utils import get_setting
@@ -202,20 +202,12 @@ def process_export(export_fields='all_fields', export_status_detail='',
     file_name_temp = 'export/directories/%s_temp.csv' % identifier
 
     with default_storage.open(file_name_temp, 'wb') as csvfile:
-        csv_writer = UnicodeWriter(csvfile, encoding='utf-8')
-        fields_names = list(field_list)
-        for i, item in enumerate(fields_names):
-            if item == 'headline':
-                fields_names[i] = 'name'
-            if item == 'body':
-                fields_names[i] = 'description'
-        csv_writer.writerow(fields_names)
-
+        csv_writer = csv.DictWriter(csvfile, fieldnames=field_list)
         directories = Directory.objects.all()
         if export_status_detail:
             directories = directories.filter(status_detail__icontains=export_status_detail)
         for directory in directories:
-            items_list = []
+            row_dict = {}
             for field_name in field_list:
                 item = getattr(directory, field_name)
                 if item is None:
@@ -232,8 +224,8 @@ def process_export(export_fields='all_fields', export_status_detail='',
                         item = 'Total: %d / Balance: %d' % (item.total, item.balance)
                     elif isinstance(item, str):
                         item = escape_csv(item)
-                items_list.append(item)
-            csv_writer.writerow(items_list)
+                row_dict[field_name] = item
+            csv_writer.writerow(row_dict)
 
     # rename the file name
     file_name = 'export/directories/%s.csv' % identifier