From 1ef2a2b3523b73a1dfac99262f5a4df919f840e0 Mon Sep 17 00:00:00 2001 From: Franco Pellegrini Date: Thu, 22 Mar 2012 01:36:25 -0300 Subject: [PATCH 1/2] Improved utf-8 strings handling for import and export for members --- .../EasyNewsletter/browser/subscribers.py | 91 ++++++++++++++++--- 1 file changed, 77 insertions(+), 14 deletions(-) diff --git a/Products/EasyNewsletter/browser/subscribers.py b/Products/EasyNewsletter/browser/subscribers.py index bfc91368..44706c9a 100644 --- a/Products/EasyNewsletter/browser/subscribers.py +++ b/Products/EasyNewsletter/browser/subscribers.py @@ -1,4 +1,6 @@ import csv +import codecs +import cStringIO import tempfile from Acquisition import aq_inner @@ -20,6 +22,66 @@ from Products.EasyNewsletter.interfaces import ISubscriberSource +class UTF8Recoder: + """ + Iterator that reads an encoded stream and reencodes the input to UTF-8 + """ + def __init__(self, f, encoding): + self.reader = codecs.getreader(encoding)(f) + + def __iter__(self): + return self + + def next(self): + return self.reader.next().encode("utf-8") + +class UnicodeReader: + """ + A CSV reader which will iterate over lines in the CSV file "f", + which is encoded in the given encoding. + """ + + def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): + f = UTF8Recoder(f, encoding) + self.reader = csv.reader(f, dialect=dialect, **kwds) + + def next(self): + row = self.reader.next() + return [unicode(s, "utf-8") for s in row] + + def __iter__(self): + return self + +class UnicodeWriter: + """ + A CSV writer which will write rows to CSV file "f", + which is encoded in the given encoding. + """ + + def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): + # Redirect output to a queue + self.queue = cStringIO.StringIO() + self.writer = csv.writer(self.queue, dialect=dialect, **kwds) + self.stream = f + self.encoder = codecs.getincrementalencoder(encoding)() + + def writerow(self, row): + self.writer.writerow([s.encode("utf-8") for s in row]) + # Fetch UTF-8 output from the queue ... + data = self.queue.getvalue() + data = data.decode("utf-8") + # ... and reencode it into the target encoding + data = self.encoder.encode(data) + # write to the target stream + self.stream.write(data) + # empty queue + self.queue.truncate(0) + + def writerows(self, rows): + for row in rows: + self.writerow(row) + + def normalize_id(astring): return getUtility(IIDNormalizer).normalize(astring) @@ -124,7 +186,7 @@ def delete(self): return False existing = self.context.objectIds() # avoid wrong id to be submitted - to_remove = [i for i in ids if i in existing] + to_remove = [i for i in ids if i in existing] self.context.manage_delObjects(to_remove) msg = _(u"subscriber/s deleted successfully") msg_manager.addStatusMessage(msg, type="info") @@ -163,7 +225,7 @@ def create_subscribers(self, csv_data=None): return self.request.response.redirect(context.absolute_url() + '/@@upload_csv') # Show error if no data has been provided in the file - reader = csv.reader(filename) + reader = UnicodeReader(filename) header = reader.next() CSV_HEADER_I18N = [self.context.translate(_(x)) for x in CSV_HEADER] if header != CSV_HEADER_I18N: @@ -179,6 +241,7 @@ def create_subscribers(self, csv_data=None): fail.append( {'failure': msg}) else: + salutation = subscriber[0] fullname = subscriber[1] email = subscriber[2] @@ -202,9 +265,9 @@ def create_subscribers(self, csv_data=None): language=lang) sub = context[id] sub.email = email - sub.fullname = fullname.decode(encoding) - sub.organization = organization.decode(encoding) - sub.salutation = salutation.decode(encoding) + sub.fullname = fullname + sub.organization = organization + sub.salutation = salutation obj = self.context.get(id, None) obj.reindexObject() # update existing @@ -215,6 +278,7 @@ def create_subscribers(self, csv_data=None): 'email': email, 'organization': organization}) except Exception, e: + import pdb;pdb.set_trace() fail.append( {'salutation': salutation, 'fullname': fullname, @@ -236,21 +300,20 @@ def __call__(self): # Create CSV file filename = tempfile.mktemp() file = open(filename, 'wb') - csvWriter = csv.writer(file, - delimiter=',', - quotechar='"', - quoting=csv.QUOTE_MINIMAL) + csvWriter = UnicodeWriter(file, + {'delimiter':',', + 'quotechar':'"', + 'quoting':csv.QUOTE_MINIMAL}) CSV_HEADER_I18N = [self.context.translate(_(x)) for x in CSV_HEADER] csvWriter.writerow(CSV_HEADER_I18N) for subscriber in ctool(portal_type = 'ENLSubscriber', path='/'.join(self.context.getPhysicalPath()), sort_on='email'): obj = subscriber.getObject() - csvWriter.writerow([ - obj.salutation.encode("utf-8"), - obj.fullname.encode("utf-8"), - obj.email, - obj.organization.encode("utf-8")]) + csvWriter.writerow([obj.salutation, + obj.fullname, + obj.email, + obj.organization]) file.close() data = open(filename, "r").read() From d74734b59d5c3a8dfdccc213eae27753f5f41a87 Mon Sep 17 00:00:00 2001 From: Franco Pellegrini Date: Thu, 22 Mar 2012 01:39:36 -0300 Subject: [PATCH 2/2] Update CHANGES.txt file --- CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 49703443..97187efc 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,9 @@ Changelog 2.6.2 (unreleased) ------------------ +- Improved handling utf-8 strings for the import and export for members + [frapell] + - Added stoneagehtml package for processing the newsletter's html before sending it by mail. This add support for css declararions that will be written directly into the html tags to improve rendering results in