Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #12 from frapell/master
Improved handling for utf-8 csv's
  • Loading branch information
frapell committed Mar 22, 2012
2 parents cf556c2 + d74734b commit 059e971
Show file tree
Hide file tree
Showing 2 changed files with 80 additions and 14 deletions.
3 changes: 3 additions & 0 deletions CHANGES.txt
Expand Up @@ -4,6 +4,9 @@ Changelog
2.6.2 (unreleased)
------------------

- Improved handling utf-8 strings for the import and export for members
[frapell]

- Added stoneagehtml package for processing the newsletter's html before
sending it by mail. This add support for css declararions that will be
written directly into the html tags to improve rendering results in
Expand Down
91 changes: 77 additions & 14 deletions Products/EasyNewsletter/browser/subscribers.py
@@ -1,4 +1,6 @@
import csv
import codecs
import cStringIO
import tempfile

from Acquisition import aq_inner
Expand All @@ -20,6 +22,66 @@
from Products.EasyNewsletter.interfaces import ISubscriberSource


class UTF8Recoder:
"""
Iterator that reads an encoded stream and reencodes the input to UTF-8
"""
def __init__(self, f, encoding):
self.reader = codecs.getreader(encoding)(f)

def __iter__(self):
return self

def next(self):
return self.reader.next().encode("utf-8")

class UnicodeReader:
"""
A CSV reader which will iterate over lines in the CSV file "f",
which is encoded in the given encoding.
"""

def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
f = UTF8Recoder(f, encoding)
self.reader = csv.reader(f, dialect=dialect, **kwds)

def next(self):
row = self.reader.next()
return [unicode(s, "utf-8") for s in row]

def __iter__(self):
return self

class UnicodeWriter:
"""
A CSV writer which will write rows to CSV file "f",
which is encoded in the given encoding.
"""

def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
# Redirect output to a queue
self.queue = cStringIO.StringIO()
self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
self.stream = f
self.encoder = codecs.getincrementalencoder(encoding)()

def writerow(self, row):
self.writer.writerow([s.encode("utf-8") for s in row])
# Fetch UTF-8 output from the queue ...
data = self.queue.getvalue()
data = data.decode("utf-8")
# ... and reencode it into the target encoding
data = self.encoder.encode(data)
# write to the target stream
self.stream.write(data)
# empty queue
self.queue.truncate(0)

def writerows(self, rows):
for row in rows:
self.writerow(row)


def normalize_id(astring):
return getUtility(IIDNormalizer).normalize(astring)

Expand Down Expand Up @@ -124,7 +186,7 @@ def delete(self):
return False
existing = self.context.objectIds()
# avoid wrong id to be submitted
to_remove = [i for i in ids if i in existing]
to_remove = [i for i in ids if i in existing]
self.context.manage_delObjects(to_remove)
msg = _(u"subscriber/s deleted successfully")
msg_manager.addStatusMessage(msg, type="info")
Expand Down Expand Up @@ -163,7 +225,7 @@ def create_subscribers(self, csv_data=None):
return self.request.response.redirect(context.absolute_url() + '/@@upload_csv')

# Show error if no data has been provided in the file
reader = csv.reader(filename)
reader = UnicodeReader(filename)
header = reader.next()
CSV_HEADER_I18N = [self.context.translate(_(x)) for x in CSV_HEADER]
if header != CSV_HEADER_I18N:
Expand All @@ -179,6 +241,7 @@ def create_subscribers(self, csv_data=None):
fail.append(
{'failure': msg})
else:

salutation = subscriber[0]
fullname = subscriber[1]
email = subscriber[2]
Expand All @@ -202,9 +265,9 @@ def create_subscribers(self, csv_data=None):
language=lang)
sub = context[id]
sub.email = email
sub.fullname = fullname.decode(encoding)
sub.organization = organization.decode(encoding)
sub.salutation = salutation.decode(encoding)
sub.fullname = fullname
sub.organization = organization
sub.salutation = salutation
obj = self.context.get(id, None)
obj.reindexObject()
# update existing
Expand All @@ -215,6 +278,7 @@ def create_subscribers(self, csv_data=None):
'email': email,
'organization': organization})
except Exception, e:
import pdb;pdb.set_trace()
fail.append(
{'salutation': salutation,
'fullname': fullname,
Expand All @@ -236,21 +300,20 @@ def __call__(self):
# Create CSV file
filename = tempfile.mktemp()
file = open(filename, 'wb')
csvWriter = csv.writer(file,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
csvWriter = UnicodeWriter(file,
{'delimiter':',',
'quotechar':'"',
'quoting':csv.QUOTE_MINIMAL})
CSV_HEADER_I18N = [self.context.translate(_(x)) for x in CSV_HEADER]
csvWriter.writerow(CSV_HEADER_I18N)
for subscriber in ctool(portal_type = 'ENLSubscriber',
path='/'.join(self.context.getPhysicalPath()),
sort_on='email'):
obj = subscriber.getObject()
csvWriter.writerow([
obj.salutation.encode("utf-8"),
obj.fullname.encode("utf-8"),
obj.email,
obj.organization.encode("utf-8")])
csvWriter.writerow([obj.salutation,
obj.fullname,
obj.email,
obj.organization])
file.close()
data = open(filename, "r").read()

Expand Down

0 comments on commit 059e971

Please sign in to comment.