@@ -120,10 +120,14 @@ class DocumentLoader: result = [{'key': doc['key'], 'revision': doc['revision'], 'id': doc['id']} for doc in documents] # insert data - data = [dict(thing_id=doc.pop('id'), - revision=doc['revision'], - data=simplejson.dumps(doc)) - for doc in documents] + try: + data = [dict(thing_id=doc.pop('id'), + revision=doc['revision'], + data=simplejson.dumps(doc)) + for doc in documents] + except UnicodeDecodeError: + print `doc` + raise self.db.multiple_insert('data', data, seqname=False) return result openlibrary/utils/bulkimport.py 0edeb357bc534174816ffc51ffbb44b047102110 Edward Betts edward@archive.org http://github.com/openlibrary/openlibrary/commit/6ebba8751b1c3807484ad290f5f6eaea12d9ef27 6ebba8751b1c3807484ad290f5f6eaea12d9ef27 2009-11-06T08:34:32-08:00 2009-11-06T08:34:32-08:00 add an exception handler 1a343b5527d8582549b961dc34c03a64aca49426 Edward Betts edward@archive.org