Apparently CSV files can have a BOM

Also update paypal support
j3k0 · Mar 31, 2018 · ce8660c · ce8660c
1 parent ef31cfa
commit ce8660c
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 6 deletions.
diff --git a/ledgerautosync/converter.py b/ledgerautosync/converter.py
@@ -396,16 +396,18 @@ def format_position(self, pos):
             dateStr = pos.date.strftime("%Y/%m/%d %H:%M:%S")
             return "P %s %s %s\n" % (dateStr, self.maybe_get_ticker(pos.security), pos.unit_price)
 
+def removeNonAscii(field):
+    return re.sub(r'[^a-zA-Z0-9 ]',r'', field)
 
 class CsvConverter(Converter):
     @staticmethod
     def make_converter(csv, name=None, **kwargs):
-        fieldset = set(csv.fieldnames)
+        fieldset = set(map(removeNonAscii, csv.fieldnames))
         for klass in CsvConverter.__subclasses__():
-            if klass.FIELDSET <= fieldset:
+            if fieldset.issubset(klass.FIELDSET):
                 return klass(csv, name=name, **kwargs)
         # Found no class, bail
-        raise Exception('Cannot determine CSV type')
+        raise Exception('Cannot determine CSV type \n')
 
     # By default, return an MD5 of the key-value pairs in the row.
     # If a better ID is available, should be overridden.
@@ -428,7 +430,7 @@ def __init__(self, csv, name=None, indent=4, ledger=None, unknownaccount=None):
 
 
 class PaypalConverter(CsvConverter):
-    FIELDSET = set(['Currency', 'Date', 'Gross', 'Item Title', 'Name', 'Net', 'Status', 'To Email Address', 'Transaction ID', 'Type'])
+    FIELDSET = set(['Status', 'Gross', 'Fee', 'Name', 'Receipt ID', 'Transaction ID', 'Reference Txn ID', 'Currency', 'To Email Address', 'Date', 'Time', 'TimeZone', 'Net', 'Type', 'From Email Address'])
 
     def __init__(self, *args, **kwargs):
         super(PaypalConverter, self).__init__(*args, **kwargs)
@@ -684,7 +686,7 @@ def convert(self, row):
             postings=postings)
 
 class WakatimeConverter(CsvConverter):
-    FIELDSET = set(["Account","Commodity", "Date","Project","Duration","ID"])
+    FIELDSET = set(["Account", "Commodity", "Date", "Project", "Duration", "ID"])
 
     def __init__(self, *args, **kwargs):
         super(WakatimeConverter, self).__init__(*args, **kwargs)

diff --git a/ledgerautosync/sync.py b/ledgerautosync/sync.py
@@ -22,6 +22,7 @@
 from ofxparse.ofxparse import InvestmentTransaction
 import logging
 import csv
+import codecs
 
 class Synchronizer(object):
     def __init__(self, lgr):
@@ -120,16 +121,29 @@ def __init__(self, lgr):
 
     def parse_file(self, path, accountname=None, unknownaccount=None):
         with open(path, 'rb') as f:
+            has_bom = f.read(3) == codecs.BOM_UTF8
+            if not(has_bom): f.seek(0)
+            else: f.seek(3)
             dialect = csv.Sniffer().sniff(f.read(1024))
-            f.seek(0)
+            if not(has_bom): f.seek(0)
+            else: f.seek(3)
             dialect.skipinitialspace = True
             reader = csv.DictReader(f, dialect=dialect)
             converter = CsvConverter.make_converter(
                 reader,
                 name=accountname,
                 ledger=self.lgr,
                 unknownaccount=unknownaccount)
+
+            # Create a new reader in case the converter modified the dialect
+            if not(has_bom): f.seek(0)
+            else: f.seek(3)
+            reader = csv.DictReader(f, dialect=dialect)
+
             return [converter.convert(row)
                     for row in reader
                     if not(self.lgr.check_transaction_by_id(
                             converter.get_csv_metadata(), converter.get_csv_id(row)))]
+            # return [converter.convert(row)
+            #         for row in reader
+            #         if not(self.is_row_synced(converter, row))]