Merge pull request #64 from zenbot/master

Handle transaction metadata in CDATA sections

ofxparse/ofxparse.py

@@ -22,13 +22,37 @@
 
 from . import mcc
 
+
+def skip_headers(fh):
+    '''
+    Prepare `fh` for parsing by BeautifulSoup by skipping its OFX
+    headers.
+    '''
+    if fh is None or isinstance(fh, six.string_types):
+        return
+    fh.seek(0)
+    header_re = re.compile(r"^\s*\w+:\s*\w+\s*$")
+    while True:
+        pos = fh.tell()
+        line = fh.readline()
+        if not line:
+            break
+        if header_re.search(line) is None:
+            fh.seek(pos)
+            return
+
+
 def soup_maker(fh):
+    skip_headers(fh)
     try:
         from bs4 import BeautifulSoup
-        return BeautifulSoup(fh)
+        soup = BeautifulSoup(fh, "xml")
+        for tag in soup.findAll():
+            tag.name = tag.name.lower()
     except ImportError:
         from BeautifulSoup import BeautifulStoneSoup
-        return BeautifulStoneSoup(fh)
+        soup = BeautifulStoneSoup(fh)
+    return soup
 
 
 def try_decode(string, encoding):
@@ -332,6 +356,7 @@ def parse(cls_, file_handle, fail_fast=True):
         ofx_obj.accounts = []
         ofx_obj.signon = None
 
+        skip_headers(ofx_file.fh)
         ofx = soup_maker(ofx_file.fh)
         if len(ofx.contents) == 0:
             raise OfxParserException('The ofx file is empty!')

tests/fixtures/suncorp.ofx

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="us-ascii"?>
+<?OFX OFXHEADER="200" VERSION="200" SECURITY="NONE" OLDFILEUID="NONE" NEWFILEUID="NONE"?>
+<OFX>
+  <SIGNONMSGSRSV1>
+    <SONRS>
+      <STATUS>
+        <CODE>0</CODE>
+        <SEVERITY>INFO</SEVERITY>
+      </STATUS>
+      <DTSERVER>20131215</DTSERVER>
+      <LANGUAGE>ENG</LANGUAGE>
+      <FI>
+        <ORG>SUNCORP</ORG>
+        <FID>484-799</FID>
+      </FI>
+    </SONRS>
+  </SIGNONMSGSRSV1>
+  <BANKMSGSRSV1>
+    <STMTTRNRS>
+      <TRNUID>1</TRNUID>
+      <STATUS>
+        <CODE>0</CODE>
+        <SEVERITY>INFO</SEVERITY>
+      </STATUS>
+      <STMTRS>
+        <CURDEF>AUD</CURDEF>
+        <BANKACCTFROM>
+          <BANKID>SUNCORP</BANKID>
+          <ACCTID>123456789</ACCTID>
+          <ACCTTYPE>CHECKING</ACCTTYPE>
+        </BANKACCTFROM>
+        <BANKTRANLIST>
+          <DTSTART>20130618</DTSTART>
+          <DTEND>20131215</DTEND>
+          <STMTTRN>
+            <TRNTYPE>DEBIT</TRNTYPE>
+            <DTPOSTED>20131215</DTPOSTED>
+            <TRNAMT>-16.85</TRNAMT>
+            <FITID>1</FITID>
+            <CHECKNUM>0</CHECKNUM>
+            <NAME><![CDATA[EFTPOS WDL HANDYWAY ALDI STORE  ]]></NAME>
+            <MEMO><![CDATA[EFTPOS WDL HANDYWAY ALDI STORE   GEELONG WEST VICAU]]></MEMO>
+          </STMTTRN>
+        </BANKTRANLIST>
+        <LEDGERBAL>
+          <BALAMT>1234.12</BALAMT>
+          <DTASOF>20131215</DTASOF>
+        </LEDGERBAL>
+        <AVAILBAL>
+          <BALAMT>1234.12</BALAMT>
+          <DTASOF>20131215</DTASOF>
+        </AVAILBAL>
+      </STMTRS>
+    </STMTTRNRS>
+  </BANKMSGSRSV1>
+</OFX>

tests/test_parse.py

@@ -1,17 +1,17 @@
 from __future__ import absolute_import
 
-from ofxparse.ofxparse import soup_maker
+import os
 from datetime import datetime, timedelta
 from decimal import Decimal
 from unittest import TestCase
 import sys
-sys.path.append('..')
+sys.path.insert(0, os.path.abspath('..'))
 
 import six
 
 from .support import open_file
 from ofxparse import OfxParser, AccountType, Account, Statement, Transaction
-from ofxparse.ofxparse import OfxFile, OfxPreprocessedFile, OfxParserException
+from ofxparse.ofxparse import OfxFile, OfxPreprocessedFile, OfxParserException, soup_maker
 
 class TestOfxPreprocessedFile(TestCase):
 
@@ -516,6 +516,22 @@ def testSecurityListSuccess(self):
         self.assertEquals(len(ofx.security_list), 7)
 
 
+class TestSuncorpBankStatement(TestCase):
+    def testCDATATransactions(self):
+        ofx = OfxParser.parse(open_file('suncorp.ofx'))
+        accounts = ofx.accounts
+        self.assertEquals(len(accounts), 1)
+        account = accounts[0]
+        transactions = account.statement.transactions
+        self.assertEquals(len(transactions), 1)
+        transaction = transactions[0]
+        self.assertEquals(transaction.payee, "EFTPOS WDL HANDYWAY ALDI STORE")
+        self.assertEquals(
+            transaction.memo,
+            "EFTPOS WDL HANDYWAY ALDI STORE   GEELONG WEST VICAU")
+        self.assertEquals(transaction.amount, Decimal("-16.85"))
+
+
 class TestAccountInfoAggregation(TestCase):
     def testForFourAccounts(self):
         ofx = OfxParser.parse(open_file('account_listing_aggregation.ofx'))