Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

First checkin

  • Loading branch information...
commit dc1cd9e2c4b9b4123ce357558872961fc89dcf25 0 parents
Evan Davey authored
1  CHANGES.txt
@@ -0,0 +1 @@
+v0.1, 2012-03-20 -- Initial release.
11 LICENSE.txt
@@ -0,0 +1,11 @@
+#Legal Notice
+
+This work is licensed under a [Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License][cc-nc-sa].
+
+![][img-cc-nc-sa]
+
+
+[cc-nc-sa]: http://creativecommons.org/licenses/by-nc-sa/3.0/
+
+[img-cc-nc-sa]: http://i.creativecommons.org/l/by-nc-sa/3.0/88x31.png
+
3  MANIFEST.in
@@ -0,0 +1,3 @@
+include *.txt *.md
+recursive-include bankdownloads *.py
+recursive-include docs *.txt
8 README.md
@@ -0,0 +1,8 @@
+#BankDownloadProcessor
+
+* Set of tools to read in and standardise bank transaction downloads
+
+* Capable of reading csv or ofx files and converting to a clean and standardised format
+
+* bankdowload-processor script compares checksums of new files to those stored in a database to allow automatic processing of new files
+
1  bankdownloads/__init__.py
@@ -0,0 +1 @@
+from bankdownload import *
207 bankdownloads/bankcsv.py
@@ -0,0 +1,207 @@
+import csv
+import sys
+import hashlib
+import time
+import os
+
+#Parent class that defines basic attributes and methods for BankCSV files
+class BankCSV:
+
+
+
+ def __init__(self,path):
+ self.bankid = ""
+ self.path = path
+ self.data =[]
+ self.accountid = ""
+
+ def __repr__(self):
+
+ ostr="Name:"+self.bankid+"\n"
+ ostr+="Acc:"+self.accountid+"\n"
+ ostr+="Path:"+self.path+"\n"
+
+ self.printdata()
+
+ return ostr
+
+ def wipeclear(self):
+
+ print "wiping " + self.bankid
+ self.bankid=''
+ self.path=''
+ self.accountid=''
+ del self.data[:]
+
+ def GetData(self):
+ return self.data
+
+ def GetAccountId(self):
+ return self.accountid
+
+ def GetBankId(self):
+ return self.bankid
+
+ def printdata(self):
+
+ for row in self.data:
+ print row
+
+ def GetAccountIdFromPath(self):
+
+ try:
+
+ filename=self.path
+ filename=os.path.basename(self.path)
+ return filename.split('-')[1]
+
+ except:
+ return ""
+
+ def GetAccountIdRabo(self):
+
+ ofile = open(self.path, "r")
+ reader = csv.reader(ofile)
+ header=reader.next()
+ accid=''
+ if len(header) == 3:
+ accid=header[1]
+ accid=accid.replace("-","")
+
+
+ return accid
+ ofile.close()
+
+
+ def format_val(self,val):
+
+ val=val.strip().replace('$','').replace(',','')
+
+ if val:
+ val=float(val)
+ return val
+ else:
+ return 0
+
+ def load(self):
+
+ # data definitions
+ csvs=[
+ {'bankid':'CBACashManagement',
+ 'header': ['Date','Description','Debit($)','Credit($)','Balance($)',''],
+ 'currency': 'AUD',
+ 'header_row': 1,
+ 'data_row': 2,
+ 'date_col':0,
+ 'date_format':"%d/%m/%Y",
+ 'credit_col':3,
+ 'memo_col': 1,
+ 'accountid':self.GetAccountIdFromPath()
+ },
+ {'bankid':'RaboPlus',
+ 'header': ['Operation number', 'Operation date', 'Description', 'Operation amount', 'Currency', 'Value date', 'Counterparty account', 'Counterparty name :', 'Communication 1 :', 'Communication 2 :', 'Operation reference'],
+ 'currency': 'AUD',
+ 'header_row': 2,
+ 'data_row': 3,
+ 'date_col':1,
+ 'date_format':"%d-%m-%Y",
+ 'credit_col':3,
+ 'memo_col': 2,
+ 'payee_col': 6,
+ 'accountid':self.GetAccountIdRabo(),
+ },
+ {'bankid':'INGDirect',
+ 'header': ['Date',' Description',' Debit',' Credit',' Balance'],
+ 'header_row': 1,
+ 'currency': 'AUD',
+ 'data_row': 2,
+ 'date_col':0,
+ 'date_format':"%d/%m/%Y",
+ 'credit_col':3,
+ 'debit_col':2,
+ 'memo_col': 1,
+ 'accountid':self.GetAccountIdFromPath(),
+ },
+ {'bankid':'FSFSuper',
+ 'header': ['Transaction Date','Payroll End Date','Employer','Transaction Type','Investment Option','Transaction Units','Unit Price','Transaction Amount'],
+ 'header_row': 1,
+ 'currency': 'AUD',
+ 'data_row': 2,
+ 'date_col':0,
+ 'date_format':"%d/%m/%Y",
+ 'credit_col':7,
+ 'memo_col': 3,
+ 'accountid':self.GetAccountIdFromPath(),
+ },
+ ]
+
+ for c in csvs:
+ f = open(self.path, "r")
+ reader = csv.reader(f)
+
+ for i in range(0,c['header_row']):
+ try:
+ header=reader.next()
+ except:
+ continue
+ i=i+1
+
+
+ if header == c['header']:
+ self.bankid=c['bankid']
+ self.accountid=c['accountid']
+ print "....found csv for %s,%s" % (self.bankid,self.accountid)
+
+ for row in reader:
+ if len(row)==0:
+ continue
+
+ dt=row[c['date_col']]
+ dt = time.strptime(dt, c['date_format'])
+ dt = time.strftime("%Y%m%d", dt)
+
+ curr=c['currency']
+ fxcurr=curr
+ fxrate=1
+
+ try:
+ payee=row[c['payee_col']].strip()
+ except:
+ payee=""
+
+ try:
+ memo=row[c['memo_col']].strip()
+ except:
+ memo=""
+
+ try:
+ debit=abs(self.format_val(row[c['debit_col']]))*-1
+ debit=float(debit)
+ except:
+ print 'Error reading debit'
+ debit=0.0
+
+ try:
+ credit=abs(self.format_val(row[c['credit_col']]))
+ credit=float(credit)
+
+ except:
+ print 'Error reading credit'
+ credit=0.0
+
+ if (debit):
+ val = round(debit,2)
+ else:
+ val = round(credit,2)
+
+ fxamount=val
+
+ transid=hashlib.md5(self.bankid+dt+str(val)+payee+memo).hexdigest()
+ self.data.append({"transid":transid,"bankid":self.bankid,"accountid":self.accountid,"date":dt,"payee":payee,"memo":memo,"value":val,"currency":curr,"fxcurrency":fxcurr,"fxamount":fxamount,"fxrate":fxrate})
+
+ f.close()
+ return
+ f.close()
+ print "**NO MATCH**"
+ return
+
255 bankdownloads/bankdownload.py
@@ -0,0 +1,255 @@
+import os
+import hashlib
+import time
+import re
+from bankdownloads.ofx import *
+from bankdownloads.bankcsv import BankCSV
+
+class BankDownload:
+ """
+ An object to represent a bank download of transactions
+ """
+
+
+ #Standard csv output headers
+ header=[['TransId','FileType','AccountId','Date','Payee','Memo','Value','Currency','FXCurrency','FXAmount','FXRate']]
+
+ def __init__(self, path):
+ self.path = path
+ self.data=[]
+ self.accountid=''
+ self.bankid=''
+ self.SetFileType()
+
+ #custom print
+ def __repr__(self):
+
+ ostr="::Bank Download::"+"\n"
+ ostr+="Path: " + self.path +"\n"
+ ostr+="Type: " + self.filetype +"\n"
+ ostr+="Bank id: " + self.bankid +"\n"
+ ostr+="Account id: " + self.accountid +"\n"
+ ostr+="Start Date: " + self.mindate +"\n"
+ ostr+="End Date: " + self.maxdate +"\n"
+ ostr+="Rows: " + str(len(self.data)) +"\n"
+ ostr+="Md5: %s" % str(self.md5()) +"\n"
+ return ostr
+
+ def SetFileType(self):
+ extension = os.path.splitext(self.path)[1]
+
+ if extension == ".ofx" or extension == ".qfx":
+ self.filetype = "ofx"
+ else:
+ self.filetype = "csv"
+
+ def printdata(self):
+
+ for row in self.data:
+ print row
+
+ #generate a filename for output based on attributes
+ def generatefilename(self):
+
+ fname = self.bankid+"_"+self.accountid
+ fname = self.maxdate+"_"+fname
+
+ return fname
+
+ def md5(self):
+ md5 = hashlib.md5()
+ with open(self.path,'rb') as f:
+ for chunk in iter(lambda: f.read(128*md5.block_size), ''):
+ md5.update(chunk)
+ return md5.hexdigest()
+
+ def writecsv(self,path):
+ ofile = open(path, "w")
+ writer = csv.writer(ofile,quoting=csv.QUOTE_NONNUMERIC)
+
+
+ #print header
+ writer.writerow(self.data[0].keys())
+
+ for row in self.data:
+
+ rowd=[]
+ for r in row.keys():
+ rowd.append(row[r])
+
+ writer.writerow(rowd)
+ ofile.close()
+ print "Created csv: " + path
+
+ def writeofx(self,path):
+ ofxexport(path,self.data)
+ print "Created ofx: " + path
+
+ def load(self):
+ #print "Loading..."
+ if self.filetype == "ofx":
+ #print "Loading ofx..."
+ r=self.loadofx()
+ else:
+ #print "Loading csv..."
+ r=self.loadcsv()
+
+ if r:
+ #print "Load failed"
+ return r
+
+ #print "Processing loaded data...."
+
+
+ if self.bankid == '':
+ self.SetBankIdFromPath()
+
+ if self.accountid == '':
+ self.SetAccountIdFromPath()
+
+ self.SetMaxDate()
+ self.SetMinDate()
+ self.processdata()
+
+ def SetBankIdFromPath(self):
+ if self.bankid == '':
+ fname=os.path.basename(self.path)
+ self.bankid=fname.split("-")[0]
+
+ def SetAccountIdFromPath(self):
+ if self.accountid == '':
+ fname=os.path.basename(self.path)
+ self.accountid=fname.split("-")[1]
+
+ def loadofx(self):
+ print "Loading data from " + self.path
+ ofx = OfxParser.parse(file(self.path))
+ #self.data=self.header
+
+ try:
+ for t in ofx.bank_account.statement.transactions:
+ val=float(t.amount)
+ payee=cleanStr(t.payee)
+ memo=cleanStr(t.memo)
+ dt=parseDate(cleanStr(t.date))
+ accid=cleanStr(ofx.bank_account.number)
+ bankid=cleanStr(ofx.bank_account.routing_number)
+ transid=cleanStr(t.id)
+ desc=payee + "-" + memo
+
+
+ self.accountid=accid
+ self.bankid=bankid
+
+ curr=cleanStr(ofx.bank_account.currency)
+
+ transid=hashlib.md5(transid+accid+dt+desc+str(val)).hexdigest()
+
+ #defaults to be overridden in subclass
+ fxrate=1
+ fxcurr=curr
+ fxamount=val
+
+ self.data.append({"transid":transid,"bankid":bankid,"accountid":accid,"date":dt,"payee":payee,"memo":memo,"value":val,"currency":curr,"fxcurrency":fxcurr,"fxamount":fxamount,"fxrate":fxrate})
+
+ return 0
+ except:
+ print 'Could not read ofx'
+ return 1
+
+ def loadcsv(self):
+
+ #print self.path
+ b=BankCSV(self.path)
+ b.load()
+ #print b.GetData()
+ self.data=b.GetData()
+ self.accountid=b.GetAccountId()
+
+ return 0
+
+ def processdata(self):
+
+
+ for idx in range(1,len(self.data)):
+
+ desc=self.data[idx]["memo"]
+ amount=self.data[idx]["value"]
+
+
+ #Foreign Currency Parse
+ #HSBC UK Format XXX ZZZ.ZZ @ A.SSSS
+
+ p = re.compile('\w{3}\s\d+.\d{2}\s@\s\d+.\d{4}')
+ m = re.search(p, desc)
+
+ if m:
+ fxcurr=desc[m.start():m.start()+3]
+ p=re.compile('@\s\d+.\d{4}')
+ fxrate=re.search(p,m.group()).group()[2:]
+ p=re.compile('\d+.\d{2}')
+ fxamount=re.search(p,m.group()).group()
+
+ fxamount=float(fxamount)
+ fxrate=float(fxrate)
+
+ self.data[idx]["fxamount"]=round(fxamount,2)
+ self.data[idx]["fxrate"]=round(fxrate,4)
+ self.data[idx]["fxcurrency"]=fxcurr
+
+
+ #HSBC Aus Format for FX VISA XXX
+ p = re.compile('VISA\s\w{3}\s\d+.\d{2}')
+ m = re.search(p, desc)
+
+ if m:
+ fxcurr=desc[m.start()+5:m.start()+8]
+ fxamount=float(desc[m.start()+9:m.end()])
+ fxrate=round(fxamount/abs(float(amount)),4)
+ self.data[idx]["fxamount"]=round(fxamount,2)
+ self.data[idx]["fxrate"]=round(fxrate,4)
+ self.data[idx]["fxcurrency"]=fxcurr
+
+ def SetMaxDate(self):
+
+ self.maxdate='0'
+ for row in self.data:
+
+ dt=row["date"]
+
+ if dt>=self.maxdate:
+ self.maxdate=dt
+
+ def SetMinDate(self):
+
+ self.mindate='999999999'
+ for row in self.data:
+ dt=row["date"]
+
+ if dt<=self.mindate:
+ self.mindate=dt
+
+def cleanStr(str):
+
+ import string
+ valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
+ str=''.join(c for c in str if c in valid_chars)
+
+ return str
+
+
+
+def parseDate(dt):
+
+ dt=dt.strip()
+
+ if len(dt) == 14:
+ dt = time.strptime(dt, "%Y%m%d000000")
+ else:
+ dt = time.strptime(dt, "%Y%m%d")
+
+ dt = time.strftime("%Y%m%d", dt)
+ return dt
+
+
+
256 bankdownloads/ofx.py
@@ -0,0 +1,256 @@
+from BeautifulSoup import BeautifulStoneSoup
+
+
+from datetime import datetime
+import time
+
+def ofxexport ( path, data):
+
+ accounts={}
+ today = datetime.now().strftime('%Y%m%d')
+ for row in data:
+
+ uacct="%s-%s" % (row["bankid"],row["accountid"])
+ acct = accounts.setdefault(uacct,{})
+
+ acct['BANKID'] = row["bankid"]
+ acct['ACCTID'] = row["accountid"]
+ acct['TODAY'] = today
+
+ acct['CURDEF'] = row["currency"]
+
+ trans=acct.setdefault('trans',[])
+
+ tran = {}
+ tran['DTPOSTED']=row["date"]
+ tran['TRNAMT']=row["value"]
+ tran['FITID']=row["transid"]
+ tran['PAYEE']=row["payee"]
+ tran['MEMO']=row["memo"]
+ tran['CHECKNUM']=''
+
+ tran['TRNTYPE'] = tran['TRNAMT'] >0 and 'CREDIT' or 'DEBIT'
+ trans.append(tran)
+
+
+ # output
+
+ out=open(path,'w')
+
+ out.write (
+ """
+ <OFX>
+ <SIGNONMSGSRSV1>
+ <SONRS>
+ <STATUS>
+ <CODE>0</CODE>
+ <SEVERITY>INFO</SEVERITY>
+ </STATUS>
+ <DTSERVER>%(DTSERVER)s</DTSERVER>
+ <LANGUAGE>ENG</LANGUAGE>
+ </SONRS>
+ </SIGNONMSGSRSV1>
+ <BANKMSGSRSV1><STMTTRNRS>
+ <TRNUID>%(TRNUID)d</TRNUID>
+ <STATUS><CODE>0</CODE><SEVERITY>INFO</SEVERITY></STATUS>
+
+ """ % {'DTSERVER':today,
+ 'TRNUID':int(time.mktime(time.localtime()))}
+ )
+
+ for acct in accounts.values():
+ out.write(
+ """
+ <STMTRS>
+ <CURDEF>%(CURDEF)s</CURDEF>
+ <BANKACCTFROM>
+ <BANKID>%(BANKID)s</BANKID>
+ <ACCTID>%(ACCTID)s</ACCTID>
+ <ACCTTYPE>CHECKING</ACCTTYPE>
+ </BANKACCTFROM>
+ <BANKTRANLIST>
+ <DTSTART>%(TODAY)s</DTSTART>
+ <DTEND>%(TODAY)s</DTEND>
+
+ """ % acct
+ )
+
+ for tran in acct['trans']:
+ out.write (
+ """
+ <STMTTRN>
+ <TRNTYPE>%(TRNTYPE)s</TRNTYPE>
+ <DTPOSTED>%(DTPOSTED)s</DTPOSTED>
+ <TRNAMT>%(TRNAMT)s</TRNAMT>
+ <FITID>%(FITID)s</FITID>
+
+ """ % tran
+ )
+ if tran['CHECKNUM'] is not None and len(tran['CHECKNUM'])>0:
+ out.write(
+ """
+ <CHECKNUM>%(CHECKNUM)s</CHECKNUM>
+ """ % tran
+ )
+ out.write(
+ """
+ <NAME>%(PAYEE)s</NAME>
+ <MEMO>%(MEMO)s</MEMO>
+ """ % tran
+ )
+ out.write(
+ """
+ </STMTTRN>
+ """
+ )
+
+ out.write (
+ """
+ </BANKTRANLIST>
+ <LEDGERBAL>
+ <BALAMT>0</BALAMT>
+ <DTASOF>%s</DTASOF>
+ </LEDGERBAL>
+ </STMTRS>
+ """ % today
+ )
+
+ out.write ( "</STMTTRNRS></BANKMSGSRSV1></OFX>" )
+ out.close()
+
+class Ofx(object):
+ pass
+
+class Account(object):
+ def __init__(self):
+ self.number = ''
+ self.routing_number = ''
+ self.statement = None
+ self.currency = ''
+
+class Statement(object):
+ def __init__(self):
+ self.start_date = ''
+ self.end_date = ''
+ self.transactions = []
+
+class Transaction(object):
+ def __init__(self):
+ self.name = ''
+ self.type = ''
+ self.date = ''
+ self.amount = ''
+ self.id = ''
+ self.memo = ''
+
+class Institution(object):
+ pass
+
+class OfxParser(object):
+ @classmethod
+ def parse(cls_, file_handle):
+ ofx_obj = Ofx()
+ ofx = BeautifulStoneSoup(file_handle)
+ stmtrs_ofx = ofx.find('stmtrs')
+ if stmtrs_ofx:
+ ofx_obj.bank_account = cls_.parseStmtrs(stmtrs_ofx)
+ return ofx_obj
+
+ @classmethod
+ def parseStmtrs(cls_, stmtrs_ofx):
+ ''' Parse the <STMTRS> tag and return an Account object. '''
+ account = Account()
+ acctid_tag = stmtrs_ofx.find('acctid')
+ if hasattr(acctid_tag, 'contents'):
+ account.number = acctid_tag.contents[0]
+ bankid_tag = stmtrs_ofx.find('bankid')
+ if hasattr(bankid_tag, 'contents'):
+ account.routing_number = bankid_tag.contents[0]
+
+ curr_tag = stmtrs_ofx.find('curdef')
+ if hasattr(curr_tag, 'contents'):
+ account.currency = curr_tag.contents[0]
+
+ if stmtrs_ofx:
+ account.statement = cls_.parseStatement(stmtrs_ofx)
+ return account
+
+ @classmethod
+ def parseStatement(cls_, stmt_ofx):
+ '''
+ Parse a statement in ofx-land and return a Statement object.
+ '''
+ statement = Statement()
+ dtstart_tag = stmt_ofx.find('dtstart')
+ if hasattr(dtstart_tag, "contents"):
+ statement.start_date = dtstart_tag.contents[0]
+ dtend_tag = stmt_ofx.find('dtend')
+ if hasattr(dtend_tag, "contents"):
+ statement.end_date = dtend_tag.contents[0].strip()
+ ledger_bal_tag = stmt_ofx.find('ledgerbal')
+ if hasattr(ledger_bal_tag, "contents"):
+ balamt_tag = ledger_bal_tag.find('balamt')
+ if stmtrs_ofx:
+ account.statement = cls_.parseStatement(stmtrs_ofx)
+ return account
+
+ @classmethod
+ def parseStatement(cls_, stmt_ofx):
+ '''
+ Parse a statement in ofx-land and return a Statement object.
+ '''
+ statement = Statement()
+ dtstart_tag = stmt_ofx.find('dtstart')
+ if hasattr(dtstart_tag, "contents"):
+ statement.start_date = dtstart_tag.contents[0]
+ dtend_tag = stmt_ofx.find('dtend')
+ if hasattr(dtend_tag, "contents"):
+ statement.end_date = dtend_tag.contents[0].strip()
+ ledger_bal_tag = stmt_ofx.find('ledgerbal')
+ if hasattr(ledger_bal_tag, "contents"):
+ balamt_tag = ledger_bal_tag.find('balamt')
+ if hasattr(balamt_tag, "contents"):
+ statement.balance = balamt_tag.contents[0]
+ avail_bal_tag = stmt_ofx.find('availbal')
+ if hasattr(avail_bal_tag, "contents"):
+ balamt_tag = avail_bal_tag.find('balamt')
+ if hasattr(balamt_tag, "contents"):
+ statement.available_balance = balamt_tag.contents[0]
+ for transaction_ofx in stmt_ofx.findAll('stmttrn'):
+ statement.transactions.append(cls_.parseTransaction(transaction_ofx))
+ return statement
+
+ @classmethod
+ def parseTransaction(cls_, txn_ofx):
+ '''
+ Parse a transaction in ofx-land and return a Transaction object.
+ '''
+ transaction = Transaction()
+
+ type_tag = txn_ofx.find('trntype')
+ if hasattr(type_tag, 'contents'):
+ transaction.type = type_tag.contents[0].lower()
+
+ name_tag = txn_ofx.find('name')
+ if hasattr(name_tag, "contents"):
+ transaction.payee = name_tag.contents[0]
+ else:
+ transaction.payee = ""
+
+ memo_tag = txn_ofx.find('memo')
+ if hasattr(memo_tag, "contents"):
+ transaction.memo = memo_tag.contents[0]
+
+ amt_tag = txn_ofx.find('trnamt')
+ if hasattr(amt_tag, "contents"):
+ transaction.amount = amt_tag.contents[0]
+
+ date_tag = txn_ofx.find('dtposted')
+ if hasattr(date_tag, "contents"):
+ transaction.date = date_tag.contents[0]
+
+ id_tag = txn_ofx.find('fitid')
+ if hasattr(id_tag, "contents"):
+ transaction.id = id_tag.contents[0]
+
+ return transaction
213 bin/bankdownload-processor
@@ -0,0 +1,213 @@
+#!/usr/bin/env python
+
+import os,sys
+import getopt
+import hashlib
+from traceback import print_exc
+from bankdownloads import BankDownload
+import MySQLdb
+
+
+config_file=os.path.join(os.path.dirname(__file__),'../etc/bankdownloads.conf')
+
+#renames files for use in openportfolio
+mapping={
+ 'ing-evan':[1,'Evan Davey'],
+ 'ing-vanessa':[1,'Vanessa Cochrane'],
+ 'ing-joint':[1,'Evan Davey & Vanessa Cochrane'],
+ 'ing-accvanessa':[2,'Vanessa Cochrane'],
+ 'cbacash-evan':[3,'Evan Davey'],
+ 'cbacash-vanessa':[3,'Vanessa Cochrane'],
+ 'rabo-142201002687000':[4,'Evan Davey'],
+ 'rabo-142201002072100':[4,'Evan Davey & Vanessa Cochrane'],
+ 'firststate-evan':[99,'Evan Davey'],
+ 'firststate-vanessa':[99,'Vanessa Cochrane'],
+
+}
+
+
+def file_md5(filename):
+ md5 = hashlib.md5()
+ with open(filename,'rb') as f:
+ for chunk in iter(lambda: f.read(128*md5.block_size), ''):
+ md5.update(chunk)
+ return md5.hexdigest()
+
+def processfiles(srchpath,outpath,db,db_host,db_user,db_password):
+
+
+ fileExtList=[".csv",".ofx",".qfx"]
+
+
+ print "Searching: " + srchpath
+ print "Output to: " + outpath
+
+ if not os.path.exists(srchpath):
+ raise Exception("Search path %s does not exist" % srchpath)
+
+ if not os.path.exists(outpath):
+ raise Exception("Output path %s does not exist" % outpath)
+
+
+ files=os.listdir(srchpath)
+
+ try:
+ print 'Establishing connection to %s@%s' % (db,db_host)
+ db=MySQLdb.connect(host=db_host,user=db_user,passwd=db_pword,db=db)
+
+ c=db.cursor(MySQLdb.cursors.DictCursor)
+ except:
+ print_exc()
+ raise Exception("Could not connect to %s at %s with user %s and password %s" % (db,db_host,db_user,db_pword))
+
+ print "Checking (%s) file/s vs the database..." % (len(files))
+
+ changed_files=[]
+ for f in files:
+ try:
+
+ if os.path.splitext(f)[1] in fileExtList:
+ chksum=file_md5(os.path.join(srchpath,f))
+
+ sql="""
+ SELECT *
+ FROM downloads
+ WHERE
+ name='%s'
+
+ """ % f
+
+ #print sql
+ c.execute(sql)
+ r=c.fetchone()
+
+ if r is None:
+ sql="""
+ INSERT INTO downloads
+ VALUES ('%s','%s',NULL,NULL)
+
+ """ % (f,chksum)
+
+ try:
+ c.execute(sql)
+ print "\tInserted record for %s,%s" % (f,chksum)
+ changed_files.append(f)
+ except:
+ print "\tError inserting record for %s" % f
+ print_exc()
+ else:
+ if r['chksum'] != chksum:
+ print "\tChecksums differ for %s" % f
+ changed_files.append(f)
+
+ sql="""
+ UPDATE downloads
+ SET chksum='%s'
+ WHERE
+ name='%s'
+ """ % (chksum,f)
+
+ try:
+ c.execute(sql)
+ print "\tUpdated checksum for %s,%s" % (f,chksum)
+ except:
+ print "\tError updating record for %s" % f
+ print_exc()
+
+ else:
+ print "\t%s unchanged" % f
+
+ except:
+ print_exc()
+
+
+
+ print "Processing (%s) changed file/s..." % (len(changed_files))
+
+ for f in changed_files:
+ mydownload = BankDownload(os.path.join(srchpath,f))
+ r=mydownload.load()
+ mydownload.printdata()
+
+ if r:
+ print "\tLoad failed for: "+ srchpath+f
+ else:
+
+ ofxfile=mydownload.generatefilename()
+
+ print "\tGenerated file name: %s" % ofxfile
+
+ dt=ofxfile.split('_')[0]
+ bank=ofxfile.split('_')[1]
+ acc=ofxfile.split('_')[2]
+
+ try:
+ k=bank+'-'+acc
+ print "\t\tLookup Key: %s" % k
+
+ i_id=mapping[k][0]
+ p_id=mapping[k][1]
+
+ nf="%s_%s_%s.ofx" % (i_id,p_id,dt)
+ print "\t\tNew file name: %s" % nf
+
+ out_f=os.path.join(OUTPUT_PATH,nf)
+ mydownload.writeofx(out_f)
+ os.chown(out_f,1028,1028)
+
+ except:
+ print "Error processing file %s" % (f)
+ print print_exc()
+ sql="""
+ DELETE FROM downloads
+ WHERE
+ name='%s' and chksum='%s'
+ """ % (f,chksum)
+
+ try:
+ c.execute(sql)
+ print "\tDeleted record for %s,%s" % (f,chksum)
+ except:
+ print "\tError deleting record for %s" % f
+ print_exc()
+
+ del mydownload
+
+
+
+ c.close()
+ db.commit()
+ db.close()
+
+ return 0
+
+if __name__ == "__main__":
+ import ConfigParser
+ from optparse import OptionParser
+
+ parser = OptionParser()
+ parser.add_option("-c", "--config", dest="config_file",
+ help="use config file FILE",type="string",metavar="FILE",default=config_file)
+
+ (options, args) = parser.parse_args()
+
+ config = ConfigParser.ConfigParser()
+
+ config.read(options.config_file)
+
+ db_user=config.get('mysql', 'user').strip("'")
+ db=config.get('mysql', 'database').strip("'")
+ db_host=config.get('mysql', 'host').strip("'")
+ db_pword=config.get('mysql', 'password').strip("'")
+
+ try:
+ inpath=args[0]
+ except:
+ inpath=config.get('paths', 'inputpath').strip("'")
+
+ try:
+ outpath=args[1]
+ except:
+ outpath=config.get('paths', 'outputpath').strip("'")
+
+ processfiles(inpath,outpath,db,db_host,db_user,db_pword)
0  docs/index.txt
No changes.
10 etc/bankdownloads.conf
@@ -0,0 +1,10 @@
+[mysql]
+user=''
+host=''
+password=''
+database=''
+
+[paths]
+inputpath=''
+outputpath=''
+
18 extra/com.cochranedavey.bankdownloadwatcher.plist
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+ <key>Label</key>
+ <string>com.cochranedavey.bankdownloadwatcher</string>
+ <key>ProgramArguments</key>
+ <array>
+ <string>/usr/local/web/django/www/production/openportfolio/tools/newfiles</string>
+ </array>
+ <key>WatchPaths</key>
+ <array>
+ <string>/Volumes/ExtDisk2-2tb/Data/Dropbox/Cochrane Davey/Finances/Bank Downloads/Original Format/</string>
+ </array>
+ <key>StandardOutPath</key>
+ <string>/dev/null</string>
+</dict>
+</plist>
10 extra/create.sql
@@ -0,0 +1,10 @@
+DROP TABLE IF EXISTS `downloads`;
+
+CREATE TABLE `downloads` (
+ `name` varchar(255) NOT NULL,
+ `chksum` varchar(255) NOT NULL,
+ `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ `updated` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
+ PRIMARY KEY (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
24 setup.py
@@ -0,0 +1,24 @@
+from distutils.core import setup
+from setuptools import find_packages
+
+
+setup(
+ name='bankdownloads',
+ version='0.1',
+ author=u'Evan Davey',
+ author_email='evan.davey@cochranedavey.com',
+ packages=['bankdownloads'],
+ scripts=['bin/bankdownload-processor'],
+ url='http://github.com/evandavey/bankdownloads',
+ license='LICENSE.txt',
+ description='Processes and standardises bank transaction downloads',
+ long_description=open('README.md').read(),
+ install_requires=[
+ 'mysql-python',
+ ],
+ data_files=[('etc', ['etc/bankdownloads.conf']),
+
+ ],
+ zip_safe=False,
+)
+
Please sign in to comment.
Something went wrong with that request. Please try again.