Skip to content

Commit

Permalink
Merge branch 'features'
Browse files Browse the repository at this point in the history 
* features:
  Bump to version 0.20.0
  [BUGFIX] Allow static ‘account’ key (fixes #19)
  Fix pylint errors
  [CHANGE] Remove ‘--split’ cli option (closes #15)
  Fix pylint errors
  Exit with message instead of raising an error
  [NEW] Add investment trxn support (closes #17)
  [CHANGE] Always return string from footer()
  [BUGFIX] Don’t error if source file isn’t found
  Rename short variables and uppercase trxn type
  Add function descriptions and fix typos
  Print help text when called with no args (fixes #10)
  • Loading branch information
@reubano
reubano committed Feb 10, 2017
2 parents 7883203 + 976087e commit 048695f
Show file tree
Hide file tree
Showing 12 changed files with 399 additions and 169 deletions.
165 changes: 101 additions & 64 deletions csv2ofx/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
from functools import partial
from datetime import datetime as dt
from operator import itemgetter
from decimal import Decimal

from builtins import *
from six.moves import filterfalse
Expand All @@ -40,23 +41,24 @@
__author__ = 'Reuben Cummings'
__description__ = 'converts a csv file of transactions to an ofx or qif file'
__email__ = 'reubano@gmail.com'
__version__ = '0.19.3'
__version__ = '0.20.0'
__license__ = 'MIT'
__copyright__ = 'Copyright 2015 Reuben Cummings'


# pylint: disable=invalid-name
md5 = lambda content: hashlib.md5(content.encode('utf-8')).hexdigest()


class Content(object):
class Content(object): # pylint: disable=too-many-instance-attributes
"""A transaction holding object
"""
def __init__(self, mapping=None, **kwargs):
""" Base content constructor
Args:
mapping (dict): bank mapper (see csv2ofx.mappings)
kwargs (dict): Keyword arguments
Kwargs:
split_header (str): Transaction field to use for the split account.
start (date): Date from which to begin including transactions.
end (date): Date from which to exclude transactions.
Expand All @@ -66,27 +68,33 @@ def __init__(self, mapping=None, **kwargs):
<csv2ofx.Content object at 0x...>
"""
mapping = mapping or {}

# pylint doesn't like dynamically set attributes...
self.amount = None
self.account = None
self.split_account = None
self.inv_split_account = None
self.id = None
[self.__setattr__(k, v) for k, v in mapping.items()]

if not hasattr(self, 'is_split'):
self.is_split = False

if kwargs.get('split_header'):
self.split_account = itemgetter(kwargs['split_header'])
else:
self.split_account = None
if not callable(self.account):
account = self.account
self.account = lambda _: account

self.start = kwargs.get('start') or dt(1970, 1, 1)
self.end = kwargs.get('end') or dt.now()

def get(self, name, tr=None, default=None):
def get(self, name, trxn=None, default=None):
""" Gets an attribute which could be either a normal attribute,
a mapping function, or a mapping attribute
Args:
name (str): The attribute.
tr (dict): The transaction. Require if `name` is a mapping function
(default: None).
trxn (dict): The transaction. Require if `name` is a mapping
function (default: None).
default (str): Value to use if `name` isn't found (default: None).
Expand All @@ -99,11 +107,11 @@ def get(self, name, tr=None, default=None):
>>> from datetime import datetime as dt
>>> from csv2ofx.mappings.mint import mapping
>>>
>>> tr = {'Transaction Type': 'debit', 'Amount': 1000.00}
>>> trxn = {'Transaction Type': 'DEBIT', 'Amount': 1000.00}
>>> start = dt(2015, 1, 1)
>>> Content(mapping, start=start).get('start') # normal attribute
datetime.datetime(2015, 1, 1, 0, 0)
>>> Content(mapping).get('amount', tr) # mapping function
>>> Content(mapping).get('amount', trxn) # mapping function
1000.0
>>> Content(mapping).get('has_header') # mapping attribute
True
Expand All @@ -117,20 +125,20 @@ def get(self, name, tr=None, default=None):
value = None

try:
value = value or attr(tr) if attr else default
value = value or attr(trxn) if attr else default
except TypeError:
value = attr
except KeyError:
value = default

return value

def skip_transaction(self, tr):
def skip_transaction(self, trxn):
""" Determines whether a transaction should be skipped (isn't in the
specified date range)
Args:
tr (dict): The transaction.
trxn (dict): The transaction.
Returns:
(bool): Whether or not to skip the transaction.
Expand All @@ -139,19 +147,19 @@ def skip_transaction(self, tr):
>>> from csv2ofx.mappings.mint import mapping
>>> from datetime import datetime as dt
>>>
>>> tr = {'Date': '06/12/10', 'Amount': 1000.00}
>>> Content(mapping, start=dt(2010, 1, 1)).skip_transaction(tr)
>>> trxn = {'Date': '06/12/10', 'Amount': 1000.00}
>>> Content(mapping, start=dt(2010, 1, 1)).skip_transaction(trxn)
False
>>> Content(mapping, start=dt(2013, 1, 1)).skip_transaction(tr)
>>> Content(mapping, start=dt(2013, 1, 1)).skip_transaction(trxn)
True
"""
return not (self.end >= parse(self.get('date', tr)) >= self.start)
return not self.end >= parse(self.get('date', trxn)) >= self.start

def convert_amount(self, tr):
def convert_amount(self, trxn):
""" Converts a string amount into a number
Args:
tr (dict): The transaction.
trxn (dict): The transaction.
Returns:
(decimal): The converted amount.
Expand All @@ -161,17 +169,17 @@ def convert_amount(self, tr):
>>> from datetime import datetime as dt
>>> from csv2ofx.mappings.mint import mapping
>>>
>>> tr = {'Date': '06/12/10', 'Amount': '$1,000'}
>>> Content(mapping, start=dt(2010, 1, 1)).convert_amount(tr)
>>> trxn = {'Date': '06/12/10', 'Amount': '$1,000'}
>>> Content(mapping, start=dt(2010, 1, 1)).convert_amount(trxn)
Decimal('1000.00')
"""
return utils.convert_amount(self.get('amount', tr))
return utils.convert_amount(self.get('amount', trxn))

def transaction_data(self, tr):
def transaction_data(self, trxn): # pylint: disable=too-many-locals
""" gets transaction data
Args:
tr (dict): the transaction
trxn (dict): the transaction
Returns:
(dict): the QIF content
Expand All @@ -180,80 +188,108 @@ def transaction_data(self, tr):
>>> import datetime
>>> from decimal import Decimal
>>> from csv2ofx.mappings.mint import mapping
>>> tr = {'Transaction Type': 'debit', 'Amount': 1000.00, \
'Date': '06/12/10', 'Description': 'payee', 'Original Description': \
'description', 'Notes': 'notes', 'Category': 'Checking', 'Account Name': \
'account'}
>>> Content(mapping).transaction_data(tr) == {
>>> trxn = {
... 'Transaction Type': 'DEBIT', 'Amount': 1000.00,
... 'Date': '06/12/10', 'Description': 'payee',
... 'Original Description': 'description', 'Notes': 'notes',
... 'Category': 'Checking', 'Account Name': 'account'}
>>> Content(mapping).transaction_data(trxn) == {
... 'account_id': 'e268443e43d93dab7ebef303bbe9642f',
... 'memo': 'description notes', 'split_account_id':
... None, 'currency': 'USD',
... 'date': datetime.datetime(2010, 6, 12, 0, 0),
... 'class': None, 'bank': 'account', 'account': 'account',
... 'split_account': None,
... 'bank_id': 'e268443e43d93dab7ebef303bbe9642f',
... 'id': 'ee86450a47899254e2faa82dca3c2cf2', 'payee': 'payee',
... 'amount': Decimal('-1000.00'), 'check_num': None,
... 'type': 'debit'}
... 'account': 'account',
... 'split_account_id': '195917574edc9b6bbeb5be9785b6a479',
... 'shares': Decimal('0'), 'payee': 'payee', 'currency': 'USD',
... 'bank': 'account', 'class': None, 'is_investment': False,
... 'date': datetime.datetime(2010, 6, 12, 0, 0),
... 'price': Decimal('0'), 'symbol': '', 'action': '',
... 'check_num': None, 'id': 'ee86450a47899254e2faa82dca3c2cf2',
... 'split_account': 'Checking', 'type': 'DEBIT',
... 'category': '', 'amount': Decimal('-1000.00'),
... 'memo': 'description notes', 'inv_split_account': None,
... 'x_action': ''}
True
"""
account = self.get('account', tr)
split_account = self.get('split_account', tr)
bank = self.get('bank', tr, account)

raw_amount = str(self.get('amount', tr))
amount = self.convert_amount(tr)
_type = self.get('type', tr)

if _type:
amount = -1 * amount if _type.lower() == 'debit' else amount
else:
account = self.get('account', trxn)
split_account = self.get('split_account', trxn)
bank = self.get('bank', trxn, account)
raw_amount = str(self.get('amount', trxn))
amount = self.convert_amount(trxn)
_type = self.get('type', trxn, '').upper()

if _type not in {'DEBIT', 'CREDIT'}:
_type = 'CREDIT' if amount > 0 else 'DEBIT'

date = self.get('date', tr)
payee = self.get('payee', tr)
desc = self.get('desc', tr)
notes = self.get('notes', tr)
date = self.get('date', trxn)
payee = self.get('payee', trxn)
desc = self.get('desc', trxn)
notes = self.get('notes', trxn)
memo = '%s %s' % (desc, notes) if desc and notes else desc or notes
check_num = self.get('check_num', tr)
check_num = self.get('check_num', trxn)
details = ''.join(filter(None, [date, raw_amount, payee, memo]))
category = self.get('category', trxn, '')
shares = Decimal(self.get('shares', trxn, 0))
symbol = self.get('symbol', trxn, '')
price = Decimal(self.get('price', trxn, 0))
invest = shares or (symbol and symbol != 'N/A') or 'invest' in category

if invest:
amount = abs(amount)
shares = shares or (amount / price) if price else shares
amount = amount or shares * price
price = price or (amount / shares) if shares else price
action = utils.get_action(category)
x_action = utils.get_action(category, True)
else:
amount = -1 * abs(amount) if _type == 'DEBIT' else abs(amount)
action = ''
x_action = ''

return {
'date': parse(date),
'currency': self.get('currency', tr, 'USD'),
'currency': self.get('currency', trxn, 'USD'),
'shares': shares,
'symbol': symbol,
'price': price,
'action': action,
'x_action': x_action,
'category': category,
'is_investment': invest,
'bank': bank,
'bank_id': self.get('bank_id', tr, md5(bank)),
'bank_id': self.get('bank_id', trxn, md5(bank)),
'account': account,
'account_id': self.get('account_id', tr, md5(account)),
'account_id': self.get('account_id', trxn, md5(account)),
'split_account': split_account,
'inv_split_account': self.get('inv_split_account', trxn),
'split_account_id': md5(split_account) if split_account else None,
'amount': amount,
'payee': payee,
'memo': memo,
'class': self.get('class', tr),
'id': self.get('id', tr, check_num) or md5(details),
'class': self.get('class', trxn),
'id': self.get('id', trxn, check_num) or md5(details),
'check_num': check_num,
'type': _type,
}

def gen_trxns(self, groups, collapse=False):
""" Generate transactions """
for grp, transactions in groups:
if self.is_split and collapse:
# group transactions by `collapse` field and sum the amounts
byaccount = group(transactions, collapse)
op = lambda values: sum(map(utils.convert_amount, values))
merger = partial(merge, pred=self.amount, op=op)
oprtn = lambda values: sum(map(utils.convert_amount, values))
merger = partial(merge, pred=self.amount, op=oprtn)
trxns = [merger(dicts) for _, dicts in byaccount]
else:
trxns = transactions

yield (grp, trxns)

def clean_trxns(self, groups):
""" Clean transactions """
for grp, trxns in groups:
_args = [trxns, self.convert_amount]

# if it's split, transactions skipping is all or none
# if it's split, transaction skipping is all or none
if self.is_split and self.skip_transaction(trxns[0]):
continue
elif self.is_split and not utils.verify_splits(*_args):
Expand All @@ -268,6 +304,7 @@ def clean_trxns(self, groups):
else:
main_pos = 0

# pylint: disable=cell-var-from-loop
keyfunc = lambda enum: enum[0] != main_pos
sorted_trxns = sorted(enumerate(filtered_trxns), key=keyfunc)
yield (grp, main_pos, sorted_trxns)
16 changes: 8 additions & 8 deletions csv2ofx/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
from .qif import QIF


parser = ArgumentParser( # pylint: disable=C0103
parser = ArgumentParser( # pylint: disable=invalid-name
description="description: csv2ofx converts a csv file to ofx and qif",
prog='csv2ofx', usage='%(prog)s [options] <source> <dest>',
formatter_class=RawTextHelpFormatter)
Expand Down Expand Up @@ -76,9 +76,6 @@
'-c', '--collapse', metavar='FIELD_NAME', help=(
'field used to combine transactions within a split for double entry '
'statements'))
parser.add_argument(
'-S', '--split', metavar='FIELD_NAME', help=(
'field used for the split account for single entry statements'))
parser.add_argument(
'-C', '--chunksize', metavar='ROWS', type=int, default=2 ** 14,
help="number of rows to process at a time")
Expand Down Expand Up @@ -135,7 +132,6 @@ def run(): # noqa: C901

okwargs = {
'def_type': args.account_type or 'Bank' if args.qif else 'CHECKING',
'split_header': args.split,
'start': parse(args.start) if args.start else None,
'end': parse(args.end) if args.end else None
}
Expand All @@ -156,7 +152,7 @@ def run(): # noqa: C901
else:
try:
mtime = p.getmtime(source.name)
except AttributeError:
except (AttributeError, FileNotFoundError):
mtime = time.time()

server_date = dt.fromtimestamp(mtime)
Expand All @@ -169,9 +165,9 @@ def run(): # noqa: C901
'overwrite': args.overwrite,
'chunksize': args.chunksize,
'encoding': args.encoding}
except:
except Exception as err: # pylint: disable=broad-except
source.close()
raise
exit(err)

dest = open(args.dest, 'w', encoding=args.encoding) if args.dest else stdout

Expand All @@ -186,6 +182,10 @@ def run(): # noqa: C901
msg += 'Check `start` and `end` options.'
else:
msg += 'Try again with `-c` option.'
except ValueError:
# csv2ofx called with no arguments
msg = 0
parser.print_help()
except Exception as err: # pylint: disable=broad-except
msg = 1
traceback.print_exc()
Expand Down
10 changes: 10 additions & 0 deletions csv2ofx/mappings/mint.py
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,20 @@
# -*- coding: utf-8 -*-
# vim: sw=4:ts=4:expandtab
# pylint: disable=invalid-name
"""
csv2ofx.mappings.mintapi
~~~~~~~~~~~~~~~~~~~~~~~~
Provides a mapping for transactions obtained via mint.com
"""
from __future__ import absolute_import

from operator import itemgetter

mapping = {
'is_split': False,
'has_header': True,
'split_account': itemgetter('Category'),
'account': itemgetter('Account Name'),
'date': itemgetter('Date'),
'type': itemgetter('Transaction Type'),
Expand Down

0 comments on commit 048695f

Please sign in to comment.