diff --git a/.travis.yml b/.travis.yml index 1ec610cb..45cc69a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,7 @@ install: - pip install tox codecov boto3 - pip freeze - virtualenv --version -- wget -N http://chromedriver.storage.googleapis.com/2.33/chromedriver_linux64.zip -P ~/ +- wget -N http://chromedriver.storage.googleapis.com/2.36/chromedriver_linux64.zip -P ~/ - unzip ~/chromedriver_linux64.zip -d ~/ - rm ~/chromedriver_linux64.zip - chmod +x ~/chromedriver diff --git a/CHANGES.rst b/CHANGES.rst index c0788638..bb4aac61 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -39,6 +39,18 @@ Unreleased Changes * When running under Docker/Gunicorn, append the decimal number of seconds taken to serve the request to the Gunicorn access log. * `Issue #184 `_ - Redact database password from ``/help`` view, and change ``/help`` view to show Version containing git commit hash for pre-release/development Docker builds. +* `Issue #183 `_ + + * Add UI link to ignore reconciling an OFXTransaction if there will not be a matching Transaction. + * Remove default values for the ``Account`` model's ``re_`` fields in preparation for actually using them. + * Replace the ``Account`` model's ``re_fee`` field with separate ``re_late_fee`` and ``re_other_fee`` fields. + * Add UI support for specifying Interest Charge, Interest Paid, Payment, Late Fee, and Other Fee regexes on each account. + * Add DB event handler on new or changed OFXTransaction, to set ``is_*`` fields according to Account ``re_*`` fields. + * Add DB event handler on change to Account model ``re_*`` fields, that triggers ``OFXTransaction.update_is_fields()`` to recalculate using the new regex. + * Change ``OFXTransaction.unreconciled`` to filter out OFXTransactions with any of the ``is_*`` set to True. + +* Upgrade chromedriver in TravisCI builds from 2.33 to 2.36, to fix failing acceptance tests caused by Ubuntu upgrade from Chrome 64 to 65. + 0.7.1 (2018-01-10) ------------------ diff --git a/biweeklybudget/alembic/versions/073142f641b3_account_remove_re_field_defaults.py b/biweeklybudget/alembic/versions/073142f641b3_account_remove_re_field_defaults.py new file mode 100644 index 00000000..ebdde82b --- /dev/null +++ b/biweeklybudget/alembic/versions/073142f641b3_account_remove_re_field_defaults.py @@ -0,0 +1,84 @@ +"""Account remove re field defaults + +Revision ID: 073142f641b3 +Revises: 08b6358a04bf +Create Date: 2018-03-08 09:25:17.673039 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import mysql +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +from sqlalchemy import Column, Integer, String + +Session = sessionmaker() + +Base = declarative_base() + +# revision identifiers, used by Alembic. +revision = '073142f641b3' +down_revision = '08b6358a04bf' +branch_labels = None +depends_on = None + + +class Account(Base): + + __tablename__ = 'accounts' + __table_args__ = ( + {'mysql_engine': 'InnoDB'} + ) + + #: Primary Key + id = Column(Integer, primary_key=True) + + #: regex for matching transactions as interest charges + re_interest_charge = Column(String(254)) + + #: regex for matching transactions as interest paid + re_interest_paid = Column(String(254)) + + #: regex for matching transactions as payments + re_payment = Column(String(254)) + + #: regex for matching transactions as late fees + re_fee = Column(String(254)) + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + for acct in session.query(Account).all(): + acct.re_interest_charge = None + acct.re_interest_paid = None + acct.re_payment = None + acct.re_fee = None + session.commit() + op.add_column( + 'accounts', + sa.Column('re_late_fee', sa.String(length=254), nullable=True) + ) + op.add_column( + 'accounts', + sa.Column('re_other_fee', sa.String(length=254), nullable=True) + ) + op.drop_column('accounts', 're_fee') + + +def downgrade(): + op.add_column( + 'accounts', + sa.Column('re_fee', mysql.VARCHAR(length=254), nullable=True) + ) + op.drop_column('accounts', 're_other_fee') + op.drop_column('accounts', 're_late_fee') + bind = op.get_bind() + session = Session(bind=bind) + for acct in session.query(Account).all(): + acct.re_interest_charge = '^(interest charge|purchase finance charge)' + acct.re_interest_paid = '^interest paid' + acct.re_payment = '^(online payment|' \ + 'internet payment|online pymt|payment)' + acct.re_fee = '^(late fee|past due fee)' + session.commit() diff --git a/biweeklybudget/alembic/versions/08b6358a04bf_txnreconcile_allow_txn_id_to_be_null.py b/biweeklybudget/alembic/versions/08b6358a04bf_txnreconcile_allow_txn_id_to_be_null.py new file mode 100644 index 00000000..2c2cf114 --- /dev/null +++ b/biweeklybudget/alembic/versions/08b6358a04bf_txnreconcile_allow_txn_id_to_be_null.py @@ -0,0 +1,34 @@ +"""TxnReconcile allow txn_id to be null + +Revision ID: 08b6358a04bf +Revises: 04e61490804b +Create Date: 2018-03-07 19:48:06.050926 + +""" +from alembic import op +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = '08b6358a04bf' +down_revision = '04e61490804b' +branch_labels = None +depends_on = None + + +def upgrade(): + op.alter_column( + 'txn_reconciles', 'txn_id', + existing_type=mysql.INTEGER(display_width=11), + nullable=True + ) + + +def downgrade(): + conn = op.get_bind() + conn.execute("SET FOREIGN_KEY_CHECKS=0") + op.alter_column( + 'txn_reconciles', 'txn_id', + existing_type=mysql.INTEGER(display_width=11), + nullable=False + ) + conn.execute("SET FOREIGN_KEY_CHECKS=1") diff --git a/biweeklybudget/db_event_handlers.py b/biweeklybudget/db_event_handlers.py index c2d74c9f..cf6254d8 100644 --- a/biweeklybudget/db_event_handlers.py +++ b/biweeklybudget/db_event_handlers.py @@ -174,6 +174,83 @@ def handle_new_or_deleted_budget_transaction(session): ) +def handle_ofx_transaction_new_or_change(session): + """ + ``before_flush`` event handler + (:py:meth:`sqlalchemy.orm.events.SessionEvents.before_flush`) + on the DB session, to handle setting the ``is_*`` fields on new or changed + OFXTransaction instances according to its Account. + + :param session: current database session + :type session: sqlalchemy.orm.session.Session + """ + for obj in session.new: + if isinstance(obj, OFXTransaction): + try: + obj.update_is_fields() + except Exception: + logger.error('Error setting OFXTransaction is_ fields', + exc_info=True) + for obj in session.dirty: + if isinstance(obj, OFXTransaction): + try: + obj.update_is_fields() + except Exception: + logger.error('Error setting OFXTransaction is_ fields', + exc_info=True) + + +def handle_account_re_change(session): + """ + Handler for change of one of: + + * :py:attr:`~.Account.re_interest_paid` + * :py:attr:`~.Account.re_interest_charge` + * :py:attr:`~.Account.re_late_fee` + * :py:attr:`~.Account.re_other_fee` + * :py:attr:`~.Account.re_payment` + + When one of these regexes is changed on an Account, we trigger a re-run + of :py:meth:`~.OFXTransaction.update_is_fields` on all OFXTransactions for + the account. + + :param session: current database session + :type session: sqlalchemy.orm.session.Session + """ + attrs = [ + 're_interest_paid', + 're_interest_charge', + 're_late_fee', + 're_other_fee', + 're_payment' + ] + for obj in session.dirty: + if not isinstance(obj, Account): + continue + changed = [] + insp = inspect(obj) + for attr in attrs: + hx = getattr(insp.attrs, attr).history + if hx is None or hx.added is None or hx.deleted is None: + continue + if len(hx.added) > 0 and len(hx.deleted) > 0: + logger.debug( + '%s %s changed from %s to %s', + obj, attr, hx.deleted, hx.added + ) + changed.append(attr) + if len(changed) < 1: + continue + logger.debug( + '%s has regex changes; triggering update_is_fields() on all child ' + 'OFXTransactions.', obj + ) + for stmt in obj.all_statements: + for txn in stmt.ofx_trans: + txn.update_is_fields() + logger.debug('Done with update_is_fields() for %s', obj) + + def validate_decimal_or_none(target, value, oldvalue, initiator): if isinstance(value, Decimal) or value is None: return @@ -234,6 +311,8 @@ def handle_before_flush(session, flush_context, instances): """ logger.debug('handle_before_flush handler') handle_new_or_deleted_budget_transaction(session) + handle_ofx_transaction_new_or_change(session) + handle_account_re_change(session) logger.debug('handle_before_flush done') diff --git a/biweeklybudget/flaskapp/static/js/accounts_modal.js b/biweeklybudget/flaskapp/static/js/accounts_modal.js index a2327893..871e2c69 100644 --- a/biweeklybudget/flaskapp/static/js/accounts_modal.js +++ b/biweeklybudget/flaskapp/static/js/accounts_modal.js @@ -84,6 +84,11 @@ function accountModalDivForm() { ) .addCheckbox('account_frm_negate_ofx', 'negate_ofx_amounts', 'Negate OFX Amounts', false) .addCheckbox('account_frm_reconcile_trans', 'reconcile_trans', 'Reconcile Transactions?', true) + .addText('account_frm_re_interest_charge', 're_interest_charge', 'Interest Charge Regex', { helpBlock: 'If specified, OFX Transactions with name/memo matching this regex will be marked as interest charges (and not reconciled).'}) + .addText('account_frm_re_interest_paid', 're_interest_paid', 'Interest Paid Regex', { helpBlock: 'If specified, OFX Transactions with name/memo matching this regex will be marked as interest payments (and not reconciled).'}) + .addText('account_frm_re_payment', 're_payment', 'Payment Regex', { helpBlock: 'If specified, OFX Transactions with name/memo matching this regex will be marked as payments (and not reconciled).'}) + .addText('account_frm_re_late_fee', 're_late_fee', 'Late Fee Regex', { helpBlock: 'If specified, OFX Transactions with name/memo matching this regex will be marked as late fees (and not reconciled).'}) + .addText('account_frm_re_other_fee', 're_other_fee', 'Other Fee Regex', { helpBlock: 'If specified, OFX Transactions with name/memo matching this regex will be marked as other fees (and not reconciled).'}) .addCurrency('account_frm_credit_limit', 'credit_limit', 'Credit Limit') .addText('account_frm_apr', 'apr', 'APR', { helpBlock: 'If you know the margin added to the Prime Rate for this card, use the Margin field instead.'}) .addText('account_frm_margin', 'prime_rate_margin', 'Margin', { helpBlock: 'If known, the margin added to the US Prime Rate to determine the APR.'}) @@ -153,6 +158,11 @@ function accountModalDivFillAndShow(msg) { $('#account_frm_reconcile_trans').prop('checked', false); } $('#account_frm_vault_creds_path').val(msg['vault_creds_path']); + if(msg['re_interest_charge'] != null) { $('#account_frm_re_interest_charge').val(msg['re_interest_charge']); } + if(msg['re_interest_paid'] != null) { $('#account_frm_re_interest_paid').val(msg['re_interest_paid']); } + if(msg['re_payment'] != null) { $('#account_frm_re_payment').val(msg['re_payment']); } + if(msg['re_late_fee'] != null) { $('#account_frm_re_late_fee').val(msg['re_late_fee']); } + if(msg['re_other_fee'] != null) { $('#account_frm_re_other_fee').val(msg['re_other_fee']); } $("#modalDiv").modal('show'); } diff --git a/biweeklybudget/flaskapp/static/js/reconcile.js b/biweeklybudget/flaskapp/static/js/reconcile.js index 82d1b60d..62981b18 100644 --- a/biweeklybudget/flaskapp/static/js/reconcile.js +++ b/biweeklybudget/flaskapp/static/js/reconcile.js @@ -245,7 +245,10 @@ function reconcileOfxDiv(trans) { div += ''; div += '
'; div += '
' + trans['fitid'] + ': ' + trans['name'] + '
'; - div += '
(make trans)
'; + div += '
'; + div += '(make trans)'; + div += '(ignore)'; + div += '
'; div += '
'; div += '\n'; return div; @@ -268,7 +271,7 @@ function clean_fitid(fitid) { */ function reconcileHandleSubmit() { $('body').find('#reconcile-msg').remove(); - if (jQuery.isEmptyObject(reconciled)) { + if (jQuery.isEmptyObject(reconciled) && jQuery.isEmptyObject(ofxIgnored)) { var container = $('#notifications-row').find('.col-lg-12'); var newdiv = $( '
' + @@ -281,7 +284,7 @@ function reconcileHandleSubmit() { $.ajax({ type: "POST", url: '/ajax/reconcile', - data: JSON.stringify(reconciled), + data: JSON.stringify({ reconciled: reconciled, ofxIgnored: ofxIgnored }), contentType: "application/json; charset=utf-8", dataType: "json", success: function(data) { @@ -482,6 +485,98 @@ function reconcileDoUnreconcileNoOfx(trans_id) { $('#trans-' + trans_id).find('a:contains("(no OFX)")').show(); } +/** + * Show the modal for reconciling an OFXTransaction without a matching + * Transaction. Calls :js:func:`ignoreOfxTransDivForm` to generate the modal form + * div content. Uses an inline function to handle the save action, which calls + * :js:func:`reconcileOfxNoTrans` to perform the reconcile action. + * + * @param {number} acct_id - the Account ID of the OFXTransaction + * @param {string} fitid - the FitID of the OFXTransaction + */ +function ignoreOfxTrans(acct_id, fitid) { + console.log("ignoreOfxTrans(" + acct_id + ", '" + fitid + "')"); + $('#modalBody').empty(); + $('#modalBody').append(ignoreOfxTransDivForm(acct_id, fitid)); + $('#modalLabel').text('Ignore OFXTransaction (' + acct_id + ', "' + fitid + '")'); + $('#modalSaveButton').off(); + $('#modalSaveButton').click(function() { + $('.has-error').each(function(index) { $(this).removeClass('has-error'); }); + var note = $('#trans_frm_note').val().trim(); + if (note == "") { + $('#trans_frm_note').parent().append('

Note cannot be empty.

'); + $('#trans_frm_note').parent().addClass('has-error'); + return; + } else { + reconcileOfxNoTrans(acct_id, fitid, note); + $("#modalDiv").modal('hide'); + } + }).show(); + $("#modalDiv").modal('show'); +} + +/** + * Generate the modal form div content for the modal to reconcile a Transaction + * without a matching OFXTransaction. Called by :js:func:`transNoOfx`. + * + * @param {number} acct_id - the Account ID of the OFXTransaction + * @param {string} fitid - the FitID of the OFXTransaction + */ +function ignoreOfxTransDivForm(acct_id, fitid) { + return new FormBuilder('transForm') + .addP('Mark OFXTransaction as reconciled with no Transaction (i.e. OFXTransaction that we don\'t care about and won\t have a Transaction for).') + .addHidden('trans_frm_acct_id', 'acct_id', acct_id) + .addHidden('trans_frm_fitid', 'fitid', fitid) + .addText('trans_frm_note', 'note', 'Note') + .render(); +} + +/** + * Reconcile an OFXTransaction without a matching Transaction. Called from + * the Save button handler in :js:func:`ignoreOfxTrans`. + */ +function reconcileOfxNoTrans(acct_id, fitid, note) { + var target = $('#ofx-' + acct_id + '-' + clean_fitid(fitid)); + var newdiv = $('
').html('

No Trans: ' + note + '

'); + var makeTransLink = $(target).find('.make-trans-link'); + $(makeTransLink).html('Unignore'); + $(newdiv).prependTo(target); + $(target).draggable('disable'); + ofxIgnored[acct_id + "%" + fitid] = note; +} + +/** + * Unreconcile a reconciled NoTrans OFXTransaction. This removes + * ``acct_id + "%" + fitid`` from the ``ofxIgnored`` variable and regenerates + * the OFXTransaction's div. + * + * @param {number} acct_id - the Account ID of the OFXTransaction + * @param {string} fitid - the FitID of the OFXTransaction + */ +function reconcileDoUnreconcileNoTrans(acct_id, fitid) { + // remove from the reconciled object + delete ofxIgnored[acct_id + "%" + fitid]; + // remove the "No Trans:" div + $('#ofx-' + acct_id + '-' + clean_fitid(fitid) + '-noTrans').remove(); + // restore the draggability + var target = $('#ofx-' + acct_id + '-' + clean_fitid(fitid)); + $.ajax({ + type: "GET", + url: "/ajax/ofx/" + acct_id + "/" + encodeURIComponent(fitid), + success: function (data) { + var newdiv = $(reconcileOfxDiv(data['txn'])); + $(target).replaceWith(newdiv); + $(newdiv).draggable({ + cursor: 'move', + // start: dragStart, + // containment: '#content-row', + revert: 'invalid', + helper: 'clone' + }); + } + }); +} + $(document).ready(function() { reconcileGetTransactions(); reconcileGetOFX(); diff --git a/biweeklybudget/flaskapp/templates/reconcile.html b/biweeklybudget/flaskapp/templates/reconcile.html index 7a161979..0ad4650b 100644 --- a/biweeklybudget/flaskapp/templates/reconcile.html +++ b/biweeklybudget/flaskapp/templates/reconcile.html @@ -20,6 +20,7 @@ // store state for reconciled transactions - dict of Trans ID to // Array of [OFX account_id, OFX fitid] var reconciled = {}; + var ofxIgnored = {}; {% endblock %} {% block body %} diff --git a/biweeklybudget/flaskapp/views/accounts.py b/biweeklybudget/flaskapp/views/accounts.py index ea3fbb5f..5adf93ee 100644 --- a/biweeklybudget/flaskapp/views/accounts.py +++ b/biweeklybudget/flaskapp/views/accounts.py @@ -40,6 +40,7 @@ from flask import render_template, jsonify from decimal import Decimal import json +import re from biweeklybudget.flaskapp.app import app from biweeklybudget.flaskapp.views.formhandlerview import FormHandlerView @@ -51,6 +52,14 @@ logger = logging.getLogger(__name__) +RE_FIELD_NAMES = [ + 're_interest_charge', + 're_interest_paid', + 're_payment', + 're_late_fee', + 're_other_fee' +] + class AccountsView(MethodView): """ @@ -151,6 +160,13 @@ def validate(self, data): errors['min_payment_class_name'].append( 'Invalid minimum payment class name' ) + for f in RE_FIELD_NAMES: + if data[f].strip() == '': + continue + try: + re.compile(data[f]) + except Exception: + errors[f].append('Invalid regular expression.') if have_errors: return errors return None @@ -219,6 +235,11 @@ def submit(self, data): account.is_active = True else: account.is_active = False + for f in RE_FIELD_NAMES: + data[f] = data[f].strip() + if data[f] == '': + data[f] = None + setattr(account, f, data[f]) logger.info('%s: %s', action, account.as_dict) db_session.add(account) db_session.commit() diff --git a/biweeklybudget/flaskapp/views/ofx.py b/biweeklybudget/flaskapp/views/ofx.py index 015050b4..db888829 100644 --- a/biweeklybudget/flaskapp/views/ofx.py +++ b/biweeklybudget/flaskapp/views/ofx.py @@ -91,13 +91,16 @@ class OfxTransAjax(MethodView): def get(self, acct_id, fitid): txn = db_session.query(OFXTransaction).get((acct_id, fitid)) stmt = txn.statement.as_dict + aname = db_session.query(Account).get(acct_id).name res = { - 'acct_name': db_session.query(Account).get(acct_id).name, + 'acct_name': aname, 'acct_id': txn.account_id, 'txn': txn.as_dict, 'stmt': stmt, 'account_amount': txn.account_amount } + res['txn']['account_name'] = aname + res['txn']['account_amount'] = txn.account_amount return jsonify(res) diff --git a/biweeklybudget/flaskapp/views/reconcile.py b/biweeklybudget/flaskapp/views/reconcile.py index 5533ee6f..12061a2d 100644 --- a/biweeklybudget/flaskapp/views/reconcile.py +++ b/biweeklybudget/flaskapp/views/reconcile.py @@ -140,16 +140,31 @@ def post(self): """ Handle POST ``/ajax/reconcile`` + Request is a JSON dict with two keys, "reconciled" and "ofxIgnored". + "reconciled" value is a dict of integer transaction ID keys, to + values which are either a string reason why the Transaction is being + reconciled as "No OFX" or a 2-item list of OFXTransaction acct_id + and fitid. + "ofxIgnored" is a dict with string keys which are strings identifying + an OFXTransaction in the form "%", and values are a + string reason why the OFXTransaction is being reconciled without a + matching Transaction. + Response is a JSON dict. Keys are ``success`` (boolean) and either ``error_message`` (string) or ``success_message`` (string). :return: JSON response """ raw = request.get_json() - data = {int(x): raw[x] for x in raw} + data = { + 'reconciled': { + int(x): raw['reconciled'][x] for x in raw['reconciled'] + }, + 'ofxIgnored': raw.get('ofxIgnored', {}) + } logger.debug('POST /ajax/reconcile: %s', data) rec_count = 0 - for trans_id in sorted(data.keys()): + for trans_id in sorted(data['reconciled'].keys()): trans = db_session.query(Transaction).get(trans_id) if trans is None: logger.error('Invalid transaction ID: %s', trans_id) @@ -157,19 +172,22 @@ def post(self): 'success': False, 'error_message': 'Invalid Transaction ID: %s' % trans_id }), 400 - if not isinstance(data[trans_id], type([])): + if not isinstance(data['reconciled'][trans_id], type([])): # it's a string; reconcile without OFX db_session.add(TxnReconcile( txn_id=trans_id, - note=data[trans_id] + note=data['reconciled'][trans_id] )) logger.info( - 'Reconcile %s as NoOFX; note=%s', trans, data[trans_id] + 'Reconcile %s as NoOFX; note=%s', + trans, data['reconciled'][trans_id] ) rec_count += 1 continue # else reconcile with OFX - ofx_key = (data[trans_id][0], data[trans_id][1]) + ofx_key = ( + data['reconciled'][trans_id][0], data['reconciled'][trans_id][1] + ) ofx = db_session.query(OFXTransaction).get(ofx_key) if ofx is None: logger.error('Invalid OFXTransaction: %s', ofx_key) @@ -181,11 +199,25 @@ def post(self): }), 400 db_session.add(TxnReconcile( txn_id=trans_id, - ofx_account_id=data[trans_id][0], - ofx_fitid=data[trans_id][1] + ofx_account_id=data['reconciled'][trans_id][0], + ofx_fitid=data['reconciled'][trans_id][1] )) logger.info('Reconcile %s with %s', trans, ofx) rec_count += 1 + # handle OFXTransactions to reconcile with no Transaction + for ofxkey in sorted(data['ofxIgnored'].keys()): + note = data['ofxIgnored'][ofxkey] + acct_id, fitid = ofxkey.split('%', 1) + db_session.add(TxnReconcile( + ofx_account_id=acct_id, + ofx_fitid=fitid, + note=note + )) + logger.info( + 'Reconcile OFXTransaction (%s, %s) as NoTransaction; note=%s', + acct_id, fitid, note + ) + rec_count += 1 try: db_session.flush() db_session.commit() diff --git a/biweeklybudget/models/account.py b/biweeklybudget/models/account.py index fd3764c2..92152f37 100644 --- a/biweeklybudget/models/account.py +++ b/biweeklybudget/models/account.py @@ -151,28 +151,19 @@ class Account(Base, ModelAsDict): ) #: regex for matching transactions as interest charges - re_interest_charge = Column( - String(254), - default='^(interest charge|purchase finance charge)' - ) + re_interest_charge = Column(String(254)) #: regex for matching transactions as interest paid - re_interest_paid = Column( - String(254), - default='^interest paid' - ) + re_interest_paid = Column(String(254)) #: regex for matching transactions as payments - re_payment = Column( - String(254), - default='^(online payment|internet payment|online pymt|payment)' - ) + re_payment = Column(String(254)) - #: regex for matching transactions as fees - re_fee = Column( - String(254), - default='^(late fee|past due fee)' - ) + #: regex for matching transactions as late fees + re_late_fee = Column(String(254)) + + #: regex for matching transactions as other fees + re_other_fee = Column(String(254)) def __repr__(self): return "" % ( diff --git a/biweeklybudget/models/ofx_transaction.py b/biweeklybudget/models/ofx_transaction.py index 0e0e8a02..45a07b4d 100644 --- a/biweeklybudget/models/ofx_transaction.py +++ b/biweeklybudget/models/ofx_transaction.py @@ -45,6 +45,7 @@ from pytz import UTC from datetime import datetime import logging +import re from decimal import Decimal from biweeklybudget.models.base import Base, ModelAsDict @@ -212,7 +213,12 @@ def unreconciled(db): return db.query(OFXTransaction).filter( OFXTransaction.reconcile.__eq__(null()), OFXTransaction.date_posted.__ge__(cutoff_date), - OFXTransaction.account.has(reconcile_trans=True) + OFXTransaction.account.has(reconcile_trans=True), + OFXTransaction.is_payment.__ne__(True), + OFXTransaction.is_late_fee.__ne__(True), + OFXTransaction.is_interest_charge.__ne__(True), + OFXTransaction.is_other_fee.__ne__(True), + OFXTransaction.is_interest_payment.__ne__(True) ) @property @@ -231,3 +237,40 @@ def first_statement_by_date(self): logger.debug('First statement for %s: %s', self, res) return res + + def update_is_fields(self): + """ + Method to update all ``is_*`` fields on this instance, given the + ``re_*`` properties of :py:attr:`~.account`. + """ + fields = { + 're_interest_charge': 'is_interest_charge', + 're_interest_paid': 'is_interest_payment', + 're_payment': 'is_payment', + 're_late_fee': 'is_late_fee', + 're_other_fee': 'is_other_fee' + } + acct = self.account + for fname in fields.values(): + if ( + fname == 'is_interest_charge' and + self.name == 'Interest Charged - MANUALLY ENTERED' + ): + continue + setattr(self, fname, False) + for acct_attr, self_attr in fields.items(): + if ( + self_attr == 'is_interest_charge' and + self.name == 'Interest Charged - MANUALLY ENTERED' + ): + continue + r_str = getattr(acct, acct_attr) + if r_str is None: + continue + try: + if re.match(r_str, self.name, re.I): + setattr(self, self_attr, True) + except Exception: + logger.error('Error performing regex comparison on %s using ' + 'Account %s field %s (%s)', self, acct, + acct_attr, r_str, exc_info=True) diff --git a/biweeklybudget/models/txn_reconcile.py b/biweeklybudget/models/txn_reconcile.py index 67b2b0ad..edcbb95f 100644 --- a/biweeklybudget/models/txn_reconcile.py +++ b/biweeklybudget/models/txn_reconcile.py @@ -54,7 +54,7 @@ class TxnReconcile(Base, ModelAsDict): id = Column(Integer, primary_key=True) #: Transaction ID - txn_id = Column(Integer, ForeignKey('transactions.id'), nullable=False) + txn_id = Column(Integer, ForeignKey('transactions.id')) #: Relationship - :py:class:`~.Transaction` transaction = relationship( diff --git a/biweeklybudget/tests/acceptance/flaskapp/views/test_accounts.py b/biweeklybudget/tests/acceptance/flaskapp/views/test_accounts.py index 525bed7c..4ab83f24 100644 --- a/biweeklybudget/tests/acceptance/flaskapp/views/test_accounts.py +++ b/biweeklybudget/tests/acceptance/flaskapp/views/test_accounts.py @@ -175,6 +175,11 @@ def test_10_verify_db(self, testdb): assert acct.interest_class_name is None assert acct.min_payment_class_name is None assert acct.is_active is True + assert acct.re_interest_charge == '^interest-charge' + assert acct.re_interest_paid == '^interest-paid' + assert acct.re_payment == '^(payment|thank you)' + assert acct.re_late_fee == '^Late Fee' + assert acct.re_other_fee == '^re-other-fee' def test_11_get_acct1_url(self, base_url, selenium): self.get(selenium, base_url + '/accounts/1') @@ -218,6 +223,23 @@ def test_11_get_acct1_url(self, base_url, selenium): assert selenium.find_element_by_id( 'account_frm_min_pay_class_name').is_displayed() is False # END CREDIT + # BEGIN REs + assert selenium.find_element_by_id( + 'account_frm_re_interest_charge' + ).get_attribute('value') == '^interest-charge' + assert selenium.find_element_by_id( + 'account_frm_re_interest_paid' + ).get_attribute('value') == '^interest-paid' + assert selenium.find_element_by_id( + 'account_frm_re_payment' + ).get_attribute('value') == '^(payment|thank you)' + assert selenium.find_element_by_id( + 'account_frm_re_late_fee' + ).get_attribute('value') == '^Late Fee' + assert selenium.find_element_by_id( + 'account_frm_re_other_fee' + ).get_attribute('value') == '^re-other-fee' + # END REs assert selenium.find_element_by_id('account_frm_active').is_selected() def test_12_edit_acct1(self, base_url, selenium): @@ -272,6 +294,11 @@ def test_20_verify_db(self, testdb): assert acct.interest_class_name is None assert acct.min_payment_class_name is None assert acct.is_active is True + assert acct.re_interest_charge is None + assert acct.re_interest_paid is None + assert acct.re_payment is None + assert acct.re_late_fee is None + assert acct.re_other_fee is None def test_21_get_acct2_click(self, base_url, selenium): self.get(selenium, base_url + '/accounts') @@ -317,6 +344,23 @@ def test_21_get_acct2_click(self, base_url, selenium): assert selenium.find_element_by_id( 'account_frm_min_pay_class_name').is_displayed() is False # END CREDIT + # BEGIN REs + assert selenium.find_element_by_id( + 'account_frm_re_interest_charge' + ).get_attribute('value') == '' + assert selenium.find_element_by_id( + 'account_frm_re_interest_paid' + ).get_attribute('value') == '' + assert selenium.find_element_by_id( + 'account_frm_re_payment' + ).get_attribute('value') == '' + assert selenium.find_element_by_id( + 'account_frm_re_late_fee' + ).get_attribute('value') == '' + assert selenium.find_element_by_id( + 'account_frm_re_other_fee' + ).get_attribute('value') == '' + # END REs assert selenium.find_element_by_id('account_frm_active').is_selected() def test_22_edit_acct2(self, base_url, selenium): @@ -344,6 +388,20 @@ def test_22_edit_acct2(self, base_url, selenium): selenium.find_element_by_id('account_frm_negate_ofx').click() selenium.find_element_by_id('account_frm_reconcile_trans').click() selenium.find_element_by_id('account_frm_active').click() + # BEGIN REs + selenium.find_element_by_id( + 'account_frm_re_interest_charge' + ).send_keys('my-re-ic$') + selenium.find_element_by_id( + 'account_frm_re_interest_paid' + ).send_keys('my-re-ip$') + selenium.find_element_by_id( + 'account_frm_re_payment' + ).send_keys('my-re-p$') + selenium.find_element_by_id( + 'account_frm_re_late_fee' + ).send_keys('my-re-lf$') + # END REs # submit the form selenium.find_element_by_id('modalSaveButton').click() self.wait_for_jquery_done(selenium) @@ -372,6 +430,11 @@ def test_23_verify_db(self, testdb): assert acct.interest_class_name is None assert acct.min_payment_class_name is None assert acct.is_active is False + assert acct.re_interest_charge == 'my-re-ic$' + assert acct.re_interest_paid == 'my-re-ip$' + assert acct.re_payment == 'my-re-p$' + assert acct.re_late_fee == 'my-re-lf$' + assert acct.re_other_fee is None def test_30_verify_db(self, testdb): acct = testdb.query(Account).get(3) @@ -390,6 +453,11 @@ def test_30_verify_db(self, testdb): assert acct.interest_class_name == 'AdbCompoundedDaily' assert acct.min_payment_class_name == 'MinPaymentAmEx' assert acct.is_active is True + assert acct.re_interest_charge == '^INTEREST CHARGED TO' + assert acct.re_interest_paid is None + assert acct.re_payment == '.*Online Payment, thank you.*' + assert acct.re_late_fee == '^Late Fee' + assert acct.re_other_fee == '^re-other-fee' def test_31_get_acct3_click(self, base_url, selenium): self.get(selenium, base_url + '/accounts') @@ -487,6 +555,11 @@ def test_33_verify_db(self, testdb): assert acct.interest_class_name == 'AdbCompoundedDaily' assert acct.min_payment_class_name == 'MinPaymentAmEx' assert acct.is_active is True + assert acct.re_interest_charge == '^INTEREST CHARGED TO' + assert acct.re_interest_paid is None + assert acct.re_payment == '.*Online Payment, thank you.*' + assert acct.re_late_fee == '^Late Fee' + assert acct.re_other_fee == '^re-other-fee' def test_40_verify_db(self, testdb): acct = testdb.query(Account).get(4) @@ -505,6 +578,11 @@ def test_40_verify_db(self, testdb): assert acct.interest_class_name == 'AdbCompoundedDaily' assert acct.min_payment_class_name == 'MinPaymentDiscover' assert acct.is_active is True + assert acct.re_interest_charge is None + assert acct.re_interest_paid is None + assert acct.re_payment is None + assert acct.re_late_fee is None + assert acct.re_other_fee is None def test_41_get_acct4_url(self, base_url, selenium): self.get(selenium, base_url + '/accounts/4') @@ -628,6 +706,11 @@ def test_43_verify_db(self, testdb): assert acct.interest_class_name == 'AdbCompoundedDaily' assert acct.min_payment_class_name == 'MinPaymentCiti' assert acct.is_active is False + assert acct.re_interest_charge is None + assert acct.re_interest_paid is None + assert acct.re_payment is None + assert acct.re_late_fee is None + assert acct.re_other_fee is None def test_50_verify_db(self, testdb): acct = testdb.query(Account).get(5) @@ -646,6 +729,11 @@ def test_50_verify_db(self, testdb): assert acct.interest_class_name is None assert acct.min_payment_class_name is None assert acct.is_active is True + assert acct.re_interest_charge is None + assert acct.re_interest_paid is None + assert acct.re_payment is None + assert acct.re_late_fee is None + assert acct.re_other_fee is None def test_51_get_acct5_click(self, base_url, selenium): self.get(selenium, base_url + '/accounts') @@ -729,6 +817,11 @@ def test_53_verify_db(self, testdb): assert acct.interest_class_name is None assert acct.min_payment_class_name is None assert acct.is_active is True + assert acct.re_interest_charge is None + assert acct.re_interest_paid is None + assert acct.re_payment is None + assert acct.re_late_fee is None + assert acct.re_other_fee is None def test_60_verify_db(self, testdb): max_id = max([ @@ -815,6 +908,11 @@ def test_62_verify_db(self, testdb): assert acct.interest_class_name is None assert acct.min_payment_class_name is None assert acct.is_active is True + assert acct.re_interest_charge is None + assert acct.re_interest_paid is None + assert acct.re_payment is None + assert acct.re_late_fee is None + assert acct.re_other_fee is None def test_70_verify_db(self, testdb): max_id = max([ @@ -927,6 +1025,11 @@ def test_72_verify_db(self, testdb): assert acct.interest_class_name == 'AdbCompoundedDaily' assert acct.min_payment_class_name == 'MinPaymentCiti' assert acct.is_active is False + assert acct.re_interest_charge is None + assert acct.re_interest_paid is None + assert acct.re_payment is None + assert acct.re_late_fee is None + assert acct.re_other_fee is None def test_80_verify_db(self, testdb): max_id = max([ @@ -1014,3 +1117,8 @@ def test_82_verify_db(self, testdb): assert acct.interest_class_name is None assert acct.min_payment_class_name is None assert acct.is_active is True + assert acct.re_interest_charge is None + assert acct.re_interest_paid is None + assert acct.re_payment is None + assert acct.re_late_fee is None + assert acct.re_other_fee is None diff --git a/biweeklybudget/tests/acceptance/flaskapp/views/test_reconcile.py b/biweeklybudget/tests/acceptance/flaskapp/views/test_reconcile.py index bb7b0d1e..6f5c208e 100644 --- a/biweeklybudget/tests/acceptance/flaskapp/views/test_reconcile.py +++ b/biweeklybudget/tests/acceptance/flaskapp/views/test_reconcile.py @@ -113,7 +113,7 @@ def clean_fitid(fitid): def ofx_div(dt_posted, amt, acct_name, acct_id, trans_type, fitid, name, - trans_id=None): + trans_id=None, ignored_reason=None): """ Return the HTML for an OFXTransaction div. @@ -132,10 +132,16 @@ def ofx_div(dt_posted, amt, acct_name, acct_id, trans_type, fitid, name, else: classes = 'reconcile reconcile-ofx ui-draggable ui-draggable-handle' _id = 'ofx-%s-%s' % (acct_id, cfitid) + if ignored_reason is not None: + classes += ' ui-draggable-disabled' s = '
' % ( classes, _id, acct_id, amt, fitid ) + if ignored_reason is not None: + s += '

No Trans: %s

' \ + '
' % (acct_id, fitid, ignored_reason) s += '
' s += '
%s
' % dt_posted.strftime('%Y-%m-%d') s += '
%s
' % fmt_currency(amt) @@ -152,11 +158,24 @@ def ofx_div(dt_posted, amt, acct_name, acct_id, trans_type, fitid, name, ) s += ': %s' % name s += '
' - if trans_id is None: - s += '
(make trans)' \ - '
' % (acct_id, fitid) + if trans_id is None and ignored_reason is None: + s += '
' + s += '(make trans)' % ( + acct_id, fitid + ) + s += '(ignore)' % ( + acct_id, fitid + ) + s += '
' + elif ignored_reason is not None: + s += '
' + s += '' \ + 'Unignore' % ( + acct_id, fitid + ) + s += '
' s += '
' s += '
' if trans_id is not None: @@ -222,7 +241,11 @@ def test_01_add_accounts(self, testdb): ofx_cat_memo_to_name=True, ofxgetter_config_json='{"foo": "bar"}', vault_creds_path='secret/foo/bar/BankOne', - acct_type=AcctType.Bank + acct_type=AcctType.Bank, + re_interest_charge='^interest-charge', + re_payment='^(payment|thank you)', + re_late_fee='^Late Fee', + re_other_fee='^re-other-fee' ) testdb.add(a) a.set_balance( @@ -536,7 +559,73 @@ def test_06_transactions(self, base_url, selenium): ] assert actual_trans == expected_trans - def test_07_ofxtrans(self, base_url, selenium): + def test_07_verify_unreconciled_ofxtrans(self, testdb): + assert len(OFXTransaction.unreconciled(testdb).all()) == 7 + + def test_08_add_ignored_ofxtrans(self, testdb): + """ + add OFXTransactions that shouldn't be listed because of their is_ fields + """ + acct = testdb.query(Account).get(1) + assert acct.re_interest_charge == '^interest-charge' + assert acct.re_interest_paid is None + assert acct.re_payment == '^(payment|thank you)' + assert acct.re_late_fee == '^Late Fee' + assert acct.re_other_fee == '^re-other-fee' + stmt = testdb.query(OFXStatement).get(1) + assert stmt.account_id == 1 + assert stmt.filename == 'a1.ofx' + testdb.add(OFXTransaction( + account=acct, + statement=stmt, + fitid='BankOne.77.1', + trans_type='Debit', + date_posted=stmt.ledger_bal_as_of, + amount=Decimal('-20.00'), + name='interest-charge BankOne.77.1' + )) + testdb.add(OFXTransaction( + account=acct, + statement=stmt, + fitid='BankOne.77.2', + trans_type='Debit', + date_posted=stmt.ledger_bal_as_of, + amount=Decimal('-20.00'), + name='payment BankOne.77.2' + )) + testdb.add(OFXTransaction( + account=acct, + statement=stmt, + fitid='BankOne.77.3', + trans_type='Debit', + date_posted=stmt.ledger_bal_as_of, + amount=Decimal('-20.00'), + name='thank you BankOne.77.3' + )) + testdb.add(OFXTransaction( + account=acct, + statement=stmt, + fitid='BankOne.77.4', + trans_type='Debit', + date_posted=stmt.ledger_bal_as_of, + amount=Decimal('-20.00'), + name='Late Fee BankOne.77.4' + )) + testdb.add(OFXTransaction( + account=acct, + statement=stmt, + fitid='BankOne.77.5', + trans_type='Debit', + date_posted=stmt.ledger_bal_as_of, + amount=Decimal('-20.00'), + name='re-other-fee BankOne.77.5' + )) + testdb.commit() + + def test_09_verify_unreconciled_ofxtrans(self, testdb): + assert len(OFXTransaction.unreconciled(testdb).all()) == 7 + + def test_10_ofxtrans(self, base_url, selenium): self.get(selenium, base_url + '/reconcile') ofxtrans_div = selenium.find_element_by_id('ofx-panel') actual_ofx = [ @@ -1337,7 +1426,7 @@ def test_06_verify_db(self, testdb): def test_07_success(self, base_url): res = requests.post( base_url + '/ajax/reconcile', - json={3: [2, 'OFX3']} + json={'reconciled': {3: [2, 'OFX3']}, 'ofxIgnored': {}} ) assert res.json() == { 'success': True, @@ -1357,7 +1446,7 @@ def test_08_verify_db(self, testdb): def test_09_invalid_trans(self, base_url, testdb): res = requests.post( base_url + '/ajax/reconcile', - json={32198: [2, 'OFX3']} + json={'reconciled': {32198: [2, 'OFX3']}, 'ofxIgnored': {}} ) assert res.json() == { 'success': False, @@ -1369,7 +1458,7 @@ def test_09_invalid_trans(self, base_url, testdb): def test_10_invalid_ofx(self, base_url, testdb): res = requests.post( base_url + '/ajax/reconcile', - json={3: [2, 'OFX338ufd']} + json={'reconciled': {3: [2, 'OFX338ufd']}, 'ofxIgnored': {}} ) assert res.json() == { 'success': False, @@ -1382,7 +1471,7 @@ def test_10_commit_exception(self, base_url): # already reconciled in test_07 res = requests.post( base_url + '/ajax/reconcile', - json={3: [2, 'OFX3']} + json={'reconciled': {3: [2, 'OFX3']}, 'ofxIgnored': {}} ) j = res.json() assert sorted(j.keys()) == ['error_message', 'success'] @@ -1405,7 +1494,7 @@ def test_11_verify_db(self, testdb): def test_12_reconcile_noOFX(self, base_url): res = requests.post( base_url + '/ajax/reconcile', - json={4: 'Foo Bar Baz'} + json={'reconciled': {4: 'Foo Bar Baz'}, 'ofxIgnored': {}} ) assert res.json() == { 'success': True, @@ -1436,7 +1525,7 @@ def test_14_verify_reconcile_modal(self, base_url, selenium, testdb): @pytest.mark.acceptance @pytest.mark.usefixtures('class_refresh_db', 'refreshdb') @pytest.mark.incremental -class TestOFXMakeTrans(AcceptanceHelper): +class TestOFXMakeTransAndIgnore(AcceptanceHelper): def get_reconciled(self, driver): """ @@ -1557,6 +1646,9 @@ def test_04_add_ofx(self, testdb): def test_06_verify_db(self, testdb): res = testdb.query(TxnReconcile).all() assert len(res) == 0 + stmts = testdb.query(OFXStatement).all() + assert len(stmts) == 1 + assert max([s.id for s in stmts]) == 1 def test_07_verify_db_transaction(self, testdb): res = testdb.query(Transaction).all() @@ -1693,6 +1785,263 @@ def test_10_verify_db_transaction(self, testdb): assert res[1].budget_transactions[0].budget_id == 2 assert res[1].budget_transactions[0].amount == Decimal('-251.23') + def test_30_add_ofx(self, testdb): + acct2 = testdb.query(Account).get(2) + stmt1 = testdb.query(OFXStatement).get(1) + testdb.add(OFXTransaction( + account=acct2, + statement=stmt1, + fitid='OFX30', + trans_type='Debit', + date_posted=datetime(2017, 4, 11, 12, 3, 4, tzinfo=UTC), + amount=Decimal('251.23'), + name='ofx2-trans30' + )) + testdb.add(OFXTransaction( + account=acct2, + statement=stmt1, + fitid='OFX31', + trans_type='Debit', + date_posted=datetime(2017, 4, 10, 12, 3, 4, tzinfo=UTC), + amount=Decimal('192.86'), + name='ofx2-trans31' + )) + testdb.flush() + testdb.commit() + + def test_31_verify_db(self, testdb): + res = testdb.query(TxnReconcile).all() + assert len(res) == 1 + assert res[0].id == 1 + assert res[0].txn_id == 2 + assert res[0].ofx_account_id == 2 + assert res[0].ofx_fitid == 'OFX2' + res = testdb.query(TxnReconcile).all() + assert len(res) == 1 + assert max([r.id for r in res]) == 1 + + def test_32_verify_columns(self, base_url, selenium): + self.get(selenium, base_url + '/reconcile') + ofxtrans_div = selenium.find_element_by_id('ofx-panel') + actual_ofx = [ + self.normalize_html(x.get_attribute('outerHTML')) + for x in ofxtrans_div.find_elements_by_class_name('reconcile-ofx') + ] + expected_ofx = [ + ofx_div( + date(2017, 4, 10), + Decimal('-192.86'), + 'BankTwo', 2, + 'Debit', + 'OFX31', + 'ofx2-trans31' + ), + ofx_div( + date(2017, 4, 11), + Decimal('-251.23'), + 'BankTwo', 2, + 'Debit', + 'OFX30', + 'ofx2-trans30' + ) + ] + assert expected_ofx == actual_ofx + + def test_33_ignore_ofx(self, base_url, selenium): + self.get(selenium, base_url + '/reconcile') + ofxdiv = selenium.find_element_by_id('ofx-2-OFX31') + link = ofxdiv.find_element_by_xpath('//a[text()="(ignore)"]') + link.click() + # test the modal population + modal, title, body = self.get_modal_parts(selenium) + self.assert_modal_displayed(modal, title, body) + assert title.text == 'Ignore OFXTransaction (2, "OFX31")' + assert body.find_element_by_id( + 'trans_frm_acct_id').get_attribute('value') == '2' + assert body.find_element_by_id( + 'trans_frm_fitid').get_attribute('value') == 'OFX31' + notes = selenium.find_element_by_id('trans_frm_note') + assert notes.get_attribute( + 'value') == '' + notes.send_keys('My Note') + # submit the form + selenium.find_element_by_id('modalSaveButton').click() + self.wait_for_jquery_done(selenium) + sleep(1) + # check that modal was hidden + modal, title, body = self.get_modal_parts(selenium, wait=False) + self.assert_modal_hidden(modal, title, body) + # check that the JS variable has been updated + res = selenium.execute_script('return JSON.stringify(ofxIgnored);') + assert json.loads(res.strip()) == {'2%OFX31': 'My Note'} + # check that the OFX div has been updated + ofxtrans_div = selenium.find_element_by_id('ofx-panel') + actual_ofx = [ + self.normalize_html(x.get_attribute('outerHTML')) + for x in ofxtrans_div.find_elements_by_class_name('reconcile-ofx') + ] + expected_ofx = [ + ofx_div( + date(2017, 4, 10), + Decimal('-192.86'), + 'BankTwo', 2, + 'Debit', + 'OFX31', + 'ofx2-trans31', + ignored_reason='My Note' + ), + ofx_div( + date(2017, 4, 11), + Decimal('-251.23'), + 'BankTwo', 2, + 'Debit', + 'OFX30', + 'ofx2-trans30' + ) + ] + assert expected_ofx == actual_ofx + # check that the OFX div is no longer draggable + ofxdiv = selenium.find_element_by_id('ofx-2-OFX31') + assert 'ui-draggable-disabled' in ofxdiv.get_attribute('class') + # wait for submit button to be visible and clickable, and click it + self.wait_for_jquery_done(selenium) + WebDriverWait(selenium, 10).until( + EC.invisibility_of_element_located((By.ID, 'modalDiv')) + ) + WebDriverWait(selenium, 10).until( + EC.element_to_be_clickable((By.ID, 'reconcile-submit')) + ) + selenium.find_element_by_id('reconcile-submit').click() + sleep(1) + self.wait_for_jquery_done(selenium) + msg = selenium.find_element_by_id('reconcile-msg') + assert msg.text == 'Successfully reconciled 1 transactions' + assert 'alert-success' in msg.get_attribute('class') + + def test_34_verify_db(self, testdb): + res = testdb.query(TxnReconcile).all() + assert len(res) == 2 + assert max([r.id for r in res]) == 2 + vals = {r.id: r for r in res} + tr = vals[2] + assert tr.txn_id is None + assert tr.ofx_account_id == 2 + assert tr.ofx_fitid == 'OFX31' + assert tr.note == 'My Note' + + def test_35_verify_columns(self, base_url, selenium): + self.get(selenium, base_url + '/reconcile') + ofxtrans_div = selenium.find_element_by_id('ofx-panel') + actual_ofx = [ + self.normalize_html(x.get_attribute('outerHTML')) + for x in ofxtrans_div.find_elements_by_class_name('reconcile-ofx') + ] + expected_ofx = [ + ofx_div( + date(2017, 4, 11), + Decimal('-251.23'), + 'BankTwo', 2, + 'Debit', + 'OFX30', + 'ofx2-trans30' + ) + ] + assert expected_ofx == actual_ofx + + def test_36_ignore_and_unignore_ofx(self, base_url, selenium): + self.get(selenium, base_url + '/reconcile') + # check that the OFX div has been updated + ofxtrans_div = selenium.find_element_by_id('ofx-panel') + actual_ofx = [ + self.normalize_html(x.get_attribute('outerHTML')) + for x in ofxtrans_div.find_elements_by_class_name('reconcile-ofx') + ] + expected_ofx = [ + ofx_div( + date(2017, 4, 11), + Decimal('-251.23'), + 'BankTwo', 2, + 'Debit', + 'OFX30', + 'ofx2-trans30' + ) + ] + assert expected_ofx == actual_ofx + # ignore + ofxdiv = selenium.find_element_by_id('ofx-2-OFX30') + link = ofxdiv.find_element_by_xpath('//a[text()="(ignore)"]') + link.click() + # test the modal population + modal, title, body = self.get_modal_parts(selenium) + self.assert_modal_displayed(modal, title, body) + assert title.text == 'Ignore OFXTransaction (2, "OFX30")' + assert body.find_element_by_id( + 'trans_frm_acct_id').get_attribute('value') == '2' + assert body.find_element_by_id( + 'trans_frm_fitid').get_attribute('value') == 'OFX30' + notes = selenium.find_element_by_id('trans_frm_note') + assert notes.get_attribute( + 'value') == '' + notes.send_keys('My Note') + # submit the form + selenium.find_element_by_id('modalSaveButton').click() + self.wait_for_jquery_done(selenium) + sleep(1) + # check that modal was hidden + modal, title, body = self.get_modal_parts(selenium, wait=False) + self.assert_modal_hidden(modal, title, body) + # check that the JS variable has been updated + res = selenium.execute_script('return JSON.stringify(ofxIgnored);') + assert json.loads(res.strip()) == {'2%OFX30': 'My Note'} + # check that the OFX div has been updated + ofxtrans_div = selenium.find_element_by_id('ofx-panel') + actual_ofx = [ + self.normalize_html(x.get_attribute('outerHTML')) + for x in ofxtrans_div.find_elements_by_class_name('reconcile-ofx') + ] + expected_ofx = [ + ofx_div( + date(2017, 4, 11), + Decimal('-251.23'), + 'BankTwo', 2, + 'Debit', + 'OFX30', + 'ofx2-trans30', + ignored_reason='My Note' + ) + ] + assert expected_ofx == actual_ofx + # check that the OFX div is no longer draggable + ofxdiv = selenium.find_element_by_id('ofx-2-OFX30') + assert 'ui-draggable-disabled' in ofxdiv.get_attribute('class') + # ok, now Unignore + ofxdiv = selenium.find_element_by_id('ofx-2-OFX30') + link = ofxdiv.find_element_by_xpath('//a[text()="Unignore"]') + link.click() + # and test that everything was reverted... + res = selenium.execute_script('return JSON.stringify(ofxIgnored);') + assert json.loads(res.strip()) == {} + # check that the OFX div has been updated + ofxtrans_div = selenium.find_element_by_id('ofx-panel') + actual_ofx = [ + self.normalize_html(x.get_attribute('outerHTML')) + for x in ofxtrans_div.find_elements_by_class_name('reconcile-ofx') + ] + expected_ofx = [ + ofx_div( + date(2017, 4, 11), + Decimal('-251.23'), + 'BankTwo', 2, + 'Debit', + 'OFX30', + 'ofx2-trans30' + ) + ] + assert expected_ofx == actual_ofx + # check that the OFX div is no longer draggable + ofxdiv = selenium.find_element_by_id('ofx-2-OFX30') + assert 'ui-draggable-disabled' not in ofxdiv.get_attribute('class') + @pytest.mark.acceptance @pytest.mark.usefixtures('class_refresh_db', 'refreshdb') diff --git a/biweeklybudget/tests/acceptance/test_db_event_handlers.py b/biweeklybudget/tests/acceptance/test_db_event_handlers.py index 116eb8ea..cd27b9c1 100644 --- a/biweeklybudget/tests/acceptance/test_db_event_handlers.py +++ b/biweeklybudget/tests/acceptance/test_db_event_handlers.py @@ -42,6 +42,8 @@ from biweeklybudget.models.transaction import Transaction from biweeklybudget.models.account import Account from biweeklybudget.models.budget_model import Budget +from biweeklybudget.models.ofx_transaction import OFXTransaction +from biweeklybudget.models.ofx_statement import OFXStatement @pytest.mark.acceptance @@ -168,3 +170,186 @@ def test_8_verify_db(self, testdb): assert periodic.is_periodic is True assert periodic.name == 'Periodic2' assert periodic.current_balance is None + + +@pytest.mark.acceptance +@pytest.mark.usefixtures('class_refresh_db', 'refreshdb') +class TestIsFieldsSet(AcceptanceHelper): + + def test_0_is_fields_set_by_ofxtxn_event_handler(self, testdb): + """ + Test for + :py:func:`~.db_event_handlers.handle_ofx_transaction_new_or_change` + """ + acct = testdb.query(Account).get(1) + acct.re_interest_paid = None + testdb.commit() + assert acct.re_interest_charge == '^interest-charge' + assert acct.re_interest_paid is None + assert acct.re_payment == '^(payment|thank you)' + assert acct.re_late_fee == '^Late Fee' + assert acct.re_other_fee == '^re-other-fee' + stmt = testdb.query(OFXStatement).get(1) + assert stmt.account_id == 1 + assert stmt.filename == '/stmt/BankOne/0' + txn = OFXTransaction( + account=acct, + statement=stmt, + fitid='BankOne-9-1', + trans_type='Credit', + date_posted=stmt.ledger_bal_as_of, + amount=Decimal('1234.56'), + name='BankOne-9-1' + ) + testdb.add(txn) + testdb.commit() + assert txn.is_payment is False + assert txn.is_late_fee is False + assert txn.is_interest_charge is False + assert txn.is_other_fee is False + assert txn.is_interest_payment is False + txn = OFXTransaction( + account=acct, + statement=stmt, + fitid='BankOne-9-2', + trans_type='Credit', + date_posted=stmt.ledger_bal_as_of, + amount=Decimal('1234.56'), + name='re-other-fee BankOne-9-2' + ) + testdb.add(txn) + testdb.commit() + assert txn.is_payment is False + assert txn.is_late_fee is False + assert txn.is_interest_charge is False + assert txn.is_other_fee is True + assert txn.is_interest_payment is False + txn = OFXTransaction( + account=acct, + statement=stmt, + fitid='BankOne-9-3', + trans_type='Credit', + date_posted=stmt.ledger_bal_as_of, + amount=Decimal('1234.56'), + name='Late Fee BankOne-9-3' + ) + testdb.add(txn) + testdb.commit() + assert txn.is_payment is False + assert txn.is_late_fee is True + assert txn.is_interest_charge is False + assert txn.is_other_fee is False + assert txn.is_interest_payment is False + txn = OFXTransaction( + account=acct, + statement=stmt, + fitid='BankOne-9-4', + trans_type='Credit', + date_posted=stmt.ledger_bal_as_of, + amount=Decimal('1234.56'), + name='payment BankOne-9-4' + ) + testdb.add(txn) + testdb.commit() + assert txn.is_payment is True + assert txn.is_late_fee is False + assert txn.is_interest_charge is False + assert txn.is_other_fee is False + assert txn.is_interest_payment is False + txn = OFXTransaction( + account=acct, + statement=stmt, + fitid='BankOne-9-5', + trans_type='Credit', + date_posted=stmt.ledger_bal_as_of, + amount=Decimal('1234.56'), + name='Thank You BankOne-9-5' + ) + testdb.add(txn) + testdb.commit() + assert txn.is_payment is True + assert txn.is_late_fee is False + assert txn.is_interest_charge is False + assert txn.is_other_fee is False + assert txn.is_interest_payment is False + txn = OFXTransaction( + account=acct, + statement=stmt, + fitid='BankOne-9-6', + trans_type='Credit', + date_posted=stmt.ledger_bal_as_of, + amount=Decimal('1234.56'), + name='interest-paid' + ) + testdb.add(txn) + testdb.commit() + assert txn.is_payment is False + assert txn.is_late_fee is False + assert txn.is_interest_charge is False + assert txn.is_other_fee is False + assert txn.is_interest_payment is False + + def test_1_account_re_change_triggers_update_is_fields(self, testdb): + """ + Test for + :py:func:`~.db_event_handlers.handle_account_re_change` + """ + acct = testdb.query(Account).get(1) + acct.re_interest_paid = None + testdb.commit() + assert acct.re_interest_charge == '^interest-charge' + assert acct.re_interest_paid is None + assert acct.re_payment == '^(payment|thank you)' + assert acct.re_late_fee == '^Late Fee' + assert acct.re_other_fee == '^re-other-fee' + stmt = testdb.query(OFXStatement).get(1) + assert stmt.account_id == 1 + assert stmt.filename == '/stmt/BankOne/0' + txn1 = testdb.query(OFXTransaction).get((1, 'BankOne-9-1')) + assert txn1.name == 'BankOne-9-1' + assert txn1.is_payment is False + assert txn1.is_late_fee is False + assert txn1.is_interest_charge is False + assert txn1.is_other_fee is False + assert txn1.is_interest_payment is False + txn2 = testdb.query(OFXTransaction).get((1, 'BankOne-9-2')) + assert txn2.name == 're-other-fee BankOne-9-2' + assert txn2.is_payment is False + assert txn2.is_late_fee is False + assert txn2.is_interest_charge is False + assert txn2.is_other_fee is True + assert txn2.is_interest_payment is False + txn3 = testdb.query(OFXTransaction).get((1, 'BankOne-9-3')) + assert txn3.name == 'Late Fee BankOne-9-3' + assert txn3.is_payment is False + assert txn3.is_late_fee is True + assert txn3.is_interest_charge is False + assert txn3.is_other_fee is False + assert txn3.is_interest_payment is False + # change the account + acct.re_interest_charge = '^BankOne-9-1$' + acct.re_payment = '^Late Fee' + acct.re_late_fee = '^foobarbaz' + testdb.commit() + # re-confirm + txn1 = testdb.query(OFXTransaction).get((1, 'BankOne-9-1')) + assert txn1.name == 'BankOne-9-1' + assert txn1.is_payment is False + assert txn1.is_late_fee is False + assert txn1.is_interest_charge is True + assert txn1.is_other_fee is False + assert txn1.is_interest_payment is False + txn2 = testdb.query(OFXTransaction).get((1, 'BankOne-9-2')) + assert txn2.name == 're-other-fee BankOne-9-2' + assert txn2.is_payment is False + assert txn2.is_late_fee is False + assert txn2.is_interest_charge is False + assert txn2.is_other_fee is True + assert txn2.is_interest_payment is False + txn3 = testdb.query(OFXTransaction).get((1, 'BankOne-9-3')) + assert txn3.name == 'Late Fee BankOne-9-3' + assert txn3.is_payment is True + assert txn3.is_late_fee is False + assert txn3.is_interest_charge is False + assert txn3.is_other_fee is False + assert txn3.is_interest_payment is False diff --git a/biweeklybudget/tests/fixtures/sampledata.py b/biweeklybudget/tests/fixtures/sampledata.py index 0d401b60..ca19be36 100644 --- a/biweeklybudget/tests/fixtures/sampledata.py +++ b/biweeklybudget/tests/fixtures/sampledata.py @@ -273,7 +273,12 @@ def _bank_one(self): ofx_cat_memo_to_name=True, ofxgetter_config_json='{"foo": "bar"}', vault_creds_path='secret/foo/bar/BankOne', - acct_type=AcctType.Bank + acct_type=AcctType.Bank, + re_interest_charge='^interest-charge', + re_interest_paid='^interest-paid', + re_payment='^(payment|thank you)', + re_late_fee='^Late Fee', + re_other_fee='^re-other-fee' ) statements = [ OFXStatement( @@ -411,7 +416,11 @@ def _credit_one(self): prime_rate_margin=Decimal('0.0050'), negate_ofx_amounts=True, interest_class_name='AdbCompoundedDaily', - min_payment_class_name='MinPaymentAmEx' + min_payment_class_name='MinPaymentAmEx', + re_interest_charge='^INTEREST CHARGED TO', + re_payment='.*Online Payment, thank you.*', + re_late_fee='^Late Fee', + re_other_fee='^re-other-fee' ) statements = [ OFXStatement( diff --git a/biweeklybudget/tests/unit/models/test_ofx_transaction.py b/biweeklybudget/tests/unit/models/test_ofx_transaction.py index 8a64237c..d6d43161 100644 --- a/biweeklybudget/tests/unit/models/test_ofx_transaction.py +++ b/biweeklybudget/tests/unit/models/test_ofx_transaction.py @@ -160,7 +160,22 @@ def test_unreconciled(self): cutoff = datetime(2017, 3, 17, 0, 0, 0, tzinfo=UTC) expected2 = OFXTransaction.date_posted.__ge__(cutoff) expected3 = OFXTransaction.account.has(reconcile_trans=True) - assert len(kall[1]) == 3 + assert len(kall[1]) == 8 assert str(expected1) == str(kall[1][0]) assert binexp_to_dict(expected2) == binexp_to_dict(kall[1][1]) assert str(kall[1][2]) == str(expected3) + assert str( + OFXTransaction.is_payment.__ne__(True) + ) == str(kall[1][3]) + assert str( + OFXTransaction.is_late_fee.__ne__(True) + ) == str(kall[1][4]) + assert str( + OFXTransaction.is_interest_charge.__ne__(True) + ) == str(kall[1][5]) + assert str( + OFXTransaction.is_other_fee.__ne__(True) + ) == str(kall[1][6]) + assert str( + OFXTransaction.is_interest_payment.__ne__(True) + ) == str(kall[1][7]) diff --git a/docs/source/jsdoc.creditPayoffErrorModal.rst b/docs/source/jsdoc.creditPayoffErrorModal.rst new file mode 100644 index 00000000..2e87abfb --- /dev/null +++ b/docs/source/jsdoc.creditPayoffErrorModal.rst @@ -0,0 +1,26 @@ +jsdoc.creditPayoffErrorModal +============================ + +File: ``biweeklybudget/flaskapp/static/js/creditPayoffErrorModal.js`` + +.. js:function:: creditPayoffErrorModal(acct_id) + + Trigger Ajax to get account OFX statement information. Ajax callback to + create the form and display the modal is :js:func:`creditPayoffErrorModalForm`. + + :param number acct_id: the ID of the Account to show data for. + + + + +.. js:function:: creditPayoffErrorModalForm(data) + + Generate the HTML for the form on the Credit Payoff Error Modal and show the + modal. This is an Ajax callback triggered by a request to + ``/ajax/account_ofx_ajax/`` in :js:func:`creditPayoffErrorModal`. + The response data is generated by :py:class:`~.AccountOfxAjax`. + + + + + diff --git a/docs/source/jsdoc.reconcile.rst b/docs/source/jsdoc.reconcile.rst index ce0045a3..099f177f 100644 --- a/docs/source/jsdoc.reconcile.rst +++ b/docs/source/jsdoc.reconcile.rst @@ -13,6 +13,30 @@ File: ``biweeklybudget/flaskapp/static/js/reconcile.js`` +.. js:function:: ignoreOfxTrans(acct_id, fitid) + + Show the modal for reconciling an OFXTransaction without a matching + Transaction. Calls :js:func:`ignoreOfxTransDivForm` to generate the modal form + div content. Uses an inline function to handle the save action, which calls + :js:func:`reconcileOfxNoTrans` to perform the reconcile action. + + :param number acct_id: the Account ID of the OFXTransaction + :param string fitid: the FitID of the OFXTransaction + + + + +.. js:function:: ignoreOfxTransDivForm(acct_id, fitid) + + Generate the modal form div content for the modal to reconcile a Transaction + without a matching OFXTransaction. Called by :js:func:`transNoOfx`. + + :param number acct_id: the Account ID of the OFXTransaction + :param string fitid: the FitID of the OFXTransaction + + + + .. js:function:: makeTransFromOfx(acct_id, fitid) Link function to create a Transaction from a specified OFXTransaction, @@ -61,6 +85,18 @@ File: ``biweeklybudget/flaskapp/static/js/reconcile.js`` +.. js:function:: reconcileDoUnreconcileNoTrans(acct_id, fitid) + + Unreconcile a reconciled NoTrans OFXTransaction. This removes + ``acct_id + "%" + fitid`` from the ``ofxIgnored`` variable and regenerates + the OFXTransaction's div. + + :param number acct_id: the Account ID of the OFXTransaction + :param string fitid: the FitID of the OFXTransaction + + + + .. js:function:: reconcileGetOFX() Show unreconciled OFX transactions in the proper div. Empty the div, then @@ -101,6 +137,15 @@ File: ``biweeklybudget/flaskapp/static/js/reconcile.js`` +.. js:function:: reconcileOfxNoTrans(acct_id, fitid, note) + + Reconcile an OFXTransaction without a matching Transaction. Called from + the Save button handler in :js:func:`ignoreOfxTrans`. + + + + + .. js:function:: reconcileShowOFX(data) Ajax callback handler for :js:func:`reconcileGetOFX`. Display the @@ -129,7 +174,8 @@ File: ``biweeklybudget/flaskapp/static/js/reconcile.js`` .. js:function:: reconcileTransDiv(trans) Generate a div for an individual Transaction, to display on the reconcile - view. + view. Called from :js:func:`reconcileShowTransactions`, + :js:func:`makeTransSaveCallback` and :js:func:`updateReconcileTrans`. :param Object trans: ajax JSON object representing one Transaction diff --git a/docs/source/jsdoc.rst b/docs/source/jsdoc.rst index 49a4b71c..79fdf9db 100644 --- a/docs/source/jsdoc.rst +++ b/docs/source/jsdoc.rst @@ -11,6 +11,7 @@ Files jsdoc.bom_items_modal jsdoc.budget_transfer_modal jsdoc.budgets_modal + jsdoc.creditPayoffErrorModal jsdoc.credit_payoffs jsdoc.custom jsdoc.formBuilder