Merge pull request #186 from /issues/183

Fixes #183 - UI to ignore OFXTransactions, and ignore is_ OFXTransactions

.travis.yml

@@ -46,7 +46,7 @@ install:
 - pip install tox codecov boto3
 - pip freeze
 - virtualenv --version
-- wget -N http://chromedriver.storage.googleapis.com/2.33/chromedriver_linux64.zip -P ~/
+- wget -N http://chromedriver.storage.googleapis.com/2.36/chromedriver_linux64.zip -P ~/
 - unzip ~/chromedriver_linux64.zip -d ~/
 - rm ~/chromedriver_linux64.zip
 - chmod +x ~/chromedriver

CHANGES.rst

@@ -39,6 +39,18 @@ Unreleased Changes
   * When running under Docker/Gunicorn, append the decimal number of seconds taken to serve the request to the Gunicorn access log.
 
 * `Issue #184 <https://github.com/jantman/biweeklybudget/issues/184>`_ - Redact database password from ``/help`` view, and change ``/help`` view to show Version containing git commit hash for pre-release/development Docker builds.
+* `Issue #183 <https://github.com/jantman/biweeklybudget/issues/183>`_
+
+  * Add UI link to ignore reconciling an OFXTransaction if there will not be a matching Transaction.
+  * Remove default values for the ``Account`` model's ``re_`` fields in preparation for actually using them.
+  * Replace the ``Account`` model's ``re_fee`` field with separate ``re_late_fee`` and ``re_other_fee`` fields.
+  * Add UI support for specifying Interest Charge, Interest Paid, Payment, Late Fee, and Other Fee regexes on each account.
+  * Add DB event handler on new or changed OFXTransaction, to set ``is_*`` fields according to Account ``re_*`` fields.
+  * Add DB event handler on change to Account model ``re_*`` fields, that triggers ``OFXTransaction.update_is_fields()`` to recalculate using the new regex.
+  * Change ``OFXTransaction.unreconciled`` to filter out OFXTransactions with any of the ``is_*`` set to True.
+
+* Upgrade chromedriver in TravisCI builds from 2.33 to 2.36, to fix failing acceptance tests caused by Ubuntu upgrade from Chrome 64 to 65.
+
 
 0.7.1 (2018-01-10)
 ------------------

biweeklybudget/alembic/versions/073142f641b3_account_remove_re_field_defaults.py

@@ -0,0 +1,84 @@
+"""Account remove re field defaults
+
+Revision ID: 073142f641b3
+Revises: 08b6358a04bf
+Create Date: 2018-03-08 09:25:17.673039
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import mysql
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy import Column, Integer, String
+
+Session = sessionmaker()
+
+Base = declarative_base()
+
+# revision identifiers, used by Alembic.
+revision = '073142f641b3'
+down_revision = '08b6358a04bf'
+branch_labels = None
+depends_on = None
+
+
+class Account(Base):
+
+    __tablename__ = 'accounts'
+    __table_args__ = (
+        {'mysql_engine': 'InnoDB'}
+    )
+
+    #: Primary Key
+    id = Column(Integer, primary_key=True)
+
+    #: regex for matching transactions as interest charges
+    re_interest_charge = Column(String(254))
+
+    #: regex for matching transactions as interest paid
+    re_interest_paid = Column(String(254))
+
+    #: regex for matching transactions as payments
+    re_payment = Column(String(254))
+
+    #: regex for matching transactions as late fees
+    re_fee = Column(String(254))
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+    for acct in session.query(Account).all():
+        acct.re_interest_charge = None
+        acct.re_interest_paid = None
+        acct.re_payment = None
+        acct.re_fee = None
+    session.commit()
+    op.add_column(
+        'accounts',
+        sa.Column('re_late_fee', sa.String(length=254), nullable=True)
+    )
+    op.add_column(
+        'accounts',
+        sa.Column('re_other_fee', sa.String(length=254), nullable=True)
+    )
+    op.drop_column('accounts', 're_fee')
+
+
+def downgrade():
+    op.add_column(
+        'accounts',
+        sa.Column('re_fee', mysql.VARCHAR(length=254), nullable=True)
+    )
+    op.drop_column('accounts', 're_other_fee')
+    op.drop_column('accounts', 're_late_fee')
+    bind = op.get_bind()
+    session = Session(bind=bind)
+    for acct in session.query(Account).all():
+        acct.re_interest_charge = '^(interest charge|purchase finance charge)'
+        acct.re_interest_paid = '^interest paid'
+        acct.re_payment = '^(online payment|' \
+                          'internet payment|online pymt|payment)'
+        acct.re_fee = '^(late fee|past due fee)'
+    session.commit()

biweeklybudget/alembic/versions/08b6358a04bf_txnreconcile_allow_txn_id_to_be_null.py

@@ -0,0 +1,34 @@
+"""TxnReconcile allow txn_id to be null
+
+Revision ID: 08b6358a04bf
+Revises: 04e61490804b
+Create Date: 2018-03-07 19:48:06.050926
+
+"""
+from alembic import op
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = '08b6358a04bf'
+down_revision = '04e61490804b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.alter_column(
+        'txn_reconciles', 'txn_id',
+        existing_type=mysql.INTEGER(display_width=11),
+        nullable=True
+    )
+
+
+def downgrade():
+    conn = op.get_bind()
+    conn.execute("SET FOREIGN_KEY_CHECKS=0")
+    op.alter_column(
+        'txn_reconciles', 'txn_id',
+        existing_type=mysql.INTEGER(display_width=11),
+        nullable=False
+    )
+    conn.execute("SET FOREIGN_KEY_CHECKS=1")

biweeklybudget/db_event_handlers.py

@@ -174,6 +174,83 @@ def handle_new_or_deleted_budget_transaction(session):
     )
 
 
+def handle_ofx_transaction_new_or_change(session):
+    """
+    ``before_flush`` event handler
+    (:py:meth:`sqlalchemy.orm.events.SessionEvents.before_flush`)
+    on the DB session, to handle setting the ``is_*`` fields on new or changed
+    OFXTransaction instances according to its Account.
+
+    :param session: current database session
+    :type session: sqlalchemy.orm.session.Session
+    """
+    for obj in session.new:
+        if isinstance(obj, OFXTransaction):
+            try:
+                obj.update_is_fields()
+            except Exception:
+                logger.error('Error setting OFXTransaction is_ fields',
+                             exc_info=True)
+    for obj in session.dirty:
+        if isinstance(obj, OFXTransaction):
+            try:
+                obj.update_is_fields()
+            except Exception:
+                logger.error('Error setting OFXTransaction is_ fields',
+                             exc_info=True)
+
+
+def handle_account_re_change(session):
+    """
+    Handler for change of one of:
+
+    * :py:attr:`~.Account.re_interest_paid`
+    * :py:attr:`~.Account.re_interest_charge`
+    * :py:attr:`~.Account.re_late_fee`
+    * :py:attr:`~.Account.re_other_fee`
+    * :py:attr:`~.Account.re_payment`
+
+    When one of these regexes is changed on an Account, we trigger a re-run
+    of :py:meth:`~.OFXTransaction.update_is_fields` on all OFXTransactions for
+    the account.
+
+    :param session: current database session
+    :type session: sqlalchemy.orm.session.Session
+    """
+    attrs = [
+        're_interest_paid',
+        're_interest_charge',
+        're_late_fee',
+        're_other_fee',
+        're_payment'
+    ]
+    for obj in session.dirty:
+        if not isinstance(obj, Account):
+            continue
+        changed = []
+        insp = inspect(obj)
+        for attr in attrs:
+            hx = getattr(insp.attrs, attr).history
+            if hx is None or hx.added is None or hx.deleted is None:
+                continue
+            if len(hx.added) > 0 and len(hx.deleted) > 0:
+                logger.debug(
+                    '%s %s changed from %s to %s',
+                    obj, attr, hx.deleted, hx.added
+                )
+                changed.append(attr)
+        if len(changed) < 1:
+            continue
+        logger.debug(
+            '%s has regex changes; triggering update_is_fields() on all child '
+            'OFXTransactions.', obj
+        )
+        for stmt in obj.all_statements:
+            for txn in stmt.ofx_trans:
+                txn.update_is_fields()
+        logger.debug('Done with update_is_fields() for %s', obj)
+
+
 def validate_decimal_or_none(target, value, oldvalue, initiator):
     if isinstance(value, Decimal) or value is None:
         return
@@ -234,6 +311,8 @@ def handle_before_flush(session, flush_context, instances):
     """
     logger.debug('handle_before_flush handler')
     handle_new_or_deleted_budget_transaction(session)
+    handle_ofx_transaction_new_or_change(session)
+    handle_account_re_change(session)
     logger.debug('handle_before_flush done')
 
 

biweeklybudget/flaskapp/static/js/accounts_modal.js

@@ -84,6 +84,11 @@ function accountModalDivForm() {
         )
         .addCheckbox('account_frm_negate_ofx', 'negate_ofx_amounts', 'Negate OFX Amounts', false)
         .addCheckbox('account_frm_reconcile_trans', 'reconcile_trans', 'Reconcile Transactions?', true)
+        .addText('account_frm_re_interest_charge', 're_interest_charge', 'Interest Charge Regex', { helpBlock: 'If specified, OFX Transactions with name/memo matching this regex will be marked as interest charges (and not reconciled).'})
+        .addText('account_frm_re_interest_paid', 're_interest_paid', 'Interest Paid Regex', { helpBlock: 'If specified, OFX Transactions with name/memo matching this regex will be marked as interest payments (and not reconciled).'})
+        .addText('account_frm_re_payment', 're_payment', 'Payment Regex', { helpBlock: 'If specified, OFX Transactions with name/memo matching this regex will be marked as payments (and not reconciled).'})
+        .addText('account_frm_re_late_fee', 're_late_fee', 'Late Fee Regex', { helpBlock: 'If specified, OFX Transactions with name/memo matching this regex will be marked as late fees (and not reconciled).'})
+        .addText('account_frm_re_other_fee', 're_other_fee', 'Other Fee Regex', { helpBlock: 'If specified, OFX Transactions with name/memo matching this regex will be marked as other fees (and not reconciled).'})
         .addCurrency('account_frm_credit_limit', 'credit_limit', 'Credit Limit')
         .addText('account_frm_apr', 'apr', 'APR', { helpBlock: 'If you know the margin added to the Prime Rate for this card, use the Margin field instead.'})
         .addText('account_frm_margin', 'prime_rate_margin', 'Margin', { helpBlock: 'If known, the margin added to the US Prime Rate to determine the APR.'})
@@ -153,6 +158,11 @@ function accountModalDivFillAndShow(msg) {
         $('#account_frm_reconcile_trans').prop('checked', false);
     }
     $('#account_frm_vault_creds_path').val(msg['vault_creds_path']);
+    if(msg['re_interest_charge'] != null) { $('#account_frm_re_interest_charge').val(msg['re_interest_charge']); }
+    if(msg['re_interest_paid'] != null) { $('#account_frm_re_interest_paid').val(msg['re_interest_paid']); }
+    if(msg['re_payment'] != null) { $('#account_frm_re_payment').val(msg['re_payment']); }
+    if(msg['re_late_fee'] != null) { $('#account_frm_re_late_fee').val(msg['re_late_fee']); }
+    if(msg['re_other_fee'] != null) { $('#account_frm_re_other_fee').val(msg['re_other_fee']); }
     $("#modalDiv").modal('show');
 }
 

biweeklybudget/flaskapp/static/js/reconcile.js

@@ -245,7 +245,10 @@ function reconcileOfxDiv(trans) {
   div += '</div>';
   div += '<div class="row"><div class="col-lg-12">';
   div += '<div style="float: left;"><a href="javascript:ofxTransModal(' + trans['account_id'] + ', \'' + trans['fitid'] + '\', false)">' + trans['fitid'] + '</a>: ' + trans['name'] + '</div>';
-  div += '<div style="float: right;" class="make-trans-link"><a href="javascript:makeTransFromOfx(' + trans['account_id'] + ', \'' + trans['fitid'] + '\')" title="Create Transaction from this OFX">(make trans)</a></div>';
+  div += '<div style="float: right;" class="make-trans-link">';
+  div += '<a href="javascript:makeTransFromOfx(' + trans['account_id'] + ', \'' + trans['fitid'] + '\')" title="Create Transaction from this OFX">(make trans)</a>';
+  div += '<a href="javascript:ignoreOfxTrans(' + trans['account_id'] + ', \'' + trans['fitid'] + '\')" title="Ignore this OFX Transaction">(ignore)</a>';
+  div += '</div>';
   div += '</div></div>';
   div += '</div>\n';
   return div;
@@ -268,7 +271,7 @@ function clean_fitid(fitid) {
  */
 function reconcileHandleSubmit() {
   $('body').find('#reconcile-msg').remove();
-  if (jQuery.isEmptyObject(reconciled)) {
+  if (jQuery.isEmptyObject(reconciled) && jQuery.isEmptyObject(ofxIgnored)) {
     var container = $('#notifications-row').find('.col-lg-12');
     var newdiv = $(
       '<div class="alert alert-warning" id="reconcile-msg">' +
@@ -281,7 +284,7 @@ function reconcileHandleSubmit() {
   $.ajax({
         type: "POST",
         url: '/ajax/reconcile',
-        data: JSON.stringify(reconciled),
+        data: JSON.stringify({ reconciled: reconciled, ofxIgnored: ofxIgnored }),
         contentType: "application/json; charset=utf-8",
         dataType: "json",
         success: function(data) {
@@ -482,6 +485,98 @@ function reconcileDoUnreconcileNoOfx(trans_id) {
   $('#trans-' + trans_id).find('a:contains("(no OFX)")').show();
 }
 
+/**
+ * Show the modal for reconciling an OFXTransaction without a matching
+ * Transaction. Calls :js:func:`ignoreOfxTransDivForm` to generate the modal form
+ * div content. Uses an inline function to handle the save action, which calls
+ * :js:func:`reconcileOfxNoTrans` to perform the reconcile action.
+ *
+ * @param {number} acct_id - the Account ID of the OFXTransaction
+ * @param {string} fitid - the FitID of the OFXTransaction
+ */
+function ignoreOfxTrans(acct_id, fitid) {
+    console.log("ignoreOfxTrans(" + acct_id + ", '" + fitid + "')");
+    $('#modalBody').empty();
+    $('#modalBody').append(ignoreOfxTransDivForm(acct_id, fitid));
+    $('#modalLabel').text('Ignore OFXTransaction (' + acct_id + ', "' + fitid + '")');
+    $('#modalSaveButton').off();
+    $('#modalSaveButton').click(function() {
+        $('.has-error').each(function(index) { $(this).removeClass('has-error'); });
+        var note = $('#trans_frm_note').val().trim();
+        if (note == "") {
+            $('#trans_frm_note').parent().append('<p class="text-danger formfeedback">Note cannot be empty.</p>');
+            $('#trans_frm_note').parent().addClass('has-error');
+            return;
+        } else {
+            reconcileOfxNoTrans(acct_id, fitid, note);
+            $("#modalDiv").modal('hide');
+        }
+    }).show();
+    $("#modalDiv").modal('show');
+}
+
+/**
+ * Generate the modal form div content for the modal to reconcile a Transaction
+ * without a matching OFXTransaction. Called by :js:func:`transNoOfx`.
+ *
+ * @param {number} acct_id - the Account ID of the OFXTransaction
+ * @param {string} fitid - the FitID of the OFXTransaction
+ */
+function ignoreOfxTransDivForm(acct_id, fitid) {
+    return new FormBuilder('transForm')
+        .addP('Mark OFXTransaction as reconciled with no Transaction (i.e. OFXTransaction that we don\'t care about and won\t have a Transaction for).')
+        .addHidden('trans_frm_acct_id', 'acct_id', acct_id)
+        .addHidden('trans_frm_fitid', 'fitid', fitid)
+        .addText('trans_frm_note', 'note', 'Note')
+        .render();
+}
+
+/**
+ * Reconcile an OFXTransaction without a matching Transaction. Called from
+ * the Save button handler in :js:func:`ignoreOfxTrans`.
+ */
+function reconcileOfxNoTrans(acct_id, fitid, note) {
+  var target = $('#ofx-' + acct_id + '-' + clean_fitid(fitid));
+  var newdiv = $('<div class="row" id="ofx-' + acct_id + '-' + clean_fitid(fitid) + '-noTrans" style="" />').html('<div class="col-lg-12"><p><strong>No Trans:</strong> ' + note + '</p></div>');
+  var makeTransLink = $(target).find('.make-trans-link');
+  $(makeTransLink).html('<a href="javascript:reconcileDoUnreconcileNoTrans(' + acct_id + ', \'' + fitid + '\')">Unignore</a>');
+  $(newdiv).prependTo(target);
+  $(target).draggable('disable');
+  ofxIgnored[acct_id + "%" + fitid] = note;
+}
+
+/**
+ * Unreconcile a reconciled NoTrans OFXTransaction. This removes
+ * ``acct_id + "%" + fitid`` from the ``ofxIgnored`` variable and regenerates
+ * the OFXTransaction's div.
+ *
+ * @param {number} acct_id - the Account ID of the OFXTransaction
+ * @param {string} fitid - the FitID of the OFXTransaction
+ */
+function reconcileDoUnreconcileNoTrans(acct_id, fitid) {
+  // remove from the reconciled object
+  delete ofxIgnored[acct_id + "%" + fitid];
+  // remove the "No Trans:" div
+  $('#ofx-' + acct_id + '-' + clean_fitid(fitid) + '-noTrans').remove();
+  // restore the draggability
+  var target = $('#ofx-' + acct_id + '-' + clean_fitid(fitid));
+  $.ajax({
+      type: "GET",
+      url: "/ajax/ofx/" + acct_id + "/" + encodeURIComponent(fitid),
+      success: function (data) {
+          var newdiv = $(reconcileOfxDiv(data['txn']));
+          $(target).replaceWith(newdiv);
+          $(newdiv).draggable({
+              cursor: 'move',
+              // start: dragStart,
+              // containment: '#content-row',
+              revert: 'invalid',
+              helper: 'clone'
+          });
+      }
+  });
+}
+
 $(document).ready(function() {
   reconcileGetTransactions();
   reconcileGetOFX();