Merge pull request #182 from /issues/105

Fixes #105 and many other fixes

.gitignore

@@ -99,4 +99,5 @@ bin/*
 .ropeproject
 
 .idea
-/.release_position.json
+/.release_position.json
+.pytest_cache

.travis.yml

@@ -12,31 +12,30 @@ addons:
       - google-chrome-stable
 
 before_install:
-  - mysql -u root -h 127.0.0.1 -e 'CREATE DATABASE budgettest;'
-  - mysql -u root -h 127.0.0.1 -e 'CREATE DATABASE budgettest27;'
-  - mysql -u root -h 127.0.0.1 -e 'CREATE DATABASE budgettest36;'
+  - mysql -u root -h 127.0.0.1 -e 'CREATE DATABASE budgettest; CREATE DATABASE budgettest27; CREATE DATABASE budgettest36; CREATE DATABASE alembicLeft; CREATE DATABASE alembicRight;'
 
 env:
   global:
-    - DB_CONNSTRING='mysql+pymysql://root@127.0.0.1:3306/budgettest?charset=utf8mb4'
     - SETTINGS_MODULE=biweeklybudget.tests.fixtures.test_settings
     - secure: "unGeawllwXvQdk1vT3kX09yrOh6as3K+4tOurIAlFA8K3osEAurs35fDMptFb4XYl/1eK05UAYzhvtMmdbjflxAfZCuwGH457LSOkm2/YcuaLI7tMzeu0tWSuOMco6nJyZCvsL1Bx2tyuQqI43uzHp5HcgQBSuVwb+wdfKR5m21oQTrWfVr11hZ6YiYzKLlyEQ3l5mXbbTPbrNPZF+MBR5u1qjn6QNcQgMk4e+1QvMXqjF/OIJfV7f2TzmsHmJvGIT9AR5ia9UKeR3pXoFScHUZVdlJBieAPYhS+cmq5s0N015xWwi8jhafenmA8jFSsw1aM4poot4dZzn42L+ItL5weyZZNkpCUIfkvwEKHfBgU8T/Pc7oQDd9U3aA/oO1M0M2Hc56AA1ic0iH9MLq2CUIAghUcXt2RkmgUmOv93HJJGQACdYEwIumWl7OWrlE8/mi56JoKgCPyWYUp4vTCcTH3fHmky4UVhVuO1l4hxotUl2xYsJ1Ugg50U2/ZBRKQ3lcpLXd+Oe12EC30KfhJ6rfO+LQDaIL1sFwfiSTsSf4LtljKiRGp4eLfLBQ5O8KhOIBQZoiP1iFHZWFWUjDIR9UZUib39A1lprcHVlCku9bD1PbMf51cET6UCsf1m7B7Qaq7sEMsGY71fFGq1UUwcursi8PNJmvmbKh+SvHjNVE="
     - secure: "sUpWjFDwysYIkjFyIg7XanJwlZV/y9+XvoAfWa0Q14zj1L+I2/t714P3RnB9+z8Ur7Y3ZP5gzVo0kIBeq+9yXJMd1noUUxAKYQ+KoLvN3ojyBG7Ezf/oMv7OIeqKq7GXaZUND54/lrN4FCS8YXTzgjiyA0GiCsB6Qfo51KJscObK4UOcKP6dJVnovEXPlvqWj8mnSfVZ7rKMIDNqP9O2cPQ/QLxQjCILT3461zZaklg6+xek/8npvzEaOrgHd6AiERB7je1nGBsnkvJq28kBEoCjeCVJuP2BW1a/ToutixnxO2YjJSjyAFsh5JOlb3l/Pu2KqSzXPv807cbqK5SNyAXD9GEroVOf66yZXraMnVu19H1BAi7hgMhOyPMj49tXtQr51lu4cUtwlBc3NBun0fYatd0ZdDD9rLZ+DWGmkOMzZQW7b9T5+dwFM79LzdqZqpOXcSdYC/vbXkwc2MXu04dO2e2RLgK7O4HUpgbi69Oe9AzXsU5vXwxGEGDiPaovIBaSYhxi/pgwJgsjB+HNQeuiRfYrCr9gbS75zg03GfRD75AXlCzm2S17u0VMqVXSELhUiCd3YnwCbP5ZR/nJahpW/c8Kca2Z7bfRECEctkIrpWg2oSDX5TQryD6nVMfzgYdnw6RErJN4vC0Ykk1ltiVwGEyKJPKpOifXVABt0Vo="
 
 matrix:
   include:
     - python: "2.7"
-      env: TOXENV=py27
+      env: TOXENV=py27 DB_CONNSTRING='mysql+pymysql://root@127.0.0.1:3306/budgettest?charset=utf8mb4'
     - python: "3.4"
-      env: TOXENV=py34
+      env: TOXENV=py34 DB_CONNSTRING='mysql+pymysql://root@127.0.0.1:3306/budgettest?charset=utf8mb4'
     - python: "3.5"
-      env: TOXENV=py35
+      env: TOXENV=py35 DB_CONNSTRING='mysql+pymysql://root@127.0.0.1:3306/budgettest?charset=utf8mb4'
     - python: "3.6"
-      env: TOXENV=py36
+      env: TOXENV=py36 DB_CONNSTRING='mysql+pymysql://root@127.0.0.1:3306/budgettest?charset=utf8mb4'
     - python: "2.7"
       env: TOXENV=acceptance27 DB_CONNSTRING='mysql+pymysql://root@127.0.0.1:3306/budgettest27?charset=utf8mb4'
     - python: "3.6"
       env: TOXENV=acceptance36 DB_CONNSTRING='mysql+pymysql://root@127.0.0.1:3306/budgettest36?charset=utf8mb4'
+    - python: "3.6"
+      env: TOXENV=migrations MYSQL_USER='root' MYSQL_HOST='127.0.0.1' MYSQL_DBNAME_LEFT='alembicLeft' MYSQL_DBNAME_RIGHT='alembicRight' DB_CONNSTRING='mysql+pymysql://root@127.0.0.1:3306/alembicLeft?charset=utf8mb4'
     - python: "3.6"
       env: TOXENV=docs
 

CHANGES.rst

@@ -5,6 +5,31 @@ Unreleased Changes
 ------------------
 
 * Fix major logic error in Credit Card Payoff calculations; interest fees were ignored for the current month/statement, resulting in "Next Payment" values significantly lower than they should be. Fixed to use the last Interest Charge retrieved via OFX (or, if no interest charges present in OFX statements, prompt users to manually enter the last Interest Charge via a new modal that will create an OFXTransaction for it) as the interest amount on the first month/statement when calculating payoffs. This fix now returns Next Payment values that aren't identical to sample cards, but are significantly closer (within 1-2%).
+* `Issue #105 <https://github.com/jantman/biweeklybudget/issues/105>`_ - Major refactor to the Transaction database model. This is transparent to users, but causes massive database and code changes. This is the first step in supporting Transaction splits between multiple budgets:
+
+  * A new BudgetTransaction model has been added, which will support a one-to-many association between Transactions and Budgets. This model associates a Transaction with a Budget, and a currency amount counted against that Budget. This first step only supports a one-to-one relationship, but a forthcoming change will implement the one-to-many budget split for Transactions.
+  * The database migration for this creates BudgetTransactions for every current Transaction, migrating data to the new format.
+  * The ``budget_id`` attribute and ``budget`` relationship of the Transaction model has been removed, as that information is now in the related BudgetTransactions.
+  * A new ``planned_budget_id`` attribute (and ``planned_budget`` relationship) has been added to the Transaction model. For Transactions that were created from ScheduledTransactions, this attribute/relationship stores the original planned budget (distinct from the actual budget now stored in BudgetTransactions).
+  * The Transaction model now has a ``budget_transactions`` back-populated property, containing a list of all associated BudgetTransactions.
+  * The Transaction model now has a ``set_budget_amounts()`` method which takes a single dict mapping either integer Budget IDs or Budget objects, to the Decimal amount of the Transaction allocated to that Budget. While the underlying API supports an arbitrary number of budgets, the UI and codebase currently only supports one.
+  * The Transaction constructor now accepts a ``budget_amounts`` keyword argument that passes its value through to ``set_budget_amounts()``, for ease of creating Transactions in one call.
+  * ``Transaction.actual_amount`` is no longer an attribute stored in the database, but now a hybrid property (read-only) generated from the sum of amounts of all related BudgetTransactions.
+  * Add support to serialize property values of models, in addition to attributes.
+
+* Relatively major and sweeping code refactors to support the above.
+* Switch tests from using deprecated pytest-capturelog to using pytest built-in log capturing.
+* Miscellaneous fixes to unit and acceptance tests, and docs build.
+* Finish converting *all* code, including tests and sample data, from using floats to Decimals.
+
+  * For purposes of testing this, add a database event handler that throws an exception if any attributes that should be set to a Decimal, are set to a float.
+
+* Acceptance test fix so that pytest-selenium can take full page screenshots with Chromedriver.
+* `PR #180 <https://github.com/jantman/biweeklybudget/pull/180>`_ - Acceptance test fix so that the testflask LiveServer fixture captures server logs, and includes them in test HTML reports (this generates a temporary file per-test-run outside of pytest's control).
+* Fix bug found where simultaneously editing the Amount and Budget of an existing Transaction against a Standing Budget would result in incorrect changes to the balances of the Budgets.
+* Add a new ``migrations`` tox environment that automatically tests all database migrations (forward and reverse) and also validates that the database schema created from the migrations matches the one created from the models.
+* Add support for writing tests of data manipulation during database migrations, and write tests for the migration in for Issue 105, above.
+* Add support for ``BIWEEKLYBUDGET_LOG_FILE`` environment variable to cause Flask application logs to go to a file *in addition to* STDOUT.
 
 0.7.1 (2018-01-10)
 ------------------

bin/db_tester.py

@@ -4,6 +4,7 @@
 import argparse
 import logging
 import atexit
+from decimal import Decimal
 
 from sqlalchemy.orm.exc import NoResultFound
 
@@ -23,7 +24,7 @@ def main():
     atexit.register(cleanup_db)
     init_db()
     logger.debug('Database initialized')
-    print("DO STUFF HERE")
+    print("DO STUFF HERE; database session is 'db_session' global")
 
 if __name__ == "__main__":
     main()

bin/t

@@ -202,6 +202,13 @@ class ToxIt(object):
             mod = self._find_test_mod_for_module(module_name, True)
             logger.debug('Discovered acceptnace test module; env=%s mod=%s',
                      env, mod)
+        else:
+            env = script_name
+            mod = module_name
+            logger.debug(
+                'Directly-specified environment name; env=%s mod=%s',
+                env, mod
+            )
         # run it
         env_config = self.env_config[env]
         for cmd in env_config['commands']:
@@ -236,6 +243,8 @@ def parse_args(argv):
     p.add_argument('-v', '--verbose', dest='verbose', action='store_true',
                    default=False,
                    help='verbose output')
+    p.add_argument('-e', '--env-name', dest='envname', action='store',
+                   type=str, default=None, help='tox environment name')
     p.add_argument('MODULE', type=str,
                    help='Module to run; can be either a code module or a '
                         'test module.')
@@ -249,5 +258,7 @@ if __name__ == "__main__":
     if args.verbose:
         logger.setLevel(logging.DEBUG)
     script_name = sys.argv[0].split('/')[-1]
+    if args.envname is not None:
+        script_name = args.envname
     script = ToxIt()
     script.run_one_module(script_name, args.MODULE, args.EXPR)

biweeklybudget/alembic/versions/04e61490804b_add_budgettransaction_model.py

@@ -0,0 +1,189 @@
+"""add BudgetTransaction model
+
+Revision ID: 04e61490804b
+Revises: 6d37400ea9cd
+Create Date: 2018-01-12 18:43:14.983133
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker, relationship
+from sqlalchemy import Column, Integer, ForeignKey, Numeric
+import logging
+
+logger = logging.getLogger(__name__)
+
+Session = sessionmaker()
+
+Base = declarative_base()
+
+# revision identifiers, used by Alembic.
+revision = '04e61490804b'
+down_revision = '6d37400ea9cd'
+branch_labels = None
+depends_on = None
+
+
+class Transaction(Base):
+
+    __tablename__ = 'transactions'
+    __table_args__ = (
+        {'mysql_engine': 'InnoDB'}
+    )
+
+    #: Primary Key
+    id = Column(Integer, primary_key=True)
+
+    #: Actual amount of the transaction
+    actual_amount = Column(Numeric(precision=10, scale=4), nullable=False)
+
+    #: Budgeted amount of the transaction, if it was budgeted ahead of time
+    #: via a :py:class:`~.ScheduledTransaction`.
+    budgeted_amount = Column(Numeric(precision=10, scale=4))
+
+    #: ID of the Budget this transaction is against
+    budget_id = Column(Integer, ForeignKey('budgets.id'))
+
+    #: ID of the Budget this transaction was planned to be funded by, if it
+    #: was planned ahead via a :py:class:`~.ScheduledTransaction`
+    planned_budget_id = Column(Integer, ForeignKey('budgets.id'))
+
+    def __repr__(self):
+        return "<Transaction(id=%s)>" % (
+            self.id
+        )
+
+
+class BudgetTransaction(Base):
+    """
+    Represents the portion (amount) of a Transaction that is allocated
+    against a specific budget. There will be one or more BudgetTransactions
+    associated with each :py:class:`~.Transaction`.
+    """
+
+    __tablename__ = 'budget_transactions'
+    __table_args__ = (
+        {'mysql_engine': 'InnoDB'}
+    )
+
+    #: Primary Key
+    id = Column(Integer, primary_key=True)
+
+    #: Amount of the transaction against this budget
+    amount = Column(Numeric(precision=10, scale=4), nullable=False)
+
+    #: ID of the Transaction this is part of
+    trans_id = Column(Integer, ForeignKey('transactions.id'))
+
+    #: Relationship - the :py:class:`~.Transaction` this is part of
+    transaction = relationship(
+        "Transaction", backref="budget_transactions", uselist=False
+    )
+
+    #: ID of the Budget this transaction is against
+    budget_id = Column(Integer, ForeignKey('budgets.id'))
+
+    def __repr__(self):
+        return "<BudgetTransaction(id=%s)>" % (
+            self.id
+        )
+
+
+class Budget(Base):
+
+    __tablename__ = 'budgets'
+    __table_args__ = (
+        {'mysql_engine': 'InnoDB'}
+    )
+
+    #: Primary Key
+    id = Column(Integer, primary_key=True)
+
+
+def upgrade():
+    op.create_table(
+        'budget_transactions',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('amount', sa.Numeric(precision=10, scale=4), nullable=False),
+        sa.Column('trans_id', sa.Integer(), nullable=True),
+        sa.Column('budget_id', sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ['budget_id'], ['budgets.id'],
+            name=op.f('fk_budget_transactions_budget_id_budgets')
+        ),
+        sa.ForeignKeyConstraint(
+            ['trans_id'], ['transactions.id'],
+            name=op.f('fk_budget_transactions_trans_id_transactions')
+        ),
+        sa.PrimaryKeyConstraint('id', name=op.f('pk_budget_transactions')),
+        mysql_engine='InnoDB'
+    )
+    op.add_column(
+        'transactions',
+        sa.Column('planned_budget_id', sa.Integer(), nullable=True)
+    )
+    op.create_foreign_key(
+        op.f('fk_transactions_planned_budget_id_budgets'),
+        'transactions', 'budgets', ['planned_budget_id'], ['id']
+    )
+    # Copy amount and budget_id from each existing Transaction to a
+    # corresponding BudgetTransaction
+    bind = op.get_bind()
+    session = Session(bind=bind)
+    for txn in session.query(Transaction).all():
+        b = BudgetTransaction(
+            amount=txn.actual_amount,
+            trans_id=txn.id,
+            budget_id=txn.budget_id
+        )
+        session.add(b)
+        if txn.budgeted_amount is not None:
+            txn.planned_budget_id = txn.budget_id
+            session.add(txn)
+    session.commit()
+    op.drop_constraint(
+        'fk_transactions_budget_id_budgets', 'transactions', type_='foreignkey'
+    )
+    op.drop_column('transactions', 'budget_id')
+    op.drop_column('transactions', 'actual_amount')
+
+
+def downgrade():
+    op.add_column(
+        'transactions',
+        sa.Column(
+            'budget_id',
+            sa.Integer(),
+            autoincrement=False,
+            nullable=True
+        )
+    )
+    op.create_foreign_key(
+        'fk_transactions_budget_id_budgets',
+        'transactions', 'budgets', ['budget_id'], ['id']
+    )
+    op.add_column(
+        'transactions',
+        sa.Column(
+            'actual_amount',
+            Numeric(precision=10, scale=4),
+            nullable=False
+        )
+    )
+    # migrate budget_transactions back to Transaction.budget_id
+    bind = op.get_bind()
+    session = Session(bind=bind)
+    for txn in session.query(Transaction).all():
+        bt = txn.budget_transactions[0]
+        txn.budget_id = bt.budget_id
+        txn.actual_amount = bt.amount
+        session.add(txn)
+    session.commit()
+    op.drop_table('budget_transactions')
+    op.drop_constraint(
+        op.f('fk_transactions_planned_budget_id_budgets'),
+        'transactions',
+        type_='foreignkey'
+    )
+    op.drop_column('transactions', 'planned_budget_id')

biweeklybudget/alembic/versions/8627a80874a6_add_vehicle_and_fuellog_models.py

@@ -17,7 +17,6 @@
 
 
 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.create_table(
         'vehicles',
         sa.Column('id', sa.Integer(), nullable=False),
@@ -71,11 +70,8 @@ def upgrade():
         sa.PrimaryKeyConstraint('id', name=op.f('pk_fuellog')),
         mysql_engine='InnoDB'
     )
-    # ### end Alembic commands ###
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('vehicles')
     op.drop_table('fuellog')
-    # ### end Alembic commands ###
+    op.drop_table('vehicles')

biweeklybudget/alembic/versions/ceb73ddf66eb_add_project_and_bomitem_models.py

@@ -48,5 +48,5 @@ def upgrade():
 
 
 def downgrade():
-    op.drop_table('projects')
     op.drop_table('bom_items')
+    op.drop_table('projects')