Merge branch 'release/v4.0'

.gitignore

@@ -43,3 +43,4 @@ output/*/index.html
 docs/_build
 /docs/doctrees/
 /docs/html/
+/docs/doctrees/

.travis.yml

@@ -2,27 +2,43 @@
 
 sudo: false
 language: python
-python: 2.7
+
 env:
-    - TOX_ENV=docs
-    - TOX_ENV=flake8
-    - TOX_ENV=py27 
-    - TOX_ENV=py33 
-    - TOX_ENV=py34 
+  global:
+    - PIP_WHEEL_DIR=$HOME/.wheels
+    - PIP_FIND_LINKS=file://$PIP_WHEEL_DIR
+
+matrix:
+  include:
+    - python: '2.7'
+      env: TOXENV=docs
+    - python: '2.7'
+      env: TOXENV=flake8
+    - python: '2.7'
+      env: TOXENV=py27
+    - python: '3.3'
+      env: TOXENV=py33
+    - python: '3.4'
+      env: TOXENV=py34
+    - python: '3.5'
+      env: TOXENV=py35
+
+cache:
+  directories:
+    - $HOME/.wheels
 
-# command to install dependencies, e.g. pip install -r requirements.txt 
+# command to install dependencies, e.g. pip install -r requirements.txt
 install:
-    - pip install -r tests/requirements.txt
-    - pip install -e .
-    - pip install tox
-    - pip install coveralls
+  - mkdir -p $PIP_WHEEL_DIR
+  - pip wheel -r tests/requirements.txt
+  - pip install -e .
+  - pip install tox coveralls
 
 script:
-    - tox -e $TOX_ENV
-    - py.test
+  - tox
 
 after_success:
-    - coveralls
+  - coveralls
 
 notifications:
   email:

AUTHORS.rst

@@ -10,7 +10,9 @@ Development Lead
 Contributors
 ------------
 
-None yet. Why not be the first? (since 2015-07-11, new format)
+* Ben Konrath (benkonrath)
+
+Why not join the club?
 
 A more up to date list can be found here: 
 https://github.com/WoLpH/mt940/graphs/contributors

README.rst

@@ -7,7 +7,7 @@ MT940
     :alt: MT940 test status
     :target: https://travis-ci.org/WoLpH/mt940
 
-.. image:: https://badge.fury.io/py/mt940.svg
+.. image:: https://badge.fury.io/py/mt-940.svg
     :alt: MT940 Pypi version
     :target: https://pypi.python.org/pypi/mt-940
 
@@ -43,13 +43,13 @@ To install the latest release:
 
 .. code-block:: bash
 
-    pip install mt940
+    pip install mt-940
 
 Or if `pip` is not available:
 
 .. code-block:: bash
 
-    easy_install mt940
+    easy_install mt-940
    
 To install the latest development release:
 
@@ -67,6 +67,8 @@ everything in all supported python versions.
 Usage
 -----
 
+Basic parsing:
+
 .. code-block:: python
 
    import mt940
@@ -83,6 +85,31 @@ Usage
        print 'Transaction: ', transaction
        pprint.pprint(transaction.data)
 
+Set opening / closing balance information on each transaction:
+
+.. code-block:: python
+
+   import mt940
+   import pprint
+
+   mt940.tags.BalanceBase.scope = mt940.models.Transaction
+
+   # The currency has to be set manually when setting the BalanceBase scope to Transaction.
+   transactions = mt940.models.Transactions(processors=dict(
+       pre_statement=[
+           mt940.processors.add_currency_pre_processor('EUR'),
+       ],
+   ))
+
+   with open('tests/jejik/abnamro.sta') as f:
+       data = f.read()
+
+   transactions.parse(data)
+
+   for transaction in transactions:
+       print 'Transaction: ', transaction
+       pprint.pprint(transaction.data)
+
 Contributing
 ------------
 

mt940/__about__.py

@@ -6,7 +6,7 @@
 statistics and manipulation.
 '''.strip()
 __email__ = 'wolph@wol.ph'
-__version__ = '3.2'
+__version__ = '4.0'
 __license__ = 'BSD'
 __copyright__ = 'Copyright 2015 Rick van Hattem (wolph)'
 __url__ = 'https://github.com/WoLpH/mt940'

mt940/models.py

@@ -134,7 +134,7 @@ class Transactions(collections.Sequence):
         post_final_opening_balance=[],
         pre_related_reference=[],
         post_related_reference=[],
-        pre_statement=[],
+        pre_statement=[processors.date_fixup_pre_processor],
         post_statement=[processors.date_cleanup_post_processor],
         pre_statement_number=[],
         post_statement_number=[],
@@ -167,6 +167,25 @@ def currency(self):
         if balance:
             return balance.amount.currency
 
+    @classmethod
+    def strip(cls, lines):
+        for line in lines:
+            # We don't like carriage returns in case of Windows files so let's
+            # just replace them with nothing
+            line = line.replace('\r', '')
+
+            # Strip trailing whitespace from lines since they cause incorrect
+            # files
+            line = line.rstrip()
+
+            # Skip separators
+            if line.strip() == '-':
+                continue
+
+            # Return actual lines
+            if line:
+                yield line
+
     def parse(self, data):
         '''Parses mt940 data, expects a string with data
 
@@ -175,9 +194,8 @@ def parse(self, data):
 
         Returns: :py:class:`list` of :py:class:`Transaction`
         '''
-        # We don't like carriage returns in case of Windows files so let's just
-        # replace them with nothing
-        data = data.replace('\r', '')
+        # Remove extraneous whitespace and such
+        data = '\n'.join(self.strip(data.split('\n')))
 
         # The pattern is a bit annoying to match by regex, even with a greedy
         # match it's difficult to get both the beginning and the end so we're
@@ -218,7 +236,13 @@ def parse(self, data):
             for processor in self.processors.get('post_%s' % tag.slug):
                 result = processor(self, tag, tag_dict, result)
 
-            if isinstance(tag, mt940.tags.Statement):
+            # Creating a new transaction for :20: and :61: tags allows the
+            # tags from :20: to :61: to be captured as part of the transaction.
+            if isinstance(tag, mt940.tags.TransactionReferenceNumber) or \
+                    isinstance(tag, mt940.tags.Statement):
+                # Transactions only get a Transaction Reference Code ID from a
+                # :61: tag which is why a new transaction is created if the
+                # 'id' has a value.
                 if transaction.data.get('id'):
                     transaction = Transaction(self, result)
                     self.transactions.append(transaction)
@@ -228,7 +252,7 @@ def parse(self, data):
                 # Combine multiple results together as one string, Rabobank has
                 # multiple :86: tags for a single transaction
                 for k, v in _compat.iteritems(result):
-                    if k in transaction.data:
+                    if k in transaction.data and hasattr(v, 'strip'):
                         transaction.data[k] += '\n%s' % v.strip()
                     else:
                         transaction.data[k] = v

mt940/processors.py

@@ -1,3 +1,6 @@
+import calendar
+
+
 def add_currency_pre_processor(currency, overwrite=True):
     def _add_currency_pre_processor(transactions, tag, tag_dict, *args):
         if 'currency' not in tag_dict or overwrite:  # pragma: no branch
@@ -8,6 +11,21 @@ def _add_currency_pre_processor(transactions, tag, tag_dict, *args):
     return _add_currency_pre_processor
 
 
+def date_fixup_pre_processor(transactions, tag, tag_dict, *args):
+    """
+    Replace illegal February 30 dates with the last day of February.
+
+    German banks use a variant of the 30/360 interest rate calculation,
+    where each month has always 30 days even February. Python's datetime
+    module won't accept such dates.
+    """
+    if tag_dict['day'] == '30' and tag_dict['month'] == '02':
+        year = int(tag_dict['year'], 10)
+        tag_dict['day'] = str(calendar.monthrange(year, 2)[1])
+        tag_dict['month'] = '02'
+    return tag_dict
+
+
 def date_cleanup_post_processor(transactions, tag, tag_dict, result):
     for k in ('day', 'month', 'year', 'entry_day', 'entry_month'):
         result.pop(k, None)

mt940/tags.py

@@ -52,13 +52,22 @@ class Tag(object):
     scope = models.Transactions
 
     def __init__(self):
-        self.re = re.compile(self.pattern, re.IGNORECASE | re.VERBOSE)
+        self.re = re.compile(self.pattern,
+                             re.IGNORECASE | re.VERBOSE | re.UNICODE)
 
     def parse(self, transactions, value):
-        logger.debug('matching (%d) %r against %r', len(value), value,
-                     self.pattern)
         match = self.re.match(value)
-        assert match is not None, 'Unable to parse %r from %r' % (self, value)
+        if match:  # pragma: no branch
+            self.logger.debug(
+                'matched (%d) "%s" against "%s", got: %r',
+                len(value), value, self.pattern, match.groupdict())
+        else:  # pragma: no cover
+            self.logger.info(
+                'matching (%d) "%s" against "%s"', len(value), value,
+                self.pattern)
+            raise RuntimeError(
+                'Unable to parse "%s" from "%s"' % (self, value),
+                self, value)
         return match.groupdict()
 
     def __call__(self, transactions, value):
@@ -69,6 +78,7 @@ def __new__(cls, *args, **kwargs):
 
         words = re.findall('([A-Z][a-z]+)', cls.__name__)
         cls.slug = '_'.join(w.lower() for w in words)
+        cls.logger = logger.getChild(cls.name)
 
         return object.__new__(cls, *args, **kwargs)
 
@@ -77,6 +87,7 @@ def __hash__(self):
 
 
 class TransactionReferenceNumber(Tag):
+
     '''Transaction reference number
 
     Pattern: 16x
@@ -86,6 +97,7 @@ class TransactionReferenceNumber(Tag):
 
 
 class RelatedReference(Tag):
+
     '''Related reference
 
     Pattern: 16x
@@ -95,6 +107,7 @@ class RelatedReference(Tag):
 
 
 class AccountIdentification(Tag):
+
     '''Account identification
 
     Pattern: 35x
@@ -104,6 +117,7 @@ class AccountIdentification(Tag):
 
 
 class StatementNumber(Tag):
+
     '''Statement number / sequence number
 
     Pattern: 5n[/5n]
@@ -116,6 +130,7 @@ class StatementNumber(Tag):
 
 
 class BalanceBase(Tag):
+
     '''Balance base
 
     Pattern: 1!a6!n3!a15d
@@ -151,6 +166,7 @@ class IntermediateOpeningBalance(BalanceBase):
 
 
 class Statement(Tag):
+
     '''Statement
 
     Pattern: 6!n[4!n]2a[1!a]15d1!a3!c16x[//16x]
@@ -170,9 +186,10 @@ class Statement(Tag):
     (?P<id>[A-Z][A-Z0-9]{3})?  # 1!a3!c Transaction Type Identification Code
     (?P<customer_reference>.{0,16})  # 16x Customer Reference
     (//(?P<bank_reference>.{0,16}))?  # [//16x] Bank Reference
-    (?P<extra_details>.{0,34})  # [34x] Supplementary Details (this will be on
-                                # a new/separate line)
-    '''
+    (\n?(?P<extra_details>.{0,34}))?  # [34x] Supplementary Details
+                                             # (this will be on a new/separate
+                                             # line)
+    $'''
 
     def __call__(self, transactions, value):
         data = super(Statement, self).__call__(transactions, value)
@@ -211,6 +228,7 @@ class ForwardAvailableBalance(BalanceBase):
 
 
 class TransactionDetails(Tag):
+
     '''Transaction details
 
     Pattern: 6x65x

tests/betterplace/empty_86.yml

@@ -25,13 +25,11 @@ processors:
   pre_intermediate_opening_balance: []
   pre_opening_balance: []
   pre_related_reference: []
-  pre_statement: []
+  pre_statement: [!!python/name:mt940.processors.date_fixup_pre_processor '']
   pre_statement_number: []
   pre_transaction_details: []
   pre_transaction_reference_number: []
 transactions:
 - !!python/object:mt940.models.Transaction
-  data: {transaction_details: '091
-
-      -'}
+  data: {transaction_details: 091}
   transactions: *id001

tests/betterplace/empty_line.yml

@@ -1,5 +1,5 @@
 &id001 !!python/object:mt940.models.Transactions
-data: {transaction_reference: TELEREPORTING}
+data: {}
 processors:
   post_account_identification: []
   post_available_balance: []
@@ -31,5 +31,5 @@ processors:
   pre_transaction_reference_number: []
 transactions:
 - !!python/object:mt940.models.Transaction
-  data: {}
+  data: {transaction_reference: TELEREPORTING}
   transactions: *id001