Merge branch 'release/4.13.0'

wolph · Jan 10, 2019 · eae26a4 · eae26a4
2 parents 0da6d71 + 976b07a
commit eae26a4
Show file tree

Hide file tree

Showing 53 changed files with 827 additions and 103 deletions.
diff --git a/mt940/__about__.py b/mt940/__about__.py
@@ -6,7 +6,7 @@
 statistics and manipulation.
 '''.strip().split())
 __email__ = 'wolph@wol.ph'
-__version__ = '4.12.2'
+__version__ = '4.13.0'
 __license__ = 'BSD'
 __copyright__ = 'Copyright 2015 Rick van Hattem (wolph)'
 __url__ = 'https://github.com/WoLpH/mt940'
diff --git a/mt940/models.py b/mt940/models.py
@@ -1,7 +1,12 @@
 import re
 import decimal
 import datetime
-import collections
+
+# python 3.8+ compatibility
+try:  # pragma: no cover
+    from collections import abc
+except ImportError:  # pragma: no cover
+    import collections as abc
 
 import mt940
 
@@ -18,8 +23,8 @@ class FixedOffset(datetime.tzinfo):
     Source: https://docs.python.org/2/library/datetime.html#tzinfo-objects
 
     >>> offset = FixedOffset(60)
-    >>> offset.utcoffset(None)
-    datetime.timedelta(0, 3600)
+    >>> offset.utcoffset(None).total_seconds()
+    3600.0
     >>> offset.dst(None)
     datetime.timedelta(0)
     >>> offset.tzname(None)
@@ -86,13 +91,15 @@ def __new__(cls, *args, **kwargs):
                 microsecond='0', )
 
             # The list makes sure this works in both Python 2 and 3
-
             for key, default in list(values.items()):
                 # Fetch the value or the default
                 value = kwargs.get(key, default)
+                assert value is not None, '%s should not be None' % key
                 # Convert the value to integer and force base 10 to make sure
                 # it doesn't get recognized as octal
-                value = int(value, 10)
+                if not isinstance(value, int):
+                    value = int(value, 10)
+
                 # Save the values again
                 values[key] = value
 
@@ -214,7 +221,7 @@ def __str__(self):
             self.date, )
 
 
-class Transactions(collections.Sequence):
+class Transactions(abc.Sequence):
     '''
     Collection of :py:class:`Transaction` objects with global properties such
     as begin and end balance
@@ -380,7 +387,7 @@ def parse(self, data):
         # match it's difficult to get both the beginning and the end so we're
         # working around it in a safer way to get everything.
         tag_re = re.compile(
-            r'^:(?P<full_tag>(?P<tag>[0-9]{2}|NS)(?P<sub_tag>[A-Z])?):',
+            r'^:\n?(?P<full_tag>(?P<tag>[0-9]{2}|NS)(?P<sub_tag>[A-Z])?):',
             re.MULTILINE)
         matches = list(tag_re.finditer(data))
 

diff --git a/mt940/processors.py b/mt940/processors.py
@@ -50,7 +50,7 @@ def mBank_set_transaction_code(transactions, tag, tag_dict, *args):
     return tag_dict
 
 
-iph_id_re = re.compile(' ID IPH: X*(?P<iph_id>\d{0,14});')
+iph_id_re = re.compile(r' ID IPH: X*(?P<iph_id>\d{0,14});')
 
 
 def mBank_set_iph_id(transactions, tag, tag_dict, *args):
@@ -66,7 +66,7 @@ def mBank_set_iph_id(transactions, tag, tag_dict, *args):
     return tag_dict
 
 
-tnr_re = re.compile('TNR:[ \n](?P<tnr>\d+\.\d+)',
+tnr_re = re.compile(r'TNR:[ \n](?P<tnr>\d+\.\d+)',
                     flags=re.MULTILINE | re.UNICODE)
 
 

diff --git a/mt940/tags.py b/mt940/tags.py
@@ -290,6 +290,7 @@ class Statement(Tag):
     (?P<status>[A-Z]?[DC])  # 2a Debit/Credit Mark
     (?P<funds_code>[A-Z])? # [1!a] Funds Code (3rd character of the currency
                             # code, if needed)
+    \n? # apparently some banks (sparkassen) incorporate newlines here
     (?P<amount>[\d,]{1,15})  # 15d Amount
     (?P<id>[A-Z][A-Z0-9 ]{3})?  # 1!a3!c Transaction Type Identification Code
     (?P<customer_reference>.{0,16})  # 16x Customer Reference
@@ -302,14 +303,28 @@ def __call__(self, transactions, value):
         data.setdefault('currency', transactions.currency)
 
         data['amount'] = models.Amount(**data)
-        data['date'] = models.Date(**data)
+        date = data['date'] = models.Date(**data)
 
         if data.get('entry_day') and data.get('entry_month'):
-            data['entry_date'] = models.Date(
+            entry_date = data['entry_date'] = models.Date(
                 day=data.get('entry_day'),
                 month=data.get('entry_month'),
                 year=str(data['date'].year),
             )
+
+            if date > entry_date and (date - entry_date).days >= 330:
+                year = 1
+            elif entry_date > date and (entry_date - date).days >= 330:
+                year = -1
+            else:
+                year = 0
+
+            data['guessed_entry_date'] = models.Date(
+                day=entry_date.day,
+                month=entry_date.month,
+                year=entry_date.year + year,
+            )
+
         return data
 
 

diff --git a/tests/betterplace/amount_formats.yml b/tests/betterplace/amount_formats.yml
@@ -77,6 +77,7 @@ tags:
         d{2})?  # [4!n] Entry Date (MMDD)\n    (?P<entry_day>\\d{2})?\n    (?P<status>[A-Z]?[DC])\
         \  # 2a Debit/Credit Mark\n    (?P<funds_code>[A-Z])? # [1!a] Funds Code (3rd\
         \ character of the currency\n                            # code, if needed)\n\
+        \    \\n? # apparently some banks (sparkassen) incorporate newlines here\n\
         \    (?P<amount>[\\d,]{1,15})  # 15d Amount\n    (?P<id>[A-Z][A-Z0-9 ]{3})?\
         \  # 1!a3!c Transaction Type Identification Code\n    (?P<customer_reference>.{0,16})\
         \  # 16x Customer Reference\n    (//(?P<bank_reference>.{0,16}))?  # [//16x]\
@@ -97,13 +98,13 @@ tags:
         S]{0,65}\\r?\\n?){0,8}[\\s\\S]{0,65}))\n    ", 98]
   34: !!python/object:mt940.tags.FloorLimitIndicator
     re: !!python/object/apply:re._compile ["^\n    (?P<currency>[A-Z]{3})  # 3!a Currency\n\
-        \    (?P<status>[DC]?)  # 2a Debit/Credit Mark\n    (?P<amount>[0-9,]{0,16})\
+        \    (?P<status>[DC ]?)  # 2a Debit/Credit Mark\n    (?P<amount>[0-9,]{0,16})\
         \  # 15d Amount (includes decimal sign, so 16)\n    $", 98]
   NS: !!python/object:mt940.tags.NonSwift
     re: !!python/object/apply:re._compile ["\n    (?P<non_swift>\n        (\\d{2}.{0,})\n\
         \        (\\n\\d{2}.{0,})*\n    )\n    $", 98]
   90: !!python/object:mt940.tags.SumEntries
-    re: &id002 !!python/object/apply:re._compile ["^\n    (?P<number>\\d+)\n    (?P<currency>.{3})\
+    re: &id002 !!python/object/apply:re._compile ["^\n    (?P<number>\\d*)\n    (?P<currency>.{3})\
         \  # 3!a Currency\n    (?P<amount>[\\d,]{1,15})  # 15d Amount\n    ", 98]
   90D: !!python/object:mt940.tags.SumDebitEntries
     re: *id002

diff --git a/tests/betterplace/currency_in_25.yml b/tests/betterplace/currency_in_25.yml
@@ -69,6 +69,7 @@ tags:
         d{2})?  # [4!n] Entry Date (MMDD)\n    (?P<entry_day>\\d{2})?\n    (?P<status>[A-Z]?[DC])\
         \  # 2a Debit/Credit Mark\n    (?P<funds_code>[A-Z])? # [1!a] Funds Code (3rd\
         \ character of the currency\n                            # code, if needed)\n\
+        \    \\n? # apparently some banks (sparkassen) incorporate newlines here\n\
         \    (?P<amount>[\\d,]{1,15})  # 15d Amount\n    (?P<id>[A-Z][A-Z0-9 ]{3})?\
         \  # 1!a3!c Transaction Type Identification Code\n    (?P<customer_reference>.{0,16})\
         \  # 16x Customer Reference\n    (//(?P<bank_reference>.{0,16}))?  # [//16x]\
@@ -89,13 +90,13 @@ tags:
         S]{0,65}\\r?\\n?){0,8}[\\s\\S]{0,65}))\n    ", 98]
   34: !!python/object:mt940.tags.FloorLimitIndicator
     re: !!python/object/apply:re._compile ["^\n    (?P<currency>[A-Z]{3})  # 3!a Currency\n\
-        \    (?P<status>[DC]?)  # 2a Debit/Credit Mark\n    (?P<amount>[0-9,]{0,16})\
+        \    (?P<status>[DC ]?)  # 2a Debit/Credit Mark\n    (?P<amount>[0-9,]{0,16})\
         \  # 15d Amount (includes decimal sign, so 16)\n    $", 98]
   NS: !!python/object:mt940.tags.NonSwift
     re: !!python/object/apply:re._compile ["\n    (?P<non_swift>\n        (\\d{2}.{0,})\n\
         \        (\\n\\d{2}.{0,})*\n    )\n    $", 98]
   90: !!python/object:mt940.tags.SumEntries
-    re: &id002 !!python/object/apply:re._compile ["^\n    (?P<number>\\d+)\n    (?P<currency>.{3})\
+    re: &id002 !!python/object/apply:re._compile ["^\n    (?P<number>\\d*)\n    (?P<currency>.{3})\
         \  # 3!a Currency\n    (?P<amount>[\\d,]{1,15})  # 15d Amount\n    ", 98]
   90D: !!python/object:mt940.tags.SumDebitEntries
     re: *id002

diff --git a/tests/betterplace/empty_86.yml b/tests/betterplace/empty_86.yml
@@ -69,6 +69,7 @@ tags:
         d{2})?  # [4!n] Entry Date (MMDD)\n    (?P<entry_day>\\d{2})?\n    (?P<status>[A-Z]?[DC])\
         \  # 2a Debit/Credit Mark\n    (?P<funds_code>[A-Z])? # [1!a] Funds Code (3rd\
         \ character of the currency\n                            # code, if needed)\n\
+        \    \\n? # apparently some banks (sparkassen) incorporate newlines here\n\
         \    (?P<amount>[\\d,]{1,15})  # 15d Amount\n    (?P<id>[A-Z][A-Z0-9 ]{3})?\
         \  # 1!a3!c Transaction Type Identification Code\n    (?P<customer_reference>.{0,16})\
         \  # 16x Customer Reference\n    (//(?P<bank_reference>.{0,16}))?  # [//16x]\
@@ -89,13 +90,13 @@ tags:
         S]{0,65}\\r?\\n?){0,8}[\\s\\S]{0,65}))\n    ", 98]
   34: !!python/object:mt940.tags.FloorLimitIndicator
     re: !!python/object/apply:re._compile ["^\n    (?P<currency>[A-Z]{3})  # 3!a Currency\n\
-        \    (?P<status>[DC]?)  # 2a Debit/Credit Mark\n    (?P<amount>[0-9,]{0,16})\
+        \    (?P<status>[DC ]?)  # 2a Debit/Credit Mark\n    (?P<amount>[0-9,]{0,16})\
         \  # 15d Amount (includes decimal sign, so 16)\n    $", 98]
   NS: !!python/object:mt940.tags.NonSwift
     re: !!python/object/apply:re._compile ["\n    (?P<non_swift>\n        (\\d{2}.{0,})\n\
         \        (\\n\\d{2}.{0,})*\n    )\n    $", 98]
   90: !!python/object:mt940.tags.SumEntries
-    re: &id002 !!python/object/apply:re._compile ["^\n    (?P<number>\\d+)\n    (?P<currency>.{3})\
+    re: &id002 !!python/object/apply:re._compile ["^\n    (?P<number>\\d*)\n    (?P<currency>.{3})\
         \  # 3!a Currency\n    (?P<amount>[\\d,]{1,15})  # 15d Amount\n    ", 98]
   90D: !!python/object:mt940.tags.SumDebitEntries
     re: *id002

diff --git a/tests/betterplace/empty_entry_date.yml b/tests/betterplace/empty_entry_date.yml
@@ -69,6 +69,7 @@ tags:
         d{2})?  # [4!n] Entry Date (MMDD)\n    (?P<entry_day>\\d{2})?\n    (?P<status>[A-Z]?[DC])\
         \  # 2a Debit/Credit Mark\n    (?P<funds_code>[A-Z])? # [1!a] Funds Code (3rd\
         \ character of the currency\n                            # code, if needed)\n\
+        \    \\n? # apparently some banks (sparkassen) incorporate newlines here\n\
         \    (?P<amount>[\\d,]{1,15})  # 15d Amount\n    (?P<id>[A-Z][A-Z0-9 ]{3})?\
         \  # 1!a3!c Transaction Type Identification Code\n    (?P<customer_reference>.{0,16})\
         \  # 16x Customer Reference\n    (//(?P<bank_reference>.{0,16}))?  # [//16x]\
@@ -89,13 +90,13 @@ tags:
         S]{0,65}\\r?\\n?){0,8}[\\s\\S]{0,65}))\n    ", 98]
   34: !!python/object:mt940.tags.FloorLimitIndicator
     re: !!python/object/apply:re._compile ["^\n    (?P<currency>[A-Z]{3})  # 3!a Currency\n\
-        \    (?P<status>[DC]?)  # 2a Debit/Credit Mark\n    (?P<amount>[0-9,]{0,16})\
+        \    (?P<status>[DC ]?)  # 2a Debit/Credit Mark\n    (?P<amount>[0-9,]{0,16})\
         \  # 15d Amount (includes decimal sign, so 16)\n    $", 98]
   NS: !!python/object:mt940.tags.NonSwift
     re: !!python/object/apply:re._compile ["\n    (?P<non_swift>\n        (\\d{2}.{0,})\n\
         \        (\\n\\d{2}.{0,})*\n    )\n    $", 98]
   90: !!python/object:mt940.tags.SumEntries
-    re: &id002 !!python/object/apply:re._compile ["^\n    (?P<number>\\d+)\n    (?P<currency>.{3})\
+    re: &id002 !!python/object/apply:re._compile ["^\n    (?P<number>\\d*)\n    (?P<currency>.{3})\
         \  # 3!a Currency\n    (?P<amount>[\\d,]{1,15})  # 15d Amount\n    ", 98]
   90D: !!python/object:mt940.tags.SumDebitEntries
     re: *id002

diff --git a/tests/betterplace/empty_line.yml b/tests/betterplace/empty_line.yml
@@ -69,6 +69,7 @@ tags:
         d{2})?  # [4!n] Entry Date (MMDD)\n    (?P<entry_day>\\d{2})?\n    (?P<status>[A-Z]?[DC])\
         \  # 2a Debit/Credit Mark\n    (?P<funds_code>[A-Z])? # [1!a] Funds Code (3rd\
         \ character of the currency\n                            # code, if needed)\n\
+        \    \\n? # apparently some banks (sparkassen) incorporate newlines here\n\
         \    (?P<amount>[\\d,]{1,15})  # 15d Amount\n    (?P<id>[A-Z][A-Z0-9 ]{3})?\
         \  # 1!a3!c Transaction Type Identification Code\n    (?P<customer_reference>.{0,16})\
         \  # 16x Customer Reference\n    (//(?P<bank_reference>.{0,16}))?  # [//16x]\
@@ -89,13 +90,13 @@ tags:
         S]{0,65}\\r?\\n?){0,8}[\\s\\S]{0,65}))\n    ", 98]
   34: !!python/object:mt940.tags.FloorLimitIndicator
     re: !!python/object/apply:re._compile ["^\n    (?P<currency>[A-Z]{3})  # 3!a Currency\n\
-        \    (?P<status>[DC]?)  # 2a Debit/Credit Mark\n    (?P<amount>[0-9,]{0,16})\
+        \    (?P<status>[DC ]?)  # 2a Debit/Credit Mark\n    (?P<amount>[0-9,]{0,16})\
         \  # 15d Amount (includes decimal sign, so 16)\n    $", 98]
   NS: !!python/object:mt940.tags.NonSwift
     re: !!python/object/apply:re._compile ["\n    (?P<non_swift>\n        (\\d{2}.{0,})\n\
         \        (\\n\\d{2}.{0,})*\n    )\n    $", 98]
   90: !!python/object:mt940.tags.SumEntries
-    re: &id002 !!python/object/apply:re._compile ["^\n    (?P<number>\\d+)\n    (?P<currency>.{3})\
+    re: &id002 !!python/object/apply:re._compile ["^\n    (?P<number>\\d*)\n    (?P<currency>.{3})\
         \  # 3!a Currency\n    (?P<amount>[\\d,]{1,15})  # 15d Amount\n    ", 98]
   90D: !!python/object:mt940.tags.SumDebitEntries
     re: *id002

diff --git a/tests/betterplace/missing_crlf_at_end.yml b/tests/betterplace/missing_crlf_at_end.yml
@@ -77,6 +77,7 @@ tags:
         d{2})?  # [4!n] Entry Date (MMDD)\n    (?P<entry_day>\\d{2})?\n    (?P<status>[A-Z]?[DC])\
         \  # 2a Debit/Credit Mark\n    (?P<funds_code>[A-Z])? # [1!a] Funds Code (3rd\
         \ character of the currency\n                            # code, if needed)\n\
+        \    \\n? # apparently some banks (sparkassen) incorporate newlines here\n\
         \    (?P<amount>[\\d,]{1,15})  # 15d Amount\n    (?P<id>[A-Z][A-Z0-9 ]{3})?\
         \  # 1!a3!c Transaction Type Identification Code\n    (?P<customer_reference>.{0,16})\
         \  # 16x Customer Reference\n    (//(?P<bank_reference>.{0,16}))?  # [//16x]\
@@ -97,13 +98,13 @@ tags:
         S]{0,65}\\r?\\n?){0,8}[\\s\\S]{0,65}))\n    ", 98]
   34: !!python/object:mt940.tags.FloorLimitIndicator
     re: !!python/object/apply:re._compile ["^\n    (?P<currency>[A-Z]{3})  # 3!a Currency\n\
-        \    (?P<status>[DC]?)  # 2a Debit/Credit Mark\n    (?P<amount>[0-9,]{0,16})\
+        \    (?P<status>[DC ]?)  # 2a Debit/Credit Mark\n    (?P<amount>[0-9,]{0,16})\
         \  # 15d Amount (includes decimal sign, so 16)\n    $", 98]
   NS: !!python/object:mt940.tags.NonSwift
     re: !!python/object/apply:re._compile ["\n    (?P<non_swift>\n        (\\d{2}.{0,})\n\
         \        (\\n\\d{2}.{0,})*\n    )\n    $", 98]
   90: !!python/object:mt940.tags.SumEntries
-    re: &id002 !!python/object/apply:re._compile ["^\n    (?P<number>\\d+)\n    (?P<currency>.{3})\
+    re: &id002 !!python/object/apply:re._compile ["^\n    (?P<number>\\d*)\n    (?P<currency>.{3})\
         \  # 3!a Currency\n    (?P<amount>[\\d,]{1,15})  # 15d Amount\n    ", 98]
   90D: !!python/object:mt940.tags.SumDebitEntries
     re: *id002