Add lru_cache to parsing for improved performance

trollsift/parser.py

@@ -21,6 +21,7 @@
 import datetime as dt
 import random
 import string
+from functools import lru_cache
 
 
 class Parser(object):
@@ -187,6 +188,7 @@ def __init__(self):
         self._cached_fields = {}
         super(RegexFormatter, self).__init__()
 
+    @lru_cache()
     def format(*args, **kwargs):
         try:
             # super() doesn't seem to work here
@@ -366,6 +368,8 @@ def _convert(convdef, stri):
     return result
 
 
+
+@lru_cache()
 def get_convert_dict(fmt):
     """Retrieve parse definition from the format string `fmt`."""
     convdef = {}
@@ -579,3 +583,14 @@ def is_one2one(fmt):
             return False
     # all checks passed, so just return True
     return True
+
+
+def purge():
+    """Clear internal caches.
+    
+    Not needed normally, but can be used to force cache clear when memory
+    is very limited.
+    
+    """
+    regex_formatter.format.cache_clear()
+    get_convert_dict.cache_clear()

trollsift/tests/integrationtests/test_parser.py

@@ -40,6 +40,18 @@ def test_parse(self):
         # Assert
         self.assertDictEqual(result, self.data)
 
+    def test_cache_clear(self):
+        """Test we can clear the internal cache properly"""
+        from trollsift.parser import purge
+        from trollsift.parser import regex_formatter
+        # Run
+        result = self.p.parse(self.string)
+        # Assert
+        self.assertDictEqual(result, self.data)
+        assert regex_formatter.format.cache_info()[-1] != 0
+        purge()
+        assert regex_formatter.format.cache_info()[-1] == 0
+
     def test_compose(self):
         # Run
         result = self.p.compose(self.data)