memoized: Convert to a class method memoizer

Earlier, the memoizer was not related to the class. Now,
the cache is specifically saved to a class. Hence, when the
class is garbage collected, the cache is deleted too.

file_metadata/generic_file.py

@@ -58,12 +58,12 @@ def config(self, key, new_defaults=()):
         except KeyError:
             return defaults[key]
 
-    @memoized(is_method=True)
+    @memoized
     def fetch(self, key=''):
         """
         Fetch data about the file based on the key provided. Provides a
         uniform location where all the conversions of filetype, reading, etc.
-        can happen efficiently and also it gets cached as it's memoized.
+        can happen efficiently and also it gets cached.
 
         :param key: The key decides what data is fetched.
         """
@@ -119,7 +119,7 @@ def analyze(self, prefix='analyze_', suffix='', methods=None):
                 data.update(getattr(self, method)())
         return data
 
-    @memoized(is_method=True)
+    @memoized
     def exiftool(self):
         """
         The exif data from the given file using ``exiftool``. The data it
@@ -173,7 +173,7 @@ def exiftool(self):
         assert len(data) == 1
         return data[0]
 
-    @memoized(is_method=True)
+    @memoized
     def mime(self):
         if hasattr(magic, "from_file"):
             # Use https://pypi.python.org/pypi/python-magic
@@ -190,7 +190,7 @@ def mime(self):
             'package python-magic (https://pypi.python.org/pypi/python-magic) '
             'nor file\'s (http://www.darwinsys.com/file/) package.')
 
-    @memoized(is_method=True)
+    @memoized
     def is_type(self, key):
         """
         Some checks on whether the file is of a spacific type. Useful for

file_metadata/image/image_file.py

@@ -58,7 +58,7 @@ def create(cls, *args, **kwargs):
             return SVGFile.create(*args, **kwargs)
         return cls(*args, **kwargs)
 
-    @memoized(is_method=True)
+    @memoized
     def fetch(self, key=''):
         if key == 'filename_raster':
             # A raster filename holds the file in a raster graphic format

file_metadata/image/jpeg_file.py

@@ -19,7 +19,7 @@ class JPEGFile(ImageFile):
     def create(cls, *args, **kwargs):
         return cls(*args, **kwargs)
 
-    @memoized(is_method=True)
+    @memoized
     def fetch(self, key=''):
         if key == 'filename_zxing':
             exif = self.exiftool()

file_metadata/image/svg_file.py

@@ -18,7 +18,7 @@ class SVGFile(ImageFile):
     def create(cls, *args, **kwargs):
         return cls(*args, **kwargs)
 
-    @memoized(is_method=True)
+    @memoized
     def fetch(self, key=''):
         if key == 'filename_raster':
             # SVG files are not raster graphics, hence we convert it to one

file_metadata/image/xcf_file.py

@@ -18,7 +18,7 @@ class XCFFile(ImageFile):
     def create(cls, *args, **kwargs):
         return cls(*args, **kwargs)
 
-    @memoized(is_method=True)
+    @memoized
     def fetch(self, key=''):
         if key == 'filename_raster':
             # XCF files are not raste graphics, hence we convert it to one

file_metadata/mixins.py

@@ -25,7 +25,7 @@
 
 class FFProbeMixin(object):
 
-    @memoized(is_method=True)
+    @memoized
     def ffprobe(self):
         """
         Read multimedia streams and give information about it using the

file_metadata/utilities.py

@@ -10,17 +10,11 @@
 import bz2
 import functools
 import hashlib
-import inspect
 import os
 import tarfile
 import tempfile
 from shutil import copyfileobj
 
-try:
-    from cPickle import dumps
-except ImportError:
-    from pickle import dumps
-
 try:
     import lzma
 except ImportError:
@@ -209,108 +203,39 @@ def app_dir(dirtype, *args):
     return os.path.join(makedirs(path, exist_ok=True), *args)
 
 
-def memoized(func=None, is_method=False, hashable=True, cache=None):
+class memoized(object):  # flake8: noqa (class names should use CapWords)
     """
-    A generic efficient memoized decorator.
-
-    :param func:
-        If not None it decorates the given callable ``func``, otherwise it
-        returns a decorator. Basically a convenience for creating a
-        decorator with the default parameters as ``@memoized`` instead
-        of ``@memoized()``.
-    :param is_method:
-        Specify whether the decorated function is going to be a class method.
-        Currently this is only used as a hint for returning an efficient
-        implementation for single argument functions (but not methods).
-    :param hashable:
-        Set to False if any parameter may be non-hashable.
-    :param cache:
-        A dict-like instance to be used as the underlying storage for
-        the memoized values. The cache instance must implement ``__getitem__``
-        and ``__setitem__``. Defaults to a new empty dict.
+    Cache the return value of a method.
+
+    This class is meant to be used as a decorator of methods. The return value
+    from a given method invocation will be cached on the instance whose method
+    was invoked. All arguments passed to a method decorated with this decorator
+    must be hashable.
+
+    If a cached method is invoked directly on its class the result will not
+    be cached. Instead the method will be invoked like a static method.
+
+    Taken from: http://code.activestate.com/recipes/
+    577452-a-memoize-decorator-for-instance-methods/
     """
-    def _args_kwargs_memoized(func, hashable=True, cache=None):
-        func.cache = cache if cache is not None else {}
-
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs):
-            if hashable:
-                key = (args, frozenset(kwargs.iteritems()))
-            else:
-                key = dumps((args, kwargs), -1)
-            try:
-                return func.cache[key]
-            except KeyError:
-                func.cache[key] = value = func(*args, **kwargs)
-                return value
-        return wrapper
-
-    def _args_memoized(func, hashable=True, cache=None):
-        func.cache = cache if cache is not None else {}
-
-        @functools.wraps(func)
-        def wrapper(*args):
-            key = args if hashable else dumps(args, -1)
-            try:
-                return func.cache[key]
-            except KeyError:
-                func.cache[key] = value = func(*args)
-                return value
-        return wrapper
-
-    def _one_arg_memoized(func, cache=None):
-        func.cache = cache if cache is not None else {}
-
-        @functools.wraps(func)
-        def wrapper(arg):
-            key = arg
-            try:
-                return func.cache[key]
-            except KeyError:
-                func.cache[key] = value = func(arg)
-                return value
-        return wrapper
-
-    def _fast_one_arg_memoized(func):
-        """
-        A fast memoize function when there is only 1 argument.
-        """
-        class MemoDict(dict):
-            def __missing__(self, key):
-                self[key] = ret = func(key)
-                return ret
-        func.cache = MemoDict()
-        return func.cache.__getitem__
-
-    def _fast_zero_arg_memoized(func):
-        """
-        Use a fast memoize function which works when there are no arguments.
-        """
-        class MemoDict(dict):
-            def __missing__(self, key):
-                self[key] = ret = func()
-                return ret
-        func.cache = MemoDict()
-        return functools.partial(func.cache.__getitem__, None)
-
-    if func is None:
-        return functools.partial(memoized, is_method=is_method,
-                                 hashable=hashable, cache=cache)
-
-    spec = inspect.getargspec(func)
-    allow_named = bool(spec.defaults)
-    if allow_named or spec.keywords:
-        return _args_kwargs_memoized(func, hashable, cache)
-
-    nargs = len(spec.args)
-    if (nargs > 1 or spec.varargs or spec.defaults or not hashable or
-            nargs == 0 and cache is not None):
-        return _args_memoized(func, hashable, cache)
-
-    if nargs == 1:
-        if is_method or cache is not None:
-            return _one_arg_memoized(func, cache)
-        else:
-            return _fast_one_arg_memoized(func)
 
-    return _fast_zero_arg_memoized(func)
+    def __init__(self, func):
+        self.func = func
+
+    def __get__(self, obj, objtype=None):
+        if obj is None:
+            return self.func
+        return functools.partial(self, obj)
+
+    def __call__(self, *args, **kw):
+        obj = args[0]
+        try:
+            cache = obj.__cache
+        except AttributeError:
+            cache = obj.__cache = {}
+        key = (self.func, args[1:], frozenset(kw.items()))
+        try:
+            res = cache[key]
+        except KeyError:
+            res = cache[key] = self.func(*args, **kw)
+        return res

tests/utilities_test.py

@@ -154,110 +154,16 @@ def test_integration(self):
 
 class MemoizedTest(unittest.TestCase):
 
-    def setUp(self):
-        self.calls = 0
-        self.arg0 = lambda: self.incr_calls()
-        self.arg1 = lambda x: self.incr_calls() or x
-        self.arg2 = lambda x, y = 0: self.incr_calls() or (x, y)
-        # self.argv = lambda x, y, z = 0: self.incr_calls() or (x, y, z)
-        self.kwargv = lambda *a: self.incr_calls() or a
-        self.argv_kwargv = lambda x, y = 0, *a, **k: (
-            self.incr_calls() or (x, y, a, k))
-
-    def incr_calls(self):
-        self.calls += 1
-
-    def multicall(self, func, args_expr, expected_calls, num_calls=10):
-        self.calls = 0
-        results = [eval("f(%s)" % args_expr, {}, {"f": func})
-                   for _ in xrange(num_calls)]
-        self.assertEqual(self.calls, expected_calls)
-        self.assertTrue(all(r == results[0] for r in results))
-        return results[0]
-
-    def assert_memoized(self, func, args_expr, deco=None):
-        deco = deco or memoized
-        self.assertEqual(self.multicall(func, args_expr, 10, 10),
-                         self.multicall(deco(func), args_expr, 1, 10))
-
-    def test_zero_arg(self):
-        for func in self.arg0, self.kwargv:
-            self.assert_memoized(func, "")
-
-    def test_one_arg_pos(self):
-        for f in self.arg1, self.arg2, self.kwargv, self.argv_kwargv:
-            self.assert_memoized(f, "1")
-
-    def test_one_arg_name(self):
-        for f in self.arg2, self.argv_kwargv:
-            self.assert_memoized(f, "x=1")
-
-    def test_two_args_pos(self):
-        for f in self.arg2, self.kwargv, self.argv_kwargv:
-            self.assert_memoized(f, "1, 2")
-
-    def test_two_args_named(self):
-        for f in self.arg2, self.argv_kwargv:
-            self.assert_memoized(f, "1, y=2")
-            self.assert_memoized(f, "x=1, y=2")
-
-    def test_varargs(self):
-        for f in self.kwargv, self.argv_kwargv:
-            self.assert_memoized(f, "*range(10)")
-
-    def test_varargs_kwargs(self):
-        self.assert_memoized(self.argv_kwargv, "x=1, z=5")
-        self.assert_memoized(self.argv_kwargv, "1, 2, 3, 4, z=5")
-
-    def test_unhashable(self):
-        deco = memoized(hashable=False)
-        for f in self.arg1, self.arg2, self.kwargv, self.argv_kwargv:
-            self.assert_memoized(f, "[2]", deco)
-            self.assert_memoized(f, "{'foo': 3}", deco)
-        for f in self.arg2, self.argv_kwargv:
-            self.assert_memoized(f, "x=[2]", deco)
-            self.assert_memoized(f, "x={'foo': 3}", deco)
-        for f in self.arg2, self.kwargv, self.argv_kwargv:
-            self.assert_memoized(f, "[2], {'foo': 3}", deco)
-        for f in self.arg2, self.argv_kwargv:
-            self.assert_memoized(f, "[2], y={'foo': 3}", deco)
-            self.assert_memoized(f, "x=[2], y={'bar': 2}", deco)
-
-        for f in self.kwargv, self.argv_kwargv:
-            self.assert_memoized(f, "[2], {'foo': 3}, 3", deco)
-
-        for f in (self.argv_kwargv,):
-            self.assert_memoized(f, "[2], {'foo': 3}, z={1,2}", deco)
-            self.assert_memoized(f, "[2], y={'foo': 3}, z={1,2}", deco)
-            self.assert_memoized(f, "x=[2], y={'foo': 3}, z={1,2}", deco)
-
-        for f in self.kwargv, self.argv_kwargv:
-            self.assert_memoized(f, "*[[] for _ in range(10)]", deco)
-
-        self.assert_memoized(self.argv_kwargv, "x=[2], z={'foo': 3}", deco)
-        self.assert_memoized(self.argv_kwargv,
-                             "1, [2], {'foo': 3}, 4, z={5,6}", deco)
-
-    def test_method(self):
-        incr_calls = self.incr_calls
-
-        class X(object):
-
-            def arg0(self):
-                incr_calls()
-
-            bad_mf0 = memoized()(arg0)
-            good_mf0 = memoized(is_method=True)(arg0)
-
-        x = X()
-        self.assertEqual(self.multicall(x.arg0, "", 10),
-                         self.multicall(x.good_mf0, "", 1))
-
-        with self.assertRaises(TypeError):
-            x.bad_mf0()
-
-    def test_default_decorator(self):
-        @memoized
-        def func():
-            return self.arg0()
-        self.multicall(func, "", 1)
+    def test_memoized_decorator(self):
+
+        class AbcClass:
+            val = 0
+
+            @memoized
+            def inc_val(self, arg):
+                self.val += 1
+                return self.val + arg
+
+        uut = AbcClass()
+        self.assertEqual(uut.inc_val(2), uut.inc_val(2))
+        self.assertNotEqual(AbcClass.inc_val(uut, 2), AbcClass.inc_val(uut, 2))