Add tests

panel/io/cache.py

@@ -147,29 +147,33 @@ def _io_hash(obj):
 for name in _FFI_TYPE_NAMES:
     _hash_funcs[name] = b'0'
 
-def _generate_hash_inner(obj, hash_funcs={}):
-
+def _find_hash_func(obj, hash_funcs={}):
     fqn_type = _get_fqn(obj)
     if fqn_type in hash_funcs:
-        hash_func = hash_funcs[fqn_type]
-        try:
-            output = hash_func(obj)
-        except BaseException as e:
-            raise ValueError(
-                f'User hash function {hash_func!r} failed for input '
-                f'type {fqn_type} with following error: '
-                f'{type(e).__name__}("{e}").'
-            )
-        return _generate_hash(output)
+        return hash_funcs[fqn_type]
+    elif fqn_type in _hash_funcs:
+        return _hash_funcs[fqn_type]
     for otype, hash_func in _hash_funcs.items():
         if isinstance(otype, str):
             if otype == fqn_type:
-                return hash_func(obj)
+                return hash_func
         elif inspect.isfunction(otype):
             if otype(obj):
-                return hash_func(obj)
+                return hash_func
         elif isinstance(obj, otype):
-            return hash_func(obj)
+            return hash_func
+
+def _generate_hash_inner(obj, hash_funcs={}):
+    hash_func = _find_hash_func(obj, hash_funcs)
+    if hash_func is not None:
+        try:
+            output = hash_func(obj)
+        except BaseException as e:
+            raise ValueError(
+                f'User hash function {hash_func!r} failed for input '
+                f'{obj!r} with following error: {type(e).__name__}("{e}").'
+            )
+        return output
     if hasattr(obj, '__reduce__'):
         h = hashlib.new("md5")
         try:
@@ -211,24 +215,28 @@ def _key(obj):
         return id(obj)
     return _INDETERMINATE
 
-def _cleanup_cache(cache, policy, max_items, ttl, time):
+def _cleanup_cache(cache, policy, max_items, time):
     """
     Deletes items in the cache if the exceed the number of items or
     their TTL (time-to-live) has expired.
     """
     while len(cache) >= max_items:
-        if policy.lower() == 'fifo':
+        if policy.lower() == 'lifo':
             key = list(cache.keys())[0]
         elif policy.lower() == 'lru':
-            key = sorted(((key, time-t) for k, (_, _, _, t) in cache.items()),
+            key = sorted(((k, -(time-t)) for k, (_, _, _, t) in cache.items()),
                          key=lambda o: o[1])[0][0]
         elif policy.lower() == 'lfu':
             key = sorted(cache.items(), key=lambda o: o[1][2])[0][0]
         del cache[key]
-    if ttl is not None:
-        for key, (_, ts, _, _) in list(cache.items()):
-            if (time-ts) > ttl:
-                del cache[key]
+
+def _cleanup_ttl(cache, ttl, time):
+    """
+    Deletes items in the cache if their TTL (time-to-live) has expired.
+    """
+    for key, (_, ts, _, _) in list(cache.items()):
+        if (time-ts) > ttl:
+            del cache[key]
 
 #---------------------------------------------------------------------
 # Public API
@@ -336,32 +344,44 @@ def wrapped_func(*args, **kwargs):
         func_hash = hashlib.sha256(_generate_hash(func_hash)).hexdigest()
 
         func_cache = state._memoize_cache.get(func_hash)
-        if func_cache is None:
+
+        empty = func_cache is None
+        if empty:
             if to_disk:
                 from diskcache import Index
                 cache = Index(os.path.join(cache_path, func_hash))
             else:
                 cache = {}
             state._memoize_cache[func_hash] = func_cache = cache
-        elif hash_value in func_cache:
+
+        if ttl is not None:
+            _cleanup_ttl(func_cache, ttl, time)
+
+        if not empty and hash_value in func_cache:
             with lock:
                 ret, ts, count, _ = func_cache[hash_value]
                 func_cache[hash_value] = (ret, ts, count+1, time)
                 return ret
 
         if max_items is not None:
-            with lock:
-                _cleanup_cache(cache, policy, max_items, ttl, time)
+            _cleanup_cache(func_cache, policy, max_items, time)
 
         ret = func(*args, **kwargs)
         with lock:
             func_cache[hash_value] = (ret, time, 0, time)
         return ret
 
     def clear():
+        global func_hash
         if func_hash is None:
             return
-        state._memoize_cache.get(func_hash, {}).clear()
+        if to_disk:
+            from diskcache import Index
+            cache = Index(os.path.join(cache_path, func_hash))
+            cache.clear()
+        else:
+            cache = state._memoize_cache.get(func_hash, {})
+        cache.clear()
     wrapped_func.clear = clear
 
     try:

panel/tests/conftest.py

@@ -230,6 +230,10 @@ def server_cleanup():
             state._thread_pool.shutdown(wait=False)
             state._thread_pool = None
 
+@pytest.fixture(autouse=True)
+def cache_cleanup():
+    state._memoize_cache.clear()
+
 @pytest.fixture
 def py_file():
     tf = tempfile.NamedTemporaryFile(mode='w', suffix='.py')

panel/tests/io/test_cache.py

@@ -0,0 +1,207 @@
+import io
+import time
+
+import numpy as np
+
+from panel.io.cache import _find_hash_func, cache
+from panel.tests.util import pd_available
+
+################
+# Test hashing #
+################
+
+def hashes_equal(v1, v2):
+    a, b = _find_hash_func(v1)(v1), _find_hash_func(v2)(v2)
+    return a == b
+
+def test_str_hash():
+    assert hashes_equal('foo', 'foo')
+    assert not hashes_equal('foo', 'bar')
+
+def test_int_hash():
+    assert hashes_equal(12, 12)
+    assert not hashes_equal(1, 2)
+
+def test_float_hash():
+    assert hashes_equal(3.14, 3.14)
+    assert not hashes_equal(1.2, 3.14)
+
+def test_bool_hash():
+    assert hashes_equal(True, True)
+    assert hashes_equal(False, False)
+    assert not hashes_equal(True, False)
+
+def test_none_hash():
+    assert hashes_equal(None, None)
+    assert not hashes_equal(None, False)
+
+def test_bytes_hash():
+    assert hashes_equal(b'0', b'0')
+    assert not hashes_equal(b'0', b'1')
+
+def test_list_hash():
+    assert hashes_equal([0], [0])
+    assert hashes_equal(['a', ['b']], ['a', ['b']])
+    assert not hashes_equal([0], [1])
+    assert not hashes_equal(['a', ['b']], ['a', ['c']])
+
+    # Recursion
+    l = [0]
+    l.append(l)
+    assert hashes_equal(l, list(l))
+
+def test_tuple_hash():
+    assert hashes_equal((0,), (0,))
+    assert hashes_equal(('a', ('b',)), ('a', ('b',)))
+    assert not hashes_equal((0,), (1,))
+    assert not hashes_equal(('a', ('b',)), ('a', ('c',)))
+
+def test_dict_hash():
+    assert hashes_equal({'a': 0}, {'a': 0})
+    assert hashes_equal({'a': {'b': 0}}, {'a': {'b': 0}})
+    assert not hashes_equal({'a': 0}, {'b': 0})
+    assert not hashes_equal({'a': 0}, {'a': 1})
+    assert not hashes_equal({'a': {'b': 0}}, {'a': {'b': 1}})
+
+    # Recursion
+    d = {'a': {}}
+    d['a'] = d
+    assert hashes_equal(d, dict(d))
+
+def test_stringio_hash():
+    sio1, sio2 = io.StringIO(), io.StringIO()
+    sio1.write('foo')
+    sio2.write('foo')
+    sio1.seek(0)
+    sio2.seek(0)
+    assert hashes_equal(sio1, sio2)
+    sio3 = io.StringIO()
+    sio3.write('bar')
+    sio3.seek(0)
+    assert not hashes_equal(sio1, sio3)
+
+def test_bytesio_hash():
+    bio1, bio2 = io.BytesIO(), io.BytesIO()
+    bio1.write(b'foo')
+    bio2.write(b'foo')
+    bio1.seek(0)
+    bio2.seek(0)
+    assert hashes_equal(bio1, bio2)
+    bio3 = io.BytesIO()
+    bio3.write(b'bar')
+    bio3.seek(0)
+    assert not hashes_equal(bio1, bio3)
+
+def test_ndarray_hash():
+    assert hashes_equal(np.array([0, 1, 2]), np.array([0, 1, 2]))
+    assert not hashes_equal(
+        np.array([0, 1, 2], dtype='uint32'),
+        np.array([0, 1, 2], dtype='float64')
+    )
+    assert not hashes_equal(
+        np.array([0, 1, 2]),
+        np.array([2, 1, 0])
+    )
+
+@pd_available
+def test_dataframe_hash():
+    import pandas as pd
+    df1, df2 = pd._testing.makeMixedDataFrame(), pd._testing.makeMixedDataFrame()
+    assert hashes_equal(df1, df2)
+    df2['A'] = df2['A'].values[::-1]
+    assert not hashes_equal(df1, df2)
+
+@pd_available
+def test_series_hash():
+    import pandas as pd
+    series1 = pd._testing.makeStringSeries()
+    series2 = series1.copy()
+    assert hashes_equal(series1, series2)
+    series2.iloc[0] = 3.14
+    assert not hashes_equal(series1, series2)
+
+def test_ufunc_hash():
+    assert hashes_equal(np.absolute, np.absolute)
+    assert not hashes_equal(np.sin, np.cos)
+
+def test_builtin_hash():
+    assert hashes_equal(max, max)
+    assert not hashes_equal(max, min)
+
+def test_module_hash():
+    assert hashes_equal(np, np)
+    assert not hashes_equal(np, io)
+
+################
+# Test caching #
+################
+
+OFFSET = {}
+
+def function_with_args(a, b):
+    global OFFSET
+    offset = OFFSET.get((a, b), 0)
+    result = a + b + offset
+    OFFSET[(a, b)] = offset + 1
+    return result
+
+def test_cache_with_args():
+    global OFFSET
+    OFFSET.clear()
+    fn = cache(function_with_args)
+    assert fn(0, 0) == 0
+    assert fn(0, 0) == 0
+
+def test_cache_with_kwargs():
+    global OFFSET
+    OFFSET.clear()
+    fn = cache(function_with_args)
+    assert fn(a=0, b=0) == 0
+    assert fn(a=0, b=0) == 0
+
+def test_cache_clear():
+    global OFFSET
+    OFFSET.clear()
+    fn = cache(function_with_args)
+    assert fn(0, 0) == 0
+    fn.clear()
+    assert fn(0, 0) == 1
+
+def test_cache_lifo():
+    global OFFSET
+    OFFSET.clear()
+    fn = cache(function_with_args, max_items=2, policy='lifo')
+    assert fn(0, 0) == 0
+    assert fn(0, 1) == 1
+    assert fn(0, 0) == 0
+    assert fn(0, 2) == 2 # (0, 0) should be evicted
+    assert fn(0, 0) == 1
+
+def test_cache_lfu():
+    global OFFSET
+    OFFSET.clear()
+    fn = cache(function_with_args, max_items=2, policy='lfu')
+    assert fn(0, 0) == 0
+    assert fn(0, 0) == 0
+    assert fn(0, 1) == 1
+    assert fn(0, 2) == 2 # (0, 1) should be evicted
+    assert fn(0, 1) == 2
+
+def test_cache_lru():
+    global OFFSET
+    OFFSET.clear()
+    fn = cache(function_with_args, max_items=3, policy='lru')
+    assert fn(0, 0) == 0
+    assert fn(0, 1) == 1
+    assert fn(0, 2) == 2
+    assert fn(0, 0) == 0
+    assert fn(0, 3) == 3 # (0, 1) should be evicted
+    assert fn(0, 1) == 2
+
+def test_cache_ttl():
+    global OFFSET
+    OFFSET.clear()
+    fn = cache(function_with_args, ttl=0.1)
+    assert fn(0, 0) == 0
+    time.sleep(0.1)
+    assert fn(0, 0) == 1