From 687fa0178ce9b517b1b5270f2888f7d11ac40f72 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Thu, 11 Aug 2022 14:15:04 +0200 Subject: [PATCH] Add tests --- panel/io/cache.py | 74 ++++++++----- panel/tests/conftest.py | 4 + panel/tests/io/test_cache.py | 207 +++++++++++++++++++++++++++++++++++ 3 files changed, 258 insertions(+), 27 deletions(-) create mode 100644 panel/tests/io/test_cache.py diff --git a/panel/io/cache.py b/panel/io/cache.py index 860d8f88d8..bc04a3f1d8 100644 --- a/panel/io/cache.py +++ b/panel/io/cache.py @@ -147,29 +147,33 @@ def _io_hash(obj): for name in _FFI_TYPE_NAMES: _hash_funcs[name] = b'0' -def _generate_hash_inner(obj, hash_funcs={}): - +def _find_hash_func(obj, hash_funcs={}): fqn_type = _get_fqn(obj) if fqn_type in hash_funcs: - hash_func = hash_funcs[fqn_type] - try: - output = hash_func(obj) - except BaseException as e: - raise ValueError( - f'User hash function {hash_func!r} failed for input ' - f'type {fqn_type} with following error: ' - f'{type(e).__name__}("{e}").' - ) - return _generate_hash(output) + return hash_funcs[fqn_type] + elif fqn_type in _hash_funcs: + return _hash_funcs[fqn_type] for otype, hash_func in _hash_funcs.items(): if isinstance(otype, str): if otype == fqn_type: - return hash_func(obj) + return hash_func elif inspect.isfunction(otype): if otype(obj): - return hash_func(obj) + return hash_func elif isinstance(obj, otype): - return hash_func(obj) + return hash_func + +def _generate_hash_inner(obj, hash_funcs={}): + hash_func = _find_hash_func(obj, hash_funcs) + if hash_func is not None: + try: + output = hash_func(obj) + except BaseException as e: + raise ValueError( + f'User hash function {hash_func!r} failed for input ' + f'{obj!r} with following error: {type(e).__name__}("{e}").' + ) + return output if hasattr(obj, '__reduce__'): h = hashlib.new("md5") try: @@ -211,24 +215,28 @@ def _key(obj): return id(obj) return _INDETERMINATE -def _cleanup_cache(cache, policy, max_items, ttl, time): +def _cleanup_cache(cache, policy, max_items, time): """ Deletes items in the cache if the exceed the number of items or their TTL (time-to-live) has expired. """ while len(cache) >= max_items: - if policy.lower() == 'fifo': + if policy.lower() == 'lifo': key = list(cache.keys())[0] elif policy.lower() == 'lru': - key = sorted(((key, time-t) for k, (_, _, _, t) in cache.items()), + key = sorted(((k, -(time-t)) for k, (_, _, _, t) in cache.items()), key=lambda o: o[1])[0][0] elif policy.lower() == 'lfu': key = sorted(cache.items(), key=lambda o: o[1][2])[0][0] del cache[key] - if ttl is not None: - for key, (_, ts, _, _) in list(cache.items()): - if (time-ts) > ttl: - del cache[key] + +def _cleanup_ttl(cache, ttl, time): + """ + Deletes items in the cache if their TTL (time-to-live) has expired. + """ + for key, (_, ts, _, _) in list(cache.items()): + if (time-ts) > ttl: + del cache[key] #--------------------------------------------------------------------- # Public API @@ -336,22 +344,27 @@ def wrapped_func(*args, **kwargs): func_hash = hashlib.sha256(_generate_hash(func_hash)).hexdigest() func_cache = state._memoize_cache.get(func_hash) - if func_cache is None: + + empty = func_cache is None + if empty: if to_disk: from diskcache import Index cache = Index(os.path.join(cache_path, func_hash)) else: cache = {} state._memoize_cache[func_hash] = func_cache = cache - elif hash_value in func_cache: + + if ttl is not None: + _cleanup_ttl(func_cache, ttl, time) + + if not empty and hash_value in func_cache: with lock: ret, ts, count, _ = func_cache[hash_value] func_cache[hash_value] = (ret, ts, count+1, time) return ret if max_items is not None: - with lock: - _cleanup_cache(cache, policy, max_items, ttl, time) + _cleanup_cache(func_cache, policy, max_items, time) ret = func(*args, **kwargs) with lock: @@ -359,9 +372,16 @@ def wrapped_func(*args, **kwargs): return ret def clear(): + global func_hash if func_hash is None: return - state._memoize_cache.get(func_hash, {}).clear() + if to_disk: + from diskcache import Index + cache = Index(os.path.join(cache_path, func_hash)) + cache.clear() + else: + cache = state._memoize_cache.get(func_hash, {}) + cache.clear() wrapped_func.clear = clear try: diff --git a/panel/tests/conftest.py b/panel/tests/conftest.py index 54027cec1c..adbfb1034b 100644 --- a/panel/tests/conftest.py +++ b/panel/tests/conftest.py @@ -230,6 +230,10 @@ def server_cleanup(): state._thread_pool.shutdown(wait=False) state._thread_pool = None +@pytest.fixture(autouse=True) +def cache_cleanup(): + state._memoize_cache.clear() + @pytest.fixture def py_file(): tf = tempfile.NamedTemporaryFile(mode='w', suffix='.py') diff --git a/panel/tests/io/test_cache.py b/panel/tests/io/test_cache.py new file mode 100644 index 0000000000..e3ca30ab88 --- /dev/null +++ b/panel/tests/io/test_cache.py @@ -0,0 +1,207 @@ +import io +import time + +import numpy as np + +from panel.io.cache import _find_hash_func, cache +from panel.tests.util import pd_available + +################ +# Test hashing # +################ + +def hashes_equal(v1, v2): + a, b = _find_hash_func(v1)(v1), _find_hash_func(v2)(v2) + return a == b + +def test_str_hash(): + assert hashes_equal('foo', 'foo') + assert not hashes_equal('foo', 'bar') + +def test_int_hash(): + assert hashes_equal(12, 12) + assert not hashes_equal(1, 2) + +def test_float_hash(): + assert hashes_equal(3.14, 3.14) + assert not hashes_equal(1.2, 3.14) + +def test_bool_hash(): + assert hashes_equal(True, True) + assert hashes_equal(False, False) + assert not hashes_equal(True, False) + +def test_none_hash(): + assert hashes_equal(None, None) + assert not hashes_equal(None, False) + +def test_bytes_hash(): + assert hashes_equal(b'0', b'0') + assert not hashes_equal(b'0', b'1') + +def test_list_hash(): + assert hashes_equal([0], [0]) + assert hashes_equal(['a', ['b']], ['a', ['b']]) + assert not hashes_equal([0], [1]) + assert not hashes_equal(['a', ['b']], ['a', ['c']]) + + # Recursion + l = [0] + l.append(l) + assert hashes_equal(l, list(l)) + +def test_tuple_hash(): + assert hashes_equal((0,), (0,)) + assert hashes_equal(('a', ('b',)), ('a', ('b',))) + assert not hashes_equal((0,), (1,)) + assert not hashes_equal(('a', ('b',)), ('a', ('c',))) + +def test_dict_hash(): + assert hashes_equal({'a': 0}, {'a': 0}) + assert hashes_equal({'a': {'b': 0}}, {'a': {'b': 0}}) + assert not hashes_equal({'a': 0}, {'b': 0}) + assert not hashes_equal({'a': 0}, {'a': 1}) + assert not hashes_equal({'a': {'b': 0}}, {'a': {'b': 1}}) + + # Recursion + d = {'a': {}} + d['a'] = d + assert hashes_equal(d, dict(d)) + +def test_stringio_hash(): + sio1, sio2 = io.StringIO(), io.StringIO() + sio1.write('foo') + sio2.write('foo') + sio1.seek(0) + sio2.seek(0) + assert hashes_equal(sio1, sio2) + sio3 = io.StringIO() + sio3.write('bar') + sio3.seek(0) + assert not hashes_equal(sio1, sio3) + +def test_bytesio_hash(): + bio1, bio2 = io.BytesIO(), io.BytesIO() + bio1.write(b'foo') + bio2.write(b'foo') + bio1.seek(0) + bio2.seek(0) + assert hashes_equal(bio1, bio2) + bio3 = io.BytesIO() + bio3.write(b'bar') + bio3.seek(0) + assert not hashes_equal(bio1, bio3) + +def test_ndarray_hash(): + assert hashes_equal(np.array([0, 1, 2]), np.array([0, 1, 2])) + assert not hashes_equal( + np.array([0, 1, 2], dtype='uint32'), + np.array([0, 1, 2], dtype='float64') + ) + assert not hashes_equal( + np.array([0, 1, 2]), + np.array([2, 1, 0]) + ) + +@pd_available +def test_dataframe_hash(): + import pandas as pd + df1, df2 = pd._testing.makeMixedDataFrame(), pd._testing.makeMixedDataFrame() + assert hashes_equal(df1, df2) + df2['A'] = df2['A'].values[::-1] + assert not hashes_equal(df1, df2) + +@pd_available +def test_series_hash(): + import pandas as pd + series1 = pd._testing.makeStringSeries() + series2 = series1.copy() + assert hashes_equal(series1, series2) + series2.iloc[0] = 3.14 + assert not hashes_equal(series1, series2) + +def test_ufunc_hash(): + assert hashes_equal(np.absolute, np.absolute) + assert not hashes_equal(np.sin, np.cos) + +def test_builtin_hash(): + assert hashes_equal(max, max) + assert not hashes_equal(max, min) + +def test_module_hash(): + assert hashes_equal(np, np) + assert not hashes_equal(np, io) + +################ +# Test caching # +################ + +OFFSET = {} + +def function_with_args(a, b): + global OFFSET + offset = OFFSET.get((a, b), 0) + result = a + b + offset + OFFSET[(a, b)] = offset + 1 + return result + +def test_cache_with_args(): + global OFFSET + OFFSET.clear() + fn = cache(function_with_args) + assert fn(0, 0) == 0 + assert fn(0, 0) == 0 + +def test_cache_with_kwargs(): + global OFFSET + OFFSET.clear() + fn = cache(function_with_args) + assert fn(a=0, b=0) == 0 + assert fn(a=0, b=0) == 0 + +def test_cache_clear(): + global OFFSET + OFFSET.clear() + fn = cache(function_with_args) + assert fn(0, 0) == 0 + fn.clear() + assert fn(0, 0) == 1 + +def test_cache_lifo(): + global OFFSET + OFFSET.clear() + fn = cache(function_with_args, max_items=2, policy='lifo') + assert fn(0, 0) == 0 + assert fn(0, 1) == 1 + assert fn(0, 0) == 0 + assert fn(0, 2) == 2 # (0, 0) should be evicted + assert fn(0, 0) == 1 + +def test_cache_lfu(): + global OFFSET + OFFSET.clear() + fn = cache(function_with_args, max_items=2, policy='lfu') + assert fn(0, 0) == 0 + assert fn(0, 0) == 0 + assert fn(0, 1) == 1 + assert fn(0, 2) == 2 # (0, 1) should be evicted + assert fn(0, 1) == 2 + +def test_cache_lru(): + global OFFSET + OFFSET.clear() + fn = cache(function_with_args, max_items=3, policy='lru') + assert fn(0, 0) == 0 + assert fn(0, 1) == 1 + assert fn(0, 2) == 2 + assert fn(0, 0) == 0 + assert fn(0, 3) == 3 # (0, 1) should be evicted + assert fn(0, 1) == 2 + +def test_cache_ttl(): + global OFFSET + OFFSET.clear() + fn = cache(function_with_args, ttl=0.1) + assert fn(0, 0) == 0 + time.sleep(0.1) + assert fn(0, 0) == 1