From b1b9405df168c3f0a375c3ae5d0c29a792d94793 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 16 Feb 2024 21:21:10 +0100 Subject: [PATCH 01/15] refactor: cleaning of using package/global defaults --- cachier/__init__.py | 4 +- cachier/config.py | 90 ++++++++++++++++++++++++++ cachier/core.py | 137 ++++++++-------------------------------- cachier/cores/base.py | 16 ++--- cachier/cores/memory.py | 5 +- cachier/cores/mongo.py | 52 +++++++++------ cachier/cores/pickle.py | 45 ++++--------- 7 files changed, 171 insertions(+), 178 deletions(-) create mode 100644 cachier/config.py diff --git a/cachier/__init__.py b/cachier/__init__.py index 1297cd7f..4a7770cc 100644 --- a/cachier/__init__.py +++ b/cachier/__init__.py @@ -1,10 +1,10 @@ -from .core import ( - cachier, +from .config import ( set_default_params, get_default_params, enable_caching, disable_caching, ) +from .core import cachier from ._version import * # noqa: F403 diff --git a/cachier/config.py b/cachier/config.py new file mode 100644 index 00000000..f2158f8b --- /dev/null +++ b/cachier/config.py @@ -0,0 +1,90 @@ +import datetime +import hashlib +import os +import pickle +from typing import Callable, Optional, Union, TypedDict, TYPE_CHECKING +from typing_extensions import Literal + +if TYPE_CHECKING: + import pymongo.collection + + +_Type_HashFunc = Callable[..., str] +_Type_Mongetter = Callable[[], "pymongo.collection.Collection"] +_Type_Backend = Literal["pickle", "mongo", "memory"] + + +def _default_hash_func(args, kwds): + # Sort the kwargs to ensure consistent ordering + sorted_kwargs = sorted(kwds.items()) + # Serialize args and sorted_kwargs using pickle or similar + serialized = pickle.dumps((args, sorted_kwargs)) + # Create a hash of the serialized data + return hashlib.sha256(serialized).hexdigest() + + +class Params(TypedDict): + caching_enabled: bool + hash_func: _Type_HashFunc + backend: _Type_Backend + mongetter: Optional[_Type_Mongetter] + stale_after: datetime.timedelta + next_time: bool + cache_dir: Union[str, os.PathLike] + pickle_reload: bool + separate_files: bool + wait_for_calc_timeout: int + allow_none: bool + + +_default_params: Params = { + "caching_enabled": True, + "hash_func": _default_hash_func, + "backend": "pickle", + "mongetter": None, + "stale_after": datetime.timedelta.max, + "next_time": False, + "cache_dir": "~/.cachier/", + "pickle_reload": True, + "separate_files": False, + "wait_for_calc_timeout": 0, + "allow_none": False, +} + + +def _update_with_defaults(param, name: str): + if param is None: + return _default_params[name] + return param + + +def set_default_params(**params): + """Configure global parameters applicable to all memoized functions. + + This function takes the same keyword parameters as the ones defined in the + decorator, which can be passed all at once or with multiple calls. + Parameters given directly to a decorator take precedence over any values + set by this function. + + Only 'stale_after', 'next_time', and 'wait_for_calc_timeout' can be changed + after the memoization decorator has been applied. Other parameters will + only have an effect on decorators applied after this function is run. + + """ + valid_params = (p for p in params.items() if p[0] in _default_params) + _default_params.update(valid_params) + + +def get_default_params(): + """Get current set of default parameters.""" + return _default_params + + +def enable_caching(): + """Enable caching globally.""" + _default_params["caching_enabled"] = True + + +def disable_caching(): + """Disable caching globally.""" + _default_params["caching_enabled"] = False diff --git a/cachier/core.py b/cachier/core.py index dcbbd888..759f3c3a 100644 --- a/cachier/core.py +++ b/cachier/core.py @@ -7,26 +7,26 @@ # http://www.opensource.org/licenses/MIT-license # Copyright (c) 2016, Shay Palachy -# python 2 compatibility - import datetime -import hashlib import inspect import os -import pickle from collections import OrderedDict from concurrent.futures import ThreadPoolExecutor from functools import wraps -from typing import TYPE_CHECKING, Callable, Literal, Optional, TypedDict, Union +from typing import Optional, Union from warnings import warn from .cores.base import RecalculationNeeded, _BaseCore from .cores.memory import _MemoryCore from .cores.mongo import _MongoCore from .cores.pickle import _PickleCore - -if TYPE_CHECKING: - import pymongo.collection +from .config import ( + _Type_Mongetter, + _Type_HashFunc, + _Type_Backend, + _default_params, + _update_with_defaults, +) MAX_WORKERS_ENVAR_NAME = "CACHIER_MAX_WORKERS" @@ -68,15 +68,6 @@ def _calc_entry(core, key, func, args, kwds): core.mark_entry_not_calculated(key) -def _default_hash_func(args, kwds): - # Sort the kwargs to ensure consistent ordering - sorted_kwargs = sorted(kwds.items()) - # Serialize args and sorted_kwargs using pickle or similar - serialized = pickle.dumps((args, sorted_kwargs)) - # Create a hash of the serialized data - return hashlib.sha256(serialized).hexdigest() - - def _convert_args_kwargs( func, _is_method: bool, args: tuple, kwds: dict ) -> dict: @@ -103,49 +94,11 @@ def _convert_args_kwargs( return OrderedDict(sorted(kwargs.items())) -class MissingMongetter(ValueError): - """Thrown when the mongetter keyword argument is missing.""" - - -HashFunc = Callable[..., str] -Mongetter = Callable[[], "pymongo.collection.Collection"] -Backend = Literal["pickle", "mongo", "memory"] - - -class Params(TypedDict): - caching_enabled: bool - hash_func: HashFunc - backend: Backend - mongetter: Optional[Mongetter] - stale_after: datetime.timedelta - next_time: bool - cache_dir: Union[str, os.PathLike] - pickle_reload: bool - separate_files: bool - wait_for_calc_timeout: int - allow_none: bool - - -_default_params: Params = { - "caching_enabled": True, - "hash_func": _default_hash_func, - "backend": "pickle", - "mongetter": None, - "stale_after": datetime.timedelta.max, - "next_time": False, - "cache_dir": "~/.cachier/", - "pickle_reload": True, - "separate_files": False, - "wait_for_calc_timeout": 0, - "allow_none": False, -} - - def cachier( - hash_func: Optional[HashFunc] = None, - hash_params: Optional[HashFunc] = None, - backend: Optional[Backend] = None, - mongetter: Optional[Mongetter] = None, + hash_func: Optional[_Type_HashFunc] = None, + hash_params: Optional[_Type_HashFunc] = None, + backend: Optional[_Type_Backend] = None, + mongetter: Optional[_Type_Mongetter] = None, stale_after: Optional[datetime.timedelta] = None, next_time: Optional[bool] = None, cache_dir: Optional[Union[str, os.PathLike]] = None, @@ -220,12 +173,17 @@ def cachier( warn(message, DeprecationWarning, stacklevel=2) hash_func = hash_params # Override the backend parameter if a mongetter is provided. - if mongetter is None: - mongetter = _default_params["mongetter"] + hash_func = _update_with_defaults(hash_func, "hash_func") + mongetter = _update_with_defaults(mongetter, "mongetter") + backend = _update_with_defaults(backend, "backend") + pickle_reload = _update_with_defaults(pickle_reload, "pickle_reload") + cache_dir = _update_with_defaults(cache_dir, "cache_dir") + separate_files = _update_with_defaults(separate_files, "separate_files") + wait_for_calc_timeout = _update_with_defaults( + wait_for_calc_timeout, "wait_for_calc_timeout" + ) if callable(mongetter): backend = "mongo" - if backend is None: - backend = _default_params["backend"] core: _BaseCore if backend == "pickle": core = _PickleCore( @@ -234,23 +192,16 @@ def cachier( cache_dir=cache_dir, separate_files=separate_files, wait_for_calc_timeout=wait_for_calc_timeout, - default_params=_default_params, ) elif backend == "mongo": - if mongetter is None: - raise MissingMongetter( - "must specify ``mongetter`` when using the mongo core" - ) core = _MongoCore( mongetter=mongetter, hash_func=hash_func, wait_for_calc_timeout=wait_for_calc_timeout, - default_params=_default_params, ) elif backend == "memory": core = _MemoryCore( - hash_func=hash_func, - default_params=_default_params, + hash_func=hash_func, wait_for_calc_timeout=wait_for_calc_timeout ) else: raise ValueError("specified an invalid core: %s" % backend) @@ -261,11 +212,7 @@ def _cachier_decorator(func): @wraps(func) def func_wrapper(*args, **kwds): nonlocal allow_none - _allow_none = ( - allow_none - if allow_none is not None - else _default_params["allow_none"] - ) + _allow_none = _update_with_defaults(allow_none, "allow_none") # print('Inside general wrapper for {}.'.format(func.__name__)) ignore_cache = kwds.pop("ignore_cache", False) overwrite_cache = kwds.pop("overwrite_cache", False) @@ -289,10 +236,10 @@ def func_wrapper(*args, **kwds): _print("Entry found.") if _allow_none or entry.get("value", None) is not None: _print("Cached result found.") - local_stale_after = ( - stale_after or _default_params["stale_after"] + local_stale_after = _update_with_defaults( + stale_after, "stale_after" ) - local_next_time = next_time or _default_params["next_time"] # noqa: E501 + local_next_time = _update_with_defaults(next_time, "next_time") now = datetime.datetime.now() if now - entry["time"] <= local_stale_after: _print("And it is fresh!") @@ -362,35 +309,3 @@ def precache_value(*args, value_to_cache, **kwds): return func_wrapper return _cachier_decorator - - -def set_default_params(**params): - """Configure global parameters applicable to all memoized functions. - - This function takes the same keyword parameters as the ones defined in the - decorator, which can be passed all at once or with multiple calls. - Parameters given directly to a decorator take precedence over any values - set by this function. - - Only 'stale_after', 'next_time', and 'wait_for_calc_timeout' can be changed - after the memoization decorator has been applied. Other parameters will - only have an effect on decorators applied after this function is run. - - """ - valid_params = (p for p in params.items() if p[0] in _default_params) - _default_params.update(valid_params) - - -def get_default_params(): - """Get current set of default parameters.""" - return _default_params - - -def enable_caching(): - """Enable caching globally.""" - _default_params["caching_enabled"] = True - - -def disable_caching(): - """Disable caching globally.""" - _default_params["caching_enabled"] = False diff --git a/cachier/cores/base.py b/cachier/cores/base.py index c1f9b7f6..adf7050c 100644 --- a/cachier/cores/base.py +++ b/cachier/cores/base.py @@ -9,6 +9,8 @@ import abc # for the _BaseCore abstract base class import inspect +from ..config import _Type_HashFunc + class RecalculationNeeded(Exception): pass @@ -17,9 +19,9 @@ class RecalculationNeeded(Exception): class _BaseCore: __metaclass__ = abc.ABCMeta - def __init__(self, hash_func, default_params): - self.default_params = default_params + def __init__(self, hash_func: _Type_HashFunc, wait_for_calc_timeout: int): self.hash_func = hash_func + self.wait_for_calc_timeout = wait_for_calc_timeout def set_func(self, func): """Sets the function this core will use. @@ -37,10 +39,7 @@ def set_func(self, func): def get_key(self, args, kwds): """Returns a unique key based on the arguments provided.""" - if self.hash_func is not None: - return self.hash_func(args, kwds) - else: - return self.default_params["hash_func"](args, kwds) + return self.hash_func(args, kwds) def get_entry(self, args, kwds): """Returns the result mapped to the given arguments in this core's @@ -56,10 +55,7 @@ def precache_value(self, args, kwds, value_to_cache): def check_calc_timeout(self, time_spent): """Raise an exception if a recalculation is needed.""" - if self.wait_for_calc_timeout is not None: - calc_timeout = self.wait_for_calc_timeout - else: - calc_timeout = self.default_params["wait_for_calc_timeout"] + calc_timeout = self.wait_for_calc_timeout if calc_timeout > 0 and (time_spent >= calc_timeout): raise RecalculationNeeded() diff --git a/cachier/cores/memory.py b/cachier/cores/memory.py index 5f6bb5e9..4d369e69 100644 --- a/cachier/cores/memory.py +++ b/cachier/cores/memory.py @@ -4,13 +4,14 @@ from datetime import datetime from .base import _BaseCore +from ..config import _Type_HashFunc class _MemoryCore(_BaseCore): """The memory core class for cachier.""" - def __init__(self, hash_func, default_params): - super().__init__(hash_func, default_params) + def __init__(self, hash_func: _Type_HashFunc, wait_for_calc_timeout: int): + super().__init__(hash_func, wait_for_calc_timeout) self.cache = {} self.lock = threading.RLock() diff --git a/cachier/cores/mongo.py b/cachier/cores/mongo.py index dff393dd..adf7a3d8 100644 --- a/cachier/cores/mongo.py +++ b/cachier/cores/mongo.py @@ -13,6 +13,7 @@ from datetime import datetime import time # to sleep when waiting on Mongo cache\ import warnings # to warn if pymongo is missing +from ..config import _Type_HashFunc, _Type_Mongetter with suppress(ImportError): from pymongo import IndexModel, ASCENDING @@ -25,21 +26,32 @@ MONGO_SLEEP_DURATION_IN_SEC = 1 +class MissingMongetter(ValueError): + """Thrown when the mongetter keyword argument is missing.""" + + class _MongoCore(_BaseCore): _INDEX_NAME = "func_1_key_1" def __init__( - self, mongetter, hash_func, wait_for_calc_timeout, default_params + self, + mongetter: _Type_Mongetter, + hash_func: _Type_HashFunc, + wait_for_calc_timeout: int, ): if "pymongo" not in sys.modules: warnings.warn( "Cachier warning: pymongo was not found. " "MongoDB cores will not function." ) # pragma: no cover - super().__init__(hash_func, default_params) + + if mongetter is None: + raise MissingMongetter( + "must specify ``mongetter`` when using the mongo core" + ) + super().__init__(hash_func, wait_for_calc_timeout) self.mongetter = mongetter self.mongo_collection = self.mongetter() - self.wait_for_calc_timeout = wait_for_calc_timeout index_inf = self.mongo_collection.index_information() if _MongoCore._INDEX_NAME not in index_inf: func1key1 = IndexModel( @@ -56,23 +68,23 @@ def get_entry_by_key(self, key): res = self.mongo_collection.find_one( {"func": _MongoCore._get_func_str(self.func), "key": key} ) - if res: - try: - entry = { - "value": pickle.loads(res["value"]), # noqa: S301 - "time": res.get("time", None), - "stale": res.get("stale", False), - "being_calculated": res.get("being_calculated", False), - } - except KeyError: - entry = { - "value": None, - "time": res.get("time", None), - "stale": res.get("stale", False), - "being_calculated": res.get("being_calculated", False), - } - return key, entry - return key, None + if not res: + return key, None + try: + entry = { + "value": pickle.loads(res["value"]), # noqa: S301 + "time": res.get("time", None), + "stale": res.get("stale", False), + "being_calculated": res.get("being_calculated", False), + } + except KeyError: + entry = { + "value": None, + "time": res.get("time", None), + "stale": res.get("stale", False), + "being_calculated": res.get("being_calculated", False), + } + return key, entry def set_entry(self, key, func_res): thebytes = pickle.dumps(func_res) diff --git a/cachier/cores/pickle.py b/cachier/cores/pickle.py index 1339fb69..efac2fd5 100644 --- a/cachier/cores/pickle.py +++ b/cachier/cores/pickle.py @@ -19,19 +19,11 @@ # Alternative: https://github.com/WoLpH/portalocker from .base import _BaseCore +from ..config import _Type_HashFunc class _PickleCore(_BaseCore): - """The pickle core class for cachier. - - Parameters - ---------- - pickle_reload : bool, optional - See core.cachier() documentation. - cache_dir : str, optional. - See core.cachier() documentation. - - """ + """The pickle core class for cachier.""" class CacheChangeHandler(PatternMatchingEventHandler): """Handles cache-file modification events.""" @@ -79,36 +71,23 @@ def on_modified(self, event): def __init__( self, - hash_func, - pickle_reload, - cache_dir, - separate_files, - wait_for_calc_timeout, - default_params, + hash_func: _Type_HashFunc, + pickle_reload: bool, + cache_dir: str, + separate_files: bool, + wait_for_calc_timeout: int, ): - super().__init__(hash_func, default_params) + super().__init__(hash_func, wait_for_calc_timeout) self.cache = None - if pickle_reload is not None: - self.reload = pickle_reload - else: - self.reload = self.default_params["pickle_reload"] - if cache_dir is not None: - self.cache_dir = os.path.expanduser(cache_dir) - else: - self.cache_dir = os.path.expanduser( - self.default_params["cache_dir"] - ) - if separate_files is not None: - self.separate_files = separate_files - else: - self.separate_files = self.default_params["separate_files"] - self.wait_for_calc_timeout = wait_for_calc_timeout + self.reload = pickle_reload + self.cache_dir = os.path.expanduser(cache_dir) + self.separate_files = separate_files self.cache_fname = None self.cache_fpath = None self.lock = threading.RLock() def _cache_fname(self): - if self.cache_fname is None: + if self.cache_fname is not None: self.cache_fname = ( f".{self.func.__module__}.{self.func.__qualname__}" ) From 446975990ed1cfd6ce7c67a69fa128244edfdbda Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 16 Feb 2024 21:30:29 +0100 Subject: [PATCH 02/15] Literal --- cachier/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cachier/config.py b/cachier/config.py index f2158f8b..5cefe7d7 100644 --- a/cachier/config.py +++ b/cachier/config.py @@ -2,8 +2,7 @@ import hashlib import os import pickle -from typing import Callable, Optional, Union, TypedDict, TYPE_CHECKING -from typing_extensions import Literal +from typing import Callable, Optional, Union, TypedDict, TYPE_CHECKING, Literal if TYPE_CHECKING: import pymongo.collection From e24fde2c68b3516967a7e5395b9fff912b4c3c54 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 16 Feb 2024 21:35:47 +0100 Subject: [PATCH 03/15] MissingMongetter --- tests/test_core_lookup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_core_lookup.py b/tests/test_core_lookup.py index 57b5f3a2..02535964 100644 --- a/tests/test_core_lookup.py +++ b/tests/test_core_lookup.py @@ -2,7 +2,7 @@ import pytest from cachier import cachier, get_default_params -from cachier.core import MissingMongetter +from cachier.cores.mongo import MissingMongetter def test_get_default_params(): From af0c6bfbafcb2e979c3eb9eff96b19db01cd6381 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 16 Feb 2024 21:47:07 +0100 Subject: [PATCH 04/15] typos --- cachier/cores/pickle.py | 2 +- tests/test_mongo_core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cachier/cores/pickle.py b/cachier/cores/pickle.py index efac2fd5..67f8b012 100644 --- a/cachier/cores/pickle.py +++ b/cachier/cores/pickle.py @@ -87,7 +87,7 @@ def __init__( self.lock = threading.RLock() def _cache_fname(self): - if self.cache_fname is not None: + if self.cache_fname is None: self.cache_fname = ( f".{self.func.__module__}.{self.func.__qualname__}" ) diff --git a/tests/test_mongo_core.py b/tests/test_mongo_core.py index 65b2339f..755263d8 100644 --- a/tests/test_mongo_core.py +++ b/tests/test_mongo_core.py @@ -268,7 +268,7 @@ def test_stalled_mongo_db_cache(): def _stalled_func(): return 1 - core = _MongoCore(_test_mongetter, None, 0, {}) + core = _MongoCore(_test_mongetter, None, 0) core.set_func(_stalled_func) core.clear_cache() with pytest.raises(RecalculationNeeded): From 76317eb591627d7cdc72c5df1024eccf176e2d9d Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 16 Feb 2024 23:13:21 +0100 Subject: [PATCH 05/15] cleaning --- cachier/cores/pickle.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/cachier/cores/pickle.py b/cachier/cores/pickle.py index 67f8b012..e9b219fa 100644 --- a/cachier/cores/pickle.py +++ b/cachier/cores/pickle.py @@ -88,22 +88,16 @@ def __init__( def _cache_fname(self): if self.cache_fname is None: - self.cache_fname = ( - f".{self.func.__module__}.{self.func.__qualname__}" - ) - self.cache_fname = self.cache_fname.replace("<", "_").replace( - ">", "_" - ) + fname = f".{self.func.__module__}.{self.func.__qualname__}" + self.cache_fname = fname.replace("<", "_").replace(">", "_") return self.cache_fname def _cache_fpath(self): if self.cache_fpath is None: - if not os.path.exists(self.cache_dir): - os.makedirs(self.cache_dir) + os.makedirs(self.cache_dir, exist_ok=True) self.cache_fpath = os.path.abspath( os.path.join( - os.path.realpath(self.cache_dir), - self._cache_fname(), + os.path.realpath(self.cache_dir), self._cache_fname() ) ) return self.cache_fpath From a7a5cc4d6dd2e0263604f4c259fffb81de10da5d Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 19 Feb 2024 10:39:22 +0100 Subject: [PATCH 06/15] types --- cachier/_types.py | 9 +++++++++ cachier/config.py | 16 +++++----------- cachier/core.py | 14 +++++++------- cachier/cores/base.py | 4 ++-- cachier/cores/memory.py | 4 ++-- cachier/cores/mongo.py | 6 +++--- cachier/cores/pickle.py | 4 ++-- 7 files changed, 30 insertions(+), 27 deletions(-) create mode 100644 cachier/_types.py diff --git a/cachier/_types.py b/cachier/_types.py new file mode 100644 index 00000000..04ad9208 --- /dev/null +++ b/cachier/_types.py @@ -0,0 +1,9 @@ +from typing import Callable, TYPE_CHECKING, Literal + +if TYPE_CHECKING: + import pymongo.collection + + +HashFunc = Callable[..., str] +Mongetter = Callable[[], "pymongo.collection.Collection"] +Backend = Literal["pickle", "mongo", "memory"] diff --git a/cachier/config.py b/cachier/config.py index 5cefe7d7..2c603943 100644 --- a/cachier/config.py +++ b/cachier/config.py @@ -2,15 +2,9 @@ import hashlib import os import pickle -from typing import Callable, Optional, Union, TypedDict, TYPE_CHECKING, Literal +from typing import Optional, Union, TypedDict -if TYPE_CHECKING: - import pymongo.collection - - -_Type_HashFunc = Callable[..., str] -_Type_Mongetter = Callable[[], "pymongo.collection.Collection"] -_Type_Backend = Literal["pickle", "mongo", "memory"] +from ._types import HashFunc, Backend, Mongetter def _default_hash_func(args, kwds): @@ -24,9 +18,9 @@ def _default_hash_func(args, kwds): class Params(TypedDict): caching_enabled: bool - hash_func: _Type_HashFunc - backend: _Type_Backend - mongetter: Optional[_Type_Mongetter] + hash_func: HashFunc + backend: Backend + mongetter: Optional[Mongetter] stale_after: datetime.timedelta next_time: bool cache_dir: Union[str, os.PathLike] diff --git a/cachier/core.py b/cachier/core.py index 759f3c3a..50225ad8 100644 --- a/cachier/core.py +++ b/cachier/core.py @@ -21,9 +21,9 @@ from .cores.mongo import _MongoCore from .cores.pickle import _PickleCore from .config import ( - _Type_Mongetter, - _Type_HashFunc, - _Type_Backend, + Mongetter, + HashFunc, + Backend, _default_params, _update_with_defaults, ) @@ -95,10 +95,10 @@ def _convert_args_kwargs( def cachier( - hash_func: Optional[_Type_HashFunc] = None, - hash_params: Optional[_Type_HashFunc] = None, - backend: Optional[_Type_Backend] = None, - mongetter: Optional[_Type_Mongetter] = None, + hash_func: Optional[HashFunc] = None, + hash_params: Optional[HashFunc] = None, + backend: Optional[Backend] = None, + mongetter: Optional[Mongetter] = None, stale_after: Optional[datetime.timedelta] = None, next_time: Optional[bool] = None, cache_dir: Optional[Union[str, os.PathLike]] = None, diff --git a/cachier/cores/base.py b/cachier/cores/base.py index adf7050c..b99214b8 100644 --- a/cachier/cores/base.py +++ b/cachier/cores/base.py @@ -9,7 +9,7 @@ import abc # for the _BaseCore abstract base class import inspect -from ..config import _Type_HashFunc +from .._types import HashFunc class RecalculationNeeded(Exception): @@ -19,7 +19,7 @@ class RecalculationNeeded(Exception): class _BaseCore: __metaclass__ = abc.ABCMeta - def __init__(self, hash_func: _Type_HashFunc, wait_for_calc_timeout: int): + def __init__(self, hash_func: HashFunc, wait_for_calc_timeout: int): self.hash_func = hash_func self.wait_for_calc_timeout = wait_for_calc_timeout diff --git a/cachier/cores/memory.py b/cachier/cores/memory.py index 4d369e69..b22566b9 100644 --- a/cachier/cores/memory.py +++ b/cachier/cores/memory.py @@ -4,13 +4,13 @@ from datetime import datetime from .base import _BaseCore -from ..config import _Type_HashFunc +from .._types import HashFunc class _MemoryCore(_BaseCore): """The memory core class for cachier.""" - def __init__(self, hash_func: _Type_HashFunc, wait_for_calc_timeout: int): + def __init__(self, hash_func: HashFunc, wait_for_calc_timeout: int): super().__init__(hash_func, wait_for_calc_timeout) self.cache = {} self.lock = threading.RLock() diff --git a/cachier/cores/mongo.py b/cachier/cores/mongo.py index adf7a3d8..cf5a32fe 100644 --- a/cachier/cores/mongo.py +++ b/cachier/cores/mongo.py @@ -13,7 +13,7 @@ from datetime import datetime import time # to sleep when waiting on Mongo cache\ import warnings # to warn if pymongo is missing -from ..config import _Type_HashFunc, _Type_Mongetter +from .._types import HashFunc, Mongetter with suppress(ImportError): from pymongo import IndexModel, ASCENDING @@ -35,8 +35,8 @@ class _MongoCore(_BaseCore): def __init__( self, - mongetter: _Type_Mongetter, - hash_func: _Type_HashFunc, + mongetter: Mongetter, + hash_func: HashFunc, wait_for_calc_timeout: int, ): if "pymongo" not in sys.modules: diff --git a/cachier/cores/pickle.py b/cachier/cores/pickle.py index e9b219fa..429f21cd 100644 --- a/cachier/cores/pickle.py +++ b/cachier/cores/pickle.py @@ -19,7 +19,7 @@ # Alternative: https://github.com/WoLpH/portalocker from .base import _BaseCore -from ..config import _Type_HashFunc +from .._types import HashFunc class _PickleCore(_BaseCore): @@ -71,7 +71,7 @@ def on_modified(self, event): def __init__( self, - hash_func: _Type_HashFunc, + hash_func: HashFunc, pickle_reload: bool, cache_dir: str, separate_files: bool, From 21acefc036a1582af250f9a2ba0c4da7ba49f97f Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 19 Feb 2024 10:41:15 +0100 Subject: [PATCH 07/15] note --- cachier/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cachier/core.py b/cachier/core.py index 50225ad8..b5cd730b 100644 --- a/cachier/core.py +++ b/cachier/core.py @@ -172,7 +172,7 @@ def cachier( ) warn(message, DeprecationWarning, stacklevel=2) hash_func = hash_params - # Override the backend parameter if a mongetter is provided. + # Update parameters with defaults if input is None hash_func = _update_with_defaults(hash_func, "hash_func") mongetter = _update_with_defaults(mongetter, "mongetter") backend = _update_with_defaults(backend, "backend") @@ -182,6 +182,7 @@ def cachier( wait_for_calc_timeout = _update_with_defaults( wait_for_calc_timeout, "wait_for_calc_timeout" ) + # Override the backend parameter if a mongetter is provided. if callable(mongetter): backend = "mongo" core: _BaseCore From cb1ffbb9c4d3cb6ec187f5312d0697e3b69294bf Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 19 Feb 2024 10:42:41 +0100 Subject: [PATCH 08/15] isort --- cachier/__init__.py | 9 ++++----- cachier/_types.py | 2 +- cachier/config.py | 4 ++-- cachier/core.py | 13 ++++++------- cachier/cores/memory.py | 2 +- cachier/cores/mongo.py | 14 +++++++------- cachier/cores/pickle.py | 3 ++- 7 files changed, 23 insertions(+), 24 deletions(-) diff --git a/cachier/__init__.py b/cachier/__init__.py index 4a7770cc..44983133 100644 --- a/cachier/__init__.py +++ b/cachier/__init__.py @@ -1,13 +1,12 @@ +from ._version import * # noqa: F403 from .config import ( - set_default_params, - get_default_params, - enable_caching, disable_caching, + enable_caching, + get_default_params, + set_default_params, ) from .core import cachier -from ._version import * # noqa: F403 - __all__ = [ "cachier", "set_default_params", diff --git a/cachier/_types.py b/cachier/_types.py index 04ad9208..0a3f873a 100644 --- a/cachier/_types.py +++ b/cachier/_types.py @@ -1,4 +1,4 @@ -from typing import Callable, TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Callable, Literal if TYPE_CHECKING: import pymongo.collection diff --git a/cachier/config.py b/cachier/config.py index 2c603943..27c37dd3 100644 --- a/cachier/config.py +++ b/cachier/config.py @@ -2,9 +2,9 @@ import hashlib import os import pickle -from typing import Optional, Union, TypedDict +from typing import Optional, TypedDict, Union -from ._types import HashFunc, Backend, Mongetter +from ._types import Backend, HashFunc, Mongetter def _default_hash_func(args, kwds): diff --git a/cachier/core.py b/cachier/core.py index b5cd730b..3b814024 100644 --- a/cachier/core.py +++ b/cachier/core.py @@ -16,18 +16,17 @@ from typing import Optional, Union from warnings import warn -from .cores.base import RecalculationNeeded, _BaseCore -from .cores.memory import _MemoryCore -from .cores.mongo import _MongoCore -from .cores.pickle import _PickleCore from .config import ( - Mongetter, - HashFunc, Backend, + HashFunc, + Mongetter, _default_params, _update_with_defaults, ) - +from .cores.base import RecalculationNeeded, _BaseCore +from .cores.memory import _MemoryCore +from .cores.mongo import _MongoCore +from .cores.pickle import _PickleCore MAX_WORKERS_ENVAR_NAME = "CACHIER_MAX_WORKERS" DEFAULT_MAX_WORKERS = 8 diff --git a/cachier/cores/memory.py b/cachier/cores/memory.py index b22566b9..b1e1bd0a 100644 --- a/cachier/cores/memory.py +++ b/cachier/cores/memory.py @@ -3,8 +3,8 @@ import threading from datetime import datetime -from .base import _BaseCore from .._types import HashFunc +from .base import _BaseCore class _MemoryCore(_BaseCore): diff --git a/cachier/cores/mongo.py b/cachier/cores/mongo.py index cf5a32fe..b74b6fb2 100644 --- a/cachier/cores/mongo.py +++ b/cachier/cores/mongo.py @@ -7,21 +7,21 @@ # http://www.opensource.org/licenses/MIT-license # Copyright (c) 2016, Shay Palachy -import sys # to make sure that pymongo was imported import pickle # for serialization of python objects -from contextlib import suppress -from datetime import datetime +import sys # to make sure that pymongo was imported import time # to sleep when waiting on Mongo cache\ import warnings # to warn if pymongo is missing +from contextlib import suppress +from datetime import datetime + from .._types import HashFunc, Mongetter with suppress(ImportError): - from pymongo import IndexModel, ASCENDING - from pymongo.errors import OperationFailure from bson.binary import Binary # to save binary data to mongodb + from pymongo import ASCENDING, IndexModel + from pymongo.errors import OperationFailure -from .base import _BaseCore, RecalculationNeeded - +from .base import RecalculationNeeded, _BaseCore MONGO_SLEEP_DURATION_IN_SEC = 1 diff --git a/cachier/cores/pickle.py b/cachier/cores/pickle.py index 4a9cf73d..2534cf55 100644 --- a/cachier/cores/pickle.py +++ b/cachier/cores/pickle.py @@ -16,9 +16,10 @@ from watchdog.events import PatternMatchingEventHandler from watchdog.observers import Observer +from .._types import HashFunc + # Alternative: https://github.com/WoLpH/portalocker from .base import _BaseCore -from .._types import HashFunc class _PickleCore(_BaseCore): From d1408a64350d779dcbe2ae86e75ec922e7295c39 Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 19 Feb 2024 17:34:20 +0100 Subject: [PATCH 09/15] move loweer --- .pre-commit-config.yaml | 6 +++--- cachier/core.py | 8 -------- cachier/cores/base.py | 9 +++++++-- cachier/cores/memory.py | 1 - cachier/cores/mongo.py | 5 +++-- cachier/cores/pickle.py | 13 ++++++++----- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 35e7b1a0..32c5d849 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -66,10 +66,10 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.2.1 hooks: + # use black formatting + - id: ruff-format + name: Black by Ruff # basic check - id: ruff name: Ruff check args: ["--fix"] - # use black formatting - - id: ruff-format - name: Black by Ruff diff --git a/cachier/core.py b/cachier/core.py index 3b814024..6612efa6 100644 --- a/cachier/core.py +++ b/cachier/core.py @@ -172,15 +172,7 @@ def cachier( warn(message, DeprecationWarning, stacklevel=2) hash_func = hash_params # Update parameters with defaults if input is None - hash_func = _update_with_defaults(hash_func, "hash_func") - mongetter = _update_with_defaults(mongetter, "mongetter") backend = _update_with_defaults(backend, "backend") - pickle_reload = _update_with_defaults(pickle_reload, "pickle_reload") - cache_dir = _update_with_defaults(cache_dir, "cache_dir") - separate_files = _update_with_defaults(separate_files, "separate_files") - wait_for_calc_timeout = _update_with_defaults( - wait_for_calc_timeout, "wait_for_calc_timeout" - ) # Override the backend parameter if a mongetter is provided. if callable(mongetter): backend = "mongo" diff --git a/cachier/cores/base.py b/cachier/cores/base.py index b99214b8..0bcbd95d 100644 --- a/cachier/cores/base.py +++ b/cachier/cores/base.py @@ -8,8 +8,10 @@ import abc # for the _BaseCore abstract base class import inspect +import threading from .._types import HashFunc +from ..config import _update_with_defaults class RecalculationNeeded(Exception): @@ -20,8 +22,9 @@ class _BaseCore: __metaclass__ = abc.ABCMeta def __init__(self, hash_func: HashFunc, wait_for_calc_timeout: int): - self.hash_func = hash_func + self.hash_func = _update_with_defaults(hash_func, "hash_func") self.wait_for_calc_timeout = wait_for_calc_timeout + self.lock = threading.RLock() def set_func(self, func): """Sets the function this core will use. @@ -55,7 +58,9 @@ def precache_value(self, args, kwds, value_to_cache): def check_calc_timeout(self, time_spent): """Raise an exception if a recalculation is needed.""" - calc_timeout = self.wait_for_calc_timeout + calc_timeout = _update_with_defaults( + self.wait_for_calc_timeout, "wait_for_calc_timeout" + ) if calc_timeout > 0 and (time_spent >= calc_timeout): raise RecalculationNeeded() diff --git a/cachier/cores/memory.py b/cachier/cores/memory.py index b1e1bd0a..de87da00 100644 --- a/cachier/cores/memory.py +++ b/cachier/cores/memory.py @@ -13,7 +13,6 @@ class _MemoryCore(_BaseCore): def __init__(self, hash_func: HashFunc, wait_for_calc_timeout: int): super().__init__(hash_func, wait_for_calc_timeout) self.cache = {} - self.lock = threading.RLock() def get_entry_by_key(self, key, reload=False): with self.lock: diff --git a/cachier/cores/mongo.py b/cachier/cores/mongo.py index b74b6fb2..997f2190 100644 --- a/cachier/cores/mongo.py +++ b/cachier/cores/mongo.py @@ -15,6 +15,7 @@ from datetime import datetime from .._types import HashFunc, Mongetter +from ..config import _update_with_defaults with suppress(ImportError): from bson.binary import Binary # to save binary data to mongodb @@ -35,8 +36,8 @@ class _MongoCore(_BaseCore): def __init__( self, - mongetter: Mongetter, hash_func: HashFunc, + mongetter: Mongetter, wait_for_calc_timeout: int, ): if "pymongo" not in sys.modules: @@ -50,7 +51,7 @@ def __init__( "must specify ``mongetter`` when using the mongo core" ) super().__init__(hash_func, wait_for_calc_timeout) - self.mongetter = mongetter + self.mongetter = _update_with_defaults(mongetter, "mongetter") self.mongo_collection = self.mongetter() index_inf = self.mongo_collection.index_information() if _MongoCore._INDEX_NAME not in index_inf: diff --git a/cachier/cores/pickle.py b/cachier/cores/pickle.py index 2534cf55..4180f934 100644 --- a/cachier/cores/pickle.py +++ b/cachier/cores/pickle.py @@ -8,7 +8,6 @@ # Copyright (c) 2016, Shay Palachy import os import pickle # for local caching -import threading from contextlib import suppress from datetime import datetime @@ -17,6 +16,7 @@ from watchdog.observers import Observer from .._types import HashFunc +from ..config import _update_with_defaults # Alternative: https://github.com/WoLpH/portalocker from .base import _BaseCore @@ -79,12 +79,15 @@ def __init__( ): super().__init__(hash_func, wait_for_calc_timeout) self.cache = None - self.reload = pickle_reload - self.cache_dir = os.path.expanduser(cache_dir) - self.separate_files = separate_files + self.reload = _update_with_defaults(pickle_reload, "pickle_reload") + self.cache_dir = os.path.expanduser( + _update_with_defaults(cache_dir, "cache_dir") + ) + self.separate_files = _update_with_defaults( + separate_files, "separate_files" + ) self.cache_fname = None self.cache_fpath = None - self.lock = threading.RLock() def _cache_fname(self): if self.cache_fname is None: From 19bcb4f3b53675a3921c439c445877897ef206ce Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 19 Feb 2024 17:39:29 +0100 Subject: [PATCH 10/15] resolve --- cachier/core.py | 2 +- cachier/cores/mongo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cachier/core.py b/cachier/core.py index 6612efa6..b6df3fc7 100644 --- a/cachier/core.py +++ b/cachier/core.py @@ -187,8 +187,8 @@ def cachier( ) elif backend == "mongo": core = _MongoCore( - mongetter=mongetter, hash_func=hash_func, + mongetter=mongetter, wait_for_calc_timeout=wait_for_calc_timeout, ) elif backend == "memory": diff --git a/cachier/cores/mongo.py b/cachier/cores/mongo.py index 997f2190..513056dd 100644 --- a/cachier/cores/mongo.py +++ b/cachier/cores/mongo.py @@ -46,12 +46,12 @@ def __init__( "MongoDB cores will not function." ) # pragma: no cover + self.mongetter = _update_with_defaults(mongetter, "mongetter") if mongetter is None: raise MissingMongetter( "must specify ``mongetter`` when using the mongo core" ) super().__init__(hash_func, wait_for_calc_timeout) - self.mongetter = _update_with_defaults(mongetter, "mongetter") self.mongo_collection = self.mongetter() index_inf = self.mongo_collection.index_information() if _MongoCore._INDEX_NAME not in index_inf: From 28fcf7cf9c889bd52358a1024a009de5c56f9aad Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 19 Feb 2024 17:45:57 +0100 Subject: [PATCH 11/15] global --- cachier/config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cachier/config.py b/cachier/config.py index 27c37dd3..5be61110 100644 --- a/cachier/config.py +++ b/cachier/config.py @@ -46,6 +46,7 @@ class Params(TypedDict): def _update_with_defaults(param, name: str): + global _default_params if param is None: return _default_params[name] return param @@ -64,20 +65,24 @@ def set_default_params(**params): only have an effect on decorators applied after this function is run. """ + global _default_params valid_params = (p for p in params.items() if p[0] in _default_params) _default_params.update(valid_params) def get_default_params(): """Get current set of default parameters.""" + global _default_params return _default_params def enable_caching(): """Enable caching globally.""" + global _default_params _default_params["caching_enabled"] = True def disable_caching(): """Disable caching globally.""" + global _default_params _default_params["caching_enabled"] = False From fcee4e03ff09af3125e67e5e8113bd1fa9d162aa Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 19 Feb 2024 17:54:08 +0100 Subject: [PATCH 12/15] nonlocal --- cachier/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cachier/config.py b/cachier/config.py index 5be61110..1a46b208 100644 --- a/cachier/config.py +++ b/cachier/config.py @@ -46,7 +46,7 @@ class Params(TypedDict): def _update_with_defaults(param, name: str): - global _default_params + nonlocal _default_params if param is None: return _default_params[name] return param @@ -65,24 +65,24 @@ def set_default_params(**params): only have an effect on decorators applied after this function is run. """ - global _default_params + nonlocal _default_params valid_params = (p for p in params.items() if p[0] in _default_params) _default_params.update(valid_params) def get_default_params(): """Get current set of default parameters.""" - global _default_params + nonlocal _default_params return _default_params def enable_caching(): """Enable caching globally.""" - global _default_params + nonlocal _default_params _default_params["caching_enabled"] = True def disable_caching(): """Disable caching globally.""" - global _default_params + nonlocal _default_params _default_params["caching_enabled"] = False From 3a1ed04a3dadb5e5dd22554f102c9a14e195e425 Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 19 Feb 2024 18:09:57 +0100 Subject: [PATCH 13/15] import --- cachier/config.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/cachier/config.py b/cachier/config.py index 1a46b208..93b1768a 100644 --- a/cachier/config.py +++ b/cachier/config.py @@ -46,9 +46,10 @@ class Params(TypedDict): def _update_with_defaults(param, name: str): - nonlocal _default_params + import cachier + if param is None: - return _default_params[name] + return cachier.config._default_params[name] return param @@ -65,24 +66,30 @@ def set_default_params(**params): only have an effect on decorators applied after this function is run. """ - nonlocal _default_params - valid_params = (p for p in params.items() if p[0] in _default_params) + import cachier + + valid_params = ( + p for p in params.items() if p[0] in cachier.config._default_params + ) _default_params.update(valid_params) def get_default_params(): """Get current set of default parameters.""" - nonlocal _default_params - return _default_params + import cachier + + return cachier.config._default_params def enable_caching(): """Enable caching globally.""" - nonlocal _default_params - _default_params["caching_enabled"] = True + import cachier + + cachier.config._default_params["caching_enabled"] = True def disable_caching(): """Disable caching globally.""" - nonlocal _default_params - _default_params["caching_enabled"] = False + import cachier + + cachier.config._default_params["caching_enabled"] = False From 3d34945be8725583cd8f348a989ed026ec526cb2 Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 19 Feb 2024 18:37:07 +0100 Subject: [PATCH 14/15] fall --- cachier/core.py | 1 + cachier/cores/mongo.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cachier/core.py b/cachier/core.py index b6df3fc7..6bd0d871 100644 --- a/cachier/core.py +++ b/cachier/core.py @@ -173,6 +173,7 @@ def cachier( hash_func = hash_params # Update parameters with defaults if input is None backend = _update_with_defaults(backend, "backend") + mongetter = _update_with_defaults(mongetter, "mongetter") # Override the backend parameter if a mongetter is provided. if callable(mongetter): backend = "mongo" diff --git a/cachier/cores/mongo.py b/cachier/cores/mongo.py index 513056dd..09cc36b1 100644 --- a/cachier/cores/mongo.py +++ b/cachier/cores/mongo.py @@ -15,7 +15,6 @@ from datetime import datetime from .._types import HashFunc, Mongetter -from ..config import _update_with_defaults with suppress(ImportError): from bson.binary import Binary # to save binary data to mongodb @@ -46,12 +45,12 @@ def __init__( "MongoDB cores will not function." ) # pragma: no cover - self.mongetter = _update_with_defaults(mongetter, "mongetter") + super().__init__(hash_func, wait_for_calc_timeout) if mongetter is None: raise MissingMongetter( "must specify ``mongetter`` when using the mongo core" ) - super().__init__(hash_func, wait_for_calc_timeout) + self.mongetter = mongetter self.mongo_collection = self.mongetter() index_inf = self.mongo_collection.index_information() if _MongoCore._INDEX_NAME not in index_inf: From 8d258d79e71eec89060f0d1355ed2dc5e334bd3f Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 19 Feb 2024 18:42:14 +0100 Subject: [PATCH 15/15] core = _MongoCore(None, _test_mongetter, 0) --- tests/test_mongo_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_mongo_core.py b/tests/test_mongo_core.py index 5e3dc7d0..93c4a2ce 100644 --- a/tests/test_mongo_core.py +++ b/tests/test_mongo_core.py @@ -267,7 +267,7 @@ def test_stalled_mongo_db_cache(): def _stalled_func(): return 1 - core = _MongoCore(_test_mongetter, None, 0) + core = _MongoCore(None, _test_mongetter, 0) core.set_func(_stalled_func) core.clear_cache() with pytest.raises(RecalculationNeeded):