diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 8d7baa2f..89e4c967 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,12 +1,59 @@ -# GitHub Copilot Custom Instructions for Cachier +# Copilot Instructions for Cachier -- Cachier is a Python package providing persistent, stale-free memoization decorators for Python functions, supporting local (pickle), cross-machine (MongoDB), and in-memory caching backends. -- Always refer to the main decorator as `@cachier`, and note that it can be configured via parameters such as `stale_after`, `backend`, `mongetter`, `cache_dir`, `pickle_reload`, `separate_files`, `wait_for_calc_timeout`, and `allow_none`. -- Arguments to cached functions must be hashable; custom hash functions can be provided via the `hash_func` parameter for unhashable arguments. +Welcome to the Cachier codebase! Please follow these guidelines to ensure code suggestions, reviews, and contributions are robust, maintainable, and compatible with our multi-backend architecture. + +## 1. Decorator and API Usage + +- The main decorator is `@cachier`. It supports parameters such as `stale_after`, `backend`, `mongetter`, `cache_dir`, `pickle_reload`, `separate_files`, `wait_for_calc_timeout`, `allow_none`, and `hash_func`. +- Arguments to cached functions must be hashable; for unhashable arguments, provide a custom hash function via the `hash_func` parameter. - The default backend is pickle-based, storing cache files in `~/.cachier/` unless otherwise specified. MongoDB and memory backends are also supported. - Cachier is thread-safe and supports per-function cache clearing via the `clear_cache()` method on decorated functions. -- Global configuration is possible via `set_default_params` and `enable_caching`/`disable_caching` functions. -- When reviewing code, ensure new features or bugfixes maintain compatibility with Python 3.9+, preserve thread safety, and follow the numpy docstring conventions for documentation. -- Tests are located in the `tests/` directory and should be run with `pytest`. MongoDB-related tests require either a mocked or live MongoDB instance. -- When discussing or generating code, prefer concise, readable, and well-documented Python code, and follow the established conventions in the codebase and README. +- Global configuration is possible via `set_default_params`, `set_global_params`, and `enable_caching`/`disable_caching`. + +## 2. Optional Dependencies and Backends + +- Cachier supports multiple backends: `pickle`, `memory`, `mongo`, and `sql`. +- Not all dependencies are required for all backends. Code and tests for optional backends (e.g., MongoDB, SQL/SQLAlchemy) **must gracefully handle missing dependencies** and should not break import or test collection for other backends. +- Only raise errors or warnings for missing dependencies when the relevant backend is actually used (not at import time). + +## 3. Testing Matrix and Markers + +- Tests are located in the `tests/` directory and should be run with `pytest`. +- Tests are marked with `@pytest.mark.` (e.g., `@pytest.mark.sql`, `@pytest.mark.mongo`, `@pytest.mark.local`). +- The CI matrix runs different backends on different OSes. Do **not** assume all tests run on all platforms. +- MongoDB-related tests require either a mocked or live MongoDB instance. +- When adding new backends that require external services (e.g., databases), update the CI matrix and use Dockerized services as in the current MongoDB and PostgreSQL setup. Exclude backends from OSes where they are not supported. + +## 4. Coverage, Linting, and Typing + +- Code must pass `mypy`, `ruff`, and `pytest`. +- Use per-file or per-line ignores for known, justified issues (e.g., SQLAlchemy model base class typing, intentional use of `pickle`). +- All new code must include full type annotations and docstrings matching the style of the existing codebase. +- All docstrings should follow numpy docstring conventions. + +## 5. Error Handling and Warnings + +- Do **not** emit warnings at import time for missing optional dependencies. Only raise errors or warnings when the relevant backend is actually used. + +## 6. Backward Compatibility + +- Maintain backward compatibility for public APIs unless a breaking change is explicitly approved. +- Cachier supports Python 3.9+. + +## 7. Documentation and Examples + +- When adding a new backend or feature, provide: + - Example usage in the README + - At least one test for each public method + - Documentation of any new configuration options - For documentation, follow numpy docstring conventions and validate changes to `README.rst` with `python setup.py checkdocs`. + +## 8. General Style + +- Prefer concise, readable, and well-documented Python code. +- Follow the existing code style and conventions for imports, docstrings, and type annotations. +- Prefer explicit, readable code over cleverness. + +______________________________________________________________________ + +Thank you for contributing to Cachier! These guidelines help ensure a robust, maintainable, and user-friendly package for everyone. diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 1dd88abd..49a59995 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -22,11 +22,13 @@ jobs: matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] os: ["ubuntu-latest", "macOS-latest", "windows-latest"] - backend: ["local", "db"] + backend: ["local", "mongodb", "postgres"] exclude: # ToDo: take if back when the connection become stable # or resolve using `InMemoryMongoClient` - - { os: "macOS-latest", backend: "db" } + - { os: "macOS-latest", backend: "mongodb" } + - { os: "macOS-latest", backend: "postgres" } + - { os: "windows-latest", backend: "postgres" } env: CACHIER_TEST_HOST: "localhost" CACHIER_TEST_PORT: "27017" @@ -50,10 +52,10 @@ jobs: - name: Unit tests (local) if: matrix.backend == 'local' - run: pytest -m "not mongo" --cov=cachier --cov-report=term --cov-report=xml:cov.xml + run: pytest -m "not mongo and not sql" --cov=cachier --cov-report=term --cov-report=xml:cov.xml - name: Setup docker (missing on MacOS) - if: runner.os == 'macOS' && matrix.backend == 'db' + if: runner.os == 'macOS' && matrix.backend == 'mongodb' run: | brew install docker colima start @@ -61,13 +63,14 @@ jobs: sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock # ToDo: find a way to cache docker images #- name: Cache Container Images - # if: matrix.backend == 'db' + # if: matrix.backend == 'mongodb' # uses: borda/cache-container-images-action@b32a5e804cb39af3c3d134fc03ab76eac0bfcfa9 # with: # prefix-key: "mongo-db" # images: mongo:latest + - name: Start MongoDB in docker - if: matrix.backend == 'db' + if: matrix.backend == 'mongodb' run: | # start MongoDB in a container docker run -d -p ${{ env.CACHIER_TEST_PORT }}:27017 --name mongodb mongo:latest @@ -75,12 +78,37 @@ jobs: sleep 5 # show running containers docker ps -a + - name: Unit tests (DB) - if: matrix.backend == 'db' + if: matrix.backend == 'mongodb' run: pytest -m "mongo" --cov=cachier --cov-report=term --cov-report=xml:cov.xml - name: Speed eval run: python tests/speed_eval.py + - name: Start PostgreSQL in docker + if: matrix.backend == 'postgres' + run: | + docker run -d \ + -e POSTGRES_USER=testuser \ + -e POSTGRES_PASSWORD=testpass \ + -e POSTGRES_DB=testdb \ + -p 5432:5432 \ + --name postgres postgres:15 + # wait for PostgreSQL to start + sleep 10 + docker ps -a + + - name: Install SQL core test dependencies (SQL/Postgres) + if: matrix.backend == 'postgres' + run: | + python -m pip install -e . -r tests/sql_requirements.txt + + - name: Unit tests (SQL/Postgres) + if: matrix.backend == 'postgres' + env: + SQLALCHEMY_DATABASE_URL: postgresql://testuser:testpass@localhost:5432/testdb + run: pytest -m sql --cov=cachier --cov-report=term --cov-report=xml:cov.xml + - name: Upload coverage to Codecov continue-on-error: true uses: codecov/codecov-action@v5 diff --git a/README.rst b/README.rst index 6daa1e55..a7715569 100644 --- a/README.rst +++ b/README.rst @@ -342,6 +342,64 @@ You can set an in-memory cache by assigning the ``backend`` parameter with ``'me Note, however, that ``cachier``'s in-memory core is simple, and has no monitoring or cap on cache size, and can thus lead to memory errors on large return values - it is mainly intended to be used with future multi-core functionality. As a rule, Python's built-in ``lru_cache`` is a much better stand-alone solution. +SQLAlchemy (SQL) Core +--------------------- + +**Note:** The SQL core requires SQLAlchemy to be installed. It is not installed by default with cachier. To use the SQL backend, run:: + + pip install SQLAlchemy + +Cachier supports a generic SQL backend via SQLAlchemy, allowing you to use SQLite, PostgreSQL, MySQL, and other databases. + +**Usage Example (SQLite in-memory):** + +.. code-block:: python + + from cachier import cachier + + @cachier(backend="sql", sql_engine="sqlite:///:memory:") + def my_func(x): + return x * 2 + +**Usage Example (PostgreSQL):** + +.. code-block:: python + + @cachier(backend="sql", sql_engine="postgresql://user:pass@localhost/dbname") + def my_func(x): + return x * 2 + +**Usage Example (MySQL):** + +.. code-block:: python + + @cachier(backend="sql", sql_engine="mysql+pymysql://user:pass@localhost/dbname") + def my_func(x): + return x * 2 + +**Configuration Options:** + +- ``sql_engine``: SQLAlchemy connection string, Engine, or callable returning an Engine. +- All other standard cachier options are supported. + +**Table Schema:** + +- ``function_id``: Unique identifier for the cached function +- ``key``: Cache key +- ``value``: Pickled result +- ``timestamp``: Datetime of cache entry +- ``stale``: Boolean, is value stale +- ``processing``: Boolean, is value being calculated +- ``completed``: Boolean, is value calculation completed + +**Limitations & Notes:** + +- Requires SQLAlchemy (install with ``pip install SQLAlchemy``) +- For production, use a persistent database (not ``:memory:``) +- Thread/process safety is handled via transactions and row-level locks +- Value serialization uses ``pickle``. **Warning:** `pickle` can execute arbitrary code during deserialization if the cache database is compromised. Ensure the cache is stored securely and consider using safer serialization methods like `json` if security is a concern. +- For best performance, ensure your DB supports row-level locking + Contributing ============ diff --git a/pyproject.toml b/pyproject.toml index 7795d5e3..6ea4bf44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,5 @@ +# === Metadata & Build System === + [build-system] requires = [ "setuptools", @@ -46,6 +48,8 @@ dependencies = [ "watchdog>=2.3.1", ] urls.Source = "https://github.com/python-cachier/cachier" +# --- setuptools --- + scripts.cachier = "cachier.__main__:cli" [tool.setuptools] @@ -63,6 +67,13 @@ include = [ ] # package names should match these glob patterns (["*"] by default) namespaces = false # to disable scanning PEP 420 namespaces (true by default) +# === Linting & Formatting === + +[tool.black] +line-length = 79 + +# --- ruff --- + [tool.ruff] target-version = "py38" line-length = 79 @@ -97,6 +108,8 @@ lint.extend-select = [ lint.ignore = [ "C901", "E203", + "F824", + "W503", ] lint.per-file-ignores."src/**/__init__.py" = [ "D104", @@ -104,6 +117,7 @@ lint.per-file-ignores."src/**/__init__.py" = [ lint.per-file-ignores."src/cachier/config.py" = [ "D100", ] +lint.per-file-ignores."src/cachier/cores/sql.py" = [ "S301" ] lint.per-file-ignores."tests/**" = [ "D100", "D101", @@ -119,6 +133,7 @@ lint.unfixable = [ "F401", ] +# --- flake8 --- #[tool.ruff.pydocstyle] ## Use Google-style docstrings. #convention = "google" @@ -134,6 +149,10 @@ wrap-summaries = 79 wrap-descriptions = 79 blank = true +# === Testing === + +# --- pytest --- + [tool.pytest.ini_options] testpaths = [ "cachier", @@ -154,8 +173,11 @@ markers = [ "mongo: test the MongoDB core", "memory: test the memory core", "pickle: test the pickle core", + "sql: test the SQL core", ] +# --- coverage --- + [tool.coverage.run] branch = true # dynamic_context = "test_function" diff --git a/src/cachier/__init__.py b/src/cachier/__init__.py index f5cff9b4..cfaeaea3 100644 --- a/src/cachier/__init__.py +++ b/src/cachier/__init__.py @@ -1,4 +1,4 @@ -from ._version import * # noqa: F403 +from ._version import __version__ from .config import ( disable_caching, enable_caching, @@ -17,4 +17,5 @@ "get_global_params", "enable_caching", "disable_caching", + "__version__", ] diff --git a/src/cachier/_version.py b/src/cachier/_version.py index 7302258e..ddce7e80 100644 --- a/src/cachier/_version.py +++ b/src/cachier/_version.py @@ -18,7 +18,8 @@ def _get_git_sha() -> str: from subprocess import DEVNULL, check_output - out = check_output(["git", "rev-parse", "--short", "HEAD"], stderr=DEVNULL) # noqa: S603, S607 + args = ["git", "rev-parse", "--short", "HEAD"] + out = check_output(args, stderr=DEVNULL) # noqa: S603 return out.decode("utf-8").strip() diff --git a/src/cachier/core.py b/src/cachier/core.py index 30fcb800..f7183ffb 100644 --- a/src/cachier/core.py +++ b/src/cachier/core.py @@ -14,7 +14,7 @@ from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta from functools import wraps -from typing import Any, Optional, Union +from typing import Any, Callable, Optional, Union from warnings import warn from .config import ( @@ -27,6 +27,7 @@ from .cores.memory import _MemoryCore from .cores.mongo import _MongoCore from .cores.pickle import _PickleCore +from .cores.sql import _SQLCore MAX_WORKERS_ENVAR_NAME = "CACHIER_MAX_WORKERS" DEFAULT_MAX_WORKERS = 8 @@ -107,6 +108,7 @@ def cachier( hash_params: Optional[HashFunc] = None, backend: Optional[Backend] = None, mongetter: Optional[Mongetter] = None, + sql_engine: Optional[Union[str, Any, Callable[[], Any]]] = None, stale_after: Optional[timedelta] = None, next_time: Optional[bool] = None, cache_dir: Optional[Union[str, os.PathLike]] = None, @@ -134,13 +136,16 @@ def cachier( hash_params : callable, optional backend : str, optional The name of the backend to use. Valid options currently include - 'pickle', 'mongo' and 'memory'. If not provided, defaults to + 'pickle', 'mongo', 'memory', and 'sql'. If not provided, defaults to 'pickle' unless the 'mongetter' argument is passed, in which case the mongo backend is automatically selected. mongetter : callable, optional A callable that takes no arguments and returns a pymongo.Collection object with writing permissions. If unset a local pickle cache is used instead. + sql_engine : str, Engine, or callable, optional + SQLAlchemy connection string, Engine, or callable returning an Engine. + Used for the SQL backend. stale_after : datetime.timedelta, optional The time delta after which a cached result is considered stale. Calls made after the result goes stale will trigger a recalculation of the @@ -208,6 +213,12 @@ def cachier( core = _MemoryCore( hash_func=hash_func, wait_for_calc_timeout=wait_for_calc_timeout ) + elif backend == "sql": + core = _SQLCore( + hash_func=hash_func, + sql_engine=sql_engine, + wait_for_calc_timeout=wait_for_calc_timeout, + ) else: raise ValueError("specified an invalid core: %s" % backend) diff --git a/src/cachier/cores/base.py b/src/cachier/cores/base.py index 25ba346d..f4dbbced 100644 --- a/src/cachier/cores/base.py +++ b/src/cachier/cores/base.py @@ -1,4 +1,5 @@ """Defines the interface of a cachier caching core.""" + # This file is part of Cachier. # https://github.com/python-cachier/cachier diff --git a/src/cachier/cores/mongo.py b/src/cachier/cores/mongo.py index 96c39bbb..e031464c 100644 --- a/src/cachier/cores/mongo.py +++ b/src/cachier/cores/mongo.py @@ -75,7 +75,9 @@ def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: ) if not res: return key, None - val = pickle.loads(res["value"]) if "value" in res else None # noqa: S301 + val = None + if "value" in res: + val = pickle.loads(res["value"]) # noqa: S301 entry = CacheEntry( value=val, time=res.get("time", None), diff --git a/src/cachier/cores/sql.py b/src/cachier/cores/sql.py new file mode 100644 index 00000000..ad4364cf --- /dev/null +++ b/src/cachier/cores/sql.py @@ -0,0 +1,288 @@ +"""A SQLAlchemy-based caching core for cachier.""" + +import pickle +import threading +from datetime import datetime +from typing import Any, Callable, Optional, Tuple, Union + +try: + from sqlalchemy import ( + Boolean, + Column, + DateTime, + Index, + LargeBinary, + String, + and_, + create_engine, + delete, + insert, + select, + update, + ) + from sqlalchemy.engine import Engine + from sqlalchemy.orm import declarative_base, sessionmaker + + SQLALCHEMY_AVAILABLE = True +except ImportError: + SQLALCHEMY_AVAILABLE = False + +from .._types import HashFunc +from ..config import CacheEntry +from .base import RecalculationNeeded, _BaseCore, _get_func_str + +if SQLALCHEMY_AVAILABLE: + Base = declarative_base() + + class CacheTable(Base): # type: ignore[misc, valid-type] + """SQLAlchemy model for cachier cache entries.""" + + __tablename__ = "cachier_cache" + id = Column(String, primary_key=True) + function_id = Column(String, index=True, nullable=False) + key = Column(String, index=True, nullable=False) + value = Column(LargeBinary, nullable=True) + timestamp = Column(DateTime, nullable=False) + stale = Column(Boolean, default=False) + processing = Column(Boolean, default=False) + completed = Column(Boolean, default=False) + __table_args__ = ( + Index("ix_func_key", "function_id", "key", unique=True), + ) + + +class _SQLCore(_BaseCore): + """SQLAlchemy-based core for Cachier, supporting SQL-based backends. + + This should work with SQLite, PostgreSQL and so on. + + """ + + def __init__( + self, + hash_func: Optional[HashFunc], + sql_engine: Optional[Union[str, "Engine", Callable[[], "Engine"]]], + wait_for_calc_timeout: Optional[int] = None, + ): + if not SQLALCHEMY_AVAILABLE: + raise ImportError( + "SQLAlchemy is required for the SQL core. " + "Install with `pip install SQLAlchemy`." + ) + super().__init__( + hash_func=hash_func, wait_for_calc_timeout=wait_for_calc_timeout + ) + self._engine = self._resolve_engine(sql_engine) + self._Session = sessionmaker(bind=self._engine) + Base.metadata.create_all(self._engine) + self._lock = threading.RLock() + self._func_str = None + + def _resolve_engine(self, sql_engine): + if isinstance(sql_engine, Engine): + return sql_engine + if isinstance(sql_engine, str): + return create_engine(sql_engine, future=True) + if callable(sql_engine): + return sql_engine() + raise ValueError( + "sql_engine must be a SQLAlchemy Engine, connection string, " + "or callable returning an Engine." + ) + + def set_func(self, func): + super().set_func(func) + self._func_str = _get_func_str(func) + + def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: + with self._lock, self._Session() as session: + row = session.execute( + select(CacheTable).where( + and_( + CacheTable.function_id == self._func_str, + CacheTable.key == key, + ) + ) + ).scalar_one_or_none() + if not row: + return key, None + value = pickle.loads(row.value) if row.value is not None else None + entry = CacheEntry( + value=value, + time=row.timestamp, + stale=row.stale, + _processing=row.processing, + _completed=row.completed, + ) + return key, entry + + def set_entry(self, key: str, func_res: Any) -> None: + with self._lock, self._Session() as session: + thebytes = pickle.dumps(func_res) + now = datetime.now() + base_insert = insert(CacheTable) + stmt = ( + base_insert.values( + id=f"{self._func_str}:{key}", + function_id=self._func_str, + key=key, + value=thebytes, + timestamp=now, + stale=False, + processing=False, + completed=True, + ).on_conflict_do_update( + index_elements=[CacheTable.function_id, CacheTable.key], + set_={ + "value": thebytes, + "timestamp": now, + "stale": False, + "processing": False, + "completed": True, + }, + ) + if hasattr(base_insert, "on_conflict_do_update") + else None + ) + # Fallback for non-SQLite/Postgres: try update, else insert + if stmt: + session.execute(stmt) + else: + row = session.execute( + select(CacheTable).where( + and_( + CacheTable.function_id == self._func_str, + CacheTable.key == key, + ) + ) + ).scalar_one_or_none() + if row: + session.execute( + update(CacheTable) + .where( + and_( + CacheTable.function_id == self._func_str, + CacheTable.key == key, + ) + ) + .values( + value=thebytes, + timestamp=now, + stale=False, + processing=False, + completed=True, + ) + ) + else: + session.add( + CacheTable( + id=f"{self._func_str}:{key}", + function_id=self._func_str, + key=key, + value=thebytes, + timestamp=now, + stale=False, + processing=False, + completed=True, + ) + ) + session.commit() + + def mark_entry_being_calculated(self, key: str) -> None: + with self._lock, self._Session() as session: + row = session.execute( + select(CacheTable).where( + and_( + CacheTable.function_id == self._func_str, + CacheTable.key == key, + ) + ) + ).scalar_one_or_none() + if row: + session.execute( + update(CacheTable) + .where( + and_( + CacheTable.function_id == self._func_str, + CacheTable.key == key, + ) + ) + .values(processing=True) + ) + else: + session.add( + CacheTable( + id=f"{self._func_str}:{key}", + function_id=self._func_str, + key=key, + value=None, + timestamp=datetime.now(), + stale=False, + processing=True, + completed=False, + ) + ) + session.commit() + + def mark_entry_not_calculated(self, key: str) -> None: + with self._lock, self._Session() as session: + session.execute( + update(CacheTable) + .where( + and_( + CacheTable.function_id == self._func_str, + CacheTable.key == key, + ) + ) + .values(processing=False) + ) + session.commit() + + def wait_on_entry_calc(self, key: str) -> Any: + import time + + time_spent = 0 + while True: + with self._lock, self._Session() as session: + row = session.execute( + select(CacheTable).where( + and_( + CacheTable.function_id == self._func_str, + CacheTable.key == key, + ) + ) + ).scalar_one_or_none() + if not row: + raise RecalculationNeeded() + if not row.processing: + return ( + pickle.loads(row.value) + if row.value is not None + else None + ) + time.sleep(1) + time_spent += 1 + self.check_calc_timeout(time_spent) + + def clear_cache(self) -> None: + with self._lock, self._Session() as session: + session.execute( + delete(CacheTable).where( + CacheTable.function_id == self._func_str + ) + ) + session.commit() + + def clear_being_calculated(self) -> None: + with self._lock, self._Session() as session: + session.execute( + update(CacheTable) + .where( + and_( + CacheTable.function_id == self._func_str, + CacheTable.processing, + ) + ) + .values(processing=False) + ) + session.commit() diff --git a/tests/sql_requirements.txt b/tests/sql_requirements.txt new file mode 100644 index 00000000..81af8428 --- /dev/null +++ b/tests/sql_requirements.txt @@ -0,0 +1,3 @@ +# for SQL core tests +SQLAlchemy +psycopg2-binary diff --git a/tests/test_general.py b/tests/test_general.py index b7a12f12..ef2be0ea 100644 --- a/tests/test_general.py +++ b/tests/test_general.py @@ -485,9 +485,8 @@ def fn_minus(a, b=2): assert cachier_(dummy_)(1) == expected, f"for {fn.__name__} wrapped" dummy_ = functools.partial(fn, b=2) - assert cachier_(dummy_)(1, b=2) == expected, ( - f"for {fn.__name__} wrapped" - ) + expected_str = f"for {fn.__name__} wrapped" + assert cachier_(dummy_)(1, b=2) == expected, expected_str assert cachier_(fn)(1, 2) == expected, f"for {fn.__name__} inline" assert cachier_(fn)(a=1, b=2) == expected, f"for {fn.__name__} inline" diff --git a/tests/test_sql_core.py b/tests/test_sql_core.py new file mode 100644 index 00000000..fdcaa04a --- /dev/null +++ b/tests/test_sql_core.py @@ -0,0 +1,331 @@ +import os +import queue +import sys +import threading +from datetime import datetime, timedelta +from random import random +from time import sleep + +import pytest + +from cachier import cachier +from cachier.cores.base import RecalculationNeeded +from cachier.cores.sql import _SQLCore + +SQL_CONN_STR = os.environ.get("SQLALCHEMY_DATABASE_URL", "sqlite:///:memory:") + + +@pytest.mark.sql +def test_sql_core_basic(): + @cachier(backend="sql", sql_engine=SQL_CONN_STR) + def f(x, y): + return random() + x + y + + f.clear_cache() + v1 = f(1, 2) + v2 = f(1, 2) + assert v1 == v2 + v3 = f(1, 2, cachier__skip_cache=True) + assert v3 != v1 + v4 = f(1, 2) + assert v4 == v1 + v5 = f(1, 2, cachier__overwrite_cache=True) + assert v5 != v1 + v6 = f(1, 2) + assert v6 == v5 + + +@pytest.mark.sql +def test_sql_core_keywords(): + @cachier(backend="sql", sql_engine=SQL_CONN_STR) + def f(x, y): + return random() + x + y + + f.clear_cache() + v1 = f(1, y=2) + v2 = f(1, y=2) + assert v1 == v2 + v3 = f(1, y=2, cachier__skip_cache=True) + assert v3 != v1 + v4 = f(1, y=2) + assert v4 == v1 + v5 = f(1, y=2, cachier__overwrite_cache=True) + assert v5 != v1 + v6 = f(1, y=2) + assert v6 == v5 + + +@pytest.mark.sql +def test_sql_stale_after(): + @cachier( + backend="sql", + sql_engine=SQL_CONN_STR, + stale_after=timedelta(seconds=2), + next_time=False, + ) + def f(x, y): + return random() + x + y + + f.clear_cache() + v1 = f(1, 2) + v2 = f(1, 2) + assert v1 == v2 + sleep(2) + v3 = f(1, 2) + assert v3 != v1 + + +@pytest.mark.sql +def test_sql_overwrite_and_skip_cache(): + @cachier(backend="sql", sql_engine=SQL_CONN_STR) + def f(x): + return random() + x + + f.clear_cache() + v1 = f(1) + v2 = f(1) + assert v1 == v2 + v3 = f(1, cachier__skip_cache=True) + assert v3 != v1 + v4 = f(1, cachier__overwrite_cache=True) + assert v4 != v1 + v5 = f(1) + assert v5 == v4 + + +@pytest.mark.sql +def test_sql_concurrency(): + @cachier(backend="sql", sql_engine=SQL_CONN_STR) + def slow_func(x): + sleep(1) + return random() + x + + slow_func.clear_cache() + res_queue = queue.Queue() + + def call(): + res = slow_func(5) + res_queue.put(res) + + t1 = threading.Thread(target=call) + t2 = threading.Thread(target=call) + t1.start() + sleep(0.2) + t2.start() + t1.join(timeout=3) + t2.join(timeout=3) + assert res_queue.qsize() == 2 + r1 = res_queue.get() + r2 = res_queue.get() + assert r1 == r2 + + +@pytest.mark.sql +def test_sql_clear_being_calculated(): + @cachier(backend="sql", sql_engine=SQL_CONN_STR) + def slow_func(x): + sleep(1) + return random() + x + + slow_func.clear_cache() + slow_func(1) + slow_func.clear_being_calculated() + # Should not raise + slow_func(1) + + +@pytest.mark.sql +def test_sql_missing_entry(): + @cachier(backend="sql", sql_engine=SQL_CONN_STR) + def f(x): + return x + + f.clear_cache() + # Should not raise + assert f(123) == 123 + + +class DummyWriteError(Exception): + pass + + +@pytest.mark.sql +def test_sql_failed_write(monkeypatch): + @cachier(backend="sql", sql_engine=SQL_CONN_STR) + def f(x): + return x + + f.clear_cache() + # Simulate DB failure by monkeypatching set_entry + orig = _SQLCore.set_entry + + def fail_set_entry(self, key, func_res): + raise DummyWriteError("fail") + + monkeypatch.setattr(_SQLCore, "set_entry", fail_set_entry) + with pytest.raises(DummyWriteError, match="fail"): + f(1) + monkeypatch.setattr(_SQLCore, "set_entry", orig) + + +@pytest.mark.sql +def test_import_cachier_without_sqlalchemy(monkeypatch): + """Test that importing cachier works when SQLAlchemy is missing. + + This should work unless SQL core is used. + + """ + # Simulate SQLAlchemy not installed + modules_backup = sys.modules.copy() + sys.modules["sqlalchemy"] = None + sys.modules["sqlalchemy.orm"] = None + sys.modules["sqlalchemy.engine"] = None + try: + import importlib # noqa: F401 + + import cachier # noqa: F401 + + # Should import fine + finally: + sys.modules.clear() + sys.modules.update(modules_backup) + + +@pytest.mark.pickle +def test_sqlcore_importerror_without_sqlalchemy(monkeypatch): + """Test that using SQL core without SQLAlchemy raises an ImportError.""" + # Simulate SQLAlchemy not installed + modules_backup = sys.modules.copy() + sys.modules["sqlalchemy"] = None + sys.modules["sqlalchemy.orm"] = None + sys.modules["sqlalchemy.engine"] = None + try: + import importlib + + sql_mod = importlib.import_module("cachier.cores.sql") + with pytest.raises(ImportError) as excinfo: + sql_mod._SQLCore(hash_func=None, sql_engine="sqlite:///:memory:") + assert "SQLAlchemy is required" in str(excinfo.value) + finally: + sys.modules.clear() + sys.modules.update(modules_backup) + + +@pytest.mark.sql +def test_sqlcore_invalid_sql_engine(): + with pytest.raises( + ValueError, match="sql_engine must be a SQLAlchemy Engine" + ): + _SQLCore(hash_func=None, sql_engine=12345) + + +@pytest.mark.sql +def test_sqlcore_get_entry_by_key_none_value(): + import pytest + + pytest.importorskip("sqlalchemy") + import cachier.cores.sql as sql_mod + from cachier.cores.sql import _SQLCore + + CacheTable = getattr(sql_mod, "CacheTable", None) + if CacheTable is None: + pytest.skip("CacheTable not available (SQLAlchemy missing)") + core = _SQLCore(hash_func=None, sql_engine=SQL_CONN_STR) + core.set_func(lambda x: x) + # Insert a row with value=None + with core._Session() as session: + session.add( + CacheTable( + id="testfunc:abc", + function_id=core._func_str, + key="abc", + value=None, + timestamp=datetime.now(), + stale=False, + processing=False, + completed=True, + ) + ) + session.commit() + key, entry = core.get_entry_by_key("abc") + assert entry is not None + assert entry.value is None + + +@pytest.mark.sql +def test_sqlcore_set_entry_fallback(monkeypatch): + from sqlalchemy.orm import Session + from sqlalchemy.sql.dml import Insert, Update + from sqlalchemy.sql.selectable import Select + + core = _SQLCore(hash_func=None, sql_engine=SQL_CONN_STR) + core.set_func(lambda x: x) + # Monkeypatch Session.execute to simulate fallback path + orig_execute = Session.execute + + def fake_execute(self, stmt, *args, **kwargs): + if isinstance(stmt, (Insert, Update)): + + class FakeInsert: + pass + + return FakeInsert() + elif isinstance(stmt, Select): + + class FakeSelectResult: + def scalar_one_or_none(self): + return None # Simulate no row found + + return FakeSelectResult() + + class Dummy: + pass + + return Dummy() + + monkeypatch.setattr(Session, "execute", fake_execute) + # Should not raise + core.set_entry("fallback", 123) + monkeypatch.setattr(Session, "execute", orig_execute) + + +@pytest.mark.sql +def test_sqlcore_wait_on_entry_calc_recalculation(): + core = _SQLCore(hash_func=None, sql_engine=SQL_CONN_STR) + core.set_func(lambda x: x) + with pytest.raises(RecalculationNeeded): + core.wait_on_entry_calc("missing_key") + + +@pytest.mark.sql +def test_sqlcore_clear_being_calculated_empty(): + core = _SQLCore(hash_func=None, sql_engine=SQL_CONN_STR) + core.set_func(lambda x: x) + # Should not raise even if nothing is being calculated + core.clear_being_calculated() + + +@pytest.mark.sql +def test_sqlcore_accepts_engine_instance(): + from sqlalchemy import create_engine + + engine = create_engine(SQL_CONN_STR) + core = _SQLCore(hash_func=None, sql_engine=engine) + core.set_func(lambda x: x) + core.set_entry("engine_test", 456) + key, entry = core.get_entry_by_key("engine_test") + assert entry.value == 456 + + +@pytest.mark.sql +def test_sqlcore_accepts_engine_callable(): + from sqlalchemy import create_engine + + def engine_factory(): + return create_engine(SQL_CONN_STR) + + core = _SQLCore(hash_func=None, sql_engine=engine_factory) + core.set_func(lambda x: x) + core.set_entry("callable_test", 789) + key, entry = core.get_entry_by_key("callable_test") + assert entry.value == 789