Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 61 additions & 2 deletions docs/concepts/file-hash-caching.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,64 @@ Calling the function more than once is safe. Orcapod logs a warning and
replaces the active cacher with the new one, so the prior cache entries
remain accessible at the same path if you use the same `db_path`.

## Controlling when the cache is written

By default, every file that passes through `CachedFileHasher` is inserted
into the cache on a miss. Two optional knobs let you restrict this.

### Read-only mode

Use `read_only=True` when you want lookups from a shared or authoritative
cache but must not add new entries to it — for example, when consuming a
cache pre-populated by `populate_hash_cache()` without polluting it with
ad-hoc entries.

```python
import orcapod as op

op.enable_file_hash_caching(db_path="/shared/cache.db", read_only=True)
```

Cache hits still work normally. On a miss, the file is hashed directly and
the result is returned to the caller — but it is never written to the cache.

### Minimum file size threshold

Use `min_cache_size_bytes` to skip the cache write overhead for small
files. For small files, the disk I/O bottleneck does not apply, so the
cache lookup and write add latency without meaningful savings.

```python
import orcapod as op

# Skip caching for files smaller than 1 MB
op.enable_file_hash_caching(min_cache_size_bytes=1_048_576)
```

Files smaller than the threshold are still hashed and the hash is returned
to the caller — they are simply not inserted into the cache. Files at or
above the threshold behave normally. Set to `None` (the default) or `0`
to disable the threshold.

### Combining both

The two knobs compose independently. `read_only=True` takes precedence:
when enabled, no entry is ever written regardless of file size.
`min_cache_size_bytes` is an additional guard that applies only when the
cacher is writable.

```python
import orcapod as op

# Read-only + skip files below 512 KB (threshold is moot when read-only,
# but harmless and documents intent)
op.enable_file_hash_caching(
db_path="/shared/cache.db",
read_only=True,
min_cache_size_bytes=524_288,
)
```

## Directory hashing (op.Directory)

When Orcapod hashes an `op.Directory`, it traverses the directory tree and
Expand Down Expand Up @@ -109,8 +167,9 @@ sqlite3 ~/.orcapod/file_hash_cache.db "DELETE FROM file_hash_cache;"

**Caching does not help much when:**

- Files are small. Disk I/O is not the bottleneck for small files, so the
cache lookup adds overhead without meaningful savings.
- **Files are small.** Disk I/O is not the bottleneck for small files, so the
cache lookup adds overhead without meaningful savings. Use
``min_cache_size_bytes`` to skip caching small files automatically.
- Files change on every run. A different `mtime_ns` or `size` means a
different key, so every access is a cache miss.
- It is the first run on a new machine or a freshly cleared cache. Every file
Expand Down
18 changes: 16 additions & 2 deletions src/orcapod/contexts/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,11 @@ def create_registry(
return JSONDataContextRegistry(contexts_dir, schema_file, default_version)


def enable_file_hash_caching(db_path: "Path | None" = None) -> None:
def enable_file_hash_caching(
db_path: "Path | None" = None,
read_only: bool = False,
min_cache_size_bytes: int | None = None,
) -> None:
"""Enable SQLite-backed file hash caching on the default Orcapod context.

Wraps the existing ``FileHandler``'s hasher in a ``CachedFileHasher``
Expand Down Expand Up @@ -257,6 +261,12 @@ def enable_file_hash_caching(db_path: "Path | None" = None) -> None:
db_path: Path to the SQLite cache database. Defaults to
``~/.orcapod/file_hash_cache.db`` or the
``ORCAPOD_HASH_CACHE_DB`` environment variable.
read_only: When ``True``, the underlying ``SqliteHashCacher`` will
not insert new entries. Lookups still work normally. Defaults
to ``False``.
min_cache_size_bytes: When set, files smaller than this byte count
are not inserted into the cache. ``None`` and ``0`` disable the
threshold. Defaults to ``None``.
"""
from orcapod.extension_types.file_type import File
from orcapod.hashing.file_hashers import CachedFileHasher
Expand Down Expand Up @@ -295,7 +305,11 @@ def enable_file_hash_caching(db_path: "Path | None" = None) -> None:

cached_file_hasher = CachedFileHasher(
file_hasher=base_hasher,
cacher=SqliteHashCacher(db_path),
cacher=SqliteHashCacher(
db_path,
read_only=read_only,
min_cache_size_bytes=min_cache_size_bytes,
),
)

registry.register(File, FileHandler(cached_file_hasher))
Expand Down
73 changes: 70 additions & 3 deletions src/orcapod/hashing/hash_cachers.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,30 @@ class InMemoryHashCacher:

No persistence, no thread-safety guarantees beyond the GIL, no eviction.
Use ``SqliteHashCacher`` for production workloads.

Args:
read_only: When ``True``, all ``put()`` calls are silent no-ops.
``get()`` still works normally. Defaults to ``False``.
min_cache_size_bytes: When set to a positive integer, files whose
``key.size`` is strictly below this threshold are not inserted.
``None`` and ``0`` disable the threshold (default behaviour).
Negative values raise ``ValueError``. Defaults to ``None``.
"""

def __init__(self) -> None:
def __init__(
self,
*,
read_only: bool = False,
min_cache_size_bytes: int | None = None,
) -> None:
if min_cache_size_bytes is not None and min_cache_size_bytes < 0:
raise ValueError(
f"min_cache_size_bytes must be None or a non-negative integer, "
f"got {min_cache_size_bytes!r}"
)
self._cache: dict[FileHashKey, ContentHash] = {}
self._read_only = read_only
self._min_cache_size_bytes = min_cache_size_bytes

def get(self, key: FileHashKey) -> ContentHash | None:
"""Return the cached ``ContentHash`` for ``key``, or ``None`` on miss.
Expand All @@ -38,16 +58,30 @@ def get(self, key: FileHashKey) -> ContentHash | None:
def put(self, key: FileHashKey, value: ContentHash) -> None:
"""Store ``value`` under ``key``.

No-ops silently when ``read_only=True`` or when ``key.size`` is below
``min_cache_size_bytes``.

Args:
key: File hash cache key.
value: ``ContentHash`` to store.
"""
if self._read_only:
return
if self._min_cache_size_bytes is not None and self._min_cache_size_bytes > 0 and key.size < self._min_cache_size_bytes:
return
self._cache[key] = value

def clear(self) -> None:
"""Remove all entries from the cache."""
self._cache.clear()

def __repr__(self) -> str:
return (
f"InMemoryHashCacher("
f"read_only={self._read_only!r}, "
f"min_cache_size_bytes={self._min_cache_size_bytes!r})"
)


class SqliteHashCacher:
"""SQLite-backed file hash cacher.
Expand All @@ -63,6 +97,12 @@ class SqliteHashCacher:
db_path: Path to the SQLite database file. Defaults to
``~/.orcapod/file_hash_cache.db`` or the
``ORCAPOD_HASH_CACHE_DB`` environment variable.
read_only: When ``True``, all ``put()`` calls are silent no-ops.
``get()`` still works normally. Defaults to ``False``.
min_cache_size_bytes: When set to a positive integer, files whose
``key.size`` is strictly below this threshold are not inserted.
``None`` and ``0`` disable the threshold (default behaviour).
Negative values raise ``ValueError``. Defaults to ``None``.

Note:
Heavy multi-writer scenarios are a known SQLite limitation. A Turso
Expand All @@ -71,12 +111,25 @@ class SqliteHashCacher:

DEFAULT_DB_PATH = Path.home() / ".orcapod" / "file_hash_cache.db"

def __init__(self, db_path: Path | None = None) -> None:
def __init__(
self,
db_path: Path | None = None,
*,
read_only: bool = False,
min_cache_size_bytes: int | None = None,
) -> None:
if min_cache_size_bytes is not None and min_cache_size_bytes < 0:
raise ValueError(
f"min_cache_size_bytes must be None or a non-negative integer, "
f"got {min_cache_size_bytes!r}"
)
self.db_path = Path(
db_path
or os.environ.get("ORCAPOD_HASH_CACHE_DB")
or self.DEFAULT_DB_PATH
)
self._read_only = read_only
self._min_cache_size_bytes = min_cache_size_bytes
self._local = threading.local()
self._ensure_schema()

Expand Down Expand Up @@ -136,12 +189,18 @@ def get(self, key: FileHashKey) -> ContentHash | None:
def put(self, key: FileHashKey, value: ContentHash) -> None:
"""Store ``value`` under ``key``.

Uses ``INSERT OR REPLACE`` so writes are idempotent.
No-ops silently when ``read_only=True`` or when ``key.size`` is below
``min_cache_size_bytes``. Uses ``INSERT OR REPLACE`` so writes are
idempotent when they do proceed.

Args:
key: File hash cache key.
value: ``ContentHash`` to store.
"""
if self._read_only:
return
if self._min_cache_size_bytes is not None and self._min_cache_size_bytes > 0 and key.size < self._min_cache_size_bytes:
return
conn = self._connection()
conn.execute(
"""
Expand All @@ -165,6 +224,14 @@ def close(self) -> None:
conn.close()
self._local.conn = None

def __repr__(self) -> str:
return (
f"SqliteHashCacher("
f"db_path={str(self.db_path)!r}, "
f"read_only={self._read_only!r}, "
f"min_cache_size_bytes={self._min_cache_size_bytes!r})"
)

def __enter__(self) -> "SqliteHashCacher":
"""Return self for use as a context manager."""
return self
Expand Down
Loading
Loading