airbyte.caches.util

Utility functions for working with caches.

 1# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 2"""Utility functions for working with caches."""
 3
 4from __future__ import annotations
 5
 6from pathlib import Path
 7
 8import ulid
 9
10from airbyte import exceptions as exc
11from airbyte.caches.duckdb import DuckDBCache
12
13
14def get_default_cache() -> DuckDBCache:
15    """Get a local cache for storing data, using the default database path.
16
17    Cache files are stored in the `.cache` directory, relative to the current
18    working directory.
19    """
20    cache_dir = Path("./.cache/default_cache")
21    return DuckDBCache(
22        db_path=cache_dir / "default_cache.duckdb",
23        cache_dir=cache_dir,
24    )
25
26
27def new_local_cache(
28    cache_name: str | None = None,
29    cache_dir: str | Path | None = None,
30    *,
31    cleanup: bool = True,
32) -> DuckDBCache:
33    """Get a local cache for storing data, using a name string to seed the path.
34
35    Args:
36        cache_name: Name to use for the cache. Defaults to None.
37        cache_dir: Root directory to store the cache in. Defaults to None.
38        cleanup: Whether to clean up temporary files. Defaults to True.
39
40    Cache files are stored in the `.cache` directory, relative to the current
41    working directory.
42    """
43    if cache_name:
44        if " " in cache_name:
45            raise exc.PyAirbyteInputError(
46                message="Cache name cannot contain spaces.",
47                input_value=cache_name,
48            )
49
50        if not cache_name.replace("_", "").isalnum():
51            raise exc.PyAirbyteInputError(
52                message="Cache name can only contain alphanumeric characters and underscores.",
53                input_value=cache_name,
54            )
55
56    cache_name = cache_name or str(ulid.ULID())
57    cache_dir = cache_dir or Path(f"./.cache/{cache_name}")
58    if not isinstance(cache_dir, Path):
59        cache_dir = Path(cache_dir)
60
61    return DuckDBCache(
62        db_path=cache_dir / f"db_{cache_name}.duckdb",
63        cache_dir=cache_dir,
64        cleanup=cleanup,
65    )
def get_default_cache() -> airbyte.caches.duckdb.DuckDBCache:
15def get_default_cache() -> DuckDBCache:
16    """Get a local cache for storing data, using the default database path.
17
18    Cache files are stored in the `.cache` directory, relative to the current
19    working directory.
20    """
21    cache_dir = Path("./.cache/default_cache")
22    return DuckDBCache(
23        db_path=cache_dir / "default_cache.duckdb",
24        cache_dir=cache_dir,
25    )

Get a local cache for storing data, using the default database path.

Cache files are stored in the .cache directory, relative to the current working directory.

def new_local_cache( cache_name: str | None = None, cache_dir: str | pathlib.Path | None = None, *, cleanup: bool = True) -> airbyte.caches.duckdb.DuckDBCache:
28def new_local_cache(
29    cache_name: str | None = None,
30    cache_dir: str | Path | None = None,
31    *,
32    cleanup: bool = True,
33) -> DuckDBCache:
34    """Get a local cache for storing data, using a name string to seed the path.
35
36    Args:
37        cache_name: Name to use for the cache. Defaults to None.
38        cache_dir: Root directory to store the cache in. Defaults to None.
39        cleanup: Whether to clean up temporary files. Defaults to True.
40
41    Cache files are stored in the `.cache` directory, relative to the current
42    working directory.
43    """
44    if cache_name:
45        if " " in cache_name:
46            raise exc.PyAirbyteInputError(
47                message="Cache name cannot contain spaces.",
48                input_value=cache_name,
49            )
50
51        if not cache_name.replace("_", "").isalnum():
52            raise exc.PyAirbyteInputError(
53                message="Cache name can only contain alphanumeric characters and underscores.",
54                input_value=cache_name,
55            )
56
57    cache_name = cache_name or str(ulid.ULID())
58    cache_dir = cache_dir or Path(f"./.cache/{cache_name}")
59    if not isinstance(cache_dir, Path):
60        cache_dir = Path(cache_dir)
61
62    return DuckDBCache(
63        db_path=cache_dir / f"db_{cache_name}.duckdb",
64        cache_dir=cache_dir,
65        cleanup=cleanup,
66    )

Get a local cache for storing data, using a name string to seed the path.

Arguments:
  • cache_name: Name to use for the cache. Defaults to None.
  • cache_dir: Root directory to store the cache in. Defaults to None.
  • cleanup: Whether to clean up temporary files. Defaults to True.

Cache files are stored in the .cache directory, relative to the current working directory.