airbyte.caches.util
Utility functions for working with caches.
1# Copyright (c) 2023 Airbyte, Inc., all rights reserved. 2"""Utility functions for working with caches.""" 3 4from __future__ import annotations 5 6from pathlib import Path 7 8import ulid 9 10from airbyte import exceptions as exc 11from airbyte.caches.duckdb import DuckDBCache 12 13 14def get_default_cache() -> DuckDBCache: 15 """Get a local cache for storing data, using the default database path. 16 17 Cache files are stored in the `.cache` directory, relative to the current 18 working directory. 19 """ 20 cache_dir = Path("./.cache/default_cache") 21 return DuckDBCache( 22 db_path=cache_dir / "default_cache.duckdb", 23 cache_dir=cache_dir, 24 ) 25 26 27def new_local_cache( 28 cache_name: str | None = None, 29 cache_dir: str | Path | None = None, 30 *, 31 cleanup: bool = True, 32) -> DuckDBCache: 33 """Get a local cache for storing data, using a name string to seed the path. 34 35 Args: 36 cache_name: Name to use for the cache. Defaults to None. 37 cache_dir: Root directory to store the cache in. Defaults to None. 38 cleanup: Whether to clean up temporary files. Defaults to True. 39 40 Cache files are stored in the `.cache` directory, relative to the current 41 working directory. 42 """ 43 if cache_name: 44 if " " in cache_name: 45 raise exc.PyAirbyteInputError( 46 message="Cache name cannot contain spaces.", 47 input_value=cache_name, 48 ) 49 50 if not cache_name.replace("_", "").isalnum(): 51 raise exc.PyAirbyteInputError( 52 message="Cache name can only contain alphanumeric characters and underscores.", 53 input_value=cache_name, 54 ) 55 56 cache_name = cache_name or str(ulid.ULID()) 57 cache_dir = cache_dir or Path(f"./.cache/{cache_name}") 58 if not isinstance(cache_dir, Path): 59 cache_dir = Path(cache_dir) 60 61 return DuckDBCache( 62 db_path=cache_dir / f"db_{cache_name}.duckdb", 63 cache_dir=cache_dir, 64 cleanup=cleanup, 65 )
15def get_default_cache() -> DuckDBCache: 16 """Get a local cache for storing data, using the default database path. 17 18 Cache files are stored in the `.cache` directory, relative to the current 19 working directory. 20 """ 21 cache_dir = Path("./.cache/default_cache") 22 return DuckDBCache( 23 db_path=cache_dir / "default_cache.duckdb", 24 cache_dir=cache_dir, 25 )
Get a local cache for storing data, using the default database path.
Cache files are stored in the .cache
directory, relative to the current
working directory.
def
new_local_cache( cache_name: str | None = None, cache_dir: str | pathlib.Path | None = None, *, cleanup: bool = True) -> airbyte.caches.duckdb.DuckDBCache:
28def new_local_cache( 29 cache_name: str | None = None, 30 cache_dir: str | Path | None = None, 31 *, 32 cleanup: bool = True, 33) -> DuckDBCache: 34 """Get a local cache for storing data, using a name string to seed the path. 35 36 Args: 37 cache_name: Name to use for the cache. Defaults to None. 38 cache_dir: Root directory to store the cache in. Defaults to None. 39 cleanup: Whether to clean up temporary files. Defaults to True. 40 41 Cache files are stored in the `.cache` directory, relative to the current 42 working directory. 43 """ 44 if cache_name: 45 if " " in cache_name: 46 raise exc.PyAirbyteInputError( 47 message="Cache name cannot contain spaces.", 48 input_value=cache_name, 49 ) 50 51 if not cache_name.replace("_", "").isalnum(): 52 raise exc.PyAirbyteInputError( 53 message="Cache name can only contain alphanumeric characters and underscores.", 54 input_value=cache_name, 55 ) 56 57 cache_name = cache_name or str(ulid.ULID()) 58 cache_dir = cache_dir or Path(f"./.cache/{cache_name}") 59 if not isinstance(cache_dir, Path): 60 cache_dir = Path(cache_dir) 61 62 return DuckDBCache( 63 db_path=cache_dir / f"db_{cache_name}.duckdb", 64 cache_dir=cache_dir, 65 cleanup=cleanup, 66 )
Get a local cache for storing data, using a name string to seed the path.
Arguments:
- cache_name: Name to use for the cache. Defaults to None.
- cache_dir: Root directory to store the cache in. Defaults to None.
- cleanup: Whether to clean up temporary files. Defaults to True.
Cache files are stored in the .cache
directory, relative to the current
working directory.