-
Notifications
You must be signed in to change notification settings - Fork 480
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: junjie.jiang <junjie.jiang@zilliz.com>
- Loading branch information
1 parent
39faf3a
commit 4fe3fa8
Showing
18 changed files
with
410 additions
and
48 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,4 @@ | ||
from gptcache.manager.scalar_data import CacheBase | ||
from gptcache.manager.vector_data import VectorBase | ||
from gptcache.manager.object_data import ObjectBase | ||
from gptcache.manager.factory import get_data_manager |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
__all__ = ["ObjectBase"] | ||
|
||
from gptcache.utils.lazy_import import LazyImport | ||
|
||
object_manager = LazyImport( | ||
"object_manager", globals(), "gptcache.manager.object_data.manager" | ||
) | ||
|
||
|
||
def ObjectBase(name: str, **kwargs): | ||
"""Generate specific ObjectStorage with the configuration. For example, setting for | ||
`ObjectBase` (with `name`) to manage LocalObjectStorage, S3 object storage. | ||
:param name: the name of the object storage, it is support 'local', 's3'. | ||
:type name: str | ||
:param path: the cache root of the LocalObjectStorage. | ||
:type path: str | ||
:param bucket: the bucket of s3. | ||
:type bucket: str | ||
:param path_prefix: s3 object prefix. | ||
:type path_prefix: str | ||
:param access_key: the access_key of s3. | ||
:type access_key: str | ||
:param secret_key: the secret_key of s3. | ||
:type secret_key: str | ||
:return: ObjectStorage. | ||
Example: | ||
.. code-block:: python | ||
from gptcache.manager import ObjectBase | ||
obj_storage = ObjectBase('local', path='./') | ||
""" | ||
return object_manager.ObjectBase.get(name, **kwargs) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
from abc import ABC, abstractmethod | ||
from typing import Any, List | ||
|
||
|
||
class ObjectBase(ABC): | ||
""" | ||
Object storage base. | ||
""" | ||
|
||
@abstractmethod | ||
def put(self, obj: Any) -> str: | ||
pass | ||
|
||
@abstractmethod | ||
def get(self, obj: str) -> Any: | ||
pass | ||
|
||
@abstractmethod | ||
def get_access_link(self, obj: str) -> str: | ||
pass | ||
|
||
@abstractmethod | ||
def delete(self, to_delete: List[str]): | ||
pass |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
from typing import Any, List | ||
import os | ||
import logging | ||
import uuid | ||
from pathlib import Path | ||
from gptcache.manager.object_data.base import ObjectBase | ||
|
||
logger = logging.getLogger() | ||
|
||
|
||
class LocalObjectStorage(ObjectBase): | ||
"""Local object storage | ||
""" | ||
|
||
def __init__(self, local_root: str): | ||
self._local_root = Path(local_root) | ||
self._local_root.mkdir(exist_ok=True) | ||
|
||
def put(self, obj: Any) -> str: | ||
f_path = self._local_root / str(uuid.uuid4()) | ||
with open(f_path, "wb") as f: | ||
f.write(obj) | ||
return str(f_path.absolute()) | ||
|
||
def get(self, obj: str) -> Any: | ||
try: | ||
with open(obj, "rb") as f: | ||
return f.read() | ||
except Exception: # pylint: disable=broad-except | ||
return None | ||
|
||
def get_access_link(self, obj: str, _: int = 3600): | ||
return obj | ||
|
||
def delete(self, to_delete: List[str]): | ||
assert isinstance(to_delete, list) | ||
for obj in to_delete: | ||
try: | ||
os.remove(obj) | ||
except Exception: # pylint: disable=broad-except | ||
logger.warning("Can not find obj: %s", obj) | ||
pass |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
from gptcache.utils.error import NotFoundStoreError | ||
|
||
|
||
class ObjectBase: | ||
""" | ||
ObjectBase to manager the object storage. | ||
""" | ||
|
||
def __init__(self): | ||
raise EnvironmentError( | ||
"CacheBase is designed to be instantiated, please using the `CacheBase.get(name)`." | ||
) | ||
|
||
@staticmethod | ||
def get(name, **kwargs): | ||
if name == "local": | ||
from gptcache.manager.object_data.local_storage import LocalObjectStorage # pylint: disable=import-outside-toplevel | ||
object_base = LocalObjectStorage(kwargs.get("path", "./local_obj")) | ||
elif name == "s3": | ||
from gptcache.manager.object_data.s3_storage import S3Storage # pylint: disable=import-outside-toplevel | ||
object_base = S3Storage(kwargs.get("path_prefix"), kwargs.get("bucket"), | ||
kwargs.get("access_key"), kwargs.get("secret_key"), | ||
kwargs.get("endpoint")) | ||
else: | ||
raise NotFoundStoreError("object store", name) | ||
return object_base |
Oops, something went wrong.