chore: 优化缓存的配置

Author: tianxiu2b2t

core/__init__.py

@@ -8,9 +8,8 @@
 import anyio
 import fastapi
 
-from core import units
-from core.abc import ResponseFileLocal, ResponseFileMemory, ResponseFileNotFound, ResponseFileRemote
-
+from tianxiu2b2t import units
+from .abc import ResponseFileLocal, ResponseFileMemory, ResponseFileNotFound, ResponseFileRemote
 from .locale import load_languages
 from .cluster import ClusterManager
 from .config import API_VERSION, VERSION, cfg
@@ -210,7 +209,6 @@ def access_log(request: fastapi.Request, response: fastapi.Response, total_time:
     def access_log(request: fastapi.Request, response: fastapi.Response, total_time: int):
         ...
 
-
 @web.app.middleware("http")
 async def auth_middleware(request: fastapi.Request, call_next):
     start_time = runtime.get_perf_counter_ns()

core/cluster.py

@@ -11,10 +11,9 @@
 import anyio
 import anyio.abc
 import pyzstd as zstd
-import tqdm
 import socketio
 
-from core import units
+from tianxiu2b2t import units
 
 from . import utils
 from .abc import BMCLAPIFile, Certificate, CertificateType, OpenBMCLAPIConfiguration, ResponseFile, SocketEmitResult

core/config.py

@@ -84,7 +84,7 @@ def concurrency_enable_cluster(self) -> bool:
 
 
 API_VERSION = "1.13.1"
-VERSION = "4.0.5"
+VERSION = "4.0.6"
 PROJECT = "PythonOpenBMCLAPI"
 USER_AGENT = f"openbmclapi-cluster/{API_VERSION} {PROJECT}/{VERSION}"
 ROOT_PATH = Path(__file__).parent.parent

core/storage/alist.py

@@ -4,12 +4,11 @@
 import urllib.parse as urlparse
 import aiohttp
 import anyio.abc
-import cachetools
+from tianxiu2b2t import units
 
-from core import utils
 from . import abc
 from ..config import USER_AGENT
-from ..logger import logger
+from .. import utils
 
 class AlistResponse:
     def __init__(
@@ -39,7 +38,10 @@ def __init__(
         self._endpoint = endpoint
         self._username = username
         self._password = password
-        self._redirect_urls: cachetools.TTLCache[str, abc.ResponseFile] = cachetools.TTLCache(maxsize=10000, ttl=60)
+        self._redirect_urls: utils.UnboundTTLCache[str, abc.ResponseFile] = utils.UnboundTTLCache(
+            maxsize=int(units.parse_number_units(kwargs.get("cache_size", "10000"))), 
+            ttl=units.parse_time_units(kwargs.get("cache_ttl", "5m"))
+        )
         self._token = None
 
     async def _get_token(self):

core/storage/s3.py

@@ -1,17 +1,16 @@
-import inspect
 from io import BytesIO
 import tempfile
 import time
-from typing import Any
 import aioboto3.session
 import anyio.abc
 import anyio.to_thread
 import aioboto3
 import urllib.parse as urlparse
+from tianxiu2b2t import units
 
-from core import logger
+from ..logger import logger
+from ..utils import UnboundTTLCache
 from . import abc
-import cachetools
 
 class S3ResponseMetadata:
     def __init__(
@@ -63,8 +62,14 @@ def __init__(
         self.custom_s3_host = kwargs.get("custom_s3_host", "")
         self.public_endpoint = kwargs.get("public_endpoint", "")
         self.session = aioboto3.Session()
-        self._cache: cachetools.TTLCache[str, abc.ResponseFile] = cachetools.TTLCache(maxsize=10000, ttl=60)
-        self._cache_files: cachetools.TTLCache[str, abc.FileInfo] = cachetools.TTLCache(maxsize=10000, ttl=60)
+        self._cache: UnboundTTLCache[str, abc.ResponseFile] = UnboundTTLCache(
+            maxsize=int(units.parse_number_units(kwargs.get("cache_size", "10000"))), 
+            ttl=units.parse_time_units(kwargs.get("cache_ttl", "5m"))
+        )
+        self._cache_files: UnboundTTLCache[str, abc.FileInfo] = UnboundTTLCache(
+            maxsize=int(units.parse_number_units(kwargs.get("cache_size", "10000"))), 
+            ttl=units.parse_time_units(kwargs.get("cache_files_ttl", "120s"))
+        )
         self._config = {
             "endpoint_url": self.endpoint,
             "aws_access_key_id": self.access_key,

core/storage/webdav.py

@@ -5,11 +5,13 @@
 import aiowebdav.client
 import anyio
 from anyio.abc._tasks import TaskGroup as TaskGroup
-import cachetools
+from tianxiu2b2t import units
 
-from core import logger, utils
-from core.config import USER_AGENT
 from . import abc
+from ..logger import logger
+from ..config import USER_AGENT
+from .. import utils
+
 import aiowebdav
 
 class WebDavStorage(abc.Storage):
@@ -22,13 +24,20 @@ def __init__(
         endpoint: str,
         username: str,
         password: str,
+        **kwargs
     ):
         super().__init__(name, path, weight)
         self.endpoint = endpoint
         self.username = username
         self.password = password
-        self._cache_files: cachetools.TTLCache[str, abc.FileInfo] = cachetools.TTLCache(maxsize=1000, ttl=60)
-        self._cache_redirects: cachetools.TTLCache[str, abc.ResponseFile] = cachetools.TTLCache(maxsize=1000, ttl=60)
+        self._cache_redirects: utils.UnboundTTLCache[str, abc.ResponseFile] = utils.UnboundTTLCache(
+            maxsize=int(units.parse_number_units(kwargs.get("cache_size", "10000"))), 
+            ttl=units.parse_time_units(kwargs.get("cache_ttl", "5m"))
+        )
+        self._cache_files: utils.UnboundTTLCache[str, abc.FileInfo] = utils.UnboundTTLCache(
+            maxsize=int(units.parse_number_units(kwargs.get("cache_size", "10000"))), 
+            ttl=units.parse_time_units(kwargs.get("cache_files_ttl", "120s"))
+        )
         self._mkdir_lock = utils.Lock()
 
         self.client = aiowebdav.client.Client({

core/units.py

@@ -1,12 +1,14 @@
 from collections import defaultdict, deque
 import hashlib
 import io
+import math
 from pathlib import Path
 import time
 from typing import Any, Awaitable, Callable, Optional, Coroutine, MutableMapping, TypeVar
 
 import anyio
 import anyio.abc
+import cachetools
 from tqdm import tqdm
 from functools import lru_cache
 
@@ -442,5 +444,13 @@ def __init__(
         self.start = start
         self.end = end
 
+class UnboundTTLCache(cachetools.TTLCache[K, V]):
+    def __init__(self, maxsize: Optional[int], ttl: float, timer=time.monotonic):
+        cachetools.TTLCache.__init__(self, maxsize or math.inf, ttl, timer)
+
+    @property
+    def maxsize(self):
+        return None
+
 runtime = Runtime()
 event = Event()