fix: 修复 S3 存储没有正确响应的问题，修复 206 的问题

Author: tianxiu2b2t

README.md

@@ -139,7 +139,7 @@ python main.py
 
 4. 开始 for Docker
 ```bash
-docker run -d --restart=always -p 6543:6543 -v /path/to/your/config.yaml:/app/config.yaml -v /path/to/your/bmclapi:/app/bmclapi --name python-openbmclapi atianxiua/python-openbmclapi:latest
+docker run -d --restart=always -p 6543:6543 -v /path/to/your/config.yaml:opt/python-openbmclapi/config/config.yaml -v /path/to/your/bmclapi:opt/python-openbmclapi/bmclapi --name python-openbmclapi atianxiua/python-openbmclapi:latest
 ```
 
 # TODO

core/__init__.py

@@ -9,7 +9,7 @@
 import fastapi
 
 from core import units
-from core.abc import ResponseFileLocal, ResponseFileMemory, ResponseFileRemote
+from core.abc import ResponseFileLocal, ResponseFileMemory, ResponseFileNotFound, ResponseFileRemote
 
 from .locale import load_languages
 from .cluster import ClusterManager
@@ -126,7 +126,8 @@ async def download(request: fastapi.Request, hash: str, s: str, e: str, name: Op
     resp_headers = {}
     if name:
         resp_headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{urlparse.quote(name)}"
-
+    resp_headers["X-BMCLAPI-Hash"] = hash
+    resp_headers["X-BMCLAPI-Size"] = str(size)
     clusters.hit(cluster_id, size or 0)
 
     if isinstance(file, ResponseFileLocal):
@@ -143,25 +144,36 @@ async def download(request: fastapi.Request, hash: str, s: str, e: str, name: Op
     elif isinstance(file, ResponseFileMemory):
         result = b''
         r = utils.parse_range(range or "")
+        if r is not None:
+            if r.start >= len(file.data) or r.end is not None and (r.end > len(file.data) or r.end < r.start):
+                return fastapi.responses.Response(
+                    status_code=416,
+                    content="Range Not Satisfiable"
+                )
         status = 200
         if r is None:
             result = file.data
         elif r.end is None:
             result = file.data[r.start:]
         else:
-            result = file.data[r.start:r.end]
+            result = file.data[r.start:r.end + 1]
         if r is not None:
             status = 206
             resp_headers["Content-Range"] = f"bytes {r.start}-{len(result) + r.start - 1}/{len(file.data)}"
+        resp_headers["Content-Length"] = str(len(result))
         return fastapi.responses.StreamingResponse(
             io.BytesIO(result),
             status_code=status,
             headers=resp_headers,
         )
+    elif isinstance(file, ResponseFileNotFound):
+        return fastapi.responses.Response(
+            status_code=404,
+            content="Not Found"
+        )
     else:
         return fastapi.responses.Response(
-            status_code=500,
-            content="Internal Server Error"
+            status_code=200,
         )
 
 if cfg.access_log:

core/abc.py

@@ -121,3 +121,12 @@ def __init__(
 
     def __repr__(self) -> str:
         return f'ResponseFileMemory(data={self.data}, size={self.size})'
+    
+class ResponseFileNotFound(ResponseFile):
+    def __init__(
+        self,
+    ):
+        super().__init__(-1)
+
+    def __repr__(self) -> str:
+        return f'ResponseFileNotFound(size={self.size})'

core/cluster.py

@@ -471,7 +471,6 @@ async def _download_file(
             with tempfile.NamedTemporaryFile(
                 dir=self._cache_dir,
             ) as tmp_file:
-                print(pbar.position)
                 try:
                     async with session.get(
                         file.path

core/storage/__init__.py

@@ -113,6 +113,7 @@ async def get_missing_files(self, bmclapi_files: set[BMCLAPIFile], muitlpbar: ut
 
         for file in await self.storage.list_download_files(muitlpbar):
             self.files[file.name] = file
+        muitlpbar.update(1)
 
         # start check
         # with concurreny

core/storage/abc.py

@@ -4,7 +4,7 @@
 import anyio.abc
 
 from core import utils
-from core.abc import BMCLAPIFile, ResponseFile, ResponseFileMemory, ResponseFileLocal, ResponseFileRemote
+from core.abc import BMCLAPIFile, ResponseFile, ResponseFileNotFound, ResponseFileMemory, ResponseFileLocal, ResponseFileRemote
 from ..logger import logger
 
 RANGE = range(0x00, 0xFF + 1)

core/storage/alist.py

@@ -80,7 +80,7 @@ async def _check(self):
                     async with session.put(
                         "/api/fs/put",
                         headers={
-                            "File-Path": str(self._path / ".check"),
+                            "File-Path": str(self._path / ".py_check"),
                         },
                         data=str(time.perf_counter_ns())
                     ) as resp:
@@ -90,7 +90,7 @@ async def _check(self):
                         data={
                             "dir": str(self._path),
                             "names": [
-                                ".check"
+                                ".py_check"
                             ]
                         }
                     ) as resp:

core/storage/local.py

@@ -3,7 +3,7 @@
 import time
 import anyio.abc
 
-from core.abc import ResponseFileLocal
+from core.abc import ResponseFile, ResponseFileLocal, ResponseFileNotFound
 from ..logger import logger
 
 from .abc import FileInfo, Storage
@@ -67,7 +67,7 @@ async def upload(
     async def _check(
         self,
     ):
-        file = Path(str(self.path)) / ".check"
+        file = Path(str(self.path)) / ".py_check"
         while 1:
             try:
                 file.write_text(str(time.perf_counter_ns()))
@@ -80,10 +80,10 @@ async def _check(
                 self.emit_status()
                 await anyio.sleep(60)
 
-    async def get_response_file(self, hash: str) -> ResponseFileLocal:
+    async def get_response_file(self, hash: str) -> ResponseFile:
         p = Path(str(self.path)) / "download" / hash[:2] / hash
         if not p.exists():
-            raise FileNotFoundError
+            return ResponseFileNotFound()
         size = p.stat().st_size
         return ResponseFileLocal(
             size=size,

core/storage/s3.py

@@ -1,4 +1,5 @@
 import inspect
+from io import BytesIO
 import tempfile
 import time
 from typing import Any
@@ -7,6 +8,8 @@
 import anyio.to_thread
 import aioboto3
 import urllib.parse as urlparse
+
+from core import logger
 from . import abc
 import cachetools
 
@@ -60,9 +63,8 @@ def __init__(
         self.custom_s3_host = kwargs.get("custom_s3_host", "")
         self.public_endpoint = kwargs.get("public_endpoint", "")
         self.session = aioboto3.Session()
-        self.list_lock = anyio.Lock()
-        self.cache_list_bucket: dict[str, abc.FileInfo] = {}
-        self.last_cache: float = 0
+        self._cache: cachetools.TTLCache[str, abc.ResponseFile] = cachetools.TTLCache(maxsize=10000, ttl=60)
+        self._cache_files: cachetools.TTLCache[str, abc.FileInfo] = cachetools.TTLCache(maxsize=10000, ttl=60)
         self._config = {
             "endpoint_url": self.endpoint,
             "aws_access_key_id": self.access_key,
@@ -80,11 +82,6 @@ async def setup(
 
         self.task_group.start_soon(self._check)
 
-    async def list_bucket(
-        self,
-    ):
-        ...
-
     async def list_files(
         self,
         path: str
@@ -148,31 +145,61 @@ async def upload(
             obj = await bucket.Object(str(self.path / path))
             await obj.upload_fileobj(tmp_file)
         return True
-        
+
 
     async def _check(
         self,
     ):
         while 1:
             try:
-                await self.list_bucket()
+                async with self.session.resource(
+                    "s3",
+                    endpoint_url=self.endpoint,
+                    aws_access_key_id=self.access_key,
+                    aws_secret_access_key=self.secret_key,
+                    region_name=self.region
+                ) as resource:
+                    bucket = await resource.Bucket(self.bucket)
+                    obj = await bucket.Object(str(self.path / ".py_check"))
+                    await obj.upload_fileobj(BytesIO(str(time.perf_counter_ns()).encode()))
+                    await obj.delete()
                 self.online = True
             except:
                 self.online = False
+                logger.traceback()
             finally:
                 self.emit_status()
-                await anyio.sleep(300)
+                await anyio.sleep(60)
 
     async def get_response_file(self, hash: str) -> abc.ResponseFile:
         cpath = str(self.path / "download" / hash[:2] / hash)
-        if cpath not in self.cache_list_bucket:
-            return abc.ResponseFile(
-                0
-            )
+        fileinfo = self._cache_files.get(hash)
+
+        if fileinfo is None:
+            async with self.session.resource(
+                "s3",
+                endpoint_url=self.endpoint,
+                aws_access_key_id=self.access_key,
+                aws_secret_access_key=self.secret_key,
+                region_name=self.region
+            ) as resource:
+                bucket = await resource.Bucket(self.bucket)
+                obj = await bucket.Object(cpath)
+                info = await obj.get()
+                fileinfo = abc.FileInfo(
+                    name=hash,
+                    size=info["ContentLength"],
+                    path=cpath
+                )
+                self._cache_files[hash] = fileinfo
+        
+        if fileinfo is None:
+            return abc.ResponseFileNotFound()
+
         if self.custom_s3_host:
             return abc.ResponseFileRemote(
                 f"{self.custom_s3_host}{cpath}",
-                self.cache_list_bucket[cpath].size
+                fileinfo.size
             )
         if self.public_endpoint:
             async with self.session.client(
@@ -204,7 +231,7 @@ async def get_response_file(self, hash: str) -> abc.ResponseFile:
                 )
                 return abc.ResponseFileRemote(
                     url,
-                    self.cache_list_bucket[cpath].size
+                    fileinfo.size
                 )
         async with self.session.resource(
             "s3",

core/storage/webdav.py

@@ -44,8 +44,8 @@ async def setup(self, task_group: TaskGroup):
     async def _check(self):
         while 1:
             try:
-                await self.client.upload_to(io.BytesIO(str(time.perf_counter_ns()).encode()), ".check")
-                await self.client.clean(".check")
+                await self.client.upload_to(io.BytesIO(str(time.perf_counter_ns()).encode()), ".py_check")
+                await self.client.clean(".py_check")
                 self.online = True
             except:
                 self.online = False