fix: 给下载文件报错加个原 Python 报错类型

Author: tianxiu2b2t

core/cluster.py

@@ -202,7 +202,6 @@ async def get_file(self, hash: str):
                     if got_hash == hash:
                         break
                     logger.terror("cluster.error.download_hash", got_hash=got_hash, hash=hash, content=body.decode("utf-8", "ignore")[:64])  
-                print(file)
         return file
 
     def get_width_storage(self, c_storage: Optional[storages.iStorage] = None) -> storages.iStorage:
@@ -474,7 +473,7 @@ async def report(self, file: File, error: Exception, resp: Optional[aiohttp.Clie
             responses.append(URLResponse(str(resp.real_url), resp.status))
             host = resp.host
         hash = msg[0] if len(msg) > 0 and type == "file" else None
-        logger.terror(f"clusters.error.downloading", type=type, file_hash=file.hash, file_size=units.format_bytes(file.size), host=host, file_path=file.path, hash=hash, responses="\n".join(("", *(str(r) for r in responses))))
+        logger.terror(f"clusters.error.downloading", type=type, error=str(error), file_hash=file.hash, file_size=units.format_bytes(file.size), host=host, file_path=file.path, hash=hash, responses="\n".join(("", *(str(r) for r in responses))))
         self.failed_hash_urls[file.hash].failed += 1
         self.failed_hash_urls[file.hash].urls.add(tuple(responses))
         if self.failed_hash_urls[file.hash].failed < 10:
@@ -1069,12 +1068,14 @@ async def _(request: aweb.Request):
         hash = request.match_info["hash"]
         query = request.query
         address = request.custom_address # type: ignore
+        user_agent = request.headers.get("User-Agent", "")
         s = query.get("s", "")
         e = query.get("e", "")
         if not check_sign(request.match_info["hash"], s, e):
             db.add_response(
                 address,
-                db.StatusType.FORBIDDEN
+                db.StatusType.FORBIDDEN,
+                user_agent
             )
             return aweb.Response(status=403)
         cluster_id = get_cluster_id_from_sign(hash, s, e)
@@ -1084,15 +1085,17 @@ async def _(request: aweb.Request):
         if cluster is None:
             db.add_response(
                 address,
-                db.StatusType.FORBIDDEN
+                db.StatusType.FORBIDDEN,
+                user_agent
             )
             return aweb.Response(status=403)
 
         file = await clusters.storage_manager.get_file(hash)
         if file is None:
             db.add_response(
                 address,
-                db.StatusType.NOT_FOUND
+                db.StatusType.NOT_FOUND,
+                user_agent
             )
             return aweb.Response(status=404)
 
@@ -1138,7 +1141,8 @@ async def _(request: aweb.Request):
         db.add_file(cluster.id, storage_name, size)
         db.add_response(
             address,
-            type
+            type,
+            user_agent
         )
         return resp
     except:

core/database.py

@@ -43,6 +43,7 @@ class ResponseStatistics:
     error: int = 0
     redirect: int = 0
     ip_tables: defaultdict[str, int] = field(default_factory=lambda: defaultdict(int))
+    user_agents: defaultdict[str, int] = field(default_factory=lambda: defaultdict(int))
 
 
 
@@ -75,6 +76,7 @@ class ResponseTable(Base):
     error = Column(String, nullable=False)
     redirect = Column(String, nullable=False)
     ip_tables = Column(LargeBinary, nullable=False)
+    user_agents = Column(LargeBinary, nullable=False)
 
 class StatusType(Enum):
     SUCCESS = "success"
@@ -137,10 +139,11 @@ def add_file(cluster: str, storage: Optional[str], bytes: int):
     FILE_CACHE[key].bytes += bytes
     FILE_CACHE[key].hits += 1
 
-def add_response(ip: str, type: StatusType):
+def add_response(ip: str, type: StatusType, user_agent: str):
     global RESPONSE_CACHE
     hour = get_hour()
     RESPONSE_CACHE[hour].ip_tables[ip] += 1
+    RESPONSE_CACHE[hour].user_agents[user_agent] += 1
     setattr(RESPONSE_CACHE[hour], type.value, getattr(RESPONSE_CACHE[hour], type.value) + 1)
 
 def get_hour():
@@ -178,7 +181,7 @@ def _commit_cluster(hour: int, cluster: str, hits: int, bytes: int):
     )
     return True
 
-def _commit_response(hour: int, ip_tables: defaultdict[str, int], success: int = 0, forbidden: int = 0, redirect: int = 0, not_found: int = 0, error: int = 0):
+def _commit_response(hour: int, ip_tables: defaultdict[str, int], user_agents: defaultdict[str, int], success: int = 0, forbidden: int = 0, redirect: int = 0, not_found: int = 0, error: int = 0):
     if ip_tables == {}:
         return False
     session = SESSION.get_session()
@@ -187,7 +190,9 @@ def _commit_response(hour: int, ip_tables: defaultdict[str, int], success: int =
     if q.count() == 0:
         session.add(r)
     origin_ip_tables: defaultdict[str, int] = defaultdict(lambda: 0)
+    origin_user_agents: defaultdict[str, int] = defaultdict(lambda: 0)
     ip_tables_data: bytes = r.ip_tables # type: ignore
+    user_agents_data: bytes = r.user_agents # type: ignore
     if ip_tables_data:
         try:
             input = utils.DataInputStream(pyzstd.decompress(ip_tables_data))
@@ -198,16 +203,34 @@ def _commit_response(hour: int, ip_tables: defaultdict[str, int], success: int =
         except:
             logger.ttraceback("database.error.unable.to.decompress.ip.tables", ip_tables_data)
             origin_ip_tables.clear()
+    if user_agents_data:
+        try:
+            input = utils.DataInputStream(pyzstd.decompress(user_agents_data))
+            for _ in range(input.read_long()):
+                user_agent = input.read_string()
+                count = input.read_long()
+                origin_user_agents[user_agent] = count
+        except:
+            logger.ttraceback("database.error.unable.to.decompress.user.agents", user_agents_data)
     for ip, count in ip_tables.items():
         origin_ip_tables[ip] += count
-    output = utils.DataOutputStream()
-    output.write_long(len(origin_ip_tables))
+    
+    for user_agent, count in user_agents.items():
+        origin_user_agents[user_agent] += count
+    ip_tables_output = utils.DataOutputStream()
+    ip_tables_output.write_long(len(origin_ip_tables))
     for ip, count in origin_ip_tables.items():
-        output.write_string(ip)
-        output.write_long(count)
+        ip_tables_output.write_string(ip)
+        ip_tables_output.write_long(count)
+    user_agents_output = utils.DataOutputStream()
+    user_agents_output.write_long(len(origin_user_agents))
+    for user_agent, count in origin_user_agents.items():
+        user_agents_output.write_string(user_agent)
+        user_agents_output.write_long(count)
     q.update(
         {
-            'ip_tables': pyzstd.compress(output.getvalue()), # type: ignore
+            'ip_tables': pyzstd.compress(ip_tables_output.getvalue()), # type: ignore
+            'user_agents': pyzstd.compress(user_agents_output.getvalue()), # type: ignore
             'success': str(int(r.success) + success), # type: ignore
             'forbidden': str(int(r.forbidden) + forbidden), # type: ignore
             'redirect': str(int(r.redirect) + redirect), # type: ignore
@@ -242,7 +265,7 @@ def commit():
         _commit_cluster(cluster[0], cluster[1], value.hits, value.bytes)
 
     for hour, value in response_cache.items():
-        _commit_response(hour, value.ip_tables, value.success, value.forbidden, value.redirect, value.not_found, value.error)
+        _commit_response(hour, value.ip_tables, value.user_agents, value.success, value.forbidden, value.redirect, value.not_found, value.error)
 
     session.commit()
     old_keys = []
@@ -263,10 +286,14 @@ def commit():
         RESPONSE_CACHE[hour].not_found -= value.not_found
         RESPONSE_CACHE[hour].error -= value.error
         ip_hits = 0
+        user_agent_hits = 0
         for ip, hits in value.ip_tables.items():
             RESPONSE_CACHE[hour].ip_tables[ip] -= hits
             ip_hits += RESPONSE_CACHE[hour].ip_tables[ip]
-        if RESPONSE_CACHE[hour].success == RESPONSE_CACHE[hour].forbidden == RESPONSE_CACHE[hour].redirect == RESPONSE_CACHE[hour].not_found == RESPONSE_CACHE[hour].error == ip_hits == 0:
+        for user_agent, hits in value.user_agents.items():
+            RESPONSE_CACHE[hour].user_agents[user_agent] -= hits
+            user_agent_hits += RESPONSE_CACHE[hour].user_agents[user_agent]
+        if RESPONSE_CACHE[hour].success == RESPONSE_CACHE[hour].forbidden == RESPONSE_CACHE[hour].redirect == RESPONSE_CACHE[hour].not_found == RESPONSE_CACHE[hour].error == ip_hits == user_agent_hits == 0:
             old_keys.append(hour)
     for key in old_keys:
         del RESPONSE_CACHE[key]

i18n/zh_cn.json

@@ -13,7 +13,7 @@
     "cluster.processbar.download_files": "下载文件",
     "i18n.prompt.failed": " 未找到 i18n 字符串",
     "cluster.warning.no_check_function": "没有找到检查函数，使用类型为 exists",
-    "clusters.error.downloading": "下载时候出错，原因 [${type}]，文件 [${file_path} ${file_hash}(${file_size})] 下载后的哈希值 [${hash}]，主机 [${host}] 下载时候的历史记录：${responses}",
+    "clusters.error.downloading": "下载时候出错，原因 [${type}] 报错 [${error}]，文件 [${file_path} ${file_hash}(${file_size})] 下载后的哈希值 [${hash}]，主机 [${host}] 下载时候的历史记录：${responses}",
     "web.debug.local_port": "内部开放端口为 [${port}]",
     "web.debug.ssl_port": "内部 SSL 开放端口为 [${port}]",
     "web.success.public_port": "当前 Web 服务器在 [${port}] 端口上开启",