diff --git a/bmclapi_dashboard/static/js/index.min.js b/bmclapi_dashboard/static/js/index.min.js index 8c293fd..ad21891 100644 --- a/bmclapi_dashboard/static/js/index.min.js +++ b/bmclapi_dashboard/static/js/index.min.js @@ -1152,14 +1152,12 @@ class WebSocketClient { class MinecraftUtils { static getVarInt(data) { let r = []; - while (true) { - if ((data & 0xFFFFFF80) === 0) { - r.push(data); - break; - } - r.push(data & 0x7F | 0x80); - data >>= 7; + data = (data << 1) ^ (data >> 63) + while ((data & ~0x7F) != 0) { + r.push((data & 0x7f) | 0x80) + data >>= 7 } + r.push(data); return r; } static getVarIntLength(data) { @@ -1295,16 +1293,15 @@ class DataInputStream extends BytesBuffer { return (new DataView(new Uint8Array(this.readBytes(4)))).getFloat32() } readVarInt() { - let i = 0; - let j = 0; - let k; - while (true) { - k = this.read(1)[0]; - i |= (k & 0x7F) << j * 7; - j += 1; - if ((k & 0x80) !== 128) break; - } - return i >= 2 ** 31 - 1 ? i - 2 ** 31 * 2 : i; + let b = this.read(1)[0] + let n = b & 0x7F + let shift = 7 + while ((b & 0x80) != 0) { + b = this.read(1)[0] + n |= (b & 0x7F) << shift + shift += 7 + } + return (n >> 1) ^ -(n & 1) } readString(maximum = null, encoding = 'utf-8') { return new TextDecoder(encoding).decode(new Uint8Array(this.read(maximum == null ? this.readVarInt() : maximum))); diff --git a/core/api.py b/core/api.py index 7b44dc7..a574afc 100644 --- a/core/api.py +++ b/core/api.py @@ -57,6 +57,9 @@ def is_url(self): if not isinstance(self.path, str): return False return self.path.startswith("http://") or self.path.startswith("https://") + + def is_path(self): + return isinstance(self.path, Path) def get_path(self) -> str | Path: return self.path diff --git a/core/cluster.py b/core/cluster.py index f3a6aa8..425a4b5 100644 --- a/core/cluster.py +++ b/core/cluster.py @@ -533,13 +533,14 @@ async def get(self, hash: str, offset: int = 0) -> File: file.cache = True return file path = Path(str(self.dir) + f"/{hash[:2]}/{hash}") - buf = io.BytesIO() - async with aiofiles.open(path, "rb") as r: - while data := await r.read(IO_BUFFER): - buf.write(data) - file = File(path, hash, buf.tell(), time.time(), time.time()) - file.set_data(buf.getbuffer()) + file = File(path, hash, 0) if CACHE_ENABLE: + buf = io.BytesIO() + async with aiofiles.open(path, "rb") as r: + while data := await r.read(IO_BUFFER): + buf.write(data) + file = File(path, hash, buf.tell(), time.time(), time.time()) + file.set_data(buf.getbuffer()) self.cache[hash] = file file.cache = False return file @@ -1443,7 +1444,7 @@ async def _(request: web.Request, hash: str): if data.is_url() and isinstance(data.get_path(), str): return web.RedirectResponse(str(data.get_path())).set_headers(name) return web.Response( - data.get_data().getbuffer(), headers=data.headers or {} + data.get_data().getbuffer() if not data.is_path() else data.get_path(), headers=data.headers or {} ).set_headers(name) dir = Path("./bmclapi_dashboard/") diff --git a/core/stats.py b/core/stats.py index b132a4d..cd62734 100644 --- a/core/stats.py +++ b/core/stats.py @@ -563,18 +563,25 @@ def stats_pro(day): f"select hour, data from access_ip where hour >= ?", t, ): - hour = ( - (q[0] + get_utc_offset()) - if not format_day - else (q[0] + get_utc_offset()) // 24 - ) - data = DataInputStream(zstd.decompress(q[1])) - for ip, c in { - data.readString(): data.readVarInt() for _ in range(data.readVarInt()) - }.items(): - if hour not in s_ip: - s_ip[hour] = defaultdict(int) - s_ip[hour][ip] += c + hour = (q[0] + get_utc_offset()) if not format_day else (q[0] + get_utc_offset()) // 24 + try: + data = DataInputStream(zstd.decompress(q[1])) + for ip, c in {data.readString(): data.readVarInt() for _ in range(data.readVarInt())}.items(): + if hour not in s_ip: + s_ip[hour] = defaultdict(int) + s_ip[hour][ip] += c + except: + new_data = DataOutputStream() + data_ip = {data.readString(old_data_read_varint(data)): old_data_read_varint(data) for _ in range(old_data_read_varint(data))} + new_data.writeVarInt(len(data_ip)) + for ip, c in data_ip.items(): + new_data.writeString(ip) + new_data.writeVarInt(c) + if hour not in s_ip: + s_ip[hour] = defaultdict(int) + s_ip[hour][ip] += c + execute("update access_ip set data = ? where hour = ?", zstd.compress(new_data.io.getbuffer()), hour) + addresses: defaultdict[location.IPInfo, int] = defaultdict(int) for ips in s_ip.values(): for address, count in ips.items(): @@ -596,3 +603,16 @@ def stats_pro(day): "bytes": file_bytes, "downloads": file_download, } + + +def old_data_read_varint(input: DataInputStream): + i: int = 0 + j: int = 0 + k: int + while 1: + k = int.from_bytes(input.read(1), byteorder="big") + i |= (k & 0x7F) << j * 7 + j += 1 + if (k & 0x80) != 128: + break + return i \ No newline at end of file diff --git a/core/utils.py b/core/utils.py index 1c79481..d8c0736 100644 --- a/core/utils.py +++ b/core/utils.py @@ -10,6 +10,7 @@ import re import sys import time +import avro from typing import ( Any, Coroutine, @@ -528,12 +529,11 @@ class MinecraftUtils: @staticmethod def getVarInt(data: int): r: bytes = b"" - while 1: - if data & 0xFFFFFF80 == 0: - r += data.to_bytes(1, "big") - break - r += (data & 0x7F | 0x80).to_bytes(1, "big") + data = (data << 1) ^ (data >> 63) + while (data & ~0x7F) != 0: + r += ((data & 0x7f) | 0x80).to_bytes(1, "big") data >>= 7 + r += data.to_bytes(1, "big") return r @staticmethod @@ -626,16 +626,14 @@ def readLong(self) -> int: return value - 2**64 if value > 2**63 - 1 else value def readVarInt(self) -> int: - i: int = 0 - j: int = 0 - k: int - while 1: - k = int.from_bytes(self.read(1), byteorder="big") - i |= (k & 0x7F) << j * 7 - j += 1 - if (k & 0x80) != 128: - break - return i + b = ord(self.read(1)) + n = b & 0x7F + shift = 7 + while (b & 0x80) != 0: + b = ord(self.read(1)) + n |= (b & 0x7F) << shift + shift += 7 + return (n >> 1) ^ -(n & 1) def readString( self, maximun: Optional[int] = None, encoding: Optional[str] = None diff --git a/core/web.py b/core/web.py index 2a32c8c..9c52fd4 100644 --- a/core/web.py +++ b/core/web.py @@ -905,7 +905,7 @@ async def __call__( async with aiofiles.open(content, "rb") as r: cur_length: int = 0 await r.seek(start_bytes, os.SEEK_SET) - while data := await r.read(min(IO_BUFFER, length - cur_length)): + while data := await r.read(max(0, min(IO_BUFFER, length - cur_length))): cur_length += len(data) client.write(data) await client.drain()