Skip to content

修复下载（添加下载错误时候返回下载地址），修复缓存可能会导致程序oom #62

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
May 5, 2024
Merged

修复下载（添加下载错误时候返回下载地址），修复缓存可能会导致程序oom #62

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
972cad2
更新下载链接跳转
tianxiu2b2t May 5, 2024
f3c39ed
更新dashboard进度条，修复一些特性问题
tianxiu2b2t May 5, 2024
199c49d
删除js一部分代码
tianxiu2b2t May 5, 2024
b526433
删除无用的控制台输出
tianxiu2b2t May 5, 2024
aad37c3
修复缓存时候数据爆炸情况
tianxiu2b2t May 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 62 additions & 16 deletions bmclapi_dashboard/static/js/index.min.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -534,7 +534,36 @@ class Application {
border-radius: 4px;
background-color: var(--border-background);
padding-left: 24px;
padding-bottom: 16px;`
padding-bottom: 16px;`,
'.tqdm': [
'display: flex;',
'margin-top: 16px',
],
".tqdm-outline": [
"display: block"
],
'.tqdm .tqdm-progressbar': [
'margin-left: 8px',
'margin-right: 8px',
'display: flex',
'flex-grow: 1;',
'align-items: center'
],
".tqdm.tqdm-outline .tqdm-progressbar": [
"margin: 0"
],
'.tqdm .tqdm-backgroundbar': [
'padding: 2px',
'margin-left: 8px',
'height: 4px',
'background: var(--background)',
'width: 100%'
],
'.tqdm .tqdm-bar': [
'width: 50%',
'height: 4px',
'background: var(--main-color)',
]
}
this.$side = this.createElement("aside").class("side")
this.$container = this.createElement("div").class("main").append(
Expand Down Expand Up @@ -1990,7 +2019,7 @@ $I18N.addLangs("zh_cn", {
"menu.master.rank": "排行榜",
"menu.config": "配置",
"menu.config.storage": "存储设置",
"tqdm": "%value%/%total%, %item%/s",
"tqdm": "%value%/%total% [%start% < %end%, %item%/s]",
"storage.webdav": "正在获取WebDav文件中",
"cluster.want_enable": "正在启用",
"cluster.enabled.trusted": "正常工作",
Expand Down Expand Up @@ -2704,6 +2733,16 @@ app.$Menu.add("dashboard", new class {
hits: app.createEcharts().style("min-height: 162px;"),
bytes: app.createEcharts().style("min-height: 162px;").setFormatter((n) => this._format_bytes(n))
}
this.pbar = app.createElement("div").class("tqdm").append(
app.createElement("p"),
app.createElement("p").class("tqdm-progressbar").append(
app.createElement("span").setText("100%"),
app.createElement("div").class("tqdm-backgroundbar").append(
app.createElement("div").class("tqdm-bar")
),
),
app.createElement("p")
)
this.page = [
app.createElement("div").class("panel").append(
app.createFlex().append(
Expand All @@ -2713,17 +2752,18 @@ app.$Menu.add("dashboard", new class {
),
app.createElement("div").append(
app.createElement("p").class("title").setI18N("dashboard.status"),
app.createElement("p").append(
app.createElement("span").class("value").setText("-"),
app.createElement("span").append(
app.createElement("span").class("value").setText(" | "),
app.createElement("span").class("value").setText(""),
app.createElement("span").class("value").setText(" "),
app.createElement("span").setText("")
),
)
app.createElement("p").class("value")
)
).minWidth(896).child(2)
).minWidth(896).child(2).addResize(() => {
this.pbar.removeClass("tqdm-outline")
var width = this.pbar.getChildren()[1].valueOf().offsetWidth
if (width >= 84) {
this.pbar.removeClass("tqdm-outline")
} else {
this.pbar.class("tqdm-outline")
}
}),
this.pbar
),
app.createElement("div").class("panel nopadding").style("margin-bottom: 0").append(
app.createFlex(true).class("flex-space-between").child(2).minWidth(512).append(
Expand Down Expand Up @@ -3696,15 +3736,21 @@ app.$Menu.add("dashboard", new class {
}
}
setStatus() {
this.page[0].getChildren()[0].getChildren()[1].getChildren()[1].getChildren()[0].setI18N(this.status.key)
this.page[0].getChildren()[0].getChildren()[1].getChildren()[1].getChildren()[1].style(`display: ${this.status.progress ? 'inline' : 'none'}`)
this.page[0].getChildren()[0].getChildren()[1].getChildren()[1].setI18N(this.status.key)
this.pbar.valueOf().style.display = `${this.status.progress ? '' : 'none'}`
if (this.status.progress) {
var value_formatter = this.status.progress.desc == "files.downloading" ? (n) => this._format_bytes(n) : (n) => n
this.page[0].getChildren()[0].getChildren()[1].getChildren()[1].getChildren()[1].getChildren()[1].setI18N(this.status.progress.desc)
this.page[0].getChildren()[0].getChildren()[1].getChildren()[1].getChildren()[1].getChildren()[3].setI18N("tqdm", {
var percent = ((this.status.progress.value / this.status.progress.total) * 100)
percent = isNaN(percent) ? 0 : percent
this.pbar.getChildren()[0].setI18N(this.status.progress.desc)
this.pbar.getChildren()[1].getChildren()[0].setText(`${percent.toFixed(0)}%`)
this.pbar.getChildren()[1].getChildren()[1].getChildren()[0].style(`width: ${percent}%`)
this.pbar.getChildren()[2].setI18N("tqdm", {
value: value_formatter(this.status.progress.value),
total: value_formatter(this.status.progress.total),
item: value_formatter(this.status.progress.speed),
start: this.status.progress.start,
end: this.status.progress.end
})
}
}
Expand Down
144 changes: 97 additions & 47 deletions core/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@
import hashlib
import io
from pathlib import Path
import time
from typing import Optional
import pyzstd as zstd
import aiofiles

from core import web
from core import logger, scheduler, unit, web
from core.config import Config
from core.const import CACHE_BUFFER_COMPRESSION_MIN_LENGTH
from core.const import CACHE_BUFFER_COMPRESSION_MIN_LENGTH, CACHE_TIME, CHECK_CACHE, CACHE_BUFFER


class FileCheckType(Enum):
Expand All @@ -27,6 +28,7 @@ class FileContentType(Enum):
DATA = "data"
URL = "url"
PATH = "path"
EMPTY = "empty"

@dataclass
class BMCLAPIFile:
Expand All @@ -47,66 +49,114 @@ def __eq__(self, other):
)
return False


@dataclass
class File:
path: Path | str
hash: str
size: int
last_hit: float = 0
last_access: float = 0
expiry: Optional[float] = None
data: Optional[io.BytesIO] = None
type: FileContentType
data: io.BytesIO | str | Path = None
expiry: float = 0
compressed: bool = False
data_length: int = 0
cache: bool = False
headers: Optional["web.Header"] = None
compressed: bool = False

def is_url(self):
if not isinstance(self.path, str):
return False
return self.path.startswith("http://") or self.path.startswith("https://")

def is_path(self):
return isinstance(self.path, Path)

def get_path(self) -> str | Path:
return self.path
def set_data(self, data: io.BytesIO | str | Path):
if isinstance(data, io.BytesIO):
length = len(data.getbuffer())
if CACHE_BUFFER_COMPRESSION_MIN_LENGTH <= length:
self.data = io.BytesIO(zstd.compress(data.getbuffer()))
self.data_length = len(self.data.getbuffer())
self.compressed = True
else:
self.data = data
self.data_length = len(data.getbuffer())
self.compressed = False
self.type = FileContentType.DATA
elif isinstance(data, str):
self.data_length = len(data)
self.data = data
self.type = FileContentType.URL
elif isinstance(data, Path):
self.data_length = len(str(data))
self.data = data
self.type = FileContentType.PATH

def get_data(self):
if not self.data:
return io.BytesIO()
if not self.compressed:
if self.compressed:
return io.BytesIO(zstd.decompress(self.data.getbuffer()))
else:
return self.data
return io.BytesIO(zstd.decompress(self.data.getbuffer()))

def set_data(self, data: io.BytesIO | memoryview | bytes):
if not isinstance(data, io.BytesIO):
data = io.BytesIO(data)
data_length = len(data.getbuffer())
if data_length >= CACHE_BUFFER_COMPRESSION_MIN_LENGTH:
compressed_data = zstd.compress(data.getbuffer())
if data_length > len(compressed_data):
self.compressed = True
self.data = io.BytesIO(compressed_data)
return
self.compressed = False
self.data = data


def is_url(self):
return self.type == FileContentType.URL
def is_path(self):
return self.type == FileContentType.PATH
def get_path(self) -> Path:
return self.data
@dataclass
class StatsCache:
total: int = 0
bytes: int = 0
data_bytes: int = 0


class Storage(metaclass=abc.ABCMeta):
def __init__(self, name, width: int) -> None:
self.name = name
self.disabled = False
self.width = width

self.cache: dict[str, File] = {}
self.cache_timer = scheduler.repeat(
self.clear_cache, delay=CHECK_CACHE, interval=CHECK_CACHE
)
def get_name(self):
return self.name

def get_cache(self, hash: str) -> Optional[File]:
file = self.cache.get(hash, None)
if file is not None:
file.cache = True
if not file.is_url():
file.expiry = time.time() + CACHE_TIME
return file

def is_cache(self, hash: str) -> Optional[File]:
return hash in self.cache

def set_cache(self, hash: str, file: File):
self.cache[hash] = file

def clear_cache(self):
hashs = set()
data = sorted(
self.cache.copy().items(),
key=lambda x: x[1].expiry, reverse=True)
size = 0
old_size = 0
for hash, file in data:
if file.type == FileContentType.EMPTY:
continue
size += file.data_length
if (size <= CACHE_BUFFER and file.expiry >= time.time()):
continue
hashs.add(hash)
old_size += file.data_length
for hash in hashs:
self.cache.pop(hash)
logger.tinfo(
"cluster.info.clear_cache.count",
name=self.name,
count=unit.format_number(len(hashs)),
size=unit.format_bytes(old_size),
)

def get_cache_stats(self) -> StatsCache:
stat = StatsCache()
for file in self.cache.values():
stat.total += 1
stat.bytes += file.size
stat.data_bytes += file.data_length
return stat

@abc.abstractmethod
async def get(self, file: str, offset: int = 0) -> File:
Expand Down Expand Up @@ -175,15 +225,15 @@ async def removes(self, hashs: list[str]) -> int:
"""
raise NotImplementedError

@abc.abstractmethod
async def get_cache_stats(self) -> StatsCache:
"""
dir: path
Getting cache files
return StatsCache
"""
raise NotImplementedError
@dataclass
class OpenbmclapiAgentConfiguration:
source: str
concurrency: int

@dataclass
class ResponseRedirects:
status: int
url: str

def get_hash(org):
if len(org) == 32:
Expand Down
Loading