Skip to content

Fix mount router. #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
SilianZ merged 5 commits into from
Mar 11, 2024
Merged

Fix mount router. #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3188e33
Fix mount router
tianxiu2b2t Mar 10, 2024
55c6925
Fix mount router
tianxiu2b2t Mar 10, 2024
43792d6
Fix statistics for hourly
tianxiu2b2t Mar 10, 2024
71cf652
Fix statistics
tianxiu2b2t Mar 10, 2024
848c038
Change Statistics Code and fix keepalive.
tianxiu2b2t Mar 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 19 additions & 19 deletions container/bmclapi_dashboard/static/js/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ const calc_more_bytes = (...values) => {
ExtendElement("h4").text("当前节点在线").valueOf(),
ExtendElement("h2").append(
ExtendElement("span").text("0").id("t-clusters-nodes").valueOf(),
ExtendElement("span").text("个").valueOf()
ExtendElement("span").text(" 个").valueOf()
).valueOf(),
ExtendElement("div").id("e-clusters-nodes").style("height: 216px; width: 100%").valueOf()
).valueOf(),
Expand All @@ -190,7 +190,7 @@ const calc_more_bytes = (...values) => {
ExtendElement("h4").text("当日全网总请求数").valueOf(),
ExtendElement("h2").append(
ExtendElement("span").text("0").id("t-clusters-req").valueOf(),
ExtendElement("span").text("万").valueOf()
ExtendElement("span").text(" 万").valueOf()
).valueOf(),
ExtendElement("div").id("e-clusters-req").style("height: 216px; width: 100%").valueOf()
).valueOf(),
Expand All @@ -214,13 +214,13 @@ const calc_more_bytes = (...values) => {
if (!("dashboard" in core_modules_locals)) {
core_modules_locals["dashboard"] = {
"refresh": () => {
axios.get("/dashboard").then(resp => {
axios.get("dashboard").then(resp => {
if (resp.status != 200) return
data = resp.data
req = Array.from({ length: 24 }, (_, __) => null)
hits = Array.from({ length: 24 }, (_, __) => null)
bandwidth = Array.from({ length: 24 }, (_, __) => null)
bytes = Array.from({ length: 24 }, (_, __) => null)
req = Array.from({ length: 24 }, (_, __) => null)
hits = Array.from({ length: 24 }, (_, __) => null)
file_bytes = Array.from({ length: 24 }, (_, __) => null)
bytes = Array.from({ length: 24 }, (_, __) => null)
days = data.days[0]
for (day of data.days) {
if (days._day < day._day)
Expand All @@ -230,30 +230,30 @@ const calc_more_bytes = (...values) => {
const hour = hourly._hour
req[hour] = (hourly.qps / 10000).toFixed(2)
hits[hour] = (hourly.hits / 10000).toFixed(2)
bandwidth[hour] = (hourly.bandwidth * 8 / 1024.0 / 1024.0).toFixed(2)
file_bytes[hour] = (hourly.bytes / Math.min(hourly.hits, 1) / 1024.0 / 1024.0 / 1024.0).toFixed(2)
bytes[hour] = (hourly.bytes / 1024.0 / 1024.0 / 1024.0).toFixed(2)
}
core_modules_locals["dashboard"]["req"] .setOption({title: {text: "每小时请求分布（万）"}, tooltip:{formatter: e => e[0].data == null ? '' : '<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">请求: </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'万</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: req}]})
core_modules_locals["dashboard"]["bytes"] .setOption({title: {text: "每小时流量分布（GiB）"}, tooltip:{formatter: e => e[0].data == null ? '' : '<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">流量: </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'GiB</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: bytes}]})
core_modules_locals["dashboard"]["hit"] .setOption({title: {text: "每小时请求文件数（万）"}, tooltip:{formatter: e => e[0].data == null ? '' : '<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">请求文件： </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'万</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: hits}]})
core_modules_locals["dashboard"]["bandwidth"] .setOption({title: {text: "每小时峰值出网带宽（Mbps）"}, tooltip:{formatter: e => e[0].data == null ? '' : '<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">带宽： </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'Mbps</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: bandwidth}]})
core_modules_locals["dashboard"]["file_bytes"] .setOption({title: {text: "每小时平均文件流量（GiB）"}, tooltip:{formatter: e => e[0].data == null ? '' : '<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">平均文件流量： </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+' GiB</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: file_bytes}]})
document.getElementById("t-d-req").innerText = (days.qps / 10000).toFixed(2)
document.getElementById("t-d-bytes").innerText = calc_bytes(days.bytes)
document.getElementById("t-d-hit").innerText = (days.hit / 10000).toFixed(2)
document.getElementById("t-d-bandwidth").innerText = (days.bandwidth * 8 / 1024.0 / 1024.0).toFixed(2) + " M"
document.getElementById("t-d-file_bytes").innerText = ((days.bytes / Math.min(days.hit, 1)) / 1024.0 / 1024.0 / 1024.0).toFixed(2) + " G"
})
},
"bandwidth": echarts.init(document.getElementById("e-d-bandwidth")),
"file_bytes": echarts.init(document.getElementById("e-d-file_bytes")),
"bytes": echarts.init(document.getElementById("e-d-bytes")),
"req": echarts.init(document.getElementById("e-d-req")),
"hit": echarts.init(document.getElementById("e-d-hit")),
"load": echarts.init(document.getElementById("e-d-cpu")),
"options": {tooltip:{trigger:"axis",axisPointer:{type:"cross",label:{backgroundColor:"#0FC6C2"}}},grid:{left:"3%",right:"4%",bottom:"3%",containLabel:!0},xAxis:{type:"category",boundaryGap:!1,data:time_hours},yAxis:{type:"value",axisLabel:{formatter:"{value}"}},series:[{name:"",type:"line",stack:"",areaStyle:{},color:"#0FC6C2",symbol:"circle",symbolSize:4,data:[],smooth:!0,animationEasing:"cubicOut",animationDelay:function(t){return 10*t}}]},
}
core_modules_locals["dashboard"]["bandwidth"].setOption(core_modules_locals["dashboard"]["options"])
core_modules_locals["dashboard"]["bytes"] .setOption(core_modules_locals["dashboard"]["options"])
core_modules_locals["dashboard"]["req"] .setOption(core_modules_locals["dashboard"]["options"])
core_modules_locals["dashboard"]["hit"] .setOption(core_modules_locals["dashboard"]["options"])
core_modules_locals["dashboard"]["file_bytes"].setOption(core_modules_locals["dashboard"]["options"])
core_modules_locals["dashboard"]["bytes"] .setOption(core_modules_locals["dashboard"]["options"])
core_modules_locals["dashboard"]["req"] .setOption(core_modules_locals["dashboard"]["options"])
core_modules_locals["dashboard"]["hit"] .setOption(core_modules_locals["dashboard"]["options"])
}
core_modules_locals["dashboard"]["timer"] = setInterval(core_modules_locals["dashboard"].refresh, 30000)
core_modules_locals["dashboard"].refresh()
Expand All @@ -262,12 +262,12 @@ const calc_more_bytes = (...values) => {
ExtendFlex().append(
ExtendElement("div").append(
ExtendElement("div").css("panel").append(
ExtendElement("h4").text("当日出网峰值带宽").valueOf(),
ExtendElement("h4").text("当日平均文件流量").valueOf(),
ExtendElement("h2").append(
ExtendElement("span").text("0 ").id("t-d-bandwidth").valueOf(),
ExtendElement("span").text("bps").valueOf()
ExtendElement("span").text("0 ").id("t-d-file_bytes").valueOf(),
ExtendElement("span").text("iB").valueOf()
).valueOf(),
ExtendElement("div").id("e-d-bandwidth").style("height: 216px; width: 100%").valueOf()
ExtendElement("div").id("e-d-file_bytes").style("height: 216px; width: 100%").valueOf()
).valueOf(),
ExtendElement("div").css("panel").append(
ExtendElement("h4").text("当日请求文件数").valueOf(),
Expand Down
55 changes: 34 additions & 21 deletions container/cluster.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@
VERSION = "1.9.7"
UA = f"openbmclapi-cluster/{VERSION} Python/{PY_VERSION}"
URL = 'https://openbmclapi.bangbang93.com/'
COUNTER = stats.counter
LAST_COUNTER = stats.last_counter
@dataclass
class BMCLAPIFile:
path: str
Expand Down Expand Up @@ -95,6 +93,10 @@ def __init__(self, dir: Path) -> None:
self.download_files = utils.Progress(5)
self.sio = socketio.AsyncClient()
self.keepalive = None
self.timeout = None
self.last_hit = 0
self.last_bytes = 0
self.last_cur = 0
async def download(self, session: aiohttp.ClientSession):
while not self.files.empty():
file = await self.files.get()
Expand Down Expand Up @@ -130,10 +132,9 @@ async def check_file(self):
pbar = tqdm(file=logger.PRINTSTDOUT, total=total, unit=' file(s)', unit_scale=True)
pbar.set_description("Checking files")
for i, file in enumerate(filelist):
filepath = str(self.dir) + f"/{file.hash[:2]}/{file.hash}"
if not os.path.exists(filepath) or os.path.getsize(filepath) != file.size:
miss.append(file)
...
#filepath = str(self.dir) + f"/{file.hash[:2]}/{file.hash}"
#if not os.path.exists(filepath) or os.path.getsize(filepath) != file.size:
# miss.append(file)
await asyncio.sleep(0)
byte += file.size
pbar.update(1)
Expand Down Expand Up @@ -187,8 +188,8 @@ async def enable(self):
})
if not web.get_ssl() and not (Path(".ssl/cert.pem").exists() and Path(".ssl/key.pem").exists()):
await self.emit("request-cert")
self.cur_counter = stats.Counters()
logger.info("Connected to the Main Server.")
self.keepalive = Timer.delay(self.keepaliveTimer, (), 5)
async def message(self, type, data):
if type == "request-cert":
cert = data[1]
Expand All @@ -208,23 +209,38 @@ async def message(self, type, data):
if self.keepalive:
self.keepalive.block()
self.keepalive = Timer.delay(self.keepaliveTimer, (), 5)
if len(data) == 2 and data[1] == True:
if data[0]:
logger.error("Error:" + data[0]['message'])
return
if data[1] == True:
logger.info("Checked! Starting the service")
return
logger.error("Error:" + data[0]['message'])
Timer.delay(self.enable)
Timer.delay(self.start_service, (), 5)
elif type == "keep-alive":
LAST_COUNTER.hit += self.cur_counter.hit
LAST_COUNTER.bytes += self.cur_counter.bytes
if self.keepalive:
self.keepalive.block()
if self.timeout:
self.timeout.block()
if data[0]:
logger.error("Error:" + data[0]['message'])
return
stats.get_counter(self.last_cur).sync_hit = self.last_hit
stats.get_counter(self.last_cur).sync_bytes = self.last_bytes
self.keepalive = Timer.delay(self.keepaliveTimer, (), 5)
async def keepaliveTimer(self):
self.cur_counter.hit = COUNTER.hit - LAST_COUNTER.hit
self.cur_counter.bytes = COUNTER.bytes - LAST_COUNTER.bytes
counter = stats.get_counter()
self.last_hit = counter.hit
self.last_bytes = counter.bytes
self.last_cur = stats.get_hour(0)
await self.emit("keep-alive", {
"time": time.time(),
"hits": self.cur_counter.hit,
"bytes": self.cur_counter.bytes
"hits": counter.hit - counter.sync_hit,
"bytes": counter.bytes - counter.sync_bytes,
})
self.timeout = Timer.delay(self.timeoutTimer, (), 30)
async def timeoutTimer(self):
Timer.delay(self.start_service, 0)
async def emit(self, channel, data = None):
await self.sio.emit(channel, data, callback=lambda x: Timer.delay(self.message, (channel, x)))
async def get_file_list(self):
Expand Down Expand Up @@ -288,8 +304,6 @@ async def init():
global storage
Timer.delay(storage.check_file)
app = web.app
def record_bandwidth(sent: int, recv: int):
COUNTER.bandwidth += sent
@app.get("/measure/{size}")
async def _(request: web.Request, size: int, s: str, e: str):
#if not config.SKIP_SIGN:
Expand All @@ -304,15 +318,14 @@ async def _(request: web.Request, hash: str, s: str, e: str):
#if not config.SKIP_SIGN:
# check_sign(request.protocol + "://" + request.host + request.path, config.CLUSTER_SECRET, s, e)
file = Path(str(storage.dir) + "/" + hash[:2] + "/" + hash)
COUNTER.qps += 1
stats.get_counter().qps += 1
if not file.exists():
return web.Response(status_code=404)
if hash not in cache:
cache[hash] = FileCache(file)
data = await cache[hash]()
COUNTER.bytes += cache[hash].size
request.client.set_log_network(record_bandwidth)
COUNTER.hit += 1
stats.get_counter().bytes += cache[hash].size
stats.get_counter().hit += 1
return data.getbuffer()
router: web.Router = web.Router("/bmcl")
dir = Path("./bmclapi_dashboard/")
Expand Down
Loading