Merge pull request #78 from bangbang93/next

feat: 下载错误上报

.github/workflows/docker-hub.yml

@@ -2,7 +2,7 @@ name: Docker Image CI
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, next ]
 
 jobs:
   build:

CHANGELOG.md

@@ -4,8 +4,43 @@ All notable changes to this project will be documented in this file. Dates are d
 
 Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 
+#### [v1.11.0-4](https://github.com/bangbang93/openbmclapi/compare/v1.11.0-3...v1.11.0-4)
+
+- fix: 修正异常上报 [`6c5e3b8`](https://github.com/bangbang93/openbmclapi/commit/6c5e3b8c1cf8f79a447c55806a9eb0bdededef1b)
+- feat: 移除noopen参数 [`fecb72a`](https://github.com/bangbang93/openbmclapi/commit/fecb72ad5a75730697212bdfd914714c4adc2e29)
+
+#### [v1.11.0-3](https://github.com/bangbang93/openbmclapi/compare/v1.11.0-2...v1.11.0-3)
+
+> 18 June 2024
+
+- refactor: 简化上报的错误 [`03400c5`](https://github.com/bangbang93/openbmclapi/commit/03400c543cd73d6abe64481cecc12a221aef93c1)
+- Release 1.11.0-3 [`750b711`](https://github.com/bangbang93/openbmclapi/commit/750b711e467b3f992672bcbf3c4c2072b79e45b0)
+
+#### [v1.11.0-2](https://github.com/bangbang93/openbmclapi/compare/v1.11.0-1...v1.11.0-2)
+
+> 18 June 2024
+
+- fix: 修正错误上报 [`ba027be`](https://github.com/bangbang93/openbmclapi/commit/ba027be00f11e1fab76aa974cd625c37b3c3079b)
+- Release 1.11.0-2 [`84ca717`](https://github.com/bangbang93/openbmclapi/commit/84ca717eda91a2f2735ec02465d162dec0224750)
+
+#### [v1.11.0-1](https://github.com/bangbang93/openbmclapi/compare/v1.11.0-0...v1.11.0-1)
+
+> 18 June 2024
+
+- fix: 应该在下载失败的时候就上报了，不应该等到重试失败 [`1f933f8`](https://github.com/bangbang93/openbmclapi/commit/1f933f89489aa808a7d02f983845c25389eb4de1)
+- Release 1.11.0-1 [`ca5fa71`](https://github.com/bangbang93/openbmclapi/commit/ca5fa715391721f327079089480d38115fc62a97)
+
+#### [v1.11.0-0](https://github.com/bangbang93/openbmclapi/compare/v1.10.10...v1.11.0-0)
+
+> 18 June 2024
+
+- feat: 下载错误时上报主控 [`19149d5`](https://github.com/bangbang93/openbmclapi/commit/19149d50d8a8cc90b72a8989d17dd95eeb9b3289)
+- Release 1.11.0-0 [`ff77a85`](https://github.com/bangbang93/openbmclapi/commit/ff77a85530b643a72e1cbc5d60f8aaabc451121f)
+
 #### [v1.10.10](https://github.com/bangbang93/openbmclapi/compare/v1.10.9...v1.10.10)
 
+> 13 June 2024
+
 - feat: 同时执行gc与启用 [`#77`](https://github.com/bangbang93/openbmclapi/pull/77)
 - feat: clean outdated files after enabled [`#75`](https://github.com/bangbang93/openbmclapi/pull/75)
 - feat: 简化emit异步写法 [`1a6c0bb`](https://github.com/bangbang93/openbmclapi/commit/1a6c0bb0cc436a9070d44719ada550708bad2237)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "openbmclapi",
-  "version": "1.10.10",
+  "version": "1.11.0-4",
   "description": "bmclapi@home",
   "bin": "dist/openbmclapi.js",
   "private": true,
@@ -39,6 +39,7 @@
     "fs-extra": "^8.1.0",
     "got": "^14.2.0",
     "http2-express-bridge": "^1.0.7",
+    "json-stringify-safe": "^5.0.1",
     "keyv": "^4.5.4",
     "keyv-file": "^0.3.0",
     "lodash-es": "^4.17.21",
@@ -51,6 +52,7 @@
     "pino-pretty": "^10.3.1",
     "pretty-bytes": "^6.1.1",
     "range-parser": "^1.2.1",
+    "serialize-error": "^11.0.3",
     "socket.io-client": "^4.7.4",
     "tail": "^2.2.6",
     "webdav": "^5.3.1",
@@ -65,6 +67,7 @@
     "@types/dotenv": "^6.1.1",
     "@types/express": "^4.17.13",
     "@types/fs-extra": "^8.0.0",
+    "@types/json-stringify-safe": "^5.0.3",
     "@types/lodash-es": "^4.17.7",
     "@types/morgan": "^1.7.36",
     "@types/ms": "^0.7.30",

src/cluster.ts

@@ -7,11 +7,12 @@ import express, {type NextFunction, type Request, type Response} from 'express'
 import {readFileSync} from 'fs'
 import fse from 'fs-extra'
 import {mkdtemp, open, readFile, rm} from 'fs/promises'
-import got, {type Got, HTTPError} from 'got'
+import got, {type Got, HTTPError, RequestError} from 'got'
 import {createServer, Server} from 'http'
 import {createSecureServer} from 'http2'
 import http2Express from 'http2-express-bridge'
 import {Agent as HttpsAgent} from 'https'
+import stringifySafe from 'json-stringify-safe'
 import {template, toString} from 'lodash-es'
 import morgan from 'morgan'
 import ms from 'ms'
@@ -169,65 +170,74 @@ export class Cluster {
       return
     }
     logger.info(`mismatch ${missingFiles.length} files, start syncing`)
-    if (process.env.FORCE_NOOPEN) {
-      syncConfig = {
-        concurrency: 1,
-        source: 'center',
-      }
-    }
     logger.info(syncConfig, '同步策略')
     const multibar = new MultiBar({
       format: ' {bar} | {filename} | {value}/{total}',
-      noTTYOutput: !process.stdout.isTTY,
+      noTTYOutput: true,
       notTTYSchedule: ms('10s'),
     })
     const totalBar = multibar.create(missingFiles.length, 0, {filename: '总文件数'})
     const parallel = syncConfig.concurrency
-    const noopen = syncConfig.source === 'center' ? '1' : ''
     let hasError = false
     await pMap(
       missingFiles,
       async (file) => {
         const bar = multibar.create(file.size, 0, {filename: file.path})
         try {
-          const res = await pRetry(
-            () => {
+          await pRetry(
+            async () => {
               bar.update(0)
-              return this.got
+              const res = await this.got
                 .get<Buffer>(file.path.substring(1), {
-                  searchParams: {
-                    noopen,
-                  },
                   retry: {
                     limit: 0,
                   },
                 })
                 .on('downloadProgress', (progress) => {
                   bar.update(progress.transferred)
                 })
+
+              const isFileCorrect = validateFile(res.body, file.hash)
+              if (!isFileCorrect) {
+                throw new RequestError(`文件${file.path}校验失败`, new Error(`文件${file.path}校验失败`), res.request)
+              }
+              await this.storage.writeFile(hashToFilename(file.hash), res.body, file)
             },
             {
               retries: 10,
-              onFailedAttempt: (e) => {
-                if (e.cause instanceof HTTPError) {
+              onFailedAttempt: async (e) => {
+                if (e instanceof HTTPError) {
                   logger.debug(
-                    {redirectUrls: e.cause.response.redirectUrls},
-                    `下载文件${file.path}失败: ${e.cause.response.statusCode}`,
+                    {redirectUrls: e.response.redirectUrls},
+                    `下载文件${file.path}失败: ${e.response.statusCode}`,
                   )
-                  logger.trace({err: e}, toString(e.cause.response.body))
+                  logger.trace({err: e}, toString(e.response.body))
                 } else {
                   logger.debug({err: e}, `下载文件${file.path}失败，正在重试`)
                 }
+
+                if (e instanceof RequestError) {
+                  const redirectUrls = e.response?.redirectUrls
+                  if (redirectUrls?.length) {
+                    const urls = [
+                      new URL(file.path, this.prefixUrl).toString(),
+                      ...redirectUrls.map((e) => e.toString()),
+                    ]
+                    await this.got
+                      .post('openbmclapi/report', {
+                        json: {
+                          urls,
+                          error: stringifySafe({message: e.message}),
+                        },
+                      })
+                      .catch((e) => {
+                        logger.error(e, '上报重定向失败')
+                      })
+                  }
+                }
               },
             },
           )
-          const isFileCorrect = validateFile(res.body, file.hash)
-          if (!isFileCorrect) {
-            hasError = true
-            logger.error({redirectUrls: res.redirectUrls}, `文件${file.path}校验失败`)
-            return
-          }
-          await this.storage.writeFile(hashToFilename(file.hash), res.body, file)
         } catch (e) {
           hasError = true
           if (e instanceof HTTPError) {

src/storage/alist-webdav.storage.ts

@@ -4,17 +4,46 @@ import Keyv from 'keyv'
 import {KeyvFile} from 'keyv-file'
 import ms from 'ms'
 import {join} from 'path'
+import {z} from 'zod'
+import {fromZodError} from 'zod-validation-error'
 import {WebdavStorage} from './webdav.storage.js'
 
+const storageConfigSchema = WebdavStorage.configSchema.extend({
+  cacheTtl: z.union([z.string().optional(), z.number().int()]).default('1h'),
+})
+
 export class AlistWebdavStorage extends WebdavStorage {
-  protected readonly redirectUrlCache = new Keyv<string>({
-    namespace: 'redirectUrl',
-    ttl: ms('1h'),
-    store: new KeyvFile({
-      filename: join(process.cwd(), 'cache', 'redirectUrl.json'),
-      writeDelay: ms('1m'),
-    }),
-  })
+  public readonly configSchema = storageConfigSchema
+
+  protected readonly redirectUrlCache: Keyv<string>
+  protected readonly storageConfig: z.infer<typeof storageConfigSchema>
+
+  constructor(storageConfig: unknown) {
+    super(storageConfig)
+    try {
+      this.storageConfig = this.configSchema.parse(storageConfig)
+    } catch (e) {
+      if (e instanceof z.ZodError) {
+        throw new Error('alist存储选项无效', {cause: fromZodError(e)})
+      } else {
+        throw new Error('alist存储选项无效', {cause: e})
+      }
+    }
+    let ttl: number
+    if (typeof this.storageConfig.cacheTtl === 'string') {
+      ttl = ms(this.storageConfig.cacheTtl)
+    } else {
+      ttl = this.storageConfig.cacheTtl
+    }
+    this.redirectUrlCache = new Keyv<string>({
+      namespace: 'redirectUrl',
+      ttl,
+      store: new KeyvFile({
+        filename: join(process.cwd(), 'cache', 'redirectUrl.json'),
+        writeDelay: ms('1m'),
+      }),
+    })
+  }
 
   public async express(hashPath: string, req: Request, res: Response): Promise<{bytes: number; hits: number}> {
     if (this.emptyFiles.has(hashPath)) {

src/storage/webdav.storage.ts

@@ -19,6 +19,7 @@ const storageConfigSchema = z.object({
 })
 
 export class WebdavStorage implements IStorage {
+  public static readonly configSchema = storageConfigSchema
   protected readonly client: WebDAVClient
   protected readonly storageConfig: z.infer<typeof storageConfigSchema>
   protected readonly basePath: string