From 7c0150f788cd67b9e82fd7d8b1c9b0f527489706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=83=AB=E5=AE=9D?= Date: Wed, 13 Mar 2024 14:41:52 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=20full=20diff=20?= =?UTF-8?q?=E5=85=A8=E9=87=8F=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/core/entity/Task.ts | 4 + app/core/service/BinarySyncerService.ts | 21 +++-- app/port/schedule/CreateSyncBinaryTask.ts | 12 ++- .../BinarySyncerService/executeTask.test.ts | 84 +++++++++++++++++++ 4 files changed, 115 insertions(+), 6 deletions(-) diff --git a/app/core/entity/Task.ts b/app/core/entity/Task.ts index 43280513..bbb434d8 100644 --- a/app/core/entity/Task.ts +++ b/app/core/entity/Task.ts @@ -50,6 +50,9 @@ export interface TriggerHookTaskData extends TaskBaseData { responseStatus?: number; } +export interface CreateSyncBinaryTaskData extends TaskBaseData { + fullDiff?: boolean; +} export interface CreateSyncPackageTaskData extends TaskBaseData { tips?: string; skipDependencies?: boolean; @@ -74,6 +77,7 @@ export interface TaskUpdateCondition { export type CreateHookTask = Task; export type TriggerHookTask = Task; export type CreateSyncPackageTask = Task; +export type CreateSyncBinaryTask = Task; export type ChangesStreamTask = Task; export class Task extends Entity { diff --git a/app/core/service/BinarySyncerService.ts b/app/core/service/BinarySyncerService.ts index e7808d1d..91a8aed9 100644 --- a/app/core/service/BinarySyncerService.ts +++ b/app/core/service/BinarySyncerService.ts @@ -14,7 +14,7 @@ import { NFSAdapter } from '../../common/adapter/NFSAdapter'; import { TaskType, TaskState } from '../../common/enum/Task'; import { downloadToTempfile } from '../../common/FileUtil'; import { BinaryRepository } from '../../repository/BinaryRepository'; -import { Task } from '../entity/Task'; +import { CreateSyncBinaryTask, Task } from '../entity/Task'; import { Binary } from '../entity/Binary'; import { TaskService } from './TaskService'; import { AbstractBinary, BinaryItem } from '../../common/adapter/binary/AbstractBinary'; @@ -106,12 +106,15 @@ export class BinarySyncerService extends AbstractService { return await this.taskService.findExecuteTask(TaskType.SyncBinary); } - public async executeTask(task: Task) { + public async executeTask(task: CreateSyncBinaryTask) { const binaryName = task.targetName as BinaryName; const binaryAdapter = await this.getBinaryAdapter(binaryName); const logUrl = `${this.config.cnpmcore.registry}/-/binary/${binaryName}/syncs/${task.taskId}/log`; let logs: string[] = []; logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start sync binary "${binaryName}" 🚧🚧🚧🚧🚧`); + if (task.data?.fullDiff) { + logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 full diff 🚧🚧🚧🚧🚧`); + } if (!binaryAdapter) { task.error = 'unknow binaryName'; logs.push(`[${isoNow()}] ❌ Synced "${binaryName}" fail, ${task.error}, log: ${logUrl}`); @@ -155,7 +158,7 @@ export class BinarySyncerService extends AbstractService { } } - private async syncDir(binaryAdapter: AbstractBinary, task: Task, dir: string, parentIndex = '', latestVersionParent = '/') { + private async syncDir(binaryAdapter: AbstractBinary, task: CreateSyncBinaryTask, dir: string, parentIndex = '', latestVersionParent = '/') { const binaryName = task.targetName as BinaryName; const result = await binaryAdapter.fetch(dir, binaryName); let hasDownloadError = false; @@ -163,7 +166,7 @@ export class BinarySyncerService extends AbstractService { if (result && result.items.length > 0) { hasItems = true; let logs: string[] = []; - const { newItems, latestVersionDir } = await this.diff(binaryName, dir, result.items, latestVersionParent); + const { newItems, latestVersionDir } = await this.diff(binaryName, dir, result.items, latestVersionParent, task.data?.fullDiff); logs.push(`[${isoNow()}][${dir}] 🚧 Syncing diff: ${result.items.length} => ${newItems.length}, Binary class: ${binaryAdapter.constructor.name}`); // re-check latest version for (const [ index, { item, reason }] of newItems.entries()) { @@ -244,7 +247,7 @@ export class BinarySyncerService extends AbstractService { // 上游可能正在发布新版本、同步流程中断，导致同步的时候，文件列表不一致 // 如果的当前目录命中 latestVersionParent 父目录，那么就再校验一下当前目录 // 如果 existsItems 为空或者经过修改，那么就不需要 revalidate 了 - private async diff(binaryName: BinaryName, dir: string, fetchItems: BinaryItem[], latestVersionParent = '/') { + private async diff(binaryName: BinaryName, dir: string, fetchItems: BinaryItem[], latestVersionParent = '/', fullDiff: boolean | undefined) { const existsItems = await this.binaryRepository.listBinaries(binaryName, dir); const existsMap = new Map(); for (const item of existsItems) { @@ -275,6 +278,14 @@ export class BinarySyncerService extends AbstractService { existsItem.sourceUrl = item.url; existsItem.ignoreDownloadStatuses = item.ignoreDownloadStatuses; existsItem.date = item.date; + } else if (fullDiff && item.isDir) { + diffItems.push({ + item: existsItem, + reason: `full diff, local: ${JSON.stringify(existsItem.date)}, remote: ${JSON.stringify(item.date)}`, + }); + existsItem.sourceUrl = item.url; + existsItem.ignoreDownloadStatuses = item.ignoreDownloadStatuses; + existsItem.date = item.date; } else if (dir.endsWith(latestVersionParent)) { const isLatestItem = sortBy(fetchItems, [ 'date' ]).pop()?.name === item.name; if (isLatestItem && existsItem.isDir) { diff --git a/app/port/schedule/CreateSyncBinaryTask.ts b/app/port/schedule/CreateSyncBinaryTask.ts index b87c7286..1a25877d 100644 --- a/app/port/schedule/CreateSyncBinaryTask.ts +++ b/app/port/schedule/CreateSyncBinaryTask.ts @@ -3,6 +3,7 @@ import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule'; import { Inject } from '@eggjs/tegg'; import { BinarySyncerService } from '../../core/service/BinarySyncerService'; import binaries, { BinaryName } from '../../../config/binaries'; +import dayjs from 'dayjs'; @Schedule({ type: ScheduleType.WORKER, @@ -28,7 +29,16 @@ export class CreateSyncBinaryTask { // 默认只同步 binaryName 的二进制，即使有不一致的 category，会在同名的 binaryName 任务中同步 // 例如 canvas 只同步 binaryName 为 canvas 的二进制，不同步 category 为 node-canvas-prebuilt 的二进制 // node-canvas-prebuilt 的二进制会在 node-canvas-prebuilt 的任务中同步 - await this.binarySyncerService.createTask(binaryName as BinaryName); + await this.binarySyncerService.createTask(binaryName as BinaryName, { + fullDiff: isBetween2and205AM(), + }); } } } +function isBetween2and205AM() { + const now = dayjs(); + const twoAM = now.startOf('day').add(2, 'hour'); + const twoAMFive = twoAM.add(5, 'minute'); + + return now.isAfter(twoAM) && now.isBefore(twoAMFive); +} diff --git a/test/core/service/BinarySyncerService/executeTask.test.ts b/test/core/service/BinarySyncerService/executeTask.test.ts index 8896a06a..cade4357 100644 --- a/test/core/service/BinarySyncerService/executeTask.test.ts +++ b/test/core/service/BinarySyncerService/executeTask.test.ts @@ -389,5 +389,89 @@ describe('test/core/service/BinarySyncerService/executeTask.test.ts', () => { assert(BinaryItems.length === 2); }); + it('should full diff all dir', async () => { + app.mockHttpclient('https://nodejs.org/dist/index.json', 'GET', { + data: await TestUtil.readFixturesFile('nodejs.org/site/index.json'), + persist: false, + }); + app.mockHttpclient('https://nodejs.org/dist/latest/docs/apilinks.json', 'GET', { + data: await TestUtil.readFixturesFile('nodejs.org/site/latest/docs/apilinks.json'), + persist: false, + }); + app.mockHttpclient('https://nodejs.org/dist/latest/docs/apilinks_old.json', 'GET', { + data: await TestUtil.readFixturesFile('nodejs.org/site/latest/docs/apilinks.json'), + persist: false, + }); + await binarySyncerService.createTask('node', {}); + let task = await binarySyncerService.findExecuteTask(); + assert(task); + mock(NodeBinary.prototype, 'fetch', async (dir: string) => { + if (dir === '/') { + return { + items: [ + { name: 'latest/', isDir: true, url: '', size: '-', date: '17-Dec-2021 23:17' }, + { name: 'old/', isDir: true, url: '', size: '-', date: '15-Dec-2021 23:17' }, + { name: 'index.json', isDir: false, url: 'https://nodejs.org/dist/index.json', size: '219862', date: '17-Dec-2021 23:16' }, + ], + }; + } + if (dir === '/latest/') { + return { + items: [ + { name: 'docs/', isDir: true, url: '', size: '-', date: '17-Dec-2021 21:31' }, + ], + }; + } + if (dir === '/old/') { + return { + items: [ + { + name: 'apilinks_old.json', + isDir: false, + url: 'https://nodejs.org/dist/latest/docs/apilinks_old.json', + size: '61606', + date: '17-Dec-2021 21:29', + }, + ], + }; + } + if (dir === '/latest/docs/') { + return { + items: [ + { name: 'apilinks.json', isDir: false, url: 'https://nodejs.org/dist/latest/docs/apilinks.json', size: '61606', date: '17-Dec-2021 21:29' }, + ], + }; + } + return { items: [] }; + }); + await binarySyncerService.executeTask(task); + app.mockAgent().assertNoPendingInterceptors(); + assert(!await TaskModel.findOne({ taskId: task.taskId })); + assert(await HistoryTaskModel.findOne({ taskId: task.taskId })); + let stream = await binarySyncerService.findTaskLog(task); + assert(stream); + let log = await TestUtil.readStreamToLog(stream); + // console.log(log); + assert(log.includes('Syncing diff: 3 => 3')); + assert(log.includes('[/] 🟢 Synced dir success')); + assert(log.includes('[/latest/] 🟢 Synced dir success')); + assert(log.includes('[/latest/docs/] 🟢 Synced dir success')); + assert(log.includes('[/old/] 🟢 Synced dir success')); + // sync again + await binarySyncerService.createTask('node', { fullDiff: true }); + task = await binarySyncerService.findExecuteTask(); + assert(task); + await binarySyncerService.executeTask(task); + stream = await binarySyncerService.findTaskLog(task); + assert(stream); + log = await TestUtil.readStreamToLog(stream); + // console.log(log); + assert(log.includes('reason: full diff')); + assert([ ...log.matchAll(/reason: full diff/g) ].length === 3); + assert(log.includes('Syncing diff: 3 => 2')); + assert(log.includes('[/] 🟢 Synced dir success')); + assert(log.includes('[/latest/] 🟢 Synced dir success')); + assert(log.includes('[/old/] 🟢 Synced dir success')); + }); }); }); From 51ad513a7aed657dd7c299c61fa1595adce0cba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=83=AB=E5=AE=9D?= Date: Sun, 7 Apr 2024 14:27:32 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20=E6=8C=87=E5=AE=9A=E5=8C=85=E8=83=BD?= =?UTF-8?q?=E5=85=A8=E9=87=8F=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/port/schedule/CreateSyncBinaryTask.ts | 2 +- config/binaries.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app/port/schedule/CreateSyncBinaryTask.ts b/app/port/schedule/CreateSyncBinaryTask.ts index 1a25877d..77b3fbdc 100644 --- a/app/port/schedule/CreateSyncBinaryTask.ts +++ b/app/port/schedule/CreateSyncBinaryTask.ts @@ -30,7 +30,7 @@ export class CreateSyncBinaryTask { // 例如 canvas 只同步 binaryName 为 canvas 的二进制，不同步 category 为 node-canvas-prebuilt 的二进制 // node-canvas-prebuilt 的二进制会在 node-canvas-prebuilt 的任务中同步 await this.binarySyncerService.createTask(binaryName as BinaryName, { - fullDiff: isBetween2and205AM(), + fullDiff: isBetween2and205AM() && binary.enableFullDiff, }); } } diff --git a/config/binaries.ts b/config/binaries.ts index f51834f0..b27b78ac 100644 --- a/config/binaries.ts +++ b/config/binaries.ts @@ -23,6 +23,7 @@ export type BinaryTaskConfig = { ignoreDownloadStatuses?: number[], }, disable?: boolean; + enableFullDiff?: boolean; }; const binaries = { @@ -41,6 +42,7 @@ const binaries = { type: BinaryType.Node, repo: 'nodejs/node', distUrl: 'https://nodejs.org/dist', + enableFullDiff: true, }, 'node-rc': { category: 'node-rc', @@ -107,6 +109,7 @@ const binaries = { type: BinaryType.Puppeteer, repo: 'puppeteer/puppeteer', distUrl: 'https://chromium-browser-snapshots.storage.googleapis.com/?delimiter=/&prefix=', + enableFullDiff: true, }, // ChromeBinary 'chrome-for-testing': { @@ -910,6 +913,7 @@ const binaries = { type: BinaryType.GitHub, repo: 'oven-sh/bun', distUrl: 'https://github.com/oven-sh/bun/releases', + enableFullDiff: true, }, } as const;