From b992cd1524d02416cb0946c8c6514dade0485f40 Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Tue, 9 Jul 2024 00:24:29 +0700 Subject: [PATCH 1/4] feat: support pull model by specific fileName --- .../infrastructure/controllers/models.controller.ts | 9 +++++---- cortex-js/src/usecases/models/models.usecases.ts | 10 ++++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/cortex-js/src/infrastructure/controllers/models.controller.ts b/cortex-js/src/infrastructure/controllers/models.controller.ts index 11fcd4f7d..14162ef1e 100644 --- a/cortex-js/src/infrastructure/controllers/models.controller.ts +++ b/cortex-js/src/infrastructure/controllers/models.controller.ts @@ -8,6 +8,7 @@ import { Delete, HttpCode, UseInterceptors, + Query, } from '@nestjs/common'; import { ModelsUsecases } from '@/usecases/models/models.usecases'; import { CreateModelDto } from '@/infrastructure/dtos/models/create-model.dto'; @@ -117,8 +118,8 @@ export class ModelsController { }) @Get('download/:modelId(*)') - downloadModel(@Param('modelId') modelId: string) { - this.modelsUsecases.pullModel(modelId, false).then(() => this.telemetryUsecases.addEventToQueue({ + downloadModel(@Param('modelId') modelId: string, @Query('fileName') fileName: string) { + this.modelsUsecases.pullModel(modelId, false, undefined, fileName).then(() => this.telemetryUsecases.addEventToQueue({ name: EventName.DOWNLOAD_MODEL, modelId, }) @@ -162,8 +163,8 @@ export class ModelsController { description: 'The unique identifier of the model.', }) @Get('pull/:modelId(*)') - pullModel(@Param('modelId') modelId: string) { - this.modelsUsecases.pullModel(modelId).then(() => this.telemetryUsecases.addEventToQueue({ + pullModel(@Param('modelId') modelId: string, @Query('fileName') fileName: string) { + this.modelsUsecases.pullModel(modelId, false, undefined, fileName).then(() => this.telemetryUsecases.addEventToQueue({ name: EventName.DOWNLOAD_MODEL, modelId, }) diff --git a/cortex-js/src/usecases/models/models.usecases.ts b/cortex-js/src/usecases/models/models.usecases.ts index 471ad47b8..13b5fc8e6 100644 --- a/cortex-js/src/usecases/models/models.usecases.ts +++ b/cortex-js/src/usecases/models/models.usecases.ts @@ -326,6 +326,7 @@ export class ModelsUsecases { selection?: ( siblings: HuggingFaceRepoSibling[], ) => Promise, + fileName?: string, ) { const existingModel = await this.findOne(modelId); if (isLocalModel(existingModel?.files)) { @@ -342,11 +343,14 @@ export class ModelsUsecases { await promises.mkdir(modelFolder, { recursive: true }).catch(() => {}); let files = (await fetchJanRepoData(modelId)).siblings; - + console.log(files); // HuggingFace GGUF Repo - Only one file is downloaded if (modelId.includes('/') && selection && files.length) { files = [await selection(files)]; } + if(modelId.includes('/') && fileName && files.length) { + files = files.filter((e) => e.rfilename === fileName); + } // Start downloading the model const toDownloads: Record = files @@ -420,7 +424,9 @@ export class ModelsUsecases { const modelEvent: ModelEvent = { model: modelId, event: 'model-downloaded', - metadata: {}, + metadata: { + ...(fileName || selection ? { file: [files] } : {}), + }, }; this.eventEmitter.emit('model.event', modelEvent); }, From 45857af907b4cd2efdfc7fb15c064bf63f8e8c12 Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Tue, 9 Jul 2024 00:25:10 +0700 Subject: [PATCH 2/4] remove log --- cortex-js/src/usecases/models/models.usecases.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cortex-js/src/usecases/models/models.usecases.ts b/cortex-js/src/usecases/models/models.usecases.ts index 13b5fc8e6..ac865724f 100644 --- a/cortex-js/src/usecases/models/models.usecases.ts +++ b/cortex-js/src/usecases/models/models.usecases.ts @@ -343,7 +343,7 @@ export class ModelsUsecases { await promises.mkdir(modelFolder, { recursive: true }).catch(() => {}); let files = (await fetchJanRepoData(modelId)).siblings; - console.log(files); + // HuggingFace GGUF Repo - Only one file is downloaded if (modelId.includes('/') && selection && files.length) { files = [await selection(files)]; From 42671d7c36083ccc328c3cacf02258cc2daed011 Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Tue, 9 Jul 2024 00:39:41 +0700 Subject: [PATCH 3/4] refactor, add failed event --- cortex-js/src/domain/models/model.event.ts | 1 + .../controllers/models.controller.ts | 24 +++++++++++++++++-- .../src/usecases/models/models.usecases.ts | 17 +++++++++---- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/cortex-js/src/domain/models/model.event.ts b/cortex-js/src/domain/models/model.event.ts index 7d2177911..a28a9afa4 100644 --- a/cortex-js/src/domain/models/model.event.ts +++ b/cortex-js/src/domain/models/model.event.ts @@ -8,6 +8,7 @@ const ModelLoadingEvents = [ 'starting-failed', 'stopping-failed', 'model-downloaded', + 'model-downloaded-failed', 'model-deleted', ] as const; export type ModelLoadingEvent = (typeof ModelLoadingEvents)[number]; diff --git a/cortex-js/src/infrastructure/controllers/models.controller.ts b/cortex-js/src/infrastructure/controllers/models.controller.ts index 14162ef1e..8c0329562 100644 --- a/cortex-js/src/infrastructure/controllers/models.controller.ts +++ b/cortex-js/src/infrastructure/controllers/models.controller.ts @@ -9,6 +9,7 @@ import { HttpCode, UseInterceptors, Query, + BadRequestException, } from '@nestjs/common'; import { ModelsUsecases } from '@/usecases/models/models.usecases'; import { CreateModelDto } from '@/infrastructure/dtos/models/create-model.dto'; @@ -27,6 +28,7 @@ import { } from '@/domain/telemetry/telemetry.interface'; import { TelemetryUsecases } from '@/usecases/telemetry/telemetry.usecases'; import { CommonResponseDto } from '../dtos/common/common-response.dto'; +import { HuggingFaceRepoSibling } from '@/domain/models/huggingface.interface'; @ApiTags('Models') @Controller('models') @@ -119,7 +121,16 @@ export class ModelsController { @Get('download/:modelId(*)') downloadModel(@Param('modelId') modelId: string, @Query('fileName') fileName: string) { - this.modelsUsecases.pullModel(modelId, false, undefined, fileName).then(() => this.telemetryUsecases.addEventToQueue({ + this.modelsUsecases.pullModel(modelId, false, (files) => { + return new Promise(async (resolve, reject) => { + const file = files + .find((e) => e.quantization != null && e.rfilename === fileName) + if(!file) { + return reject(new BadRequestException('File not found')); + } + return resolve(file); + }); + }).then(() => this.telemetryUsecases.addEventToQueue({ name: EventName.DOWNLOAD_MODEL, modelId, }) @@ -164,7 +175,16 @@ export class ModelsController { }) @Get('pull/:modelId(*)') pullModel(@Param('modelId') modelId: string, @Query('fileName') fileName: string) { - this.modelsUsecases.pullModel(modelId, false, undefined, fileName).then(() => this.telemetryUsecases.addEventToQueue({ + this.modelsUsecases.pullModel(modelId, false, (files) => { + return new Promise(async (resolve, reject) => { + const file = files + .find((e) => e.quantization != null && e.rfilename === fileName) + if(!file) { + return reject(new BadRequestException('File not found')); + } + return resolve(file); + }); + }).then(() => this.telemetryUsecases.addEventToQueue({ name: EventName.DOWNLOAD_MODEL, modelId, }) diff --git a/cortex-js/src/usecases/models/models.usecases.ts b/cortex-js/src/usecases/models/models.usecases.ts index ac865724f..bf1f32dfd 100644 --- a/cortex-js/src/usecases/models/models.usecases.ts +++ b/cortex-js/src/usecases/models/models.usecases.ts @@ -326,7 +326,6 @@ export class ModelsUsecases { selection?: ( siblings: HuggingFaceRepoSibling[], ) => Promise, - fileName?: string, ) { const existingModel = await this.findOne(modelId); if (isLocalModel(existingModel?.files)) { @@ -346,10 +345,18 @@ export class ModelsUsecases { // HuggingFace GGUF Repo - Only one file is downloaded if (modelId.includes('/') && selection && files.length) { + try{ files = [await selection(files)]; - } - if(modelId.includes('/') && fileName && files.length) { - files = files.filter((e) => e.rfilename === fileName); + } catch (e) { + const modelEvent: ModelEvent = { + model: modelId, + event: 'model-downloaded-failed', + metadata: { + error: e.message || e, + }, + }; + this.eventEmitter.emit('model.event', modelEvent); + } } // Start downloading the model @@ -425,7 +432,7 @@ export class ModelsUsecases { model: modelId, event: 'model-downloaded', metadata: { - ...(fileName || selection ? { file: [files] } : {}), + ...(selection ? { file: [files] } : {}), }, }; this.eventEmitter.emit('model.event', modelEvent); From 3c56187639618a7fa8a5809ac58a306fdb896d53 Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Tue, 9 Jul 2024 01:14:06 +0700 Subject: [PATCH 4/4] refactor --- cortex-js/src/infrastructure/controllers/models.controller.ts | 4 ++-- cortex-js/src/usecases/models/models.usecases.ts | 2 +- cortex-js/src/utils/model-check.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cortex-js/src/infrastructure/controllers/models.controller.ts b/cortex-js/src/infrastructure/controllers/models.controller.ts index 8c0329562..c9b1e4ca5 100644 --- a/cortex-js/src/infrastructure/controllers/models.controller.ts +++ b/cortex-js/src/infrastructure/controllers/models.controller.ts @@ -124,7 +124,7 @@ export class ModelsController { this.modelsUsecases.pullModel(modelId, false, (files) => { return new Promise(async (resolve, reject) => { const file = files - .find((e) => e.quantization != null && e.rfilename === fileName) + .find((e) => e.quantization && e.rfilename === fileName) if(!file) { return reject(new BadRequestException('File not found')); } @@ -178,7 +178,7 @@ export class ModelsController { this.modelsUsecases.pullModel(modelId, false, (files) => { return new Promise(async (resolve, reject) => { const file = files - .find((e) => e.quantization != null && e.rfilename === fileName) + .find((e) => e.quantization && e.rfilename === fileName) if(!file) { return reject(new BadRequestException('File not found')); } diff --git a/cortex-js/src/usecases/models/models.usecases.ts b/cortex-js/src/usecases/models/models.usecases.ts index bf1f32dfd..52375b7f9 100644 --- a/cortex-js/src/usecases/models/models.usecases.ts +++ b/cortex-js/src/usecases/models/models.usecases.ts @@ -345,7 +345,7 @@ export class ModelsUsecases { // HuggingFace GGUF Repo - Only one file is downloaded if (modelId.includes('/') && selection && files.length) { - try{ + try { files = [await selection(files)]; } catch (e) { const modelEvent: ModelEvent = { diff --git a/cortex-js/src/utils/model-check.ts b/cortex-js/src/utils/model-check.ts index 59b0e2842..53ef84d47 100644 --- a/cortex-js/src/utils/model-check.ts +++ b/cortex-js/src/utils/model-check.ts @@ -21,7 +21,7 @@ export const checkModelCompatibility = async (modelId: string, spinner?: ora.Ora process.exit(1); } - try{ + try { const version = await getCudaVersion(); const [currentMajor, currentMinor] = version.split('.').map(Number); const [requiredMajor, requiredMinor] = MIN_CUDA_VERSION.split('.').map(Number);