diff --git a/cortex-js/src/app.module.ts b/cortex-js/src/app.module.ts index 748ca9bb9..3120e90c1 100644 --- a/cortex-js/src/app.module.ts +++ b/cortex-js/src/app.module.ts @@ -6,20 +6,19 @@ import { DevtoolsModule } from '@nestjs/devtools-integration'; import { DatabaseModule } from './infrastructure/database/database.module'; import { ChatModule } from './usecases/chat/chat.module'; import { AssistantsModule } from './usecases/assistants/assistants.module'; -import { InferenceSettingsModule } from './usecases/inference-settings/inference-settings.module'; import { ExtensionModule } from './infrastructure/repositories/extensions/extension.module'; import { CortexModule } from './usecases/cortex/cortex.module'; import { ConfigModule } from '@nestjs/config'; +import { env } from 'node:process'; @Module({ imports: [ DevtoolsModule.register({ - http: process.env.NODE_ENV !== 'production', + http: env.NODE_ENV !== 'production', }), ConfigModule.forRoot({ isGlobal: true, - envFilePath: - process.env.NODE_ENV === 'production' ? '.env' : '.env.development', + envFilePath: env.NODE_ENV !== 'production' ? '.env.development' : '.env', }), DatabaseModule, MessagesModule, @@ -27,7 +26,6 @@ import { ConfigModule } from '@nestjs/config'; ModelsModule, ChatModule, AssistantsModule, - InferenceSettingsModule, CortexModule, ExtensionModule, ], diff --git a/cortex-js/src/infrastructure/commanders/inquirer/init.questions.ts b/cortex-js/src/infrastructure/commanders/inquirer/init.questions.ts index 2aadbe490..04df43158 100644 --- a/cortex-js/src/infrastructure/commanders/inquirer/init.questions.ts +++ b/cortex-js/src/infrastructure/commanders/inquirer/init.questions.ts @@ -1,4 +1,5 @@ import { Question, QuestionSet } from 'nest-commander'; +import { platform } from 'node:process'; @QuestionSet({ name: 'create-init-questions' }) export class CreateInitQuestions { @@ -8,7 +9,7 @@ export class CreateInitQuestions { name: 'runMode', default: 'CPU', choices: ['CPU', 'GPU'], - when: () => process.platform !== 'darwin', + when: () => platform !== 'darwin', }) parseRunMode(val: string) { return val; @@ -31,7 +32,7 @@ export class CreateInitQuestions { message: 'Select CPU instructions set', name: 'instructions', choices: ['AVX2', 'AVX', 'AVX-512'], - when: () => process.platform !== 'darwin', + when: () => platform !== 'darwin', }) parseContent(val: string) { return val; diff --git a/cortex-js/src/infrastructure/commanders/pull.command.ts b/cortex-js/src/infrastructure/commanders/pull.command.ts index 546458fc8..8770843cd 100644 --- a/cortex-js/src/infrastructure/commanders/pull.command.ts +++ b/cortex-js/src/infrastructure/commanders/pull.command.ts @@ -34,9 +34,10 @@ export class PullCommand extends CommandRunner { super(); } - async run(input: string[]): Promise { + async run(input: string[]) { if (input.length < 1) { - return Promise.reject('Model ID is required'); + console.error('Model ID is required'); + exit(1); } const modelId = input[0]; @@ -46,7 +47,7 @@ export class PullCommand extends CommandRunner { const bar = new SingleBar({}, Presets.shades_classic); bar.start(100, 0); - await this.modelsUsecases.downloadModel({ modelId }, (progress) => { + await this.modelsUsecases.downloadModel(modelId, (progress) => { bar.update(progress); }); console.log('\nDownload complete!'); diff --git a/cortex-js/src/infrastructure/commanders/start.command.ts b/cortex-js/src/infrastructure/commanders/start.command.ts index e47803f17..e4bcd484e 100644 --- a/cortex-js/src/infrastructure/commanders/start.command.ts +++ b/cortex-js/src/infrastructure/commanders/start.command.ts @@ -40,19 +40,7 @@ export class StartCommand extends CommandRunner { } private async startModel(modelId: string) { - // TODO: NamH remove these hardcoded value - const settings: ModelSettingParams = { - cpu_threads: 10, - ctx_len: 2048, - embedding: false, - prompt_template: - '{system_message}\n### Instruction: {prompt}\n### Response:', - system_prompt: '', - user_prompt: '\n### Instruction: ', - ai_prompt: '\n### Response:', - ngl: 100, - }; - return this.modelsUsecases.startModel(modelId, settings); + return this.modelsUsecases.startModel(modelId); } private async getModelOrStop(modelId: string): Promise { diff --git a/cortex-js/src/infrastructure/controllers/inference-settings.controller.spec.ts b/cortex-js/src/infrastructure/controllers/inference-settings.controller.spec.ts deleted file mode 100644 index 05097ddae..000000000 --- a/cortex-js/src/infrastructure/controllers/inference-settings.controller.spec.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { InferenceSettingsController } from './inference-settings.controller'; -import { InferenceSettingsUsecases } from '@/usecases/inference-settings/inference-settings.usecases'; - -describe('InferenceSettingsController', () => { - let controller: InferenceSettingsController; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - controllers: [InferenceSettingsController], - providers: [InferenceSettingsUsecases], - }).compile(); - - controller = module.get( - InferenceSettingsController, - ); - }); - - it('should be defined', () => { - expect(controller).toBeDefined(); - }); -}); diff --git a/cortex-js/src/infrastructure/controllers/inference-settings.controller.ts b/cortex-js/src/infrastructure/controllers/inference-settings.controller.ts deleted file mode 100644 index 805a4c1b3..000000000 --- a/cortex-js/src/infrastructure/controllers/inference-settings.controller.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { - Controller, - Get, - Post, - Body, - Patch, - Param, - Delete, -} from '@nestjs/common'; -import { InferenceSettingsUsecases } from '@/usecases/inference-settings/inference-settings.usecases'; -import { CreateInferenceSettingDto } from '@/infrastructure/dtos/inference-settings/create-inference-setting.dto'; -import { UpdateInferenceSettingDto } from '@/infrastructure/dtos/inference-settings/update-inference-setting.dto'; -import { ApiTags } from '@nestjs/swagger'; - -@ApiTags('Inference Settings') -@Controller('inference-settings') -export class InferenceSettingsController { - constructor( - private readonly inferenceSettingsService: InferenceSettingsUsecases, - ) {} - - @Post() - create(@Body() createInferenceSettingDto: CreateInferenceSettingDto) { - return this.inferenceSettingsService.create(createInferenceSettingDto); - } - - @Get() - findAll() { - return this.inferenceSettingsService.findAll(); - } - - @Get(':id') - findOne(@Param('id') id: string) { - return this.inferenceSettingsService.findOne(id); - } - - @Patch(':id') - update( - @Param('id') id: string, - @Body() updateInferenceSettingDto: UpdateInferenceSettingDto, - ) { - return this.inferenceSettingsService.update(id, updateInferenceSettingDto); - } - - @Delete(':id') - remove(@Param('id') id: string) { - return this.inferenceSettingsService.remove(id); - } -} diff --git a/cortex-js/src/infrastructure/controllers/models.controller.ts b/cortex-js/src/infrastructure/controllers/models.controller.ts index 36c6f1eb3..2983d1c31 100644 --- a/cortex-js/src/infrastructure/controllers/models.controller.ts +++ b/cortex-js/src/infrastructure/controllers/models.controller.ts @@ -13,23 +13,22 @@ import { CreateModelDto } from '@/infrastructure/dtos/models/create-model.dto'; import { UpdateModelDto } from '@/infrastructure/dtos/models/update-model.dto'; import { ApiResponse, ApiTags } from '@nestjs/swagger'; import { StartModelSuccessDto } from '@/infrastructure/dtos/models/start-model-success.dto'; -import { DownloadModelDto } from '@/infrastructure/dtos/models/download-model.dto'; import { ModelSettingParamsDto } from '../dtos/models/model-setting-params.dto'; @ApiTags('Models') @Controller('models') export class ModelsController { - constructor(private readonly modelsService: ModelsUsecases) {} + constructor(private readonly modelsUsecases: ModelsUsecases) {} @Post() create(@Body() createModelDto: CreateModelDto) { - return this.modelsService.create(createModelDto); + return this.modelsUsecases.create(createModelDto); } @HttpCode(200) @ApiResponse({ status: 200, - description: 'The model has been loaded successfully.', + description: 'The model has been started successfully.', type: StartModelSuccessDto, }) @Post(':modelId/start') @@ -37,31 +36,42 @@ export class ModelsController { @Param('modelId') modelId: string, @Body() settings: ModelSettingParamsDto, ) { - return this.modelsService.startModel(modelId, settings); + return this.modelsUsecases.startModel(modelId, settings); } - @Post('download') - downloadModel(@Body() downloadModelDto: DownloadModelDto) { - return this.modelsService.downloadModel(downloadModelDto); + @HttpCode(200) + @ApiResponse({ + status: 200, + description: 'The model has been stopped successfully.', + type: StartModelSuccessDto, + }) + @Post(':modelId/stop') + stopModel(@Param('modelId') modelId: string) { + return this.modelsUsecases.stopModel(modelId); + } + + @Get('download/:modelId') + downloadModel(@Param('modelId') modelId: string) { + return this.modelsUsecases.downloadModel(modelId); } @Get() findAll() { - return this.modelsService.findAll(); + return this.modelsUsecases.findAll(); } @Get(':id') findOne(@Param('id') id: string) { - return this.modelsService.findOne(id); + return this.modelsUsecases.findOne(id); } @Patch(':id') update(@Param('id') id: string, @Body() updateModelDto: UpdateModelDto) { - return this.modelsService.update(id, updateModelDto); + return this.modelsUsecases.update(id, updateModelDto); } @Delete(':id') remove(@Param('id') id: string) { - return this.modelsService.remove(id); + return this.modelsUsecases.remove(id); } } diff --git a/cortex-js/src/infrastructure/database/database.module.ts b/cortex-js/src/infrastructure/database/database.module.ts index ace7da1c9..016066219 100644 --- a/cortex-js/src/infrastructure/database/database.module.ts +++ b/cortex-js/src/infrastructure/database/database.module.ts @@ -4,7 +4,6 @@ import { sqliteDatabaseProviders } from './sqlite-database.providers'; import { modelProviders } from './providers/model.providers'; import { assistantProviders } from './providers/assistant.providers'; import { messageProviders } from './providers/message.providers'; -import { inferenceSettingProviders } from './providers/inference-setting.providers'; @Module({ providers: [ @@ -13,14 +12,12 @@ import { inferenceSettingProviders } from './providers/inference-setting.provide ...modelProviders, ...assistantProviders, ...messageProviders, - ...inferenceSettingProviders, ], exports: [ ...threadProviders, ...modelProviders, ...assistantProviders, ...messageProviders, - ...inferenceSettingProviders, ], }) export class DatabaseModule {} diff --git a/cortex-js/src/infrastructure/database/providers/inference-setting.providers.ts b/cortex-js/src/infrastructure/database/providers/inference-setting.providers.ts deleted file mode 100644 index 621d25fd8..000000000 --- a/cortex-js/src/infrastructure/database/providers/inference-setting.providers.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { InferenceSettingEntity } from '@/infrastructure/entities/inference-setting.entity'; -import { DataSource } from 'typeorm'; - -export const inferenceSettingProviders = [ - { - provide: 'INFERENCE_SETTING_REPOSITORY', - useFactory: (dataSource: DataSource) => - dataSource.getRepository(InferenceSettingEntity), - inject: ['DATA_SOURCE'], - }, -]; diff --git a/cortex-js/src/infrastructure/dtos/cortex/start-cortex.dto.ts b/cortex-js/src/infrastructure/dtos/cortex/start-cortex.dto.ts index 73d9ffb28..4745a3b3d 100644 --- a/cortex-js/src/infrastructure/dtos/cortex/start-cortex.dto.ts +++ b/cortex-js/src/infrastructure/dtos/cortex/start-cortex.dto.ts @@ -1,10 +1,22 @@ +import { ApiProperty } from '@nestjs/swagger'; import { IsIP, IsNumber, IsString, Max, Min } from 'class-validator'; +import { defaultCortexCppHost, defaultCortexCppPort } from 'constant'; export class StartCortexDto { + @ApiProperty({ + name: 'host', + description: 'Cortexcpp host', + default: defaultCortexCppHost, + }) @IsString() @IsIP() host: string; + @ApiProperty({ + name: 'port', + description: 'Cortexcpp port', + default: defaultCortexCppPort, + }) @IsNumber() @Min(0) @Max(65535) diff --git a/cortex-js/src/infrastructure/dtos/inference-settings/controller-props.dto.ts b/cortex-js/src/infrastructure/dtos/inference-settings/controller-props.dto.ts deleted file mode 100644 index 15c4601f3..000000000 --- a/cortex-js/src/infrastructure/dtos/inference-settings/controller-props.dto.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { IsOptional, IsString } from 'class-validator'; -import { ControllerProps } from '@/domain/models/inference-setting.interface'; - -export class ControllerPropsDto implements ControllerProps { - @IsString() - placeholder: string; - - @IsString() - value: string; - - @IsOptional() - @IsString() - type?: string; -} diff --git a/cortex-js/src/infrastructure/dtos/inference-settings/create-inference-setting.dto.ts b/cortex-js/src/infrastructure/dtos/inference-settings/create-inference-setting.dto.ts deleted file mode 100644 index ba34ec81c..000000000 --- a/cortex-js/src/infrastructure/dtos/inference-settings/create-inference-setting.dto.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { Type } from 'class-transformer'; -import { IsArray, IsString, ValidateNested } from 'class-validator'; -import { InferenceSetting } from '@/domain/models/inference-setting.interface'; -import { InferenceSettingDocumentDto } from './inference-setting-document.dto'; - -export class CreateInferenceSettingDto implements Partial { - @IsString() - inferenceId: string; - - @IsArray() - @ValidateNested({ each: true }) - @Type(() => InferenceSettingDocumentDto) - settings: InferenceSettingDocumentDto[]; -} diff --git a/cortex-js/src/infrastructure/dtos/inference-settings/inference-setting-document.dto.ts b/cortex-js/src/infrastructure/dtos/inference-settings/inference-setting-document.dto.ts deleted file mode 100644 index bca718c6a..000000000 --- a/cortex-js/src/infrastructure/dtos/inference-settings/inference-setting-document.dto.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { IsString, ValidateNested } from 'class-validator'; -import { InferenceSettingDocument } from '@/domain/models/inference-setting.interface'; -import { ControllerPropsDto } from './controller-props.dto'; - -export class InferenceSettingDocumentDto implements InferenceSettingDocument { - @IsString() - key: string; - - @IsString() - extensionName: string; - - @IsString() - title: string; - - @IsString() - description: string; - - @IsString() - controllerType: string; - - @ValidateNested() - controllerProps: ControllerPropsDto; -} diff --git a/cortex-js/src/infrastructure/dtos/inference-settings/update-inference-setting.dto.ts b/cortex-js/src/infrastructure/dtos/inference-settings/update-inference-setting.dto.ts deleted file mode 100644 index 026dffded..000000000 --- a/cortex-js/src/infrastructure/dtos/inference-settings/update-inference-setting.dto.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { PartialType } from '@nestjs/swagger'; -import { CreateInferenceSettingDto } from './create-inference-setting.dto'; - -export class UpdateInferenceSettingDto extends PartialType(CreateInferenceSettingDto) {} diff --git a/cortex-js/src/infrastructure/dtos/models/download-model.dto.ts b/cortex-js/src/infrastructure/dtos/models/download-model.dto.ts deleted file mode 100644 index 46ea3c12d..000000000 --- a/cortex-js/src/infrastructure/dtos/models/download-model.dto.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { IsString } from 'class-validator'; - -export class DownloadModelDto { - @IsString() - modelId: string; -} diff --git a/cortex-js/src/infrastructure/entities/inference-setting.entity.ts b/cortex-js/src/infrastructure/entities/inference-setting.entity.ts deleted file mode 100644 index 9b6d0cb41..000000000 --- a/cortex-js/src/infrastructure/entities/inference-setting.entity.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { - InferenceSetting, - InferenceSettingDocument, -} from '@/domain/models/inference-setting.interface'; -import { Column, Entity, PrimaryColumn } from 'typeorm'; - -@Entity('inference_setting') -export class InferenceSettingEntity implements InferenceSetting { - @PrimaryColumn() - inferenceId: string; - - @Column({ type: 'simple-json' }) - settings: InferenceSettingDocument[]; -} diff --git a/cortex-js/src/infrastructure/providers/cortex/cortex.provider.ts b/cortex-js/src/infrastructure/providers/cortex/cortex.provider.ts index 27a645c36..73405218c 100644 --- a/cortex-js/src/infrastructure/providers/cortex/cortex.provider.ts +++ b/cortex-js/src/infrastructure/providers/cortex/cortex.provider.ts @@ -6,6 +6,7 @@ import { Model, ModelSettingParams } from '@/domain/models/model.interface'; import { ConfigService } from '@nestjs/config'; import { HttpService } from '@nestjs/axios'; import { defaultCortexCppHost, defaultCortexCppPort } from 'constant'; +import { readdirSync } from 'node:fs'; /** * A class that implements the InferenceExtension interface from the @janhq/core package. @@ -36,13 +37,15 @@ export default class CortexProvider extends OAIEngineExtension { resolve('./models'); const modelFolderFullPath = join(modelsContainerDir, model.id); - //TODO: recheck this - const modelBinaryLocalPath = join( - modelFolderFullPath, - basename(model.sources[0].url), - ); + const ggufFiles = readdirSync(modelFolderFullPath).filter((file) => { + return file.endsWith('.gguf'); + }); - // TODO: NamH check if the binary is there + if (ggufFiles.length === 0) { + throw new Error('Model binary not found'); + } + + const modelBinaryLocalPath = join(modelFolderFullPath, ggufFiles[0]); const cpuThreadCount = 1; // TODO: NamH Math.max(1, nitroResourceProbe.numCpuPhysicalCore); const modelSettings = { diff --git a/cortex-js/src/usecases/inference-settings/inference-settings.module.ts b/cortex-js/src/usecases/inference-settings/inference-settings.module.ts deleted file mode 100644 index d7ca6d05e..000000000 --- a/cortex-js/src/usecases/inference-settings/inference-settings.module.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { Module } from '@nestjs/common'; -import { InferenceSettingsUsecases } from './inference-settings.usecases'; -import { InferenceSettingsController } from '@/infrastructure/controllers/inference-settings.controller'; -import { DatabaseModule } from '@/infrastructure/database/database.module'; - -@Module({ - imports: [DatabaseModule], - controllers: [InferenceSettingsController], - providers: [InferenceSettingsUsecases], - exports: [InferenceSettingsUsecases], -}) -export class InferenceSettingsModule {} diff --git a/cortex-js/src/usecases/inference-settings/inference-settings.usecases.spec.ts b/cortex-js/src/usecases/inference-settings/inference-settings.usecases.spec.ts deleted file mode 100644 index a47dd23b2..000000000 --- a/cortex-js/src/usecases/inference-settings/inference-settings.usecases.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { InferenceSettingsUsecases } from './inference-settings.usecases'; - -describe('InferenceSettingsService', () => { - let service: InferenceSettingsUsecases; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [InferenceSettingsUsecases], - }).compile(); - - service = module.get(InferenceSettingsUsecases); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); diff --git a/cortex-js/src/usecases/inference-settings/inference-settings.usecases.ts b/cortex-js/src/usecases/inference-settings/inference-settings.usecases.ts deleted file mode 100644 index 8c512aa72..000000000 --- a/cortex-js/src/usecases/inference-settings/inference-settings.usecases.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { Inject, Injectable } from '@nestjs/common'; -import { CreateInferenceSettingDto } from '@/infrastructure/dtos/inference-settings/create-inference-setting.dto'; -import { UpdateInferenceSettingDto } from '@/infrastructure/dtos/inference-settings/update-inference-setting.dto'; -import { Repository } from 'typeorm'; -import { InferenceSettingEntity } from '@/infrastructure/entities/inference-setting.entity'; - -@Injectable() -export class InferenceSettingsUsecases { - constructor( - @Inject('INFERENCE_SETTING_REPOSITORY') - private inferenceSettingRepository: Repository, - ) {} - - create(createInferenceSettingDto: CreateInferenceSettingDto) { - return this.inferenceSettingRepository.insert(createInferenceSettingDto); - } - - findAll() { - return this.inferenceSettingRepository.find(); - } - - findOne(id: string) { - return this.inferenceSettingRepository.findOne({ - where: { inferenceId: id }, - }); - } - - update(id: string, updateInferenceSettingDto: UpdateInferenceSettingDto) { - return this.inferenceSettingRepository.update( - id, - updateInferenceSettingDto, - ); - } - - remove(id: string) { - return this.inferenceSettingRepository.delete(id); - } -} diff --git a/cortex-js/src/usecases/models/models.usecases.ts b/cortex-js/src/usecases/models/models.usecases.ts index 9faa5d5d8..0a468ebb6 100644 --- a/cortex-js/src/usecases/models/models.usecases.ts +++ b/cortex-js/src/usecases/models/models.usecases.ts @@ -3,11 +3,7 @@ import { UpdateModelDto } from '@/infrastructure/dtos/models/update-model.dto'; import { ModelEntity } from '@/infrastructure/entities/model.entity'; import { BadRequestException, Inject, Injectable } from '@nestjs/common'; import { Repository } from 'typeorm'; -import { - Model, - ModelFormat, - ModelSettingParams, -} from '@/domain/models/model.interface'; +import { Model, ModelFormat } from '@/domain/models/model.interface'; import { ModelNotFoundException } from '@/infrastructure/exception/model-not-found.exception'; import { join, basename } from 'path'; import { @@ -18,7 +14,6 @@ import { rmdirSync, } from 'fs'; import { StartModelSuccessDto } from '@/infrastructure/dtos/models/start-model-success.dto'; -import { DownloadModelDto } from '@/infrastructure/dtos/models/download-model.dto'; import { ConfigService } from '@nestjs/config'; import { ExtensionRepository } from '@/domain/repositories/extension.interface'; import { EngineExtension } from '@/domain/abstracts/engine.abstract'; @@ -92,7 +87,7 @@ export class ModelsUsecases { async startModel( modelId: string, - settings: ModelSettingParamsDto, + settings?: ModelSettingParamsDto, ): Promise { const model = await this.getModelOrThrow(modelId); const extensions = (await this.extensionRepository.findAll()) ?? []; @@ -155,11 +150,8 @@ export class ModelsUsecases { }); } - async downloadModel( - downloadModelDto: DownloadModelDto, - callback?: (progress: number) => void, - ) { - const model = await this.getModelOrThrow(downloadModelDto.modelId); + async downloadModel(modelId: string, callback?: (progress: number) => void) { + const model = await this.getModelOrThrow(modelId); if (model.format === ModelFormat.API) { throw new BadRequestException('Cannot download remote model');