From a3689b063c032d2f271633a13cec85f9718fe7e2 Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Thu, 1 Aug 2024 01:56:38 +0700 Subject: [PATCH 1/4] feat: support martian, nvidia engine --- cortex-js/src/extensions/extensions.module.ts | 4 ++ cortex-js/src/extensions/martian.engine.ts | 49 +++++++++++++++++++ cortex-js/src/extensions/nvidia.engine.ts | 49 +++++++++++++++++++ .../commanders/types/engine.interface.ts | 4 ++ .../resources-manager.service.ts | 2 +- 5 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 cortex-js/src/extensions/martian.engine.ts create mode 100644 cortex-js/src/extensions/nvidia.engine.ts diff --git a/cortex-js/src/extensions/extensions.module.ts b/cortex-js/src/extensions/extensions.module.ts index d1dbeacca..690a739f5 100644 --- a/cortex-js/src/extensions/extensions.module.ts +++ b/cortex-js/src/extensions/extensions.module.ts @@ -9,6 +9,8 @@ import { EventEmitter2 } from '@nestjs/event-emitter'; import AnthropicEngineExtension from './anthropic.engine'; import OpenRouterEngineExtension from './openrouter.engine'; import CoHereEngineExtension from './cohere.engine'; +import MartianEngineExtension from './martian.engine'; +import NvidiaEngineExtension from './nvidia.engine'; const provider = { provide: 'EXTENSIONS_PROVIDER', @@ -24,6 +26,8 @@ const provider = { new AnthropicEngineExtension(httpService, configUsecases, eventEmitter), new OpenRouterEngineExtension(httpService, configUsecases, eventEmitter), new CoHereEngineExtension(httpService, configUsecases, eventEmitter), + new MartianEngineExtension(httpService, configUsecases, eventEmitter), + new NvidiaEngineExtension(httpService, configUsecases, eventEmitter), ], }; diff --git a/cortex-js/src/extensions/martian.engine.ts b/cortex-js/src/extensions/martian.engine.ts new file mode 100644 index 000000000..18ac19d8f --- /dev/null +++ b/cortex-js/src/extensions/martian.engine.ts @@ -0,0 +1,49 @@ +import { HttpService } from '@nestjs/axios'; +import { OAIEngineExtension } from '../domain/abstracts/oai.abstract'; +import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { EngineStatus } from '@/domain/abstracts/engine.abstract'; + +/** + * A class that implements the InferenceExtension interface from the @janhq/core package. + * The class provides methods for initializing and stopping a model, and for making inference requests. + * It also subscribes to events emitted by the @janhq/core package and handles new message requests. + */ +export default class MartianEngineExtension extends OAIEngineExtension { + apiUrl = 'https://withmartian.com/api/openai/v1/chat/completions'; + name = 'martian'; + productName = 'Martian Inference Engine'; + description = 'This extension enables Martian chat completion API calls'; + version = '0.0.1'; + apiKey?: string; + + constructor( + protected readonly httpService: HttpService, + protected readonly configsUsecases: ConfigsUsecases, + protected readonly eventEmmitter: EventEmitter2, + ) { + super(httpService); + + eventEmmitter.on('config.updated', async (data) => { + if (data.engine === this.name) { + this.apiKey = data.value; + this.status = + (this.apiKey?.length ?? 0) > 0 + ? EngineStatus.READY + : EngineStatus.MISSING_CONFIGURATION; + } + }); + } + + async onLoad() { + const configs = (await this.configsUsecases.getGroupConfigs( + this.name, + )) as unknown as { apiKey: string }; + + this.apiKey = configs?.apiKey; + this.status = + (this.apiKey?.length ?? 0) > 0 + ? EngineStatus.READY + : EngineStatus.MISSING_CONFIGURATION; + } +} diff --git a/cortex-js/src/extensions/nvidia.engine.ts b/cortex-js/src/extensions/nvidia.engine.ts new file mode 100644 index 000000000..caec1a09c --- /dev/null +++ b/cortex-js/src/extensions/nvidia.engine.ts @@ -0,0 +1,49 @@ +import { HttpService } from '@nestjs/axios'; +import { OAIEngineExtension } from '../domain/abstracts/oai.abstract'; +import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { EngineStatus } from '@/domain/abstracts/engine.abstract'; + +/** + * A class that implements the InferenceExtension interface from the @janhq/core package. + * The class provides methods for initializing and stopping a model, and for making inference requests. + * It also subscribes to events emitted by the @janhq/core package and handles new message requests. + */ +export default class NvidiaEngineExtension extends OAIEngineExtension { + apiUrl = 'https://integrate.api.nvidia.com/v1/chat/completions'; + name = 'nvidia'; + productName = 'Nvidia Inference Engine'; + description = 'This extension enables Nvidia chat completion API calls'; + version = '0.0.1'; + apiKey?: string; + + constructor( + protected readonly httpService: HttpService, + protected readonly configsUsecases: ConfigsUsecases, + protected readonly eventEmmitter: EventEmitter2, + ) { + super(httpService); + + eventEmmitter.on('config.updated', async (data) => { + if (data.engine === this.name) { + this.apiKey = data.value; + this.status = + (this.apiKey?.length ?? 0) > 0 + ? EngineStatus.READY + : EngineStatus.MISSING_CONFIGURATION; + } + }); + } + + async onLoad() { + const configs = (await this.configsUsecases.getGroupConfigs( + this.name, + )) as unknown as { apiKey: string }; + + this.apiKey = configs?.apiKey; + this.status = + (this.apiKey?.length ?? 0) > 0 + ? EngineStatus.READY + : EngineStatus.MISSING_CONFIGURATION; + } +} diff --git a/cortex-js/src/infrastructure/commanders/types/engine.interface.ts b/cortex-js/src/infrastructure/commanders/types/engine.interface.ts index fcd52b861..c3562ecdb 100644 --- a/cortex-js/src/infrastructure/commanders/types/engine.interface.ts +++ b/cortex-js/src/infrastructure/commanders/types/engine.interface.ts @@ -10,6 +10,8 @@ export enum Engines { anthropic = 'anthropic', openrouter = 'openrouter', cohere = 'cohere', + martian = 'martian', + nvidia = 'nvidia', } export const EngineNamesMap: { @@ -27,4 +29,6 @@ export const RemoteEngines: Engines[] = [ Engines.anthropic, Engines.openrouter, Engines.cohere, + Engines.martian, + Engines.nvidia, ]; diff --git a/cortex-js/src/infrastructure/services/resources-manager/resources-manager.service.ts b/cortex-js/src/infrastructure/services/resources-manager/resources-manager.service.ts index 266015742..da7a0c402 100644 --- a/cortex-js/src/infrastructure/services/resources-manager/resources-manager.service.ts +++ b/cortex-js/src/infrastructure/services/resources-manager/resources-manager.service.ts @@ -21,7 +21,7 @@ export class ResourcesManagerService { return { mem: memInfo, cpu: { - usage: cpuUsage.avgLoad, + usage: Number(cpuUsage.currentLoad.toFixed(2)), }, }; } From f670ca280c03350736b20b6604eec0c674305744 Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Thu, 1 Aug 2024 11:02:01 +0700 Subject: [PATCH 2/4] fix: fix claude, command-r chat --- .../src/domain/abstracts/oai.abstract.ts | 4 +-- cortex-js/src/extensions/anthropic.engine.ts | 10 +++++-- cortex-js/src/extensions/cohere.engine.ts | 30 ++++++++++++------- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/cortex-js/src/domain/abstracts/oai.abstract.ts b/cortex-js/src/domain/abstracts/oai.abstract.ts index eef2e1c16..ae1a31773 100644 --- a/cortex-js/src/domain/abstracts/oai.abstract.ts +++ b/cortex-js/src/domain/abstracts/oai.abstract.ts @@ -54,7 +54,7 @@ export abstract class OAIEngineExtension extends EngineExtension { const transformedLines = []; for (const line of lines) { if (line.trim().length > 0) { - const transformedLine = transformResponse(line); + const transformedLine = transformResponse(line, true); if (transformedLine) { transformedLines.push(`data: ${transformedLine}\n\n`); } @@ -65,6 +65,6 @@ export abstract class OAIEngineExtension extends EngineExtension { }); return response.data.pipe(lineStream); } - return this.transformResponse(response.data); + return this.transformResponse(response.data, false); } } diff --git a/cortex-js/src/extensions/anthropic.engine.ts b/cortex-js/src/extensions/anthropic.engine.ts index f5774eaad..392644cb0 100644 --- a/cortex-js/src/extensions/anthropic.engine.ts +++ b/cortex-js/src/extensions/anthropic.engine.ts @@ -3,7 +3,7 @@ import { HttpService } from '@nestjs/axios'; import { OAIEngineExtension } from '../domain/abstracts/oai.abstract'; import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; import { EventEmitter2 } from '@nestjs/event-emitter'; -import _ from 'lodash'; +import { pick } from 'lodash'; import { EngineStatus } from '@/domain/abstracts/engine.abstract'; /** @@ -59,7 +59,13 @@ export default class AnthropicEngineExtension extends OAIEngineExtension { } transformPayload = (data: any): any => { - return _.pick(data, ['messages', 'model', 'stream', 'max_tokens']); + const system = data.messages.find((m: any) => m.role === 'system'); + const messages = data.messages.filter((m: any) => m.role !== 'system'); + return { + system: system.content, + messages, + ...pick(data, ['model', 'stream', 'max_tokens']), + }; }; transformResponse = (data: any) => { diff --git a/cortex-js/src/extensions/cohere.engine.ts b/cortex-js/src/extensions/cohere.engine.ts index 505fb7a5b..2f717b189 100644 --- a/cortex-js/src/extensions/cohere.engine.ts +++ b/cortex-js/src/extensions/cohere.engine.ts @@ -98,17 +98,27 @@ export default class CoHereEngineExtension extends OAIEngineExtension { return convertedData; }; - transformResponse = (data: any) => { + transformResponse = (data: any, stream: boolean) => { const text = typeof data === 'object' ? data.text : (JSON.parse(data).text ?? ''); - return JSON.stringify({ - choices: [ - { - delta: { - content: text, - }, - }, - ], - }); + return stream + ? JSON.stringify({ + choices: [ + { + delta: { + content: text, + }, + }, + ], + }) + : { + choices: [ + { + message: { + content: text, + }, + }, + ], + }; }; } From 4cd5d478fb65c26aa47f0dd4ce924bb78b1e4fb3 Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Thu, 1 Aug 2024 11:13:39 +0700 Subject: [PATCH 3/4] chore: improve error handling --- .../src/infrastructure/controllers/chat.controller.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cortex-js/src/infrastructure/controllers/chat.controller.ts b/cortex-js/src/infrastructure/controllers/chat.controller.ts index 9c8da76a2..ad04ab015 100644 --- a/cortex-js/src/infrastructure/controllers/chat.controller.ts +++ b/cortex-js/src/infrastructure/controllers/chat.controller.ts @@ -33,7 +33,7 @@ export class ChatController { @Body() createChatDto: CreateChatCompletionDto, @Res() res: Response, ) { - let { stream } = createChatDto; + const { stream } = createChatDto; this.chatService .inference(createChatDto, extractCommonHeaders(headers)) .then((response) => { @@ -50,6 +50,9 @@ export class ChatController { let errorMessage; if (!stream) { const data = error.response?.data; + if (!data) { + return res.status(500).send(error.message || 'An error occurred'); + } return res .status(statusCode) .send( @@ -60,8 +63,10 @@ export class ChatController { ); } const streamResponse = error.response?.data; + if (!streamResponse) { + return res.status(500).send(error.message || 'An error occurred'); + } let data = ''; - streamResponse.on('data', (chunk: any) => { data += chunk; }); From 78cf9cf6449b1e118a3e28e0bbbadd2863215fb7 Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Thu, 1 Aug 2024 11:27:36 +0700 Subject: [PATCH 4/4] chore: change lodash import --- cortex-js/src/domain/abstracts/oai.abstract.ts | 4 ++-- cortex-js/src/extensions/cohere.engine.ts | 2 -- cortex-js/src/extensions/openrouter.engine.ts | 2 -- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/cortex-js/src/domain/abstracts/oai.abstract.ts b/cortex-js/src/domain/abstracts/oai.abstract.ts index ae1a31773..09b2fbbf1 100644 --- a/cortex-js/src/domain/abstracts/oai.abstract.ts +++ b/cortex-js/src/domain/abstracts/oai.abstract.ts @@ -2,7 +2,7 @@ import { HttpService } from '@nestjs/axios'; import { EngineExtension } from './engine.abstract'; import stream, { Transform } from 'stream'; import { firstValueFrom } from 'rxjs'; -import _ from 'lodash'; +import { omit } from 'lodash'; export abstract class OAIEngineExtension extends EngineExtension { abstract apiUrl: string; @@ -22,7 +22,7 @@ export abstract class OAIEngineExtension extends EngineExtension { ? this.transformPayload(createChatDto) : createChatDto; const { stream: isStream } = payload; - const additionalHeaders = _.omit(headers, [ + const additionalHeaders = omit(headers, [ 'content-type', 'authorization', 'content-length', diff --git a/cortex-js/src/extensions/cohere.engine.ts b/cortex-js/src/extensions/cohere.engine.ts index 2f717b189..9c719c05c 100644 --- a/cortex-js/src/extensions/cohere.engine.ts +++ b/cortex-js/src/extensions/cohere.engine.ts @@ -1,9 +1,7 @@ -import stream from 'stream'; import { HttpService } from '@nestjs/axios'; import { OAIEngineExtension } from '../domain/abstracts/oai.abstract'; import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; import { EventEmitter2 } from '@nestjs/event-emitter'; -import _ from 'lodash'; import { EngineStatus } from '@/domain/abstracts/engine.abstract'; import { ChatCompletionMessage } from '@/infrastructure/dtos/chat/chat-completion-message.dto'; diff --git a/cortex-js/src/extensions/openrouter.engine.ts b/cortex-js/src/extensions/openrouter.engine.ts index b7123963f..95bd1b8b6 100644 --- a/cortex-js/src/extensions/openrouter.engine.ts +++ b/cortex-js/src/extensions/openrouter.engine.ts @@ -1,9 +1,7 @@ -import stream from 'stream'; import { HttpService } from '@nestjs/axios'; import { OAIEngineExtension } from '../domain/abstracts/oai.abstract'; import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; import { EventEmitter2 } from '@nestjs/event-emitter'; -import _ from 'lodash'; import { EngineStatus } from '@/domain/abstracts/engine.abstract'; /**