From 4e33cf19ba0746bebd1306065454422fd5cb5e13 Mon Sep 17 00:00:00 2001 From: Louis Le Date: Tue, 9 Jul 2024 11:56:07 +0700 Subject: [PATCH] fix: putenv on win32 --- cortex-cpp/addon.cc | 42 ++++++++----------- cortex-js/package.json | 2 +- .../engines/engines-init.command.ts | 27 +++++++++--- .../types/init-options.interface.ts | 1 + .../commanders/usecases/init.cli.usecases.ts | 27 ++++++------ .../src/usecases/cortex/cortex.usecases.ts | 8 ++-- cortex-js/src/utils/cortex-cpp.ts | 2 +- 7 files changed, 59 insertions(+), 50 deletions(-) diff --git a/cortex-cpp/addon.cc b/cortex-cpp/addon.cc index c98d6848e..42db5f153 100644 --- a/cortex-cpp/addon.cc +++ b/cortex-cpp/addon.cc @@ -2,12 +2,12 @@ #include #include +#include #include // for PATH_MAX #include #include "cortex-common/cortexpythoni.h" #include "utils/cortex_utils.h" #include "utils/dylib.h" -#include #if defined(__APPLE__) && defined(__MACH__) #include // for dirname() @@ -22,15 +22,15 @@ #error "Unsupported platform!" #endif +static Napi::Env* s_env = nullptr; + void start() { int thread_num = 1; std::string host = "127.0.0.1"; int port = 3929; std::string uploads_folder_path; - int logical_cores = std::thread::hardware_concurrency(); int drogon_thread_num = std::max(thread_num, logical_cores); - // cortex_utils::nitro_logo(); #ifdef CORTEX_CPP_VERSION LOG_INFO << "cortex-cpp version: " << CORTEX_CPP_VERSION; #else @@ -57,40 +57,32 @@ void stop() { drogon::app().quit(); } -Napi::Value Start(const Napi::CallbackInfo &info) -{ +void exitCallback() { + Napi::TypeError::New(*s_env, "Process Exited!").ThrowAsJavaScriptException(); +} + +Napi::Value Start(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); - LOG_INFO << "set env"; - Napi::Object objs = info[0].As().ToObject(); - Napi::Array props = objs.GetPropertyNames(); + s_env = &env; - for (unsigned int i = 0; i < props.Length(); i++) - { - Napi::Value key = props.Get(i); - LOG_INFO << key.ToString().Utf8Value(); - LOG_INFO << objs.Get(key).ToString().Utf8Value(); - setenv(key.ToString().Utf8Value().c_str(), objs.Get(key).ToString().Utf8Value().c_str(), 1); - } + // Register exitCallback with atexit + std::atexit(exitCallback); start(); - return Napi::String::New(env, "Server started successfully"); + return env.Undefined(); } -Napi::Value Stop(const Napi::CallbackInfo &info) -{ +Napi::Value Stop(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); stop(); return Napi::String::New(env, "Server stopped successfully"); } -Napi::Object Init(Napi::Env env, Napi::Object exports) -{ - exports.Set(Napi::String::New(env, "start"), - Napi::Function::New(env, Start)); - exports.Set(Napi::String::New(env, "stop"), - Napi::Function::New(env, Start)); - return exports; +Napi::Object Init(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "start"), Napi::Function::New(env, Start)); + exports.Set(Napi::String::New(env, "stop"), Napi::Function::New(env, Start)); + return exports; } NODE_API_MODULE(cortex-cpp, Init) \ No newline at end of file diff --git a/cortex-js/package.json b/cortex-js/package.json index c39d180fa..f396f9a37 100644 --- a/cortex-js/package.json +++ b/cortex-js/package.json @@ -52,7 +52,7 @@ "class-transformer": "^0.5.1", "class-validator": "^0.14.1", "cli-progress": "^3.12.0", - "cortex-cpp": "^0.4.23", + "cortex-cpp": "^0.4.24", "cortexso-node": "^0.0.4", "cpu-instructions": "^0.0.11", "decompress": "^4.2.1", diff --git a/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts b/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts index 1763d8757..2bbfe0a9f 100644 --- a/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts +++ b/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts @@ -1,4 +1,4 @@ -import { CommandRunner, SubCommand } from 'nest-commander'; +import { CommandRunner, Option, SubCommand } from 'nest-commander'; import { SetCommandContext } from '../decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; import { InitCliUsecases } from '../usecases/init.cli.usecases'; @@ -25,10 +25,16 @@ export class EnginesInitCommand extends CommandRunner { super(); } - async run(passedParams: string[]): Promise { + async run( + passedParams: string[], + options: { vulkan: boolean }, + ): Promise { const engine = passedParams[0]; - const options = passedParams.includes(Engines.llamaCPP) - ? await this.initUsecases.defaultInstallationOptions() + const params = passedParams.includes(Engines.llamaCPP) + ? { + ...(await this.initUsecases.defaultInstallationOptions()), + ...options, + } : {}; const configs = await this.fileManagerService.getConfig(); @@ -40,14 +46,23 @@ export class EnginesInitCommand extends CommandRunner { } return this.initUsecases .installEngine( - options, + params, engine.includes('@') ? engine.split('@')[1] : 'latest', engine, - true, + true ) .then(() => console.log('Engine installed successfully!')) .catch((e) => console.error('Install engine failed with reason: %s', e.message ?? e), ); } + + @Option({ + flags: '-vk, --vulkan', + description: 'Install Vulkan engine', + defaultValue: false, + }) + parseVulkan() { + return true; + } } diff --git a/cortex-js/src/infrastructure/commanders/types/init-options.interface.ts b/cortex-js/src/infrastructure/commanders/types/init-options.interface.ts index 2631da0bd..55444f04e 100644 --- a/cortex-js/src/infrastructure/commanders/types/init-options.interface.ts +++ b/cortex-js/src/infrastructure/commanders/types/init-options.interface.ts @@ -4,4 +4,5 @@ export interface InitOptions { instructions?: 'AVX' | 'AVX2' | 'AVX512' | undefined; cudaVersion?: '11' | '12'; silent?: boolean; + vulkan?: boolean; } diff --git a/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts b/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts index 8163ec307..45b56ec71 100644 --- a/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts +++ b/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts @@ -62,7 +62,7 @@ export class InitCliUsecases { options?: InitOptions, version: string = 'latest', engine: string = 'default', - force: boolean = true, + force: boolean = false, ): Promise => { // Use default option if not defined if (!options && engine === Engines.llamaCPP) { @@ -70,18 +70,17 @@ export class InitCliUsecases { } const configs = await this.fileManagerService.getConfig(); - if (configs.initialized && !force) return; const engineSpinner = ora('Installing engine...').start(); // Ship Llama.cpp engine by default if ( !existsSync( - join( - await this.fileManagerService.getCortexCppEnginePath(), - Engines.llamaCPP, - ), + join(await this.fileManagerService.getCortexCppEnginePath(), engine), ) || - (engine === Engines.llamaCPP && force) - ) + force + ) { + const isVulkan = + engine === Engines.llamaCPP && + (options?.vulkan || options?.gpuType !== 'Nvidia'); await this.installAcceleratedEngine(version, engine, [ process.platform === 'win32' ? '-windows' @@ -91,17 +90,15 @@ export class InitCliUsecases { // CPU Instructions - CPU | GPU Non-Vulkan options?.instructions && (options?.runMode === 'CPU' || - (options?.runMode === 'GPU' && options?.gpuType !== 'Nvidia')) + (options?.runMode === 'GPU' && !isVulkan)) ? `-${options?.instructions?.toLowerCase()}` : '', // Cuda - options?.runMode === 'GPU' && options?.gpuType === 'Nvidia' + options?.runMode === 'GPU' && options?.gpuType === 'Nvidia' && !isVulkan ? `cuda-${options.cudaVersion ?? '12'}` : '', // Vulkan - options?.runMode === 'GPU' && options?.gpuType !== 'Nvidia' - ? '-vulkan' - : '', + isVulkan ? '-vulkan' : '', // Arch engine !== Engines.tensorrtLLM @@ -110,11 +107,13 @@ export class InitCliUsecases { : '-amd64' : '', ]); + } if ( (engine === Engines.llamaCPP || engine === Engines.tensorrtLLM) && options?.runMode === 'GPU' && - options?.gpuType === 'Nvidia' + options?.gpuType === 'Nvidia' && + !options?.vulkan ) await this.installCudaToolkitDependency(options?.cudaVersion); diff --git a/cortex-js/src/usecases/cortex/cortex.usecases.ts b/cortex-js/src/usecases/cortex/cortex.usecases.ts index 73225ba8f..8e5dc535c 100644 --- a/cortex-js/src/usecases/cortex/cortex.usecases.ts +++ b/cortex-js/src/usecases/cortex/cortex.usecases.ts @@ -38,7 +38,8 @@ export class CortexUsecases { }; } - const dataFolderPath = await this.fileManagerService.getDataFolderPath(); + const engineDir = await this.fileManagerService.getCortexCppEnginePath(); + const dataFolderPath = await this.fileManagerService.getDataFolderPath() const writer = openSync(await this.fileManagerService.getLogPath(), 'a+'); // go up one level to get the binary folder, have to also work on windows @@ -50,10 +51,10 @@ export class CortexUsecases { ...process.env, CUDA_VISIBLE_DEVICES: '0', ENGINE_PATH: dataFolderPath, - PATH: (process.env.PATH || '').concat(delimiter, dataFolderPath), + PATH: (process.env.PATH || '').concat(delimiter, engineDir), LD_LIBRARY_PATH: (process.env.LD_LIBRARY_PATH || '').concat( delimiter, - dataFolderPath, + engineDir, ), // // Vulkan - Support 1 device at a time for now // ...(executableOptions.vkVisibleDevices?.length > 0 && { @@ -61,6 +62,7 @@ export class CortexUsecases { // }), }, }); + this.cortexProcess.unref(); // Await for the /healthz status ok return new Promise((resolve, reject) => { diff --git a/cortex-js/src/utils/cortex-cpp.ts b/cortex-js/src/utils/cortex-cpp.ts index 8baafcd76..909304cfc 100644 --- a/cortex-js/src/utils/cortex-cpp.ts +++ b/cortex-js/src/utils/cortex-cpp.ts @@ -1,3 +1,3 @@ import * as cortexCPP from 'cortex-cpp'; -cortexCPP.start(process.env); +cortexCPP.start();