fix: putenv on win32

janhq · Jul 9, 2024 · 4e33cf1 · 4e33cf1
1 parent 60440b3
commit 4e33cf1
Show file tree

Hide file tree

Showing 7 changed files with 59 additions and 50 deletions.
diff --git a/cortex-cpp/addon.cc b/cortex-cpp/addon.cc
@@ -2,12 +2,12 @@
 
 #include <drogon/HttpAppFramework.h>
 #include <drogon/drogon.h>
+#include <stdlib.h>
 #include <climits>  // for PATH_MAX
 #include <iostream>
 #include "cortex-common/cortexpythoni.h"
 #include "utils/cortex_utils.h"
 #include "utils/dylib.h"
-#include <stdlib.h>
 
 #if defined(__APPLE__) && defined(__MACH__)
 #include <libgen.h>  // for dirname()
@@ -22,15 +22,15 @@
 #error "Unsupported platform!"
 #endif
 
+static Napi::Env* s_env = nullptr;
+
 void start() {
   int thread_num = 1;
   std::string host = "127.0.0.1";
   int port = 3929;
   std::string uploads_folder_path;
-
   int logical_cores = std::thread::hardware_concurrency();
   int drogon_thread_num = std::max(thread_num, logical_cores);
-  // cortex_utils::nitro_logo();
 #ifdef CORTEX_CPP_VERSION
   LOG_INFO << "cortex-cpp version: " << CORTEX_CPP_VERSION;
 #else
@@ -57,40 +57,32 @@ void stop() {
   drogon::app().quit();
 }
 
-Napi::Value Start(const Napi::CallbackInfo &info)
-{
+void exitCallback() {
+  Napi::TypeError::New(*s_env, "Process Exited!").ThrowAsJavaScriptException();
+}
+
+Napi::Value Start(const Napi::CallbackInfo& info) {
   Napi::Env env = info.Env();
 
-  LOG_INFO << "set env";
-  Napi::Object objs = info[0].As<Napi::Value>().ToObject();
-  Napi::Array props = objs.GetPropertyNames();
+  s_env = &env;
 
-  for (unsigned int i = 0; i < props.Length(); i++)
-  {
-      Napi::Value key = props.Get(i);
-      LOG_INFO << key.ToString().Utf8Value();
-      LOG_INFO << objs.Get(key).ToString().Utf8Value();
-      setenv(key.ToString().Utf8Value().c_str(), objs.Get(key).ToString().Utf8Value().c_str(), 1);
-  }
+  // Register exitCallback with atexit
+  std::atexit(exitCallback);
 
   start();
-  return Napi::String::New(env, "Server started successfully");
+  return env.Undefined();
 }
 
-Napi::Value Stop(const Napi::CallbackInfo &info)
-{
+Napi::Value Stop(const Napi::CallbackInfo& info) {
   Napi::Env env = info.Env();
   stop();
   return Napi::String::New(env, "Server stopped successfully");
 }
 
-Napi::Object Init(Napi::Env env, Napi::Object exports)
-{
-    exports.Set(Napi::String::New(env, "start"),
-                Napi::Function::New(env, Start));
-    exports.Set(Napi::String::New(env, "stop"),
-                Napi::Function::New(env, Start));
-    return exports;
+Napi::Object Init(Napi::Env env, Napi::Object exports) {
+  exports.Set(Napi::String::New(env, "start"), Napi::Function::New(env, Start));
+  exports.Set(Napi::String::New(env, "stop"), Napi::Function::New(env, Start));
+  return exports;
 }
 
 NODE_API_MODULE(cortex-cpp, Init)
diff --git a/cortex-js/package.json b/cortex-js/package.json
@@ -52,7 +52,7 @@
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
     "cli-progress": "^3.12.0",
-    "cortex-cpp": "^0.4.23",
+    "cortex-cpp": "^0.4.24",
     "cortexso-node": "^0.0.4",
     "cpu-instructions": "^0.0.11",
     "decompress": "^4.2.1",

diff --git a/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts b/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts
@@ -1,4 +1,4 @@
-import { CommandRunner, SubCommand } from 'nest-commander';
+import { CommandRunner, Option, SubCommand } from 'nest-commander';
 import { SetCommandContext } from '../decorators/CommandContext';
 import { ContextService } from '@/infrastructure/services/context/context.service';
 import { InitCliUsecases } from '../usecases/init.cli.usecases';
@@ -25,10 +25,16 @@ export class EnginesInitCommand extends CommandRunner {
     super();
   }
 
-  async run(passedParams: string[]): Promise<void> {
+  async run(
+    passedParams: string[],
+    options: { vulkan: boolean },
+  ): Promise<void> {
     const engine = passedParams[0];
-    const options = passedParams.includes(Engines.llamaCPP)
-      ? await this.initUsecases.defaultInstallationOptions()
+    const params = passedParams.includes(Engines.llamaCPP)
+      ? {
+          ...(await this.initUsecases.defaultInstallationOptions()),
+          ...options,
+        }
       : {};
 
     const configs = await this.fileManagerService.getConfig();
@@ -40,14 +46,23 @@ export class EnginesInitCommand extends CommandRunner {
     }
     return this.initUsecases
       .installEngine(
-        options,
+        params,
         engine.includes('@') ? engine.split('@')[1] : 'latest',
         engine,
-        true,
+        true
       )
       .then(() => console.log('Engine installed successfully!'))
       .catch((e) =>
         console.error('Install engine failed with reason: %s', e.message ?? e),
       );
   }
+
+  @Option({
+    flags: '-vk, --vulkan',
+    description: 'Install Vulkan engine',
+    defaultValue: false,
+  })
+  parseVulkan() {
+    return true;
+  }
 }
diff --git a/cortex-js/src/infrastructure/commanders/types/init-options.interface.ts b/cortex-js/src/infrastructure/commanders/types/init-options.interface.ts
@@ -4,4 +4,5 @@ export interface InitOptions {
   instructions?: 'AVX' | 'AVX2' | 'AVX512' | undefined;
   cudaVersion?: '11' | '12';
   silent?: boolean;
+  vulkan?: boolean;
 }
diff --git a/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts b/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts
@@ -62,26 +62,25 @@ export class InitCliUsecases {
     options?: InitOptions,
     version: string = 'latest',
     engine: string = 'default',
-    force: boolean = true,
+    force: boolean = false,
   ): Promise<any> => {
     // Use default option if not defined
     if (!options && engine === Engines.llamaCPP) {
       options = await this.defaultInstallationOptions();
     }
     const configs = await this.fileManagerService.getConfig();
 
-    if (configs.initialized && !force) return;
     const engineSpinner = ora('Installing engine...').start();
     // Ship Llama.cpp engine by default
     if (
       !existsSync(
-        join(
-          await this.fileManagerService.getCortexCppEnginePath(),
-          Engines.llamaCPP,
-        ),
+        join(await this.fileManagerService.getCortexCppEnginePath(), engine),
       ) ||
-      (engine === Engines.llamaCPP && force)
-    )
+      force
+    ) {
+      const isVulkan =
+        engine === Engines.llamaCPP &&
+        (options?.vulkan || options?.gpuType !== 'Nvidia');
       await this.installAcceleratedEngine(version, engine, [
         process.platform === 'win32'
           ? '-windows'
@@ -91,17 +90,15 @@ export class InitCliUsecases {
         // CPU Instructions - CPU | GPU Non-Vulkan
         options?.instructions &&
         (options?.runMode === 'CPU' ||
-          (options?.runMode === 'GPU' && options?.gpuType !== 'Nvidia'))
+          (options?.runMode === 'GPU' && !isVulkan))
           ? `-${options?.instructions?.toLowerCase()}`
           : '',
         // Cuda
-        options?.runMode === 'GPU' && options?.gpuType === 'Nvidia'
+        options?.runMode === 'GPU' && options?.gpuType === 'Nvidia' && !isVulkan
           ? `cuda-${options.cudaVersion ?? '12'}`
           : '',
         // Vulkan
-        options?.runMode === 'GPU' && options?.gpuType !== 'Nvidia'
-          ? '-vulkan'
-          : '',
+        isVulkan ? '-vulkan' : '',
 
         // Arch
         engine !== Engines.tensorrtLLM
@@ -110,11 +107,13 @@ export class InitCliUsecases {
             : '-amd64'
           : '',
       ]);
+    }
 
     if (
       (engine === Engines.llamaCPP || engine === Engines.tensorrtLLM) &&
       options?.runMode === 'GPU' &&
-      options?.gpuType === 'Nvidia'
+      options?.gpuType === 'Nvidia' &&
+      !options?.vulkan
     )
       await this.installCudaToolkitDependency(options?.cudaVersion);
 

diff --git a/cortex-js/src/usecases/cortex/cortex.usecases.ts b/cortex-js/src/usecases/cortex/cortex.usecases.ts
@@ -38,7 +38,8 @@ export class CortexUsecases {
       };
     }
 
-    const dataFolderPath = await this.fileManagerService.getDataFolderPath();
+    const engineDir = await this.fileManagerService.getCortexCppEnginePath();
+    const dataFolderPath = await this.fileManagerService.getDataFolderPath()
 
     const writer = openSync(await this.fileManagerService.getLogPath(), 'a+');
     // go up one level to get the binary folder, have to also work on windows
@@ -50,17 +51,18 @@ export class CortexUsecases {
         ...process.env,
         CUDA_VISIBLE_DEVICES: '0',
         ENGINE_PATH: dataFolderPath,
-        PATH: (process.env.PATH || '').concat(delimiter, dataFolderPath),
+        PATH: (process.env.PATH || '').concat(delimiter, engineDir),
         LD_LIBRARY_PATH: (process.env.LD_LIBRARY_PATH || '').concat(
           delimiter,
-          dataFolderPath,
+          engineDir,
         ),
         // // Vulkan - Support 1 device at a time for now
         // ...(executableOptions.vkVisibleDevices?.length > 0 && {
         //   GGML_VULKAN_DEVICE: executableOptions.vkVisibleDevices[0],
         // }),
       },
     });
+    this.cortexProcess.unref();
 
     // Await for the /healthz status ok
     return new Promise<CortexOperationSuccessfullyDto>((resolve, reject) => {

diff --git a/cortex-js/src/utils/cortex-cpp.ts b/cortex-js/src/utils/cortex-cpp.ts
@@ -1,3 +1,3 @@
 import * as cortexCPP from 'cortex-cpp';
 
-cortexCPP.start(process.env);
+cortexCPP.start();