From d327e0cd3d190dc8de5a6032acca3d859cec32e9 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 4 Jul 2024 18:58:46 +0700 Subject: [PATCH 1/3] fix: gate onnx and tensorrt-llm run on darwin --- .../commanders/shortcuts/run.command.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts index 0b5598f0c..121348466 100644 --- a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts +++ b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts @@ -54,6 +54,10 @@ export class RunCommand extends CommandRunner { exit(1); } } + + // Check model compatibility on this machine + await checkModelCompatibility(modelId); + // If not exist // Try Pull if (!(await this.modelsCliUsecases.getModel(modelId))) { @@ -73,16 +77,11 @@ export class RunCommand extends CommandRunner { !Array.isArray(existingModel.files) || /^(http|https):\/\/[^/]+\/.*/.test(existingModel.files[0]) ) { - checkingSpinner.fail( - `Model is not available` - ); + checkingSpinner.fail(`Model is not available`); process.exit(1); } checkingSpinner.succeed('Model found'); - // Check model compatibility on this machine - await checkModelCompatibility(modelId); - const engine = existingModel.engine || Engines.llamaCPP; // Pull engine if not exist if ( From 70739881134147ea4cd74d3d6ef0eabfb4dd9d35 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 4 Jul 2024 18:58:46 +0700 Subject: [PATCH 2/3] fix: gate onnx and tensorrt-llm run on darwin --- .../commanders/shortcuts/run.command.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts index 0b5598f0c..121348466 100644 --- a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts +++ b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts @@ -54,6 +54,10 @@ export class RunCommand extends CommandRunner { exit(1); } } + + // Check model compatibility on this machine + await checkModelCompatibility(modelId); + // If not exist // Try Pull if (!(await this.modelsCliUsecases.getModel(modelId))) { @@ -73,16 +77,11 @@ export class RunCommand extends CommandRunner { !Array.isArray(existingModel.files) || /^(http|https):\/\/[^/]+\/.*/.test(existingModel.files[0]) ) { - checkingSpinner.fail( - `Model is not available` - ); + checkingSpinner.fail(`Model is not available`); process.exit(1); } checkingSpinner.succeed('Model found'); - // Check model compatibility on this machine - await checkModelCompatibility(modelId); - const engine = existingModel.engine || Engines.llamaCPP; // Pull engine if not exist if ( From a1ab371eb1e4562a40ac18e17fe903037e3ffeed Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Thu, 4 Jul 2024 19:47:44 +0700 Subject: [PATCH 3/3] format log --- .../commanders/shortcuts/run.command.ts | 4 ++-- cortex-js/src/utils/model-check.ts | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts index 121348466..41b6c9abe 100644 --- a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts +++ b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts @@ -56,12 +56,12 @@ export class RunCommand extends CommandRunner { } // Check model compatibility on this machine - await checkModelCompatibility(modelId); + await checkModelCompatibility(modelId, checkingSpinner); // If not exist // Try Pull if (!(await this.modelsCliUsecases.getModel(modelId))) { - checkingSpinner.succeed('Model not found. Attempting to pull...'); + checkingSpinner.succeed(); await this.modelsCliUsecases.pullModel(modelId).catch((e: Error) => { if (e instanceof ModelNotFoundException) checkingSpinner.fail('Model does not exist.'); diff --git a/cortex-js/src/utils/model-check.ts b/cortex-js/src/utils/model-check.ts index 1606cf111..59b0e2842 100644 --- a/cortex-js/src/utils/model-check.ts +++ b/cortex-js/src/utils/model-check.ts @@ -1,15 +1,23 @@ import { MIN_CUDA_VERSION } from "@/infrastructure/constants/cortex"; import { getCudaVersion } from "./cuda"; +import ora from "ora"; -export const checkModelCompatibility = async (modelId: string) => { +export const checkModelCompatibility = async (modelId: string, spinner?: ora.Ora) => { + function log(message: string) { + if (spinner) { + spinner.fail(message); + } else { + console.error(message); + } + } if (modelId.includes('onnx') && process.platform !== 'win32') { - console.error('The ONNX engine does not support this OS yet.'); + log('The ONNX engine does not support this OS yet.'); process.exit(1); } if (modelId.includes('tensorrt-llm') ) { if(process.platform === 'darwin'){ - console.error('Tensorrt-LLM models are not supported on this OS'); + log('Tensorrt-LLM models are not supported on this OS'); process.exit(1); } @@ -19,11 +27,12 @@ export const checkModelCompatibility = async (modelId: string) => { const [requiredMajor, requiredMinor] = MIN_CUDA_VERSION.split('.').map(Number); const isMatchRequired = currentMajor > requiredMajor || (currentMajor === requiredMajor && currentMinor >= requiredMinor); if (!isMatchRequired) { - console.error(`CUDA version ${version} is not compatible with TensorRT-LLM models. Required version: ${MIN_CUDA_VERSION}`); + log(`CUDA version ${version} is not compatible with TensorRT-LLM models. Required version: ${MIN_CUDA_VERSION}`) process.exit(1); } } catch (e) { console.error(e.message ?? e); + log(e.message ?? e); process.exit(1); }