Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

Commit 31f5e14

Browse files
louis-janlouis-jan
authored
Cortex run attempt to pull model (#662)
1 parent 86b9758 commit 31f5e14

25 files changed

+446
-153
lines changed

‎cortex-js/src/command.module.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import { FileManagerModule } from './file-manager/file-manager.module';
2727
import { PSCommand } from './infrastructure/commanders/ps.command';
2828
import { KillCommand } from './infrastructure/commanders/kill.command';
2929
import { PresetCommand } from './infrastructure/commanders/presets.command';
30+
import { EmbeddingCommand } from './infrastructure/commanders/embeddings.command';
3031

3132
@Module({
3233
imports: [
@@ -54,6 +55,7 @@ import { PresetCommand } from './infrastructure/commanders/presets.command';
5455
PSCommand,
5556
KillCommand,
5657
PresetCommand,
58+
EmbeddingCommand,
5759

5860
// Questions
5961
InitRunModeQuestions,

‎cortex-js/src/infrastructure/commanders/chat.command.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,14 @@ export class ChatCommand extends CommandRunner {
2929
async run(_input: string[], options: ChatOptions): Promise<void> {
3030
let modelId = _input[0];
3131
// First attempt to get message from input or options
32-
let message = _input[1] ?? options.message;
32+
// Extract input from 1 to end of array
33+
let message = options.message ?? _input.slice(1).join(' ');
3334

3435
// Check for model existing
3536
if (!modelId || !(await this.modelsUsecases.findOne(modelId))) {
3637
// Model ID is not provided
3738
// first input might be message input
38-
message = _input[0] ?? options.message;
39+
message = _input.length ? _input.join(' ') : options.message ?? '';
3940
// If model ID is not provided, prompt user to select from running models
4041
const models = await this.psCliUsecases.getModels();
4142
if (models.length === 1) {

‎cortex-js/src/infrastructure/commanders/cortex-command.commander.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import { PSCommand } from './ps.command';
99
import { KillCommand } from './kill.command';
1010
import pkg from '@/../package.json';
1111
import { PresetCommand } from './presets.command';
12+
import { EmbeddingCommand } from './embeddings.command';
1213

1314
interface CortexCommandOptions {
1415
version: boolean;
@@ -24,6 +25,7 @@ interface CortexCommandOptions {
2425
PSCommand,
2526
KillCommand,
2627
PresetCommand,
28+
EmbeddingCommand,
2729
],
2830
description: 'Cortex CLI',
2931
})

‎cortex-js/src/infrastructure/commanders/embeddings.command.ts‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
import {
2+
CommandRunner,
3+
InquirerService,
4+
Option,
5+
SubCommand,
6+
} from 'nest-commander';
7+
import { ModelsUsecases } from '@/usecases/models/models.usecases';
8+
import { ModelStat, PSCliUsecases } from './usecases/ps.cli.usecases';
9+
import { ChatCliUsecases } from './usecases/chat.cli.usecases';
10+
import { inspect } from 'util';
11+
12+
interface EmbeddingCommandOptions {
13+
encoding_format?: string;
14+
input?: string;
15+
dimensions?: number;
16+
}
17+
18+
@SubCommand({
19+
name: 'embeddings',
20+
description: 'Creates an embedding vector representing the input text.',
21+
})
22+
export class EmbeddingCommand extends CommandRunner {
23+
constructor(
24+
private readonly chatCliUsecases: ChatCliUsecases,
25+
private readonly modelsUsecases: ModelsUsecases,
26+
private readonly psCliUsecases: PSCliUsecases,
27+
private readonly inquirerService: InquirerService,
28+
) {
29+
super();
30+
}
31+
async run(_input: string[], options: EmbeddingCommandOptions): Promise<void> {
32+
let modelId = _input[0];
33+
// First attempt to get message from input or options
34+
let input: string | string[] = options.input ?? _input.splice(1);
35+
36+
// Check for model existing
37+
if (!modelId || !(await this.modelsUsecases.findOne(modelId))) {
38+
// Model ID is not provided
39+
// first input might be message input
40+
input = _input ?? options.input;
41+
// If model ID is not provided, prompt user to select from running models
42+
const models = await this.psCliUsecases.getModels();
43+
if (models.length === 1) {
44+
modelId = models[0].modelId;
45+
} else if (models.length > 0) {
46+
modelId = await this.modelInquiry(models);
47+
} else {
48+
console.error('Model ID is required');
49+
process.exit(1);
50+
}
51+
}
52+
53+
return this.chatCliUsecases
54+
.embeddings(modelId, input)
55+
.then((res) =>
56+
inspect(res, { showHidden: false, depth: null, colors: true }),
57+
)
58+
.then(console.log)
59+
.catch(console.error);
60+
}
61+
62+
modelInquiry = async (models: ModelStat[]) => {
63+
const { model } = await this.inquirerService.inquirer.prompt({
64+
type: 'list',
65+
name: 'model',
66+
message: 'Select running model to chat with:',
67+
choices: models.map((e) => ({
68+
name: e.modelId,
69+
value: e.modelId,
70+
})),
71+
});
72+
return model;
73+
};
74+
75+
@Option({
76+
flags: '-i, --input <input>',
77+
description:
78+
'Input text to embed, encoded as a string or array of tokens. To embed multiple inputs in a single request, pass an array of strings or array of token arrays.',
79+
})
80+
parseInput(value: string) {
81+
return value;
82+
}
83+
84+
@Option({
85+
flags: '-e, --encoding_format <encoding_format>',
86+
description:
87+
'Encoding format for the embeddings. Supported formats are float and int.',
88+
})
89+
parseEncodingFormat(value: string) {
90+
return value;
91+
}
92+
93+
@Option({
94+
flags: '-d, --dimensions <dimensions>',
95+
description:
96+
'The number of dimensions the resulting output embeddings should have. Only supported in some models.',
97+
})
98+
parseDimensionsFormat(value: string) {
99+
return value;
100+
}
101+
}

‎cortex-js/src/infrastructure/commanders/models/model-pull.command.ts‎

Lines changed: 8 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
1-
import { CommandRunner, InquirerService, SubCommand } from 'nest-commander';
1+
import { CommandRunner, SubCommand } from 'nest-commander';
22
import { exit } from 'node:process';
33
import { ModelsCliUsecases } from '../usecases/models.cli.usecases';
4-
import { RepoDesignation, listFiles } from '@huggingface/hub';
54
import { ModelNotFoundException } from '@/infrastructure/exception/model-not-found.exception';
65

76
@SubCommand({
@@ -10,12 +9,7 @@ import { ModelNotFoundException } from '@/infrastructure/exception/model-not-fou
109
description: 'Download a model. Working with HuggingFace model id.',
1110
})
1211
export class ModelPullCommand extends CommandRunner {
13-
private janHqModelPrefix = 'janhq';
14-
15-
constructor(
16-
private readonly inquirerService: InquirerService,
17-
private readonly modelsCliUsecases: ModelsCliUsecases,
18-
) {
12+
constructor(private readonly modelsCliUsecases: ModelsCliUsecases) {
1913
super();
2014
}
2115

@@ -25,80 +19,14 @@ export class ModelPullCommand extends CommandRunner {
2519
exit(1);
2620
}
2721

28-
const branches = /[:/]/.test(input[0])
29-
? undefined
30-
: await this.tryToGetBranches(input[0]);
31-
32-
await this.modelsCliUsecases
33-
.pullModel(
34-
!branches ? input[0] : await this.handleJanHqModel(input[0], branches),
35-
)
36-
.catch((e: Error) => {
37-
if (e instanceof ModelNotFoundException)
38-
console.error('Model does not exist.');
39-
else console.error(e);
40-
exit(1);
41-
});
22+
await this.modelsCliUsecases.pullModel(input[0]).catch((e: Error) => {
23+
if (e instanceof ModelNotFoundException)
24+
console.error('Model does not exist.');
25+
else console.error(e);
26+
exit(1);
27+
});
4228

4329
console.log('\nDownload complete!');
4430
exit(0);
4531
}
46-
47-
private async tryToGetBranches(input: string): Promise<any> {
48-
try {
49-
// try to append with janhq/ if it's not already
50-
const sanitizedInput = input.trim().startsWith(this.janHqModelPrefix)
51-
? input
52-
: `${this.janHqModelPrefix}/${input}`;
53-
54-
const repo: RepoDesignation = {
55-
type: 'model',
56-
name: sanitizedInput,
57-
};
58-
59-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
60-
for await (const _fileInfo of listFiles({ repo })) {
61-
break;
62-
}
63-
64-
const response = await fetch(
65-
`https://huggingface.co/api/models/${sanitizedInput}/refs`,
66-
);
67-
const data = await response.json();
68-
const branches: string[] = data.branches.map((branch: any) => {
69-
return branch.name;
70-
});
71-
72-
return branches;
73-
} catch (err) {
74-
return undefined;
75-
}
76-
}
77-
78-
private async versionInquiry(tags: string[]): Promise<string> {
79-
const { tag } = await this.inquirerService.inquirer.prompt({
80-
type: 'list',
81-
name: 'tag',
82-
message: 'Select version',
83-
choices: tags,
84-
});
85-
86-
return tag;
87-
}
88-
89-
private async handleJanHqModel(repoName: string, branches: string[]) {
90-
let selectedTag = branches[0];
91-
92-
if (branches.length > 1) {
93-
selectedTag = await this.versionInquiry(branches);
94-
}
95-
96-
const revision = selectedTag;
97-
if (!revision) {
98-
console.error("Can't find model revision.");
99-
exit(1);
100-
}
101-
// Return parsed model Id
102-
return `${repoName}:${revision}`;
103-
}
10432
}

‎cortex-js/src/infrastructure/commanders/models/model-start.command.ts‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
import { exit } from 'node:process';
88
import { ModelsCliUsecases } from '../usecases/models.cli.usecases';
99
import { CortexUsecases } from '@/usecases/cortex/cortex.usecases';
10+
import { isLocalModel } from '../utils/normalize-model-id';
1011

1112
type ModelStartOptions = {
1213
attach: boolean;
@@ -52,9 +53,7 @@ export class ModelStartCommand extends CommandRunner {
5253

5354
modelInquiry = async () => {
5455
const models = (await this.modelsCliUsecases.listAllModels()).filter(
55-
(model) =>
56-
Array.isArray(model.files) &&
57-
!/^(http|https):\/\/[^/]+\/.*/.test(model.files[0]),
56+
(model) => isLocalModel(model.files),
5857
);
5958
if (!models.length) throw 'No models found';
6059
const { model } = await this.inquirerService.inquirer.prompt({

‎cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import { exit } from 'node:process';
99
import { ChatCliUsecases } from '../usecases/chat.cli.usecases';
1010
import { defaultCortexCppHost, defaultCortexCppPort } from 'constant';
1111
import { ModelsCliUsecases } from '../usecases/models.cli.usecases';
12+
import { isLocalModel } from '../utils/normalize-model-id';
13+
import { ModelNotFoundException } from '@/infrastructure/exception/model-not-found.exception';
1214

1315
type RunOptions = {
1416
threadId?: string;
@@ -17,7 +19,7 @@ type RunOptions = {
1719

1820
@SubCommand({
1921
name: 'run',
20-
description: 'EXPERIMENTAL: Shortcut to start a model and chat',
22+
description: 'Shortcut to start a model and chat',
2123
})
2224
export class RunCommand extends CommandRunner {
2325
constructor(
@@ -40,6 +42,18 @@ export class RunCommand extends CommandRunner {
4042
}
4143
}
4244

45+
// If not exist
46+
// Try Pull
47+
if (!(await this.modelsCliUsecases.getModel(modelId))) {
48+
console.log(`Model ${modelId} not found. Try pulling model...`);
49+
await this.modelsCliUsecases.pullModel(modelId).catch((e: Error) => {
50+
if (e instanceof ModelNotFoundException)
51+
console.error('Model does not exist.');
52+
else console.error(e);
53+
exit(1);
54+
});
55+
}
56+
4357
return this.cortexUsecases
4458
.startCortex(false, defaultCortexCppHost, defaultCortexCppPort)
4559
.then(() => this.modelsCliUsecases.startModel(modelId, options.preset))
@@ -64,9 +78,7 @@ export class RunCommand extends CommandRunner {
6478

6579
modelInquiry = async () => {
6680
const models = (await this.modelsCliUsecases.listAllModels()).filter(
67-
(model) =>
68-
Array.isArray(model.files) &&
69-
!/^(http|https):\/\/[^/]+\/.*/.test(model.files[0]),
81+
(model) => isLocalModel(model.files),
7082
);
7183
if (!models.length) throw 'No models found';
7284
const { model } = await this.inquirerService.inquirer.prompt({

‎cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,30 @@ export class ChatCliUsecases {
221221
}
222222
}
223223

224+
/**
225+
* Creates an embedding vector representing the input text.
226+
* @param model Embedding model ID.
227+
* @param input Input text to embed, encoded as a string or array of tokens. To embed multiple inputs in a single request, pass an array of strings or array of token arrays.
228+
* @param encoding_format Encoding format for the embeddings. Supported formats are 'float' and 'int'.
229+
* @param dimensions The number of dimensions the resulting output embeddings should have. Only supported in some models.
230+
* @param host Cortex CPP host.
231+
* @param port Cortex CPP port.
232+
* @returns Embedding vector.
233+
*/
234+
embeddings(
235+
model: string,
236+
input: string | string[],
237+
encoding_format: string = 'float',
238+
dimensions?: number,
239+
) {
240+
return this.chatUsecases.embeddings({
241+
model,
242+
input,
243+
encoding_format,
244+
dimensions,
245+
});
246+
}
247+
224248
private async getOrCreateNewThread(
225249
modelId: string,
226250
threadId?: string,

‎cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,10 @@ import { FileManagerService } from '@/file-manager/file-manager.service';
1111
import { rm } from 'fs/promises';
1212
import { exec } from 'child_process';
1313
import { appPath } from '../utils/app-path';
14+
import { CORTEX_RELEASES_URL, CUDA_DOWNLOAD_URL } from '../../constants/cortex';
1415

1516
@Injectable()
1617
export class InitCliUsecases {
17-
private readonly CORTEX_RELEASES_URL =
18-
'https://api.github.com/repos/janhq/cortex/releases';
19-
private readonly CUDA_DOWNLOAD_URL =
20-
'https://catalog.jan.ai/dist/cuda-dependencies/<version>/<platform>/cuda.tar.gz';
21-
2218
constructor(
2319
private readonly httpService: HttpService,
2420
private readonly fileManagerService: FileManagerService,
@@ -30,7 +26,7 @@ export class InitCliUsecases {
3026
): Promise<any> => {
3127
const res = await firstValueFrom(
3228
this.httpService.get(
33-
this.CORTEX_RELEASES_URL + `${version === 'latest' ? '/latest' : ''}`,
29+
CORTEX_RELEASES_URL + `${version === 'latest' ? '/latest' : ''}`,
3430
{
3531
headers: {
3632
'X-GitHub-Api-Version': '2022-11-28',
@@ -182,7 +178,7 @@ export class InitCliUsecases {
182178
const platform = process.platform === 'win32' ? 'windows' : 'linux';
183179

184180
const dataFolderPath = await this.fileManagerService.getDataFolderPath();
185-
const url = this.CUDA_DOWNLOAD_URL.replace(
181+
const url = CUDA_DOWNLOAD_URL.replace(
186182
'<version>',
187183
options.cudaVersion === '11' ? '11.7' : '12.0',
188184
).replace('<platform>', platform);

0 commit comments

Comments
 (0)