-
Notifications
You must be signed in to change notification settings - Fork 2.1k
/
llama_cpp.ts
121 lines (101 loc) Β· 3.24 KB
/
llama_cpp.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
import { LlamaModel, LlamaContext, LlamaChatSession } from "node-llama-cpp";
import {
LLM,
type BaseLLMCallOptions,
type BaseLLMParams,
} from "@langchain/core/language_models/llms";
import { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager";
import { GenerationChunk } from "@langchain/core/outputs";
import {
LlamaBaseCppInputs,
createLlamaModel,
createLlamaContext,
createLlamaSession,
} from "../utils/llama_cpp.js";
/**
* Note that the modelPath is the only required parameter. For testing you
* can set this in the environment variable `LLAMA_PATH`.
*/
export interface LlamaCppInputs extends LlamaBaseCppInputs, BaseLLMParams {}
export interface LlamaCppCallOptions extends BaseLLMCallOptions {
/** The maximum number of tokens the response should contain. */
maxTokens?: number;
/** A function called when matching the provided token array */
onToken?: (tokens: number[]) => void;
}
/**
* To use this model you need to have the `node-llama-cpp` module installed.
* This can be installed using `npm install -S node-llama-cpp` and the minimum
* version supported in version 2.0.0.
* This also requires that have a locally built version of Llama2 installed.
*/
export class LlamaCpp extends LLM<LlamaCppCallOptions> {
lc_serializable = true;
static inputs: LlamaCppInputs;
maxTokens?: number;
temperature?: number;
topK?: number;
topP?: number;
trimWhitespaceSuffix?: boolean;
_model: LlamaModel;
_context: LlamaContext;
_session: LlamaChatSession;
static lc_name() {
return "LlamaCpp";
}
constructor(inputs: LlamaCppInputs) {
super(inputs);
this.maxTokens = inputs?.maxTokens;
this.temperature = inputs?.temperature;
this.topK = inputs?.topK;
this.topP = inputs?.topP;
this.trimWhitespaceSuffix = inputs?.trimWhitespaceSuffix;
this._model = createLlamaModel(inputs);
this._context = createLlamaContext(this._model, inputs);
this._session = createLlamaSession(this._context);
}
_llmType() {
return "llama2_cpp";
}
/** @ignore */
async _call(
prompt: string,
options?: this["ParsedCallOptions"]
): Promise<string> {
try {
const promptOptions = {
onToken: options?.onToken,
maxTokens: this?.maxTokens,
temperature: this?.temperature,
topK: this?.topK,
topP: this?.topP,
trimWhitespaceSuffix: this?.trimWhitespaceSuffix,
};
const completion = await this._session.prompt(prompt, promptOptions);
return completion;
} catch (e) {
throw new Error("Error getting prompt completion.");
}
}
async *_streamResponseChunks(
prompt: string,
_options: this["ParsedCallOptions"],
runManager?: CallbackManagerForLLMRun
): AsyncGenerator<GenerationChunk> {
const promptOptions = {
temperature: this?.temperature,
topK: this?.topK,
topP: this?.topP,
};
const stream = await this.caller.call(async () =>
this._context.evaluate(this._context.encode(prompt), promptOptions)
);
for await (const chunk of stream) {
yield new GenerationChunk({
text: this._context.decode([chunk]),
generationInfo: {},
});
await runManager?.handleLLMNewToken(this._context.decode([chunk]) ?? "");
}
}
}