-
Notifications
You must be signed in to change notification settings - Fork 1.9k
/
list.ts
197 lines (171 loc) Β· 5.94 KB
/
list.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
import { BaseMessage } from "../messages/index.js";
import { OutputParserException } from "./base.js";
import { BaseTransformOutputParser } from "./transform.js";
/**
* Class to parse the output of an LLM call to a list.
* @augments BaseOutputParser
*/
export abstract class ListOutputParser extends BaseTransformOutputParser<
string[]
> {
re?: RegExp;
async *_transform(
inputGenerator: AsyncGenerator<string | BaseMessage>
): AsyncGenerator<string[]> {
let buffer = "";
for await (const input of inputGenerator) {
if (typeof input === "string") {
// add current chunk to buffer
buffer += input;
} else {
// extract message content and add to buffer
buffer += input.content;
}
// get parts in buffer
if (!this.re) {
const parts = await this.parse(buffer);
if (parts.length > 1) {
// if there are multiple parts, yield all but the last one
for (const part of parts.slice(0, -1)) {
yield [part];
}
// keep the last part in the buffer
buffer = parts[parts.length - 1];
}
} else {
// if there is a regex, get all matches
const matches = [...buffer.matchAll(this.re)];
if (matches.length > 1) {
let doneIdx = 0;
// if there are multiple matches, yield all but the last one
for (const match of matches.slice(0, -1)) {
yield [match[1]];
doneIdx += (match.index ?? 0) + match[0].length;
}
// keep the last match in the buffer
buffer = buffer.slice(doneIdx);
}
}
}
// yield the last part
for (const part of await this.parse(buffer)) {
yield [part];
}
}
}
/**
* Class to parse the output of an LLM call as a comma-separated list.
* @augments ListOutputParser
*/
export class CommaSeparatedListOutputParser extends ListOutputParser {
static lc_name() {
return "CommaSeparatedListOutputParser";
}
lc_namespace = ["langchain_core", "output_parsers", "list"];
lc_serializable = true;
/**
* Parses the given text into an array of strings, using a comma as the
* separator. If the parsing fails, throws an OutputParserException.
* @param text The text to parse.
* @returns An array of strings obtained by splitting the input text at each comma.
*/
async parse(text: string): Promise<string[]> {
try {
return text
.trim()
.split(",")
.map((s) => s.trim());
} catch (e) {
throw new OutputParserException(`Could not parse output: ${text}`, text);
}
}
/**
* Provides instructions on the expected format of the response for the
* CommaSeparatedListOutputParser.
* @returns A string containing instructions on the expected format of the response.
*/
getFormatInstructions(): string {
return `Your response should be a list of comma separated values, eg: \`foo, bar, baz\``;
}
}
/**
* Class to parse the output of an LLM call to a list with a specific length and separator.
* @augments ListOutputParser
*/
export class CustomListOutputParser extends ListOutputParser {
lc_namespace = ["langchain_core", "output_parsers", "list"];
private length: number | undefined;
private separator: string;
constructor({ length, separator }: { length?: number; separator?: string }) {
super(...arguments);
this.length = length;
this.separator = separator || ",";
}
/**
* Parses the given text into an array of strings, using the specified
* separator. If the parsing fails or the number of items in the list
* doesn't match the expected length, throws an OutputParserException.
* @param text The text to parse.
* @returns An array of strings obtained by splitting the input text at each occurrence of the specified separator.
*/
async parse(text: string): Promise<string[]> {
try {
const items = text
.trim()
.split(this.separator)
.map((s) => s.trim());
if (this.length !== undefined && items.length !== this.length) {
throw new OutputParserException(
`Incorrect number of items. Expected ${this.length}, got ${items.length}.`
);
}
return items;
} catch (e) {
if (Object.getPrototypeOf(e) === OutputParserException.prototype) {
throw e;
}
throw new OutputParserException(`Could not parse output: ${text}`);
}
}
/**
* Provides instructions on the expected format of the response for the
* CustomListOutputParser, including the number of items and the
* separator.
* @returns A string containing instructions on the expected format of the response.
*/
getFormatInstructions(): string {
return `Your response should be a list of ${
this.length === undefined ? "" : `${this.length} `
}items separated by "${this.separator}" (eg: \`foo${this.separator} bar${
this.separator
} baz\`)`;
}
}
export class NumberedListOutputParser extends ListOutputParser {
static lc_name() {
return "NumberedListOutputParser";
}
lc_namespace = ["langchain_core", "output_parsers", "list"];
lc_serializable = true;
getFormatInstructions(): string {
return `Your response should be a numbered list with each item on a new line. For example: \n\n1. foo\n\n2. bar\n\n3. baz`;
}
re = /\d+\.\s([^\n]+)/g;
async parse(text: string): Promise<string[]> {
return [...(text.matchAll(this.re) ?? [])].map((m) => m[1]);
}
}
export class MarkdownListOutputParser extends ListOutputParser {
static lc_name() {
return "NumberedListOutputParser";
}
lc_namespace = ["langchain_core", "output_parsers", "list"];
lc_serializable = true;
getFormatInstructions(): string {
return `Your response should be a numbered list with each item on a new line. For example: \n\n1. foo\n\n2. bar\n\n3. baz`;
}
re = /^\s*[-*]\s([^\n]+)$/gm;
async parse(text: string): Promise<string[]> {
return [...(text.matchAll(this.re) ?? [])].map((m) => m[1]);
}
}