-
Notifications
You must be signed in to change notification settings - Fork 2.1k
/
html_to_text.ts
44 lines (41 loc) · 1.25 KB
/
html_to_text.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
import { htmlToText, type HtmlToTextOptions } from "html-to-text";
import {
MappingDocumentTransformer,
Document,
} from "@langchain/core/documents";
/**
* A transformer that converts HTML content to plain text.
* @example
* ```typescript
* const loader = new CheerioWebBaseLoader("https://example.com/some-page");
* const docs = await loader.load();
*
* const splitter = new RecursiveCharacterTextSplitter({
* maxCharacterCount: 1000,
* });
* const transformer = new HtmlToTextTransformer();
*
* // The sequence of text splitting followed by HTML to text transformation
* const sequence = splitter.pipe(transformer);
*
* // Processing the loaded documents through the sequence
* const newDocuments = await sequence.invoke(docs);
*
* console.log(newDocuments);
* ```
*/
export class HtmlToTextTransformer extends MappingDocumentTransformer {
static lc_name() {
return "HtmlToTextTransformer";
}
constructor(protected options: HtmlToTextOptions = {}) {
super(options);
}
async _transformDocument(document: Document): Promise<Document> {
const extractedContent = htmlToText(document.pageContent, this.options);
return new Document({
pageContent: extractedContent,
metadata: { ...document.metadata },
});
}
}