diff --git a/.changeset/curly-shoes-give.md b/.changeset/curly-shoes-give.md new file mode 100644 index 000000000..ee12ceb40 --- /dev/null +++ b/.changeset/curly-shoes-give.md @@ -0,0 +1,5 @@ +--- +"llamaindex": patch +--- + +feat: support jina ai embedding and reranker diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 334846656..bd7214ba4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [18.x, 20.x, 21.x] + node-version: [18.x, 20.x, 22.x] name: E2E on Node.js ${{ matrix.node-version }} runs-on: ubuntu-latest steps: @@ -26,7 +26,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version-file: ".nvmrc" + node-version: ${{ matrix.node-version }} cache: "pnpm" - name: Install dependencies run: pnpm install @@ -37,7 +37,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [18.x, 20.x, 21.x] + node-version: [18.x, 20.x, 22.x] name: Test on Node.js ${{ matrix.node-version }} runs-on: ubuntu-latest diff --git a/README.md b/README.md index ee04959a7..4cce2c051 100644 --- a/README.md +++ b/README.md @@ -114,14 +114,21 @@ Add the following config to your `next.config.js` to ignore specific packages in /** @type {import('next').NextConfig} */ const nextConfig = { experimental: { - serverComponentsExternalPackages: ["pdf2json", "@zilliz/milvus2-sdk-node"], + serverComponentsExternalPackages: [ + "pdf2json", + "@zilliz/milvus2-sdk-node", + "sharp", + "onnxruntime-node", + ], }, webpack: (config) => { - config.resolve.alias = { - ...config.resolve.alias, - sharp$: false, - "onnxruntime-node$": false, - }; + config.externals.push({ + pdf2json: "commonjs pdf2json", + "@zilliz/milvus2-sdk-node": "commonjs @zilliz/milvus2-sdk-node", + sharp: "commonjs sharp", + "onnxruntime-node": "commonjs onnxruntime-node", + }); + return config; }, }; diff --git a/apps/docs/docs/modules/embeddings/available_embeddings/jinaai.md b/apps/docs/docs/modules/embeddings/available_embeddings/jinaai.md new file mode 100644 index 000000000..d6d52263e --- /dev/null +++ b/apps/docs/docs/modules/embeddings/available_embeddings/jinaai.md @@ -0,0 +1,21 @@ +# Jina AI + +To use Jina AI embeddings, you need to import `JinaAIEmbedding` from `llamaindex`. + +```ts +import { JinaAIEmbedding, Settings } from "llamaindex"; + +Settings.embedModel = new JinaAIEmbedding(); + +const document = new Document({ text: essay, id_: "essay" }); + +const index = await VectorStoreIndex.fromDocuments([document]); + +const queryEngine = index.asQueryEngine(); + +const query = "What is the meaning of life?"; + +const results = await queryEngine.query({ + query, +}); +``` diff --git a/apps/docs/docs/modules/node_postprocessors/jinaai_reranker.md b/apps/docs/docs/modules/node_postprocessors/jinaai_reranker.md new file mode 100644 index 000000000..902c77b76 --- /dev/null +++ b/apps/docs/docs/modules/node_postprocessors/jinaai_reranker.md @@ -0,0 +1,71 @@ +# Jina AI Reranker + +The Jina AI Reranker is a postprocessor that uses the Jina AI Reranker API to rerank the results of a search query. + +## Setup + +Firstly, you will need to install the `llamaindex` package. + +```bash +pnpm install llamaindex +``` + +Now, you will need to sign up for an API key at [Jina AI](https://jina.ai/reranker). Once you have your API key you can import the necessary modules and create a new instance of the `JinaAIReranker` class. + +```ts +import { + JinaAIReranker, + Document, + OpenAI, + VectorStoreIndex, + Settings, +} from "llamaindex"; +``` + +## Load and index documents + +For this example, we will use a single document. In a real-world scenario, you would have multiple documents to index. + +```ts +const document = new Document({ text: essay, id_: "essay" }); + +Settings.llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.1 }); + +const index = await VectorStoreIndex.fromDocuments([document]); +``` + +## Increase similarity topK to retrieve more results + +The default value for `similarityTopK` is 2. This means that only the most similar document will be returned. To retrieve more results, you can increase the value of `similarityTopK`. + +```ts +const retriever = index.asRetriever(); +retriever.similarityTopK = 5; +``` + +## Create a new instance of the JinaAIReranker class + +Then you can create a new instance of the `JinaAIReranker` class and pass in the number of results you want to return. +The Jina AI Reranker API key is set in the `JINAAI_API_KEY` environment variable. + +```bash +export JINAAI_API_KEY= +``` + +```ts +const nodePostprocessor = new JinaAIReranker({ + topN: 5, +}); +``` + +## Create a query engine with the retriever and node postprocessor + +```ts +const queryEngine = index.asQueryEngine({ + retriever, + nodePostprocessors: [nodePostprocessor], +}); + +// log the response +const response = await queryEngine.query("Where did the author grown up?"); +``` diff --git a/apps/docs/docusaurus.config.js b/apps/docs/docusaurus.config.js index c42794dce..c1766686d 100644 --- a/apps/docs/docusaurus.config.js +++ b/apps/docs/docusaurus.config.js @@ -163,7 +163,7 @@ const config = { "docusaurus-plugin-typedoc", { entryPoints: ["../../packages/core/src/index.ts"], - tsconfig: "../../packages/core/tsconfig.json", + tsconfig: "../../tsconfig.json", readme: "none", sourceLinkTemplate: "https://github.com/run-llama/LlamaIndexTS/blob/{gitRevision}/{path}#L{line}", diff --git a/apps/docs/package.json b/apps/docs/package.json index 0f888034e..9bf8bfc4c 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -20,6 +20,7 @@ "@llamaindex/examples": "workspace:*", "@mdx-js/react": "^3.0.1", "clsx": "^2.1.0", + "llamaindex": "workspace:*", "postcss": "^8.4.38", "prism-react-renderer": "^2.3.1", "raw-loader": "^4.0.2", diff --git a/examples/multimodal/load.ts b/examples/multimodal/load.ts index 15c845b8f..3ed94e30b 100644 --- a/examples/multimodal/load.ts +++ b/examples/multimodal/load.ts @@ -4,7 +4,6 @@ import { VectorStoreIndex, storageContextFromDefaults, } from "llamaindex"; -import { DocStoreStrategy } from "llamaindex/ingestion/strategies/index"; import * as path from "path"; @@ -32,7 +31,6 @@ async function generateDatasource() { }); await VectorStoreIndex.fromDocuments(documents, { storageContext, - docStoreStrategy: DocStoreStrategy.NONE, }); }); console.log(`Storage successfully generated in ${ms / 1000}s.`); diff --git a/examples/readers/package.json b/examples/readers/package.json index 8202eb59c..3889180ed 100644 --- a/examples/readers/package.json +++ b/examples/readers/package.json @@ -3,20 +3,21 @@ "private": true, "type": "module", "scripts": { - "start": "node --loader ts-node/esm ./src/simple-directory-reader.ts", - "start:csv": "node --loader ts-node/esm ./src/csv.ts", - "start:docx": "node --loader ts-node/esm ./src/docx.ts", - "start:html": "node --loader ts-node/esm ./src/html.ts", - "start:markdown": "node --loader ts-node/esm ./src/markdown.ts", - "start:pdf": "node --loader ts-node/esm ./src/pdf.ts", - "start:llamaparse": "node --loader ts-node/esm ./src/llamaparse.ts" + "start": "node --import tsx ./src/simple-directory-reader.ts", + "start:csv": "node --import tsx ./src/csv.ts", + "start:docx": "node --import tsx ./src/docx.ts", + "start:html": "node --import tsx ./src/html.ts", + "start:markdown": "node --import tsx ./src/markdown.ts", + "start:pdf": "node --import tsx ./src/pdf.ts", + "start:llamaparse": "node --import tsx ./src/llamaparse.ts", + "start:notion": "node --import tsx ./src/notion.ts" }, "dependencies": { "llamaindex": "*" }, "devDependencies": { "@types/node": "^20.12.7", - "ts-node": "^10.9.2", - "typescript": "^5.4.3" + "tsx": "^4.7.2", + "typescript": "^5.4.5" } } diff --git a/examples/readers/src/notion.ts b/examples/readers/src/notion.ts index d6450af4b..439e92700 100644 --- a/examples/readers/src/notion.ts +++ b/examples/readers/src/notion.ts @@ -7,7 +7,7 @@ import { createInterface } from "node:readline/promises"; program .argument("[page]", "Notion page id (must be provided)") - .action(async (page, _options, command) => { + .action(async (page, _options) => { // Initializing a client if (!process.env.NOTION_TOKEN) { @@ -55,7 +55,7 @@ program .filter((page) => page !== null); console.log("Found pages:"); console.table(pages); - console.log(`To run, run ts-node ${command.name()} [page id]`); + console.log(`To run, run with [page id]`); return; } } diff --git a/packages/core/package.json b/packages/core/package.json index 18409fbab..a38c95727 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -12,7 +12,6 @@ "@llamaindex/cloud": "0.0.5", "@llamaindex/env": "workspace:*", "@mistralai/mistralai": "^0.1.3", - "@notionhq/client": "^2.2.15", "@pinecone-database/pinecone": "^2.2.0", "@qdrant/js-client-rest": "^1.8.2", "@types/lodash": "^4.17.0", @@ -31,7 +30,7 @@ "mammoth": "^1.7.1", "md-utils-ts": "^2.0.0", "mongodb": "^6.5.0", - "notion-md-crawler": "^0.0.2", + "notion-md-crawler": "^1.0.0", "ollama": "^0.5.0", "openai": "^4.38.0", "papaparse": "^5.4.1", @@ -45,7 +44,11 @@ "wikipedia": "^2.1.2", "wink-nlp": "^1.14.3" }, + "peerDependencies": { + "@notionhq/client": "^2.2.15" + }, "devDependencies": { + "@notionhq/client": "^2.2.15", "@swc/cli": "^0.3.12", "@swc/core": "^1.4.16", "concurrently": "^8.2.2", diff --git a/packages/core/src/Node.ts b/packages/core/src/Node.ts index bb7e43c69..8d8d045ab 100644 --- a/packages/core/src/Node.ts +++ b/packages/core/src/Node.ts @@ -326,6 +326,37 @@ export class ImageNode extends TextNode { const absPath = path.resolve(this.id_); return new URL(`file://${absPath}`); } + + // Calculates the image part of the hash + private generateImageHash() { + const hashFunction = createSHA256(); + + if (this.image instanceof Blob) { + // TODO: ideally we should use the blob's content to calculate the hash: + // hashFunction.update(new Uint8Array(await this.image.arrayBuffer())); + // as this is async, we're using the node's ID for the time being + hashFunction.update(this.id_); + } else if (this.image instanceof URL) { + hashFunction.update(this.image.toString()); + } else if (typeof this.image === "string") { + hashFunction.update(this.image); + } else { + throw new Error( + `Unknown image type: ${typeof this.image}. Can't calculate hash`, + ); + } + + return hashFunction.digest(); + } + + generateHash() { + const hashFunction = createSHA256(); + // calculates hash based on hash of both components (image and text) + hashFunction.update(super.generateHash()); + hashFunction.update(this.generateImageHash()); + + return hashFunction.digest(); + } } export class ImageDocument extends ImageNode { diff --git a/packages/core/src/embeddings/JinaAIEmbedding.ts b/packages/core/src/embeddings/JinaAIEmbedding.ts new file mode 100644 index 000000000..9e2b95f5e --- /dev/null +++ b/packages/core/src/embeddings/JinaAIEmbedding.ts @@ -0,0 +1,29 @@ +import { getEnv } from "@llamaindex/env"; +import { OpenAIEmbedding } from "./OpenAIEmbedding.js"; + +export class JinaAIEmbedding extends OpenAIEmbedding { + constructor(init?: Partial) { + const { + apiKey = getEnv("JINAAI_API_KEY"), + additionalSessionOptions = {}, + model = "jina-embeddings-v2-base-en", + ...rest + } = init ?? {}; + + if (!apiKey) { + throw new Error( + "Set Jina AI API Key in JINAAI_API_KEY env variable. Get one for free or top up your key at https://jina.ai/embeddings", + ); + } + + additionalSessionOptions.baseURL = + additionalSessionOptions.baseURL ?? "https://api.jina.ai/v1"; + + super({ + apiKey, + additionalSessionOptions, + model, + ...rest, + }); + } +} diff --git a/packages/core/src/embeddings/index.ts b/packages/core/src/embeddings/index.ts index 005f1aaa3..5e56493a6 100644 --- a/packages/core/src/embeddings/index.ts +++ b/packages/core/src/embeddings/index.ts @@ -1,5 +1,6 @@ export * from "./ClipEmbedding.js"; export * from "./HuggingFaceEmbedding.js"; +export * from "./JinaAIEmbedding.js"; export * from "./MistralAIEmbedding.js"; export * from "./MultiModalEmbedding.js"; export { OllamaEmbedding } from "./OllamaEmbedding.js"; diff --git a/packages/core/src/ingestion/IngestionPipeline.ts b/packages/core/src/ingestion/IngestionPipeline.ts index 775f17123..d93867ce2 100644 --- a/packages/core/src/ingestion/IngestionPipeline.ts +++ b/packages/core/src/ingestion/IngestionPipeline.ts @@ -94,20 +94,20 @@ export class IngestionPipeline { documents?: Document[], nodes?: BaseNode[], ): Promise { - const inputNodes: BaseNode[] = []; + const inputNodes: BaseNode[][] = []; if (documents) { - inputNodes.push(...documents); + inputNodes.push(documents); } if (nodes) { - inputNodes.push(...nodes); + inputNodes.push(nodes); } if (this.documents) { - inputNodes.push(...this.documents); + inputNodes.push(this.documents); } if (this.reader) { - inputNodes.push(...(await this.reader.loadData())); + inputNodes.push(await this.reader.loadData()); } - return inputNodes; + return inputNodes.flat(); } async run( diff --git a/packages/core/src/postprocessors/rerankers/JinaAIReranker.ts b/packages/core/src/postprocessors/rerankers/JinaAIReranker.ts new file mode 100644 index 000000000..68bb8af00 --- /dev/null +++ b/packages/core/src/postprocessors/rerankers/JinaAIReranker.ts @@ -0,0 +1,89 @@ +import { getEnv } from "@llamaindex/env"; +import type { NodeWithScore } from "../../Node.js"; +import { MetadataMode } from "../../Node.js"; +import type { BaseNodePostprocessor } from "../types.js"; + +interface JinaAIRerankerResult { + index: number; + document?: { + text?: string; + }; + relevance_score: number; +} + +export class JinaAIReranker implements BaseNodePostprocessor { + model: string = "jina-reranker-v1-base-en"; + topN?: number; + apiKey?: string = undefined; + + constructor(init?: Partial) { + this.topN = init?.topN ?? 2; + this.model = init?.model ?? "jina-reranker-v1-base-en"; + this.apiKey = getEnv("JINAAI_API_KEY"); + + if (!this.apiKey) { + throw new Error( + "Set Jina AI API Key in JINAAI_API_KEY env variable. Get one for free or top up your key at https://jina.ai/reranker", + ); + } + } + + async rerank( + query: string, + documents: string[], + topN: number | undefined = this.topN, + ): Promise { + const url = "https://api.jina.ai/v1/rerank"; + const headers = { + "Content-Type": "application/json", + Authorization: `Bearer ${this.apiKey}`, + }; + const data = { + model: this.model, + query: query, + documents: documents, + top_n: topN, + }; + + try { + const response = await fetch(url, { + method: "POST", + headers: headers, + body: JSON.stringify(data), + }); + const jsonData = await response.json(); + + return jsonData.results; + } catch (error) { + console.error("Error while reranking:", error); + throw new Error("Failed to rerank documents due to an API error"); + } + } + + async postprocessNodes( + nodes: NodeWithScore[], + query?: string, + ): Promise { + if (nodes.length === 0) { + return []; + } + + if (query === undefined) { + throw new Error("JinaAIReranker requires a query"); + } + + const documents = nodes.map((n) => n.node.getContent(MetadataMode.ALL)); + const results = await this.rerank(query, documents, this.topN); + const newNodes: NodeWithScore[] = []; + + for (const result of results) { + const node = nodes[result.index]; + newNodes.push({ + node: node.node, + score: result.relevance_score, + }); + } + + return newNodes; + } +} diff --git a/packages/core/src/postprocessors/rerankers/index.ts b/packages/core/src/postprocessors/rerankers/index.ts index d0e25d182..20f48359f 100644 --- a/packages/core/src/postprocessors/rerankers/index.ts +++ b/packages/core/src/postprocessors/rerankers/index.ts @@ -1 +1,2 @@ export * from "./CohereRerank.js"; +export * from "./JinaAIReranker.js"; diff --git a/packages/core/src/readers/NotionReader.ts b/packages/core/src/readers/NotionReader.ts index bbac5d08f..4c645e7c2 100644 --- a/packages/core/src/readers/NotionReader.ts +++ b/packages/core/src/readers/NotionReader.ts @@ -1,21 +1,9 @@ -import type { Client } from "@notionhq/client"; -import type { Crawler, Pages } from "notion-md-crawler"; +import type { Crawler, CrawlerOptions, Page } from "notion-md-crawler"; import { crawler, pageToString } from "notion-md-crawler"; import { Document } from "../Node.js"; import type { BaseReader } from "./type.js"; -type OptionalSerializers = Parameters[number]["serializers"]; - -/** - * Options for initializing the NotionReader class - * @typedef {Object} NotionReaderOptions - * @property {Client} client - The Notion Client object for API interactions - * @property {OptionalSerializers} [serializers] - Option to customize serialization. See [the url](https://github.com/TomPenguin/notion-md-crawler/tree/main) for details. - */ -type NotionReaderOptions = { - client: Client; - serializers?: OptionalSerializers; -}; +type NotionReaderOptions = Pick; /** * Notion pages are retrieved recursively and converted to Document objects. @@ -25,7 +13,7 @@ type NotionReaderOptions = { * Please refer to [this document](https://www.notion.so/help/create-integrations-with-the-notion-api) for details. */ export class NotionReader implements BaseReader { - private crawl: ReturnType; + private readonly crawl: ReturnType; /** * Constructor for the NotionReader class @@ -37,10 +25,10 @@ export class NotionReader implements BaseReader { /** * Converts Pages to an array of Document objects - * @param {Pages} pages - The Notion pages to convert (Return value of `loadPages`) + * @param {Page} pages - The Notion pages to convert (Return value of `loadPages`) * @returns {Document[]} An array of Document objects */ - toDocuments(pages: Pages): Document[] { + toDocuments(pages: Page[]): Document[] { return Object.values(pages).map((page) => { const text = pageToString(page); return new Document({ @@ -54,10 +42,21 @@ export class NotionReader implements BaseReader { /** * Loads recursively the Notion page with the specified root page ID. * @param {string} rootPageId - The root Notion page ID - * @returns {Promise} A Promise that resolves to a Pages object(Convertible with the `toDocuments` method) + * @returns {Promise} A Promise that resolves to a Pages object(Convertible with the `toDocuments` method) */ - async loadPages(rootPageId: string): Promise { - return this.crawl(rootPageId); + async loadPages(rootPageId: string): Promise { + const iter = this.crawl(rootPageId); + const pages: Page[] = []; + for await (const result of iter) { + if (result.success) { + pages.push(result.page); + } else { + console.error( + `Failed to load page (${result.failure.parentId}): ${result.failure.reason}`, + ); + } + } + return pages; } /** diff --git a/packages/core/src/readers/SimpleDirectoryReader.edge.ts b/packages/core/src/readers/SimpleDirectoryReader.edge.ts index face1fdd3..70123b153 100644 --- a/packages/core/src/readers/SimpleDirectoryReader.edge.ts +++ b/packages/core/src/readers/SimpleDirectoryReader.edge.ts @@ -1,6 +1,6 @@ import type { CompleteFileSystem } from "@llamaindex/env"; import { defaultFS, path } from "@llamaindex/env"; -import { Document } from "../Node.js"; +import { Document, type Metadata } from "../Node.js"; import { walk } from "../storage/FileSystem.js"; import { TextFileReader } from "./TextFileReader.js"; import type { BaseReader } from "./type.js"; @@ -89,6 +89,7 @@ export class SimpleDirectoryReader implements BaseReader { } const fileDocs = await reader.loadData(filePath, fs); + fileDocs.forEach(addMetaData(filePath)); // Observer can still cancel addition of the resulting docs from this file if (this.doObserverCheck("file", filePath, ReaderStatus.COMPLETE)) { @@ -124,3 +125,10 @@ export class SimpleDirectoryReader implements BaseReader { return true; } } + +function addMetaData(filePath: string): (doc: Document) => void { + return (doc: Document) => { + doc.metadata["file_path"] = path.resolve(filePath); + doc.metadata["file_name"] = path.basename(filePath); + }; +} diff --git a/packages/core/tests/Document.test.ts b/packages/core/tests/Document.test.ts index 73da731df..d4a6704a9 100644 --- a/packages/core/tests/Document.test.ts +++ b/packages/core/tests/Document.test.ts @@ -1,4 +1,4 @@ -import { Document } from "llamaindex/Node"; +import { Document, ImageDocument } from "llamaindex/Node"; import { describe, expect, test } from "vitest"; describe("Document", () => { @@ -6,4 +6,16 @@ describe("Document", () => { const doc = new Document({ text: "text", id_: "docId" }); expect(doc).toBeDefined(); }); + + test("should generate different hash for different image contents", () => { + const imageNode1 = new ImageDocument({ + id_: "image", + image: "data:image/png;base64,sample_image_content1", + }); + const imageNode2 = new ImageDocument({ + id_: "image", + image: "data:image/png;base64,sample_image_content2", + }); + expect(imageNode1.hash).not.toBe(imageNode2.hash); + }); }); diff --git a/packages/core/tests/ingestion/ingestion-pipeline.test.ts b/packages/core/tests/ingestion/ingestion-pipeline.test.ts new file mode 100644 index 000000000..746741803 --- /dev/null +++ b/packages/core/tests/ingestion/ingestion-pipeline.test.ts @@ -0,0 +1,22 @@ +import { Document } from "llamaindex/Node"; +import { IngestionPipeline } from "llamaindex/ingestion/IngestionPipeline"; +import { test } from "vitest"; + +// Refs: https://github.com/run-llama/LlamaIndexTS/pull/760 +test("load large data should not cause RangeError #760", async () => { + const pipeline = new IngestionPipeline({ + reader: { + loadData: async () => { + return Array.from( + { length: 1e6 }, + (_, i) => + new Document({ + id_: `${i}`, + text: "some text", + }), + ); + }, + }, + }); + await pipeline.prepareInput(); +}); diff --git a/packages/edge/package.json b/packages/edge/package.json index 274dc56e8..2915237c9 100644 --- a/packages/edge/package.json +++ b/packages/edge/package.json @@ -11,7 +11,6 @@ "@llamaindex/cloud": "0.0.5", "@llamaindex/env": "workspace:*", "@mistralai/mistralai": "^0.1.3", - "@notionhq/client": "^2.2.15", "@pinecone-database/pinecone": "^2.2.0", "@qdrant/js-client-rest": "^1.8.2", "@types/lodash": "^4.17.0", @@ -30,7 +29,7 @@ "mammoth": "^1.7.1", "md-utils-ts": "^2.0.0", "mongodb": "^6.5.0", - "notion-md-crawler": "^0.0.2", + "notion-md-crawler": "^1.0.0", "ollama": "^0.5.0", "openai": "^4.38.0", "papaparse": "^5.4.1", @@ -82,5 +81,17 @@ "update:deps": "node scripts/update-deps.js", "build:core": "pnpm --filter llamaindex build && cp -r ../core/dist . && rm -rf dist/cjs", "build": "pnpm run update:deps && pnpm run build:core && pnpm copy" + }, + "devDependencies": { + "@notionhq/client": "^2.2.15", + "@swc/cli": "^0.3.12", + "@swc/core": "^1.4.16", + "concurrently": "^8.2.2", + "glob": "^10.3.12", + "madge": "^7.0.0", + "typescript": "^5.4.5" + }, + "peerDependencies": { + "@notionhq/client": "^2.2.15" } } diff --git a/packages/edge/scripts/update-deps.js b/packages/edge/scripts/update-deps.js index f8c913d52..49d87278a 100644 --- a/packages/edge/scripts/update-deps.js +++ b/packages/edge/scripts/update-deps.js @@ -10,6 +10,8 @@ const edgePackagePath = path.join(process.cwd(), "package.json"); const edgePackage = readJson(edgePackagePath); const corePackage = readJson(corePackagePath); edgePackage.dependencies = corePackage.dependencies; +edgePackage.devDependencies = corePackage.devDependencies; +edgePackage.peerDependencies = corePackage.peerDependencies; edgePackage.version = corePackage.version; writeJson(edgePackagePath, edgePackage); execSync("pnpm install --lockfile-only", { stdio: "inherit" }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a0a8004b2..983587ebc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -70,6 +70,9 @@ importers: clsx: specifier: ^2.1.0 version: 2.1.0 + llamaindex: + specifier: workspace:* + version: link:../../packages/core postcss: specifier: ^8.4.38 version: 8.4.38 @@ -178,12 +181,12 @@ importers: '@types/node': specifier: ^20.12.7 version: 20.12.7 - ts-node: - specifier: ^10.9.2 - version: 10.9.2(@swc/core@1.4.16(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.3) + tsx: + specifier: ^4.7.2 + version: 4.7.2 typescript: - specifier: ^5.4.3 - version: 5.4.3 + specifier: ^5.4.5 + version: 5.4.5 packages/core: dependencies: @@ -208,9 +211,6 @@ importers: '@mistralai/mistralai': specifier: ^0.1.3 version: 0.1.3(encoding@0.1.13) - '@notionhq/client': - specifier: ^2.2.15 - version: 2.2.15(encoding@0.1.13) '@pinecone-database/pinecone': specifier: ^2.2.0 version: 2.2.0 @@ -266,8 +266,8 @@ importers: specifier: ^6.5.0 version: 6.5.0 notion-md-crawler: - specifier: ^0.0.2 - version: 0.0.2(encoding@0.1.13) + specifier: ^1.0.0 + version: 1.0.0(encoding@0.1.13) ollama: specifier: ^0.5.0 version: 0.5.0 @@ -305,6 +305,9 @@ importers: specifier: ^1.14.3 version: 1.14.3 devDependencies: + '@notionhq/client': + specifier: ^2.2.15 + version: 2.2.15(encoding@0.1.13) '@swc/cli': specifier: ^0.3.12 version: 0.3.12(@swc/core@1.4.16(@swc/helpers@0.5.2))(chokidar@3.6.0) @@ -371,9 +374,6 @@ importers: '@mistralai/mistralai': specifier: ^0.1.3 version: 0.1.3(encoding@0.1.13) - '@notionhq/client': - specifier: ^2.2.15 - version: 2.2.15(encoding@0.1.13) '@pinecone-database/pinecone': specifier: ^2.2.0 version: 2.2.0 @@ -429,8 +429,8 @@ importers: specifier: ^6.5.0 version: 6.5.0 notion-md-crawler: - specifier: ^0.0.2 - version: 0.0.2(encoding@0.1.13) + specifier: ^1.0.0 + version: 1.0.0(encoding@0.1.13) ollama: specifier: ^0.5.0 version: 0.5.0 @@ -467,6 +467,28 @@ importers: wink-nlp: specifier: ^1.14.3 version: 1.14.3 + devDependencies: + '@notionhq/client': + specifier: ^2.2.15 + version: 2.2.15(encoding@0.1.13) + '@swc/cli': + specifier: ^0.3.12 + version: 0.3.12(@swc/core@1.4.16(@swc/helpers@0.5.2))(chokidar@3.6.0) + '@swc/core': + specifier: ^1.4.16 + version: 1.4.16(@swc/helpers@0.5.2) + concurrently: + specifier: ^8.2.2 + version: 8.2.2 + glob: + specifier: ^10.3.12 + version: 10.3.12 + madge: + specifier: ^7.0.0 + version: 7.0.0(typescript@5.4.5) + typescript: + specifier: ^5.4.5 + version: 5.4.5 packages/edge/e2e/test-edge-runtime: dependencies: @@ -6125,8 +6147,8 @@ packages: resolution: {integrity: sha512-IO9QvjUMWxPQQhs60oOu10CRkWCiZzSUkzbXGGV9pviYl1fXYcvkzQ5jV9z8Y6un8ARoVRl4EtC6v6jNqbaJ/w==} engines: {node: '>=14.16'} - notion-md-crawler@0.0.2: - resolution: {integrity: sha512-lE3/DFMrg7GSbl1sBfDuLVLyxw+yjdarPVm1JGfQ6eONEbNGgO+BdZxpwwZQ1uYeEJurAXMXb/AXT8GKYjKAyg==} + notion-md-crawler@1.0.0: + resolution: {integrity: sha512-mdB6zn/i32qO2C7X7wZLDpWvFryO3bPYMuBfFgmTPomnfEtIejdQJNVaZzw2GapM82lfWZ5dfsZp3s3UL4p1Fg==} npm-run-path@2.0.2: resolution: {integrity: sha512-lJxZYlT4DW/bRUtFh1MQIWqmLwQfAxnqWG4HhEdjMlkrJYnJn0Jrr2u3mgxqaWsdiBc76TYkTG/mhrnYTuzfHw==} @@ -15765,7 +15787,7 @@ snapshots: normalize-url@8.0.1: {} - notion-md-crawler@0.0.2(encoding@0.1.13): + notion-md-crawler@1.0.0(encoding@0.1.13): dependencies: '@notionhq/client': 2.2.15(encoding@0.1.13) md-utils-ts: 2.0.0 @@ -17780,26 +17802,6 @@ snapshots: ts-graphviz@1.8.2: {} - ts-node@10.9.2(@swc/core@1.4.16(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.3): - dependencies: - '@cspotcode/source-map-support': 0.8.1 - '@tsconfig/node10': 1.0.9 - '@tsconfig/node12': 1.0.11 - '@tsconfig/node14': 1.0.3 - '@tsconfig/node16': 1.0.4 - '@types/node': 20.12.7 - acorn: 8.11.3 - acorn-walk: 8.3.2 - arg: 4.1.3 - create-require: 1.1.1 - diff: 4.0.2 - make-error: 1.3.6 - typescript: 5.4.3 - v8-compile-cache-lib: 3.0.1 - yn: 3.1.1 - optionalDependencies: - '@swc/core': 1.4.16(@swc/helpers@0.5.2) - ts-node@10.9.2(@swc/core@1.4.16(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5): dependencies: '@cspotcode/source-map-support': 0.8.1