diff --git a/packages/components/nodes/vectorstores/OpenSearch_Existing/OpenSearch_existing.ts b/packages/components/nodes/vectorstores/OpenSearch_Existing/OpenSearch_existing.ts new file mode 100644 index 00000000000..7aeac91928c --- /dev/null +++ b/packages/components/nodes/vectorstores/OpenSearch_Existing/OpenSearch_existing.ts @@ -0,0 +1,95 @@ +import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { OpenSearchVectorStore } from 'langchain/vectorstores/opensearch' +import { Embeddings } from 'langchain/embeddings/base' +import { Client } from '@opensearch-project/opensearch' +import { getBaseClasses } from '../../../src/utils' + +class OpenSearch_Existing_VectorStores implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'OpenSearch Load Existing Index' + this.name = 'openSearchExistingIndex' + this.type = 'OpenSearch' + this.icon = 'opensearch.png' + this.category = 'Vector Stores' + this.description = 'Load existing index from OpenSearch (i.e: Document has been upserted)' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.inputs = [ + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'OpenSearch URL', + name: 'opensearchURL', + type: 'string', + placeholder: 'http://127.0.0.1:9200' + }, + { + label: 'Index Name', + name: 'indexName', + type: 'string' + }, + { + label: 'Top K', + name: 'topK', + description: 'Number of top results to fetch. Default to 4', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'OpenSearch Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'OpenSearch Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(OpenSearchVectorStore)] + } + ] + } + + async init(nodeData: INodeData): Promise { + const embeddings = nodeData.inputs?.embeddings as Embeddings + const opensearchURL = nodeData.inputs?.opensearchURL as string + const indexName = nodeData.inputs?.indexName as string + const output = nodeData.outputs?.output as string + const topK = nodeData.inputs?.topK as string + const k = topK ? parseInt(topK, 10) : 4 + + const client = new Client({ + nodes: [opensearchURL] + }) + + const vectorStore = new OpenSearchVectorStore(embeddings, { + client, + indexName + }) + + if (output === 'retriever') { + const retriever = vectorStore.asRetriever(k) + return retriever + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } +} + +module.exports = { nodeClass: OpenSearch_Existing_VectorStores } diff --git a/packages/components/nodes/vectorstores/OpenSearch_Existing/opensearch.png b/packages/components/nodes/vectorstores/OpenSearch_Existing/opensearch.png new file mode 100644 index 00000000000..3fdcfd3f09e Binary files /dev/null and b/packages/components/nodes/vectorstores/OpenSearch_Existing/opensearch.png differ diff --git a/packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts b/packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts new file mode 100644 index 00000000000..8d54392b0e7 --- /dev/null +++ b/packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts @@ -0,0 +1,110 @@ +import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { OpenSearchVectorStore } from 'langchain/vectorstores/opensearch' +import { Embeddings } from 'langchain/embeddings/base' +import { Document } from 'langchain/document' +import { Client } from '@opensearch-project/opensearch' +import { flatten } from 'lodash' +import { getBaseClasses } from '../../../src/utils' + +class OpenSearchUpsert_VectorStores implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'OpenSearch Upsert Document' + this.name = 'openSearchUpsertDocument' + this.type = 'OpenSearch' + this.icon = 'opensearch.png' + this.category = 'Vector Stores' + this.description = 'Upsert documents to OpenSearch' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.inputs = [ + { + label: 'Document', + name: 'document', + type: 'Document', + list: true + }, + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'OpenSearch URL', + name: 'opensearchURL', + type: 'string', + placeholder: 'http://127.0.0.1:9200' + }, + { + label: 'Index Name', + name: 'indexName', + type: 'string' + }, + { + label: 'Top K', + name: 'topK', + description: 'Number of top results to fetch. Default to 4', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'OpenSearch Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'OpenSearch Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(OpenSearchVectorStore)] + } + ] + } + + async init(nodeData: INodeData): Promise { + const docs = nodeData.inputs?.document as Document[] + const embeddings = nodeData.inputs?.embeddings as Embeddings + const opensearchURL = nodeData.inputs?.opensearchURL as string + const indexName = nodeData.inputs?.indexName as string + const output = nodeData.outputs?.output as string + const topK = nodeData.inputs?.topK as string + const k = topK ? parseInt(topK, 10) : 4 + + const flattenDocs = docs && docs.length ? flatten(docs) : [] + const finalDocs = [] + for (let i = 0; i < flattenDocs.length; i += 1) { + finalDocs.push(new Document(flattenDocs[i])) + } + + const client = new Client({ + nodes: [opensearchURL] + }) + + const vectorStore = await OpenSearchVectorStore.fromDocuments(finalDocs, embeddings, { + client, + indexName: indexName + }) + + if (output === 'retriever') { + const retriever = vectorStore.asRetriever(k) + return retriever + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } +} + +module.exports = { nodeClass: OpenSearchUpsert_VectorStores } diff --git a/packages/components/nodes/vectorstores/OpenSearch_Upsert/opensearch.png b/packages/components/nodes/vectorstores/OpenSearch_Upsert/opensearch.png new file mode 100644 index 00000000000..3fdcfd3f09e Binary files /dev/null and b/packages/components/nodes/vectorstores/OpenSearch_Upsert/opensearch.png differ diff --git a/packages/components/package.json b/packages/components/package.json index df0589e7582..23b68ddb9c0 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -20,6 +20,7 @@ "@dqbd/tiktoken": "^1.0.7", "@getzep/zep-js": "^0.3.1", "@huggingface/inference": "^2.6.1", + "@opensearch-project/opensearch": "^1.2.0", "@pinecone-database/pinecone": "^0.0.12", "@qdrant/js-client-rest": "^1.2.2", "@supabase/supabase-js": "^2.21.0",