diff --git a/docs/core_docs/docs/modules/model_io/concepts.mdx b/docs/core_docs/docs/modules/model_io/concepts.mdx index 698b9e5ef83..a75af7ef0bf 100644 --- a/docs/core_docs/docs/modules/model_io/concepts.mdx +++ b/docs/core_docs/docs/modules/model_io/concepts.mdx @@ -106,4 +106,4 @@ There are a few parsers dedicated to working with OpenAI function calling. They ### Agent Output Parsers -[Agents](/docs/modules/agents/) are systems that use language models to determine what steps to take. The output of a language model therefore needs to be parsed into some schema that can represent what actions (if any) are to be taken. AgentOutputParsers are responsible for taking raw LLM or ChatModel output and converting it to that schema. The logic inside these output parsers can differ depending on the model and prompting strategy being used. +[Agents](../agents) are systems that use language models to determine what steps to take. The output of a language model therefore needs to be parsed into some schema that can represent what actions (if any) are to be taken. AgentOutputParsers are responsible for taking raw LLM or ChatModel output and converting it to that schema. The logic inside these output parsers can differ depending on the model and prompting strategy being used. diff --git a/docs/core_docs/docs/use_cases/api.mdx b/docs/core_docs/docs/use_cases/api.mdx index 3b3cc62a148..e3bbb6fc720 100644 --- a/docs/core_docs/docs/use_cases/api.mdx +++ b/docs/core_docs/docs/use_cases/api.mdx @@ -14,7 +14,7 @@ If you are just getting started and you have relatively simple APIs, you should Chains are a sequence of predetermined steps, so they are good to get started with as they give you more control and let you understand what is happening better. -- [OpenAPI Chain](/docs/modules/chains/additional/openai_functions/openapi) +- [OpenAPI Chain](../modules/chains/additional/openai_functions/openapi) ## Agents diff --git a/docs/core_docs/docs/use_cases/tabular.mdx b/docs/core_docs/docs/use_cases/tabular.mdx index 8bb3b4ac994..fd58093125c 100644 --- a/docs/core_docs/docs/use_cases/tabular.mdx +++ b/docs/core_docs/docs/use_cases/tabular.mdx @@ -14,7 +14,7 @@ If you are just getting started, and you have relatively small/simple tabular da Chains are a sequence of predetermined steps, so they are good to get started with as they give you more control and let you understand what is happening better. -- [SQL Database Chain](/docs/modules/chains/popular/sqlite) +- [SQL Database Chain](../modules/chains/popular/sqlite) ## Agents diff --git a/docs/core_docs/docusaurus.config.js b/docs/core_docs/docusaurus.config.js index 8e5d48adaa7..80ef40d2767 100644 --- a/docs/core_docs/docusaurus.config.js +++ b/docs/core_docs/docusaurus.config.js @@ -13,8 +13,6 @@ const mdxComponentsPath = path.resolve(__dirname, "docs", "mdx_components"); const baseLightCodeBlockTheme = require("prism-react-renderer/themes/vsLight"); const baseDarkCodeBlockTheme = require("prism-react-renderer/themes/vsDark"); -const baseUrl = "/v0.1/"; - /** @type {import('@docusaurus/types').Config} */ const config = { title: "🦜️🔗 Langchain", @@ -24,8 +22,7 @@ const config = { url: "https://js.langchain.com", // Set the // pathname under which your site is served // For GitHub pages deployment, it is often '//' - baseUrl: baseUrl, - trailingSlash: true, + baseUrl: "/", onBrokenLinks: "throw", onBrokenMarkdownLinks: "throw", @@ -133,12 +130,6 @@ const config = { themeConfig: /** @type {import('@docusaurus/preset-classic').ThemeConfig} */ ({ - // TODO(brace) -- Uncomment once 0.2 is merged to main & site is live! - // announcementBar: { - // content: - // 'LangChain v0.2 is coming soon! Preview the new docs here.', - // isCloseable: true, - // }, prism: { theme: { ...baseLightCodeBlockTheme, @@ -308,7 +299,7 @@ const config = { }), scripts: [ - baseUrl + "js/google_analytics.js", + "/js/google_analytics.js", { src: "https://www.googletagmanager.com/gtag/js?id=G-TVSL7JBE9Y", async: true, diff --git a/docs/core_docs/package.json b/docs/core_docs/package.json index 10d1e62392d..21fdf21005c 100644 --- a/docs/core_docs/package.json +++ b/docs/core_docs/package.json @@ -6,7 +6,7 @@ "docusaurus": "docusaurus", "start": "yarn build:typedoc && yarn quarto && rimraf ./docs/api && NODE_OPTIONS=--max-old-space-size=7168 docusaurus start", "build": "yarn clean && yarn build:typedoc && yarn quarto && rimraf ./build && NODE_OPTIONS=--max-old-space-size=7168 DOCUSAURUS_SSR_CONCURRENCY=4 docusaurus build", - "build:vercel": "yarn clean && yarn build:typedoc:vercel && bash ./scripts/vercel_build.sh && yarn quarto:vercel && rimraf ./build && NODE_OPTIONS=--max-old-space-size=7168 DOCUSAURUS_SSR_CONCURRENCY=4 docusaurus build && mv build v0.1 && mkdir build && mv v0.1 build && mv build/v0.1/404.html build", + "build:vercel": "yarn clean && yarn build:typedoc:vercel && bash ./scripts/vercel_build.sh && yarn quarto:vercel && rimraf ./build && NODE_OPTIONS=--max-old-space-size=7168 DOCUSAURUS_SSR_CONCURRENCY=4 docusaurus build", "build:typedoc": "yarn workspace api_refs build", "build:typedoc:vercel": "yarn workspace api_refs build:vercel", "swizzle": "docusaurus swizzle", diff --git a/docs/core_docs/vercel.json b/docs/core_docs/vercel.json index bf652f8c97a..81784bfc7a3 100644 --- a/docs/core_docs/vercel.json +++ b/docs/core_docs/vercel.json @@ -1,541 +1,524 @@ { - "buildCommand": "yarn build:vercel", - "outputDirectory": "build", - "trailingSlash": true, - "rewrites": [ - { - "source": "/v0.2/:path(.*/?)*", - "destination": "https://langchainjs-v02.vercel.app/v0.2/:path*" - } - ], "redirects": [ { - "source": "/docs/:path*(/?)", - "destination": "/v0.1/docs/:path*/" - }, - { - "source": "(/?)", - "destination": "/v0.1/docs/get_started/introduction/" - }, - { - "source": "/v0.1(/?)", - "destination": "/v0.1/docs/get_started/introduction/" + "source": "/docs(/?)", + "destination": "/docs/get_started/introduction/" }, { - "source": "/v0.1/docs/getting-started/install(/?)", - "destination": "/v0.1/docs/get_started/installation/" + "source": "/docs/getting-started/install(/?)", + "destination": "/docs/get_started/installation/" }, { - "source": "/v0.1/docs/getting-started/guide-llm(/?)", - "destination": "/v0.1/docs/get_started/quickstart/" + "source": "/docs/getting-started/guide-llm(/?)", + "destination": "/docs/get_started/quickstart/" }, { - "source": "/v0.1/docs/getting-started/guide-chat(/?)", - "destination": "/v0.1/docs/get_started/quickstart/" + "source": "/docs/getting-started/guide-chat(/?)", + "destination": "/docs/get_started/quickstart/" }, { - "source": "/v0.1/docs/modules/schema/:path*(/?)", - "destination": "/v0.1/docs/get_started/quickstart/" + "source": "/docs/modules/schema/:path*(/?)", + "destination": "/docs/get_started/quickstart/" }, { - "source": "/v0.1/docs/modules/models(/?)", - "destination": "/v0.1/docs/modules/model_io/" + "source": "/docs/modules/models(/?)", + "destination": "/docs/modules/model_io/" }, { - "source": "/v0.1/docs/modules/model_io/models/chat/(.*)", - "destination": "/v0.1/docs/modules/model_io/chat/" + "source": "/docs/modules/model_io/models/chat/(.*)", + "destination": "/docs/modules/model_io/chat/" }, { - "source": "/v0.1/docs/modules/model_io/models/chat(/?)", - "destination": "/v0.1/docs/modules/model_io/chat/" + "source": "/docs/modules/model_io/models/chat(/?)", + "destination": "/docs/modules/model_io/chat/" }, { - "source": "/v0.1/docs/modules/models/chat(/?)", - "destination": "/v0.1/docs/modules/model_io/chat/" + "source": "/docs/modules/models/chat(/?)", + "destination": "/docs/modules/model_io/chat/" }, { - "source": "/v0.1/docs/modules/models/chat/integrations(/?)", - "destination": "/v0.1/docs/integrations/chat/" + "source": "/docs/modules/models/chat/integrations(/?)", + "destination": "/docs/integrations/chat/" }, { - "source": "/v0.1/docs/modules/models/chat/additional_functionality(/?)", - "destination": "/v0.1/docs/modules/model_io/chat/" + "source": "/docs/modules/models/chat/additional_functionality(/?)", + "destination": "/docs/modules/model_io/chat/" }, { - "source": "/v0.1/docs/modules/models/embeddings(/?)", - "destination": "/v0.1/docs/modules/data_connection/text_embedding/" + "source": "/docs/modules/models/embeddings(/?)", + "destination": "/docs/modules/data_connection/text_embedding/" }, { - "source": "/v0.1/docs/modules/models/embeddings/integrations(/?)", - "destination": "/v0.1/docs/integrations/text_embedding/" + "source": "/docs/modules/models/embeddings/integrations(/?)", + "destination": "/docs/integrations/text_embedding/" }, { - "source": "/v0.1/docs/modules/models/embeddings/additional_functionality(/?)", - "destination": "/v0.1/docs/modules/data_connection/text_embedding/" + "source": "/docs/modules/models/embeddings/additional_functionality(/?)", + "destination": "/docs/modules/data_connection/text_embedding/" }, { - "source": "/v0.1/docs/modules/data_connection/text_embedding/how_to/:path*(/?)", - "destination": "/v0.1/docs/modules/data_connection/text_embedding/:path/" + "source": "/docs/modules/data_connection/text_embedding/how_to/:path*(/?)", + "destination": "/docs/modules/data_connection/text_embedding/:path/" }, { - "source": "/v0.1/docs/modules/models/llms(/?)", - "destination": "/v0.1/docs/modules/model_io/llms/" + "source": "/docs/modules/models/llms(/?)", + "destination": "/docs/modules/model_io/llms/" }, { - "source": "/v0.1/docs/modules/model_io/models/llms/(.*)", - "destination": "/v0.1/docs/modules/model_io/llms/" + "source": "/docs/modules/model_io/models/llms/(.*)", + "destination": "/docs/modules/model_io/llms/" }, { - "source": "/v0.1/docs/modules/model_io/models/llms(/?)", - "destination": "/v0.1/docs/modules/model_io/llms/" + "source": "/docs/modules/model_io/models/llms(/?)", + "destination": "/docs/modules/model_io/llms/" }, { - "source": "/v0.1/docs/modules/models/llms/integrations(/?)", - "destination": "/v0.1/docs/integrations/llms/" + "source": "/docs/modules/models/llms/integrations(/?)", + "destination": "/docs/integrations/llms/" }, { - "source": "/v0.1/docs/modules/models/llms/additional_functionality(/?)", - "destination": "/v0.1/docs/modules/model_io/llms/" + "source": "/docs/modules/models/llms/additional_functionality(/?)", + "destination": "/docs/modules/model_io/llms/" }, { - "source": "/v0.1/docs/modules/prompts(/?)", - "destination": "/v0.1/docs/modules/model_io/prompts/" + "source": "/docs/modules/prompts(/?)", + "destination": "/docs/modules/model_io/prompts/" }, { - "source": "/v0.1/docs/modules/prompts/prompt_templates(/?)", - "destination": "/v0.1/docs/modules/model_io/prompts/prompt_templates/" + "source": "/docs/modules/prompts/prompt_templates(/?)", + "destination": "/docs/modules/model_io/prompts/prompt_templates/" }, { - "source": "/v0.1/docs/modules/prompts/prompt_templates/prompt_composition(/?)", - "destination": "/v0.1/docs/modules/model_io/prompts/prompt_templates/prompt_composition/" + "source": "/docs/modules/prompts/prompt_templates/prompt_composition(/?)", + "destination": "/docs/modules/model_io/prompts/prompt_templates/prompt_composition/" }, { - "source": "/v0.1/docs/modules/prompts/prompt_templates/additional_functionality(/?)", - "destination": "/v0.1/docs/modules/model_io/prompts/prompt_templates/" + "source": "/docs/modules/prompts/prompt_templates/additional_functionality(/?)", + "destination": "/docs/modules/model_io/prompts/prompt_templates/" }, { - "source": "/v0.1/docs/modules/prompts/output_parsers(/?)", - "destination": "/v0.1/docs/modules/model_io/output_parsers/" + "source": "/docs/modules/prompts/output_parsers(/?)", + "destination": "/docs/modules/model_io/output_parsers/" }, { - "source": "/v0.1/docs/modules/prompts/example_selectors(/?)", - "destination": "/v0.1/docs/modules/model_io/prompts/example_selector_types/" + "source": "/docs/modules/prompts/example_selectors(/?)", + "destination": "/docs/modules/model_io/prompts/example_selector_types/" }, { - "source": "/v0.1/docs/modules/model_io/prompts/example_selector(/?)", - "destination": "/v0.1/docs/modules/model_io/prompts/example_selector_types/" + "source": "/docs/modules/model_io/prompts/example_selector(/?)", + "destination": "/docs/modules/model_io/prompts/example_selector_types/" }, { - "source": "/v0.1/docs/modules/indexes(/?)", - "destination": "/v0.1/docs/modules/data_connection/" + "source": "/docs/modules/indexes(/?)", + "destination": "/docs/modules/data_connection/" }, { - "source": "/v0.1/docs/modules/indexes/document_loaders(/?)", - "destination": "/v0.1/docs/modules/data_connection/document_loaders/" + "source": "/docs/modules/indexes/document_loaders(/?)", + "destination": "/docs/modules/data_connection/document_loaders/" }, { - "source": "/v0.1/docs/modules/indexes/document_loaders/examples(/?)", - "destination": "/v0.1/docs/modules/data_connection/document_loaders/" + "source": "/docs/modules/indexes/document_loaders/examples(/?)", + "destination": "/docs/modules/data_connection/document_loaders/" }, { - "source": "/v0.1/docs/modules/data_connection/document_loaders/how_to/:path*(/?)", - "destination": "/v0.1/docs/modules/data_connection/document_loaders/:path*/" + "source": "/docs/modules/data_connection/document_loaders/how_to/:path*(/?)", + "destination": "/docs/modules/data_connection/document_loaders/:path*/" }, { - "source": "/v0.1/docs/modules/indexes/document_loaders/examples/file_loaders/:path*(/?)", + "source": "/docs/modules/indexes/document_loaders/examples/file_loaders/:path*(/?)", "destination": "docs/integrations/document_loaders/file_loaders/:path*/" }, { - "source": "/v0.1/docs/modules/indexes/document_loaders/examples/web_loaders/:path*(/?)", - "destination": "/v0.1/docs/integrations/document_loaders/web_loaders/:path*/" + "source": "/docs/modules/indexes/document_loaders/examples/web_loaders/:path*(/?)", + "destination": "/docs/integrations/document_loaders/web_loaders/:path*/" }, { - "source": "/v0.1/docs/modules/indexes/document_transformers(/?)", - "destination": "/v0.1/docs/modules/data_connection/document_transformers/" + "source": "/docs/modules/indexes/document_transformers(/?)", + "destination": "/docs/modules/data_connection/document_transformers/" }, { - "source": "/v0.1/docs/modules/indexes/text_splitters(/?)", - "destination": "/v0.1/docs/modules/data_connection/document_transformers/" + "source": "/docs/modules/indexes/text_splitters(/?)", + "destination": "/docs/modules/data_connection/document_transformers/" }, { - "source": "/v0.1/docs/modules/indexes/text_splitters/examples/:path*(/?)", - "destination": "/v0.1/docs/modules/data_connection/document_transformers/:path*/" + "source": "/docs/modules/indexes/text_splitters/examples/:path*(/?)", + "destination": "/docs/modules/data_connection/document_transformers/:path*/" }, { - "source": "/v0.1/docs/modules/data_connection/document_transformers/text_splitters/:path*/", - "destination": "/v0.1/docs/modules/data_connection/document_transformers/:path*/" + "source": "/docs/modules/data_connection/document_transformers/text_splitters/:path*/", + "destination": "/docs/modules/data_connection/document_transformers/:path*/" }, { - "source": "/v0.1/docs/modules/indexes/vector_stores(/?)", - "destination": "/v0.1/docs/modules/data_connection/vectorstores/" + "source": "/docs/modules/indexes/vector_stores(/?)", + "destination": "/docs/modules/data_connection/vectorstores/" }, { - "source": "/v0.1/docs/modules/indexes/vector_stores/integrations/:path*(/?)", - "destination": "/v0.1/docs/integrations/vectorstores/:path*/" + "source": "/docs/modules/indexes/vector_stores/integrations/:path*(/?)", + "destination": "/docs/integrations/vectorstores/:path*/" }, { - "source": "/v0.1/docs/modules/indexes/retrievers(/?)", - "destination": "/v0.1/docs/modules/data_connection/retrievers/" + "source": "/docs/modules/indexes/retrievers(/?)", + "destination": "/docs/modules/data_connection/retrievers/" }, { - "source": "/v0.1/docs/modules/indexes/retrievers/self_query(/?)", - "destination": "/v0.1/docs/modules/data_connection/retrievers/self_query/" + "source": "/docs/modules/indexes/retrievers/self_query(/?)", + "destination": "/docs/modules/data_connection/retrievers/self_query/" }, { - "source": "/v0.1/docs/modules/indexes/retrievers/self_query/examples/:path*(/?)", - "destination": "/v0.1/docs/modules/data_connection/retrievers/self_query/:path*/" + "source": "/docs/modules/indexes/retrievers/self_query/examples/:path*(/?)", + "destination": "/docs/modules/data_connection/retrievers/self_query/:path*/" }, { - "source": "/v0.1/docs/modules/data_connection/retrievers/how_to/self_query(/?)", - "destination": "/v0.1/docs/modules/data_connection/retrievers/self_query/" + "source": "/docs/modules/data_connection/retrievers/how_to/self_query(/?)", + "destination": "/docs/modules/data_connection/retrievers/self_query/" }, { - "source": "/v0.1/docs/modules/data_connection/retrievers/how_to/self_query/:path*(/?)", - "destination": "/v0.1/docs/modules/data_connection/retrievers/self_query/:path*/" + "source": "/docs/modules/data_connection/retrievers/how_to/self_query/:path*(/?)", + "destination": "/docs/modules/data_connection/retrievers/self_query/:path*/" }, { - "source": "/v0.1/docs/modules/indexes/retrievers/:path*(/?)", - "destination": "/v0.1/docs/integrations/retrievers/:path*/" + "source": "/docs/modules/indexes/retrievers/:path*(/?)", + "destination": "/docs/integrations/retrievers/:path*/" }, { - "source": "/v0.1/docs/modules/memory/examples/:path*(/?)", - "destination": "/v0.1/docs/integrations/memory/:path*/" + "source": "/docs/modules/memory/examples/:path*(/?)", + "destination": "/docs/integrations/memory/:path*/" }, { - "source": "/v0.1/docs/modules/chains/llm_chain(/?)", - "destination": "/v0.1/docs/modules/chains/foundational/llm_chain/" + "source": "/docs/modules/chains/llm_chain(/?)", + "destination": "/docs/modules/chains/foundational/llm_chain/" }, { - "source": "/v0.1/docs/modules/chains/index_related_chains(/?)", - "destination": "/v0.1/docs/modules/chains/document/" + "source": "/docs/modules/chains/index_related_chains(/?)", + "destination": "/docs/modules/chains/document/" }, { - "source": "/v0.1/docs/modules/chains/index_related_chains/document_qa(/?)", - "destination": "/v0.1/docs/modules/chains/document/" + "source": "/docs/modules/chains/index_related_chains/document_qa(/?)", + "destination": "/docs/modules/chains/document/" }, { - "source": "/v0.1/docs/modules/chains/index_related_chains/retrieval_qa(/?)", + "source": "/docs/modules/chains/index_related_chains/retrieval_qa(/?)", "destination": "docs/modules/chains/popular/vector_db_qa/" }, { - "source": "/v0.1/docs/modules/chains/index_related_chains/conversational_retrieval(/?)", - "destination": "/v0.1/docs/modules/chains/popular/chat_vector_db/" + "source": "/docs/modules/chains/index_related_chains/conversational_retrieval(/?)", + "destination": "/docs/modules/chains/popular/chat_vector_db/" }, { - "source": "/v0.1/docs/modules/chains/sequential_chain(/?)", - "destination": "/v0.1/docs/modules/chains/foundational/sequential_chains/" + "source": "/docs/modules/chains/sequential_chain(/?)", + "destination": "/docs/modules/chains/foundational/sequential_chains/" }, { - "source": "/v0.1/docs/modules/chains/openai_functions(/?)", - "destination": "/v0.1/docs/modules/chains/additional/openai_functions/" + "source": "/docs/modules/chains/openai_functions(/?)", + "destination": "/docs/modules/chains/additional/openai_functions/" }, { - "source": "/v0.1/docs/modules/chains/openai_functions/extraction(/?)", - "destination": "/v0.1/docs/modules/chains/additional/openai_functions/extraction/" + "source": "/docs/modules/chains/openai_functions/extraction(/?)", + "destination": "/docs/modules/chains/additional/openai_functions/extraction/" }, { - "source": "/v0.1/docs/modules/chains/openai_functions/openapi(/?)", - "destination": "/v0.1/docs/modules/chains/additional/openai_functions/openapi/" + "source": "/docs/modules/chains/openai_functions/openapi(/?)", + "destination": "/docs/modules/chains/additional/openai_functions/openapi/" }, { - "source": "/v0.1/docs/modules/chains/openai_functions/structured_output(/?)", - "destination": "/v0.1/docs/modules/chains/popular/structured_output/" + "source": "/docs/modules/chains/openai_functions/structured_output(/?)", + "destination": "/docs/modules/chains/popular/structured_output/" }, { - "source": "/v0.1/docs/modules/chains/openai_functions/tagging(/?)", - "destination": "/v0.1/docs/modules/chains/additional/openai_functions/tagging/" + "source": "/docs/modules/chains/openai_functions/tagging(/?)", + "destination": "/docs/modules/chains/additional/openai_functions/tagging/" }, { - "source": "/v0.1/docs/modules/chains/other_chains(/?)", - "destination": "/v0.1/docs/modules/chains/additional/" + "source": "/docs/modules/chains/other_chains(/?)", + "destination": "/docs/modules/chains/additional/" }, { - "source": "/v0.1/docs/modules/chains/other_chains/analyze_document(/?)", - "destination": "/v0.1/docs/modules/chains/additional/analyze_document/" + "source": "/docs/modules/chains/other_chains/analyze_document(/?)", + "destination": "/docs/modules/chains/additional/analyze_document/" }, { - "source": "/v0.1/docs/modules/chains/other_chains/api_chain(/?)", - "destination": "/v0.1/docs/modules/chains/popular/api/" + "source": "/docs/modules/chains/other_chains/api_chain(/?)", + "destination": "/docs/modules/chains/popular/api/" }, { - "source": "/v0.1/docs/modules/chains/other_chains/constitutional_chain(/?)", - "destination": "/v0.1/docs/modules/chains/additional/constitutional_chain/" + "source": "/docs/modules/chains/other_chains/constitutional_chain(/?)", + "destination": "/docs/modules/chains/additional/constitutional_chain/" }, { - "source": "/v0.1/docs/modules/chains/other_chains/moderation_chain(/?)", - "destination": "/v0.1/docs/modules/chains/additional/moderation/" + "source": "/docs/modules/chains/other_chains/moderation_chain(/?)", + "destination": "/docs/modules/chains/additional/moderation/" }, { - "source": "/v0.1/docs/modules/chains/other_chains/multi_prompt_chain(/?)", - "destination": "/v0.1/docs/modules/chains/additional/multi_prompt_router/" + "source": "/docs/modules/chains/other_chains/multi_prompt_chain(/?)", + "destination": "/docs/modules/chains/additional/multi_prompt_router/" }, { - "source": "/v0.1/docs/modules/chains/other_chains/multi_retrieval_qa_chain(/?)", - "destination": "/v0.1/docs/modules/chains/additional/multi_retrieval_qa_router/" + "source": "/docs/modules/chains/other_chains/multi_retrieval_qa_chain(/?)", + "destination": "/docs/modules/chains/additional/multi_retrieval_qa_router/" }, { - "source": "/v0.1/docs/modules/chains/other_chains/sql(/?)", - "destination": "/v0.1/docs/modules/chains/popular/sqlite/" + "source": "/docs/modules/chains/other_chains/sql(/?)", + "destination": "/docs/modules/chains/popular/sqlite/" }, { - "source": "/v0.1/docs/modules/chains/other_chains/summarization(/?)", - "destination": "/v0.1/docs/modules/chains/popular/summarize/" + "source": "/docs/modules/chains/other_chains/summarization(/?)", + "destination": "/docs/modules/chains/popular/summarize/" }, { - "source": "/v0.1/docs/modules/chains/prompt_selectors(/?)", - "destination": "/v0.1/docs/modules/model_io/prompts/prompt_selectors/" + "source": "/docs/modules/chains/prompt_selectors(/?)", + "destination": "/docs/modules/model_io/prompts/prompt_selectors/" }, { - "source": "/v0.1/docs/modules/agents/agents(/?)", - "destination": "/v0.1/docs/modules/agents/" + "source": "/docs/modules/agents/agents(/?)", + "destination": "/docs/modules/agents/" }, { - "source": "/v0.1/docs/modules/agents/agents/action(/?)", - "destination": "/v0.1/docs/modules/agents/" + "source": "/docs/modules/agents/agents/action(/?)", + "destination": "/docs/modules/agents/" }, { - "source": "/v0.1/docs/modules/agents/agents/action/llm_mrkl(/?)", - "destination": "/v0.1/docs/modules/agents/agent_types/react/" + "source": "/docs/modules/agents/agents/action/llm_mrkl(/?)", + "destination": "/docs/modules/agents/agent_types/react/" }, { - "source": "/v0.1/docs/modules/agents/agents/action/chat_mrkl(/?)", - "destination": "/v0.1/docs/modules/agents/agent_types/react/" + "source": "/docs/modules/agents/agents/action/chat_mrkl(/?)", + "destination": "/docs/modules/agents/agent_types/react/" }, { - "source": "/v0.1/docs/modules/agents/agents/action/conversational_agent(/?)", - "destination": "/v0.1/docs/modules/agents/agent_types/chat_conversation_agent/" + "source": "/docs/modules/agents/agents/action/conversational_agent(/?)", + "destination": "/docs/modules/agents/agent_types/chat_conversation_agent/" }, { - "source": "/v0.1/docs/modules/agents/agents/action/structured_chat(/?)", - "destination": "/v0.1/docs/modules/agents/agent_types/structured_chat/" + "source": "/docs/modules/agents/agents/action/structured_chat(/?)", + "destination": "/docs/modules/agents/agent_types/structured_chat/" }, { - "source": "/v0.1/docs/modules/agents/agents/action/openai(/?)", - "destination": "/v0.1/docs/modules/agents/agent_types/openai_functions_agent/" + "source": "/docs/modules/agents/agents/action/openai(/?)", + "destination": "/docs/modules/agents/agent_types/openai_functions_agent/" }, { "source": "docs/modules/agents/agents/plan_execute(/?)", - "destination": "/v0.1/docs/modules/agents/agent_types/plan_and_execute/" + "destination": "/docs/modules/agents/agent_types/plan_and_execute/" }, { - "source": "/v0.1/docs/modules/agents/agents/custom(/?)", - "destination": "/v0.1/docs/modules/agents/how_to/custom_llm_agent/" + "source": "/docs/modules/agents/agents/custom(/?)", + "destination": "/docs/modules/agents/how_to/custom_llm_agent/" }, { - "source": "/v0.1/docs/modules/agents/agents/custom/custom_prompt_chat(/?)", - "destination": "/v0.1/docs/modules/agents/how_to/custom_llm_chat_agent/" + "source": "/docs/modules/agents/agents/custom/custom_prompt_chat(/?)", + "destination": "/docs/modules/agents/how_to/custom_llm_chat_agent/" }, { - "source": "/v0.1/docs/modules/agents/agents/custom/custom_agent_chat(/?)", - "destination": "/v0.1/docs/modules/agents/how_to/custom_llm_chat_agent/" + "source": "/docs/modules/agents/agents/custom/custom_agent_chat(/?)", + "destination": "/docs/modules/agents/how_to/custom_llm_chat_agent/" }, { - "source": "/v0.1/docs/modules/agents/agents/custom/custom_agent_llm(/?)", - "destination": "/v0.1/docs/modules/agents/how_to/custom_llm_agent/" + "source": "/docs/modules/agents/agents/custom/custom_agent_llm(/?)", + "destination": "/docs/modules/agents/how_to/custom_llm_agent/" }, { - "source": "/v0.1/docs/modules/agents/executor(/?)", - "destination": "/v0.1/docs/modules/agents/" + "source": "/docs/modules/agents/executor(/?)", + "destination": "/docs/modules/agents/" }, { - "source": "/v0.1/docs/modules/agents/executor/getting-started(/?)", - "destination": "/v0.1/docs/modules/agents/" + "source": "/docs/modules/agents/executor/getting-started(/?)", + "destination": "/docs/modules/agents/" }, { - "source": "/v0.1/docs/modules/agents/tools/integrations(/?)", - "destination": "/v0.1/docs/modules/agents/tools/integrations/" + "source": "/docs/modules/agents/tools/integrations(/?)", + "destination": "/docs/modules/agents/tools/integrations/" }, { - "source": "/v0.1/docs/modules/agents/tools/agents_with_vectorstores(/?)", - "destination": "/v0.1/docs/modules/agents/tools/how_to/agents_with_vectorstores/" + "source": "/docs/modules/agents/tools/agents_with_vectorstores(/?)", + "destination": "/docs/modules/agents/tools/how_to/agents_with_vectorstores/" }, { - "source": "/v0.1/docs/modules/agents/tools/aiplugin-tool(/?)", - "destination": "/v0.1/docs/modules/agents/tools/integrations/aiplugin-tool/" + "source": "/docs/modules/agents/tools/aiplugin-tool(/?)", + "destination": "/docs/modules/agents/tools/integrations/aiplugin-tool/" }, { - "source": "/v0.1/docs/modules/agents/tools/lambda_agent(/?)", - "destination": "/v0.1/docs/modules/agents/tools/integrations/lambda_agent/" + "source": "/docs/modules/agents/tools/lambda_agent(/?)", + "destination": "/docs/modules/agents/tools/integrations/lambda_agent/" }, { - "source": "/v0.1/docs/modules/agents/tools/webbrowser(/?)", - "destination": "/v0.1/docs/modules/agents/tools/integrations/webbrowser/" + "source": "/docs/modules/agents/tools/webbrowser(/?)", + "destination": "/docs/modules/agents/tools/integrations/webbrowser/" }, { - "source": "/v0.1/docs/modules/agents/tools/zapier_agent(/?)", - "destination": "/v0.1/docs/modules/agents/tools/integrations/zapier_agent/" + "source": "/docs/modules/agents/tools/zapier_agent(/?)", + "destination": "/docs/modules/agents/tools/integrations/zapier_agent/" }, { - "source": "/v0.1/docs/modules/agents/additional_functionality(/?)", - "destination": "/v0.1/docs/modules/agents/" + "source": "/docs/modules/agents/additional_functionality(/?)", + "destination": "/docs/modules/agents/" }, { - "source": "/v0.1/docs/production/callbacks(/?)", - "destination": "/v0.1/docs/modules/callbacks/" + "source": "/docs/production/callbacks(/?)", + "destination": "/docs/modules/callbacks/" }, { - "source": "/v0.1/docs/production/callbacks/create-handlers(/?)", - "destination": "/v0.1/docs/modules/callbacks/how_to/create_handlers/" + "source": "/docs/production/callbacks/create-handlers(/?)", + "destination": "/docs/modules/callbacks/how_to/create_handlers/" }, { - "source": "/v0.1/docs/production/callbacks/creating-subclasses(/?)", - "destination": "/v0.1/docs/modules/callbacks/how_to/creating_subclasses/" + "source": "/docs/production/callbacks/creating-subclasses(/?)", + "destination": "/docs/modules/callbacks/how_to/creating_subclasses/" }, { - "source": "/v0.1/docs/production/deployment(/?)", - "destination": "/v0.1/docs/get_started/introduction/" + "source": "/docs/production/deployment(/?)", + "destination": "/docs/get_started/introduction/" }, { - "source": "/v0.1/docs/production/tracing(/?)", + "source": "/docs/production/tracing(/?)", "destination": "https://docs.smith.langchain.com/" }, { - "source": "/v0.1/docs/modules/data_connection/retrievers/integrations/parent-document-retriever", - "destination": "/v0.1/docs/modules/data_connection/retrievers/how_to/parent-document-retriever" + "source": "/docs/modules/data_connection/retrievers/integrations/parent-document-retriever", + "destination": "/docs/modules/data_connection/retrievers/how_to/parent-document-retriever" }, { - "source": "/v0.1/docs/modules/data_connection/caching_embeddings(/?)", - "destination": "/v0.1/docs/modules/data_connection/text_embedding/caching_embeddings" + "source": "/docs/modules/data_connection/caching_embeddings(/?)", + "destination": "/docs/modules/data_connection/text_embedding/caching_embeddings" }, { - "source": "/v0.1/docs/guides/expression_language/cookbook(/?)", - "destination": "/v0.1/docs/expression_language/cookbook/" + "source": "/docs/guides/expression_language/cookbook(/?)", + "destination": "/docs/expression_language/cookbook/" }, { - "source": "/v0.1/docs/guides/expression_language/interface(/?)", - "destination": "/v0.1/docs/expression_language/interface/" + "source": "/docs/guides/expression_language/interface(/?)", + "destination": "/docs/expression_language/interface/" }, { - "source": "/v0.1/docs/additional_resources/scrimba(/?)", - "destination": "/v0.1/docs/additional_resources/tutorials/" + "source": "/docs/additional_resources/scrimba(/?)", + "destination": "/docs/additional_resources/tutorials/" }, { - "source": "/v0.1/docs/additional_resources/expression_language_cheatsheet(/?)", - "destination": "/v0.1/docs/additional_resources/tutorials/" + "source": "/docs/additional_resources/expression_language_cheatsheet(/?)", + "destination": "/docs/additional_resources/tutorials/" }, { - "source": "/v0.1/docs/modules/model_io/models/llms/integrations(/.*)?", - "destination": "/v0.1/docs/integrations/llms$1" + "source": "/docs/modules/model_io/models/llms/integrations(/.*)?", + "destination": "/docs/integrations/llms$1" }, { - "source": "/v0.1/docs/modules/model_io/models/chat/integrations(/.*)?", - "destination": "/v0.1/docs/integrations/chat$1" + "source": "/docs/modules/model_io/models/chat/integrations(/.*)?", + "destination": "/docs/integrations/chat$1" }, { - "source": "/v0.1/docs/modules/data_connection/document_loaders/integrations(/.*)?", - "destination": "/v0.1/docs/integrations/document_loaders$1" + "source": "/docs/modules/data_connection/document_loaders/integrations(/.*)?", + "destination": "/docs/integrations/document_loaders$1" }, { - "source": "/v0.1/docs/modules/data_connection/document_transformers/integrations(/.*)?", - "destination": "/v0.1/docs/integrations/document_transformers$1" + "source": "/docs/modules/data_connection/document_transformers/integrations(/.*)?", + "destination": "/docs/integrations/document_transformers$1" }, { - "source": "/v0.1/docs/modules/data_connection/text_embedding/integrations(/.*)?", - "destination": "/v0.1/docs/integrations/text_embedding$1" + "source": "/docs/modules/data_connection/text_embedding/integrations(/.*)?", + "destination": "/docs/integrations/text_embedding$1" }, { - "source": "/v0.1/docs/modules/data_connection/vectorstores/integrations(/.*)?", - "destination": "/v0.1/docs/integrations/vectorstores$1" + "source": "/docs/modules/data_connection/vectorstores/integrations(/.*)?", + "destination": "/docs/integrations/vectorstores$1" }, { - "source": "/v0.1/docs/modules/data_connection/retrievers/integrations(/.*)?", - "destination": "/v0.1/docs/integrations/retrievers$1" + "source": "/docs/modules/data_connection/retrievers/integrations(/.*)?", + "destination": "/docs/integrations/retrievers$1" }, { - "source": "/v0.1/docs/modules/memory/integrations(/.*)?", - "destination": "/v0.1/docs/integrations/chat_memory$1" + "source": "/docs/modules/memory/integrations(/.*)?", + "destination": "/docs/integrations/chat_memory$1" }, { - "source": "/v0.1/docs/modules/agents/tools/integrations(/.*)?", - "destination": "/v0.1/docs/integrations/tools$1" + "source": "/docs/modules/agents/tools/integrations(/.*)?", + "destination": "/docs/integrations/tools$1" }, { - "source": "/v0.1/docs/modules/agents/toolkits(/.*)?", - "destination": "/v0.1/docs/integrations/toolkits$1" + "source": "/docs/modules/agents/toolkits(/.*)?", + "destination": "/docs/integrations/toolkits$1" }, { - "source": "/v0.1/docs/api/:slug", + "source": "/docs/api/:slug", "destination": "https://api.js.langchain.com/modules/:slug.html" }, { - "source": "/v0.1/docs/api/:slug1/classes/:slug2", + "source": "/docs/api/:slug1/classes/:slug2", "destination": "https://api.js.langchain.com/classes/:slug1.:slug2.html" }, { - "source": "/v0.1/docs/api/:slug1/functions/:slug2", + "source": "/docs/api/:slug1/functions/:slug2", "destination": "https://api.js.langchain.com/functions/:slug1.:slug2.html" }, { - "source": "/v0.1/docs/api/:slug1/interfaces/:slug2", + "source": "/docs/api/:slug1/interfaces/:slug2", "destination": "https://api.js.langchain.com/interfaces/:slug1.:slug2.html" }, { - "source": "/v0.1/docs/api/:slug1/types/:slug2", + "source": "/docs/api/:slug1/types/:slug2", "destination": "https://api.js.langchain.com/types/:slug1.:slug2.html" }, { - "source": "/v0.1/docs/api/:slug1/variables/:slug2", + "source": "/docs/api/:slug1/variables/:slug2", "destination": "https://api.js.langchain.com/variables/:slug1.:slug2.html" }, { - "source": "/v0.1/docs/modules/model_io/models/llms/how_to/:slug", - "destination": "/v0.1/docs/modules/model_io/llms/:slug" + "source": "/docs/modules/model_io/models/llms/how_to/:slug", + "destination": "/docs/modules/model_io/llms/:slug" }, { - "source": "/v0.1/docs/modules/model_io/models/chat/how_to/:slug", - "destination": "/v0.1/docs/modules/model_io/chat/:slug" + "source": "/docs/modules/model_io/models/chat/how_to/:slug", + "destination": "/docs/modules/model_io/chat/:slug" }, { - "source": "/v0.1/docs/modules/model_io/output_parsers/bytes(/.*)?", - "destination": "/v0.1/docs/modules/model_io/output_parsers/types/bytes/" + "source": "/docs/modules/model_io/output_parsers/bytes(/.*)?", + "destination": "/docs/modules/model_io/output_parsers/types/bytes/" }, { - "source": "/v0.1/docs/modules/model_io/output_parsers/string(/.*)?", - "destination": "/v0.1/docs/modules/model_io/output_parsers/types/string/" + "source": "/docs/modules/model_io/output_parsers/string(/.*)?", + "destination": "/docs/modules/model_io/output_parsers/types/string/" }, { - "source": "/v0.1/docs/modules/model_io/output_parsers/combining_output_parser(/.*)?", - "destination": "/v0.1/docs/modules/model_io/output_parsers/types/combining_output_parser/" + "source": "/docs/modules/model_io/output_parsers/combining_output_parser(/.*)?", + "destination": "/docs/modules/model_io/output_parsers/types/combining_output_parser/" }, { - "source": "/v0.1/docs/modules/model_io/output_parsers/comma_separated(/.*)?", - "destination": "/v0.1/docs/modules/model_io/output_parsers/types/comma_separated/" + "source": "/docs/modules/model_io/output_parsers/comma_separated(/.*)?", + "destination": "/docs/modules/model_io/output_parsers/types/comma_separated/" }, { - "source": "/v0.1/docs/modules/model_io/output_parsers/custom_list_parser(/.*)?", - "destination": "/v0.1/docs/modules/model_io/output_parsers/types/custom_list_parser/" + "source": "/docs/modules/model_io/output_parsers/custom_list_parser(/.*)?", + "destination": "/docs/modules/model_io/output_parsers/types/custom_list_parser/" }, { - "source": "/v0.1/docs/modules/model_io/output_parsers/http_response(/.*)?", - "destination": "/v0.1/docs/modules/model_io/output_parsers/types/http_response/" + "source": "/docs/modules/model_io/output_parsers/http_response(/.*)?", + "destination": "/docs/modules/model_io/output_parsers/types/http_response/" }, { - "source": "/v0.1/docs/modules/model_io/output_parsers/json_functions(/.*)?", - "destination": "/v0.1/docs/modules/model_io/output_parsers/types/json_functions/" + "source": "/docs/modules/model_io/output_parsers/json_functions(/.*)?", + "destination": "/docs/modules/model_io/output_parsers/types/json_functions/" }, { - "source": "/v0.1/docs/modules/model_io/output_parsers/output_fixing_parser(/.*)?", - "destination": "/v0.1/docs/modules/model_io/output_parsers/types/output_fixing_parser/" + "source": "/docs/modules/model_io/output_parsers/output_fixing_parser(/.*)?", + "destination": "/docs/modules/model_io/output_parsers/types/output_fixing_parser/" }, { - "source": "/v0.1/docs/modules/model_io/output_parsers/structured(/.*)?", - "destination": "/v0.1/docs/modules/model_io/output_parsers/types/structured/" + "source": "/docs/modules/model_io/output_parsers/structured(/.*)?", + "destination": "/docs/modules/model_io/output_parsers/types/structured/" }, { - "source": "/v0.1/docs/modules/agents/tools/how_to/dynamic(/?)", - "destination": "/v0.1/docs/modules/agents/tools/dynamic/" + "source": "/docs/modules/agents/tools/how_to/dynamic(/?)", + "destination": "/docs/modules/agents/tools/dynamic/" }, { - "source": "/v0.1/docs/modules/chains/foundational/sequential_chains(/?)", - "destination": "/v0.1/docs/expression_language/" + "source": "/docs/modules/chains/foundational/sequential_chains(/?)", + "destination": "/docs/expression_language/" }, { - "source": "/v0.1/docs/modules/memory/how_to(/.*)?", - "destination": "/v0.1/docs/modules/memory/types$1" + "source": "/docs/modules/memory/how_to(/.*)?", + "destination": "/docs/modules/memory/types$1" }, { - "source": "/v0.1/docs/integrations/chat/anthropic_functions(/?)", - "destination": "/v0.1/docs/integrations/chat/anthropic_tools/" + "source": "/docs/integrations/chat/anthropic_functions(/?)", + "destination": "/docs/integrations/chat/anthropic_tools/" } ] } diff --git a/examples/src/indexes/vector_stores/azure_cosmosdb/azure_cosmosdb.ts b/examples/src/indexes/vector_stores/azure_cosmosdb/azure_cosmosdb.ts index 7a26cf3244b..aeeff4a6d17 100644 --- a/examples/src/indexes/vector_stores/azure_cosmosdb/azure_cosmosdb.ts +++ b/examples/src/indexes/vector_stores/azure_cosmosdb/azure_cosmosdb.ts @@ -25,15 +25,14 @@ const store = await AzureCosmosDBVectorStore.fromDocuments( { databaseName: "langchain", collectionName: "documents", + indexOptions: { + numLists: 100, + dimensions: 1536, + similarity: AzureCosmosDBSimilarityType.COS, + }, } ); -// Create the index -const numLists = 100; -const dimensions = 1536; -const similarity = AzureCosmosDBSimilarityType.COS; -await store.createIndex(numLists, dimensions, similarity); - // Performs a similarity search const resultDocuments = await store.similaritySearch( "What did the president say about Ketanji Brown Jackson?" diff --git a/langchain-core/src/runnables/base.ts b/langchain-core/src/runnables/base.ts index 4cc635a753d..ec9047475d1 100644 --- a/langchain-core/src/runnables/base.ts +++ b/langchain-core/src/runnables/base.ts @@ -38,6 +38,12 @@ import { _RootEventFilter, isRunnableInterface } from "./utils.js"; import { AsyncLocalStorageProviderSingleton } from "../singletons/index.js"; import { Graph } from "./graph.js"; import { convertToHttpEventStream } from "./wrappers.js"; +import { + consumeAsyncIterableInContext, + consumeIteratorInContext, + isAsyncIterable, + isIterator, +} from "./iter.js"; export { type RunnableInterface, RunnableBatchOptions }; @@ -1999,6 +2005,44 @@ export class RunnableLambda extends Runnable< recursionLimit: (childConfig.recursionLimit ?? DEFAULT_RECURSION_LIMIT) - 1, }); + } else if (isAsyncIterable(output)) { + let finalOutput: RunOutput | undefined; + for await (const chunk of consumeAsyncIterableInContext( + childConfig, + output + )) { + if (finalOutput === undefined) { + finalOutput = chunk as RunOutput; + } else { + // Make a best effort to gather, for any type that supports concat. + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + finalOutput = concat(finalOutput, chunk as any); + } catch (e) { + finalOutput = chunk as RunOutput; + } + } + } + output = finalOutput as typeof output; + } else if (isIterator(output)) { + let finalOutput: RunOutput | undefined; + for (const chunk of consumeIteratorInContext( + childConfig, + output + )) { + if (finalOutput === undefined) { + finalOutput = chunk as RunOutput; + } else { + // Make a best effort to gather, for any type that supports concat. + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + finalOutput = concat(finalOutput, chunk as any); + } catch (e) { + finalOutput = chunk as RunOutput; + } + } + } + output = finalOutput as typeof output; } resolve(output); } catch (e) { @@ -2068,6 +2112,14 @@ export class RunnableLambda extends Runnable< for await (const chunk of stream) { yield chunk; } + } else if (isAsyncIterable(output)) { + for await (const chunk of consumeAsyncIterableInContext(config, output)) { + yield chunk as RunOutput; + } + } else if (isIterator(output)) { + for (const chunk of consumeIteratorInContext(config, output)) { + yield chunk as RunOutput; + } } else { yield output; } diff --git a/langchain-core/src/runnables/iter.ts b/langchain-core/src/runnables/iter.ts new file mode 100644 index 00000000000..156fc999ece --- /dev/null +++ b/langchain-core/src/runnables/iter.ts @@ -0,0 +1,57 @@ +import { AsyncLocalStorageProviderSingleton } from "../singletons/index.js"; +import { RunnableConfig } from "./config.js"; + +export function isIterator(thing: unknown): thing is IterableIterator { + return ( + typeof thing === "object" && + thing !== null && + typeof (thing as Generator)[Symbol.iterator] === "function" && + // avoid detecting array/set as iterator + typeof (thing as Generator).next === "function" + ); +} + +export function isAsyncIterable( + thing: unknown +): thing is AsyncIterable { + return ( + typeof thing === "object" && + thing !== null && + typeof (thing as AsyncIterable)[Symbol.asyncIterator] === + "function" + ); +} + +export function* consumeIteratorInContext( + context: Partial | undefined, + iter: IterableIterator +): IterableIterator { + const storage = AsyncLocalStorageProviderSingleton.getInstance(); + while (true) { + const { value, done } = storage.run(context, iter.next.bind(iter)); + if (done) { + break; + } else { + yield value; + } + } +} + +export async function* consumeAsyncIterableInContext( + context: Partial | undefined, + iter: AsyncIterable +): AsyncIterableIterator { + const storage = AsyncLocalStorageProviderSingleton.getInstance(); + const iterator = iter[Symbol.asyncIterator](); + while (true) { + const { value, done } = await storage.run( + context, + iterator.next.bind(iter) + ); + if (done) { + break; + } else { + yield value; + } + } +} diff --git a/langchain-core/src/runnables/tests/runnable.test.ts b/langchain-core/src/runnables/tests/runnable.test.ts index e38128bf933..55165b2fbce 100644 --- a/langchain-core/src/runnables/tests/runnable.test.ts +++ b/langchain-core/src/runnables/tests/runnable.test.ts @@ -197,6 +197,84 @@ test("RunnableLambda that returns a runnable should invoke the runnable", async expect(result).toEqual("testing"); }); +test("RunnableLambda that returns an async iterator should consume it", async () => { + const runnable = new RunnableLambda({ + async *func() { + yield "test"; + yield "ing"; + }, + }); + const result = await runnable.invoke({}); + expect(result).toEqual("testing"); + const chunks = []; + const stream = await runnable.stream({}); + for await (const chunk of stream) { + chunks.push(chunk); + } + expect(chunks).toEqual(["test", "ing"]); +}); + +test("RunnableLambda that returns an async iterable should consume it", async () => { + const runnable = new RunnableLambda({ + func() { + return new ReadableStream({ + async start(controller) { + controller.enqueue("test"); + controller.enqueue("ing"); + controller.close(); + }, + }); + }, + }); + const result = await runnable.invoke({}); + expect(result).toEqual("testing"); + const chunks = []; + const stream = await runnable.stream({}); + for await (const chunk of stream) { + chunks.push(chunk); + } + expect(chunks).toEqual(["test", "ing"]); +}); + +test("RunnableLambda that returns a promise for async iterable should consume it", async () => { + const runnable = new RunnableLambda({ + async func() { + return new ReadableStream({ + async start(controller) { + controller.enqueue("test"); + controller.enqueue("ing"); + controller.close(); + }, + }); + }, + }); + const result = await runnable.invoke({}); + expect(result).toEqual("testing"); + const chunks = []; + const stream = await runnable.stream({}); + for await (const chunk of stream) { + chunks.push(chunk); + } + expect(chunks).toEqual(["test", "ing"]); +}); + +test("RunnableLambda that returns an iterator should consume it", async () => { + const runnable = new RunnableLambda({ + *func() { + yield "test"; + yield "ing"; + }, + }); + const result = await runnable.invoke({}); + expect(result).toEqual("testing"); + const chunks = []; + const stream = await runnable.stream({}); + for await (const chunk of stream) { + chunks.push(chunk); + } + expect(chunks).toEqual(["test", "ing"]); +}); + test("RunnableLambda that returns a streaming runnable should stream output from the inner runnable", async () => { const runnable = new RunnableLambda({ func: () => new FakeStreamingLLM({}), diff --git a/langchain-core/src/singletons/index.ts b/langchain-core/src/singletons/index.ts index 94dcaba145e..845ae09ea97 100644 --- a/langchain-core/src/singletons/index.ts +++ b/langchain-core/src/singletons/index.ts @@ -3,7 +3,7 @@ export interface AsyncLocalStorageInterface { getStore: () => any | undefined; - run: (store: any, callback: () => any) => any; + run: (store: any, callback: () => T) => T; } export class MockAsyncLocalStorage implements AsyncLocalStorageInterface { @@ -11,8 +11,8 @@ export class MockAsyncLocalStorage implements AsyncLocalStorageInterface { return undefined; } - run(_store: any, callback: () => any): any { - callback(); + run(_store: any, callback: () => T): T { + return callback(); } } diff --git a/libs/langchain-community/src/vectorstores/azure_cosmosdb.ts b/libs/langchain-community/src/vectorstores/azure_cosmosdb.ts index 16983cd008c..bc8545368d5 100644 --- a/libs/langchain-community/src/vectorstores/azure_cosmosdb.ts +++ b/libs/langchain-community/src/vectorstores/azure_cosmosdb.ts @@ -4,13 +4,14 @@ import { Document as MongoDBDocument, MongoClient, Db, + Filter, } from "mongodb"; import type { EmbeddingsInterface } from "@langchain/core/embeddings"; import { MaxMarginalRelevanceSearchOptions, VectorStore, } from "@langchain/core/vectorstores"; -import { Document } from "@langchain/core/documents"; +import { Document, DocumentInterface } from "@langchain/core/documents"; import { maximalMarginalRelevance } from "@langchain/core/utils/math"; import { getEnvironmentVariable } from "@langchain/core/utils/env"; @@ -28,6 +29,26 @@ export const AzureCosmosDBSimilarityType = { export type AzureCosmosDBSimilarityType = (typeof AzureCosmosDBSimilarityType)[keyof typeof AzureCosmosDBSimilarityType]; +/** Azure Cosmos DB Index Options. */ +export type AzureCosmosDBIndexOptions = { + /** Skips automatic index creation. */ + readonly skipCreate?: boolean; + /** Number of clusters that the inverted file (IVF) index uses to group the vector data. */ + readonly numLists?: number; + /** Number of dimensions for vector similarity. */ + readonly dimensions?: number; + /** Similarity metric to use with the IVF index. */ + readonly similarity?: AzureCosmosDBSimilarityType; +}; + +/** Azure Cosmos DB Delete Parameters. */ +export type AzureCosmosDBDeleteParams = { + /** List of IDs for the documents to be removed. */ + readonly ids?: string | string[]; + /** MongoDB filter object or list of IDs for the documents to be removed. */ + readonly filter?: Filter; +}; + /** * Configuration options for the `AzureCosmosDBVectorStore` constructor. */ @@ -39,6 +60,7 @@ export interface AzureCosmosDBConfig { readonly indexName?: string; readonly textKey?: string; readonly embeddingKey?: string; + readonly indexOptions?: AzureCosmosDBIndexOptions; } /** @@ -60,6 +82,8 @@ export class AzureCosmosDBVectorStore extends VectorStore { }; } + private connectPromise: Promise; + private readonly initPromise: Promise; private readonly client: MongoClient | undefined; @@ -74,6 +98,8 @@ export class AzureCosmosDBVectorStore extends VectorStore { readonly embeddingKey: string; + private readonly indexOptions: AzureCosmosDBIndexOptions; + _vectorstoreType(): string { return "azure_cosmosdb"; } @@ -105,6 +131,7 @@ export class AzureCosmosDBVectorStore extends VectorStore { this.indexName = dbConfig.indexName ?? "vectorSearchIndex"; this.textKey = dbConfig.textKey ?? "textContent"; this.embeddingKey = dbConfig.embeddingKey ?? "vectorContent"; + this.indexOptions = dbConfig.indexOptions ?? {}; // Start initialization, but don't wait for it to finish here this.initPromise = this.init(client, databaseName, collectionName).catch( @@ -169,7 +196,9 @@ export class AzureCosmosDBVectorStore extends VectorStore { * Using a numLists value of 1 is akin to performing brute-force search, * which has limited performance * @param dimensions Number of dimensions for vector similarity. - * The maximum number of supported dimensions is 2000 + * The maximum number of supported dimensions is 2000. + * If no number is provided, it will be determined automatically by + * embedding a short text. * @param similarity Similarity metric to use with the IVF index. * Possible options are: * - CosmosDBSimilarityType.COS (cosine distance) @@ -179,10 +208,17 @@ export class AzureCosmosDBVectorStore extends VectorStore { */ async createIndex( numLists = 100, - dimensions = 1536, + dimensions: number | undefined = undefined, similarity: AzureCosmosDBSimilarityType = AzureCosmosDBSimilarityType.COS ): Promise { - await this.initPromise; + await this.connectPromise; + + let vectorLength = dimensions; + + if (vectorLength === undefined) { + const queryEmbedding = await this.embeddings.embedQuery("test"); + vectorLength = queryEmbedding.length; + } const createIndexCommands = { createIndexes: this.collection.collectionName, @@ -194,7 +230,7 @@ export class AzureCosmosDBVectorStore extends VectorStore { kind: "vector-ivf", numLists, similarity, - dimensions, + dimensions: vectorLength, }, }, ], @@ -205,19 +241,33 @@ export class AzureCosmosDBVectorStore extends VectorStore { /** * Removes specified documents from the AzureCosmosDBVectorStore. - * @param ids IDs of the documents to be removed. If no IDs are specified, - * all documents will be removed. + * If no IDs or filter are specified, all documents will be removed. + * @param params Parameters for the delete operation. * @returns A promise that resolves when the documents have been removed. */ - async delete(ids?: string[]): Promise { + async delete( + params: AzureCosmosDBDeleteParams | string[] = {} + ): Promise { await this.initPromise; - if (ids) { - const objectIds = ids.map((id) => new ObjectId(id)); - await this.collection.deleteMany({ _id: { $in: objectIds } }); + let ids: string | string[] | undefined; + let filter: AzureCosmosDBDeleteParams["filter"]; + if (Array.isArray(params)) { + ids = params; } else { - await this.collection.deleteMany({}); + ids = params.ids; + filter = params.filter; + } + const idsArray = Array.isArray(ids) ? ids : [ids]; + const deleteIds = ids && idsArray.length > 0 ? idsArray : undefined; + let deleteFilter = filter ?? {}; + + if (deleteIds) { + const objectIds = deleteIds.map((id) => new ObjectId(id)); + deleteFilter = { _id: { $in: objectIds }, ...deleteFilter }; } + + await this.collection.deleteMany(deleteFilter); } /** @@ -236,27 +286,31 @@ export class AzureCosmosDBVectorStore extends VectorStore { * Method for adding vectors to the AzureCosmosDBVectorStore. * @param vectors Vectors to be added. * @param documents Corresponding documents to be added. - * @returns A promise that resolves when the vectors and documents have been added. + * @returns A promise that resolves to the added documents IDs. */ - async addVectors(vectors: number[][], documents: Document[]): Promise { + async addVectors( + vectors: number[][], + documents: DocumentInterface[] + ): Promise { const docs = vectors.map((embedding, idx) => ({ [this.textKey]: documents[idx].pageContent, [this.embeddingKey]: embedding, ...documents[idx].metadata, })); await this.initPromise; - await this.collection.insertMany(docs); + const result = await this.collection.insertMany(docs); + return Object.values(result.insertedIds).map((id) => String(id)); } /** * Method for adding documents to the AzureCosmosDBVectorStore. It first converts * the documents to texts and then adds them as vectors. * @param documents The documents to add. - * @returns A promise that resolves when the documents have been added. + * @returns A promise that resolves to the added documents IDs. */ - async addDocuments(documents: Document[]): Promise { + async addDocuments(documents: DocumentInterface[]): Promise { const texts = documents.map(({ pageContent }) => pageContent); - await this.addVectors( + return this.addVectors( await this.embeddings.embedDocuments(texts), documents ); @@ -355,9 +409,21 @@ export class AzureCosmosDBVectorStore extends VectorStore { databaseName: string, collectionName: string ): Promise { - await client.connect(); - this.database = client.db(databaseName); - this.collection = this.database.collection(collectionName); + this.connectPromise = (async () => { + await client.connect(); + this.database = client.db(databaseName); + this.collection = this.database.collection(collectionName); + })(); + + // Unless skipCreate is set, create the index + // This operation is no-op if the index already exists + if (!this.indexOptions.skipCreate) { + await this.createIndex( + this.indexOptions.numLists, + this.indexOptions.dimensions, + this.indexOptions.similarity + ); + } } /** diff --git a/libs/langchain-community/src/vectorstores/tests/azure_cosmosdb.int.test.ts b/libs/langchain-community/src/vectorstores/tests/azure_cosmosdb.int.test.ts index 7e222890229..b408bfbc3b1 100644 --- a/libs/langchain-community/src/vectorstores/tests/azure_cosmosdb.int.test.ts +++ b/libs/langchain-community/src/vectorstores/tests/azure_cosmosdb.int.test.ts @@ -48,13 +48,18 @@ describe.skip("AzureCosmosDBVectorStore", () => { process.env.AZURE_COSMOSDB_CONNECTION_STRING! ); await client.connect(); - const collection = client.db(DATABASE_NAME).collection(COLLECTION_NAME); + const db = client.db(DATABASE_NAME); + const collection = await db.createCollection(COLLECTION_NAME); // Make sure the database is empty await collection.deleteMany({}); // Delete any existing index - await collection.dropIndex(INDEX_NAME); + try { + await collection.dropIndex(INDEX_NAME); + } catch { + // Ignore error if the index does not exist + } await client.close(); }); @@ -64,6 +69,9 @@ describe.skip("AzureCosmosDBVectorStore", () => { databaseName: DATABASE_NAME, collectionName: COLLECTION_NAME, indexName: INDEX_NAME, + indexOptions: { + numLists: 1, + }, }); expect(vectorStore).toBeDefined(); @@ -75,9 +83,6 @@ describe.skip("AzureCosmosDBVectorStore", () => { { pageContent: "The house is open", metadata: { d: 1, e: 2 } }, ]); - // Make sure the index is created - await vectorStore.createIndex(1); - const results: Document[] = await vectorStore.similaritySearch( "sandwich", 1 @@ -110,12 +115,12 @@ describe.skip("AzureCosmosDBVectorStore", () => { databaseName: DATABASE_NAME, collectionName: COLLECTION_NAME, indexName: INDEX_NAME, + indexOptions: { + numLists: 1, + }, } ); - // Make sure the index is created - await vectorStore.createIndex(1); - const output = await vectorStore.maxMarginalRelevanceSearch("foo", { k: 10, fetchK: 20, @@ -160,4 +165,90 @@ describe.skip("AzureCosmosDBVectorStore", () => { await vectorStore.close(); }); + + test("deletes documents by id", async () => { + const vectorStore = new AzureCosmosDBVectorStore(new OpenAIEmbeddings(), { + databaseName: DATABASE_NAME, + collectionName: COLLECTION_NAME, + indexName: INDEX_NAME, + indexOptions: { + numLists: 1, + }, + }); + + const ids = await vectorStore.addDocuments([ + { pageContent: "This book is about politics", metadata: { a: 1 } }, + { + pageContent: "The is the house of parliament", + metadata: { d: 1, e: 2 }, + }, + ]); + + // Delete document matching specified ids + await vectorStore.delete({ ids: ids.slice(0, 1) }); + + const results = await vectorStore.similaritySearch("politics", 10); + + expect(results.length).toEqual(1); + expect(results[0].pageContent).toEqual("The is the house of parliament"); + + await vectorStore.close(); + }); + + test("deletes documents by filter", async () => { + const vectorStore = new AzureCosmosDBVectorStore(new OpenAIEmbeddings(), { + databaseName: DATABASE_NAME, + collectionName: COLLECTION_NAME, + indexName: INDEX_NAME, + indexOptions: { + numLists: 1, + }, + }); + + await vectorStore.addDocuments([ + { pageContent: "This book is about politics", metadata: { a: 1 } }, + { + pageContent: "The is the house of parliament", + metadata: { d: 1, e: 2 }, + }, + ]); + + // Delete document matching the filter + await vectorStore.delete({ filter: { a: 1 } }); + + const results = await vectorStore.similaritySearch("politics", 10); + + expect(results.length).toEqual(1); + expect(results[0].pageContent).toEqual("The is the house of parliament"); + + await vectorStore.close(); + }); + + test("deletes all documents", async () => { + const vectorStore = new AzureCosmosDBVectorStore(new OpenAIEmbeddings(), { + databaseName: DATABASE_NAME, + collectionName: COLLECTION_NAME, + indexName: INDEX_NAME, + indexOptions: { + numLists: 1, + }, + }); + + await vectorStore.addDocuments([ + { pageContent: "This book is about politics", metadata: { a: 1 } }, + { + pageContent: "The is the house of parliament", + metadata: { d: 1, e: 2 }, + }, + ]); + + // Delete all documents + await vectorStore.delete(); + + const results = await vectorStore.similaritySearch("politics", 10); + + expect(results.length).toEqual(0); + + await vectorStore.close(); + }); }); diff --git a/libs/langchain-community/src/vectorstores/tests/azure_cosmosdb.test.ts b/libs/langchain-community/src/vectorstores/tests/azure_cosmosdb.test.ts index 5f881460ec7..126f747979f 100644 --- a/libs/langchain-community/src/vectorstores/tests/azure_cosmosdb.test.ts +++ b/libs/langchain-community/src/vectorstores/tests/azure_cosmosdb.test.ts @@ -18,7 +18,9 @@ const createMockClient = () => ({ }), dropIndex: jest.fn(), deleteMany: jest.fn(), - insertMany: jest.fn(), + insertMany: jest.fn().mockImplementation((docs: any) => ({ + insertedIds: docs.map((_: any, i: any) => `id${i}`), + })), aggregate: jest.fn().mockReturnValue({ map: jest.fn().mockReturnValue({ toArray: jest @@ -75,7 +77,6 @@ test("AzureCosmosDBVectorStore manages its index", async () => { client: client as any, }); - await store.createIndex(); const indexExists = await store.checkIndexExists(); const mockDb = client.db(); @@ -103,10 +104,15 @@ test("AzureCosmosDBVectorStore deletes documents", async () => { expect(mockCollection.deleteMany).toHaveBeenCalledTimes(1); expect(mockCollection.deleteMany).toHaveBeenCalledWith({}); - await store.delete(["id1234567890", "id2345678901"]); + await store.delete({ ids: ["id1234567890", "id2345678901"] }); expect(mockCollection.deleteMany).toHaveBeenCalledTimes(2); expect(mockCollection.deleteMany.mock.calls[1][0]).toMatchObject({ _id: {} }); + + await store.delete({ filter: { a: 1 } }); + + expect(mockCollection.deleteMany).toHaveBeenCalledTimes(3); + expect(mockCollection.deleteMany.mock.calls[2][0]).toMatchObject({ a: 1 }); }); test("AzureCosmosDBVectorStore adds vectors", async () => {