text-splitters[minor], langchain[minor], community[patch], templates,…

… docs: langchain-text-splitters 0.0.1 (#18346)
langchain-ai · Mar 1, 2024 · 5efb5c0 · 5efb5c0
1 parent 7891934
commit 5efb5c0
Show file tree

Hide file tree

Showing 226 changed files with 6,628 additions and 1,982 deletions.
diff --git a/.github/scripts/check_diff.py b/.github/scripts/check_diff.py
@@ -5,6 +5,7 @@
 
 LANGCHAIN_DIRS = [
     "libs/core",
+    "libs/text-splitters",
     "libs/community",
     "libs/langchain",
     "libs/experimental",

diff --git a/.github/scripts/get_min_versions.py b/.github/scripts/get_min_versions.py
@@ -4,7 +4,7 @@
 from packaging.version import parse as parse_version
 import re
 
-MIN_VERSION_LIBS = ["langchain-core", "langchain-community", "langchain"]
+MIN_VERSION_LIBS = ["langchain-core", "langchain-community", "langchain", "langchain-text-splitters"]
 
 
 def get_min_version(version: str) -> str:

diff --git a/cookbook/Multi_modal_RAG.ipynb b/cookbook/Multi_modal_RAG.ipynb
@@ -116,7 +116,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.text_splitter import CharacterTextSplitter\n",
+    "from langchain_text_splitters import CharacterTextSplitter\n",
     "from unstructured.partition.pdf import partition_pdf\n",
     "\n",
     "\n",

diff --git a/cookbook/advanced_rag_eval.ipynb b/cookbook/advanced_rag_eval.ipynb
@@ -68,7 +68,7 @@
     "pdf_pages = loader.load()\n",
     "\n",
     "# Split\n",
-    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
     "\n",
     "text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=0)\n",
     "all_splits_pypdf = text_splitter.split_documents(pdf_pages)\n",

diff --git a/cookbook/agent_vectorstore.ipynb b/cookbook/agent_vectorstore.ipynb
@@ -28,9 +28,9 @@
    "outputs": [],
    "source": [
     "from langchain.chains import RetrievalQA\n",
-    "from langchain.text_splitter import CharacterTextSplitter\n",
     "from langchain_community.vectorstores import Chroma\n",
     "from langchain_openai import OpenAI, OpenAIEmbeddings\n",
+    "from langchain_text_splitters import CharacterTextSplitter\n",
     "\n",
     "llm = OpenAI(temperature=0)"
    ]

diff --git a/cookbook/autogpt/marathon_times.ipynb b/cookbook/autogpt/marathon_times.ipynb
@@ -227,8 +227,8 @@
     "    BaseCombineDocumentsChain,\n",
     "    load_qa_with_sources_chain,\n",
     ")\n",
-    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
     "from langchain.tools import BaseTool, DuckDuckGoSearchRun\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
     "from pydantic import Field\n",
     "\n",
     "\n",

diff --git a/cookbook/code-analysis-deeplake.ipynb b/cookbook/code-analysis-deeplake.ipynb
@@ -24,7 +24,7 @@
    "source": [
     "1. Prepare data:\n",
     "   1. Upload all python project files using the `langchain_community.document_loaders.TextLoader`. We will call these files the **documents**.\n",
-    "   2. Split all documents to chunks using the `langchain.text_splitter.CharacterTextSplitter`.\n",
+    "   2. Split all documents to chunks using the `langchain_text_splitters.CharacterTextSplitter`.\n",
     "   3. Embed chunks and upload them into the DeepLake using `langchain.embeddings.openai.OpenAIEmbeddings` and `langchain_community.vectorstores.DeepLake`\n",
     "2. Question-Answering:\n",
     "   1. Build a chain from `langchain.chat_models.ChatOpenAI` and `langchain.chains.ConversationalRetrievalChain`\n",
@@ -621,7 +621,7 @@
     }
    ],
    "source": [
-    "from langchain.text_splitter import CharacterTextSplitter\n",
+    "from langchain_text_splitters import CharacterTextSplitter\n",
     "\n",
     "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
     "texts = text_splitter.split_documents(docs)\n",

diff --git a/cookbook/deeplake_semantic_search_over_chat.ipynb b/cookbook/deeplake_semantic_search_over_chat.ipynb
@@ -52,12 +52,12 @@
     "import os\n",
     "\n",
     "from langchain.chains import RetrievalQA\n",
-    "from langchain.text_splitter import (\n",
+    "from langchain_community.vectorstores import DeepLake\n",
+    "from langchain_openai import OpenAI, OpenAIEmbeddings\n",
+    "from langchain_text_splitters import (\n",
     "    CharacterTextSplitter,\n",
     "    RecursiveCharacterTextSplitter,\n",
     ")\n",
-    "from langchain_community.vectorstores import DeepLake\n",
-    "from langchain_openai import OpenAI, OpenAIEmbeddings\n",
     "\n",
     "os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"OpenAI API Key:\")\n",
     "activeloop_token = getpass.getpass(\"Activeloop Token:\")\n",

diff --git a/cookbook/fireworks_rag.ipynb b/cookbook/fireworks_rag.ipynb
@@ -132,7 +132,7 @@
     "data = loader.load()\n",
     "\n",
     "# Split\n",
-    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
     "\n",
     "text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=0)\n",
     "all_splits = text_splitter.split_documents(data)\n",

diff --git a/cookbook/hypothetical_document_embeddings.ipynb b/cookbook/hypothetical_document_embeddings.ipynb
@@ -170,8 +170,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.text_splitter import CharacterTextSplitter\n",
     "from langchain_community.vectorstores import Chroma\n",
+    "from langchain_text_splitters import CharacterTextSplitter\n",
     "\n",
     "with open(\"../../state_of_the_union.txt\") as f:\n",
     "    state_of_the_union = f.read()\n",

diff --git a/cookbook/nomic_embedding_rag.ipynb b/cookbook/nomic_embedding_rag.ipynb
@@ -124,7 +124,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.text_splitter import CharacterTextSplitter\n",
+    "from langchain_text_splitters import CharacterTextSplitter\n",
     "\n",
     "text_splitter = CharacterTextSplitter.from_tiktoken_encoder(\n",
     "    chunk_size=7500, chunk_overlap=100\n",

diff --git a/cookbook/openai_functions_retrieval_qa.ipynb b/cookbook/openai_functions_retrieval_qa.ipynb
@@ -20,10 +20,10 @@
    "outputs": [],
    "source": [
     "from langchain.chains import RetrievalQA\n",
-    "from langchain.text_splitter import CharacterTextSplitter\n",
     "from langchain_community.document_loaders import TextLoader\n",
     "from langchain_community.vectorstores import Chroma\n",
-    "from langchain_openai import OpenAIEmbeddings"
+    "from langchain_openai import OpenAIEmbeddings\n",
+    "from langchain_text_splitters import CharacterTextSplitter"
    ]
   },
   {

diff --git a/cookbook/qianfan_baidu_elasticesearch_RAG.ipynb b/cookbook/qianfan_baidu_elasticesearch_RAG.ipynb
@@ -59,13 +59,13 @@
     "from baidubce.auth.bce_credentials import BceCredentials\n",
     "from baidubce.bce_client_configuration import BceClientConfiguration\n",
     "from langchain.chains.retrieval_qa import RetrievalQA\n",
-    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
     "from langchain_community.document_loaders.baiducloud_bos_directory import (\n",
     "    BaiduBOSDirectoryLoader,\n",
     ")\n",
     "from langchain_community.embeddings.huggingface import HuggingFaceEmbeddings\n",
     "from langchain_community.llms.baidu_qianfan_endpoint import QianfanLLMEndpoint\n",
-    "from langchain_community.vectorstores import BESVectorStore"
+    "from langchain_community.vectorstores import BESVectorStore\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter"
    ]
   },
   {

diff --git a/cookbook/rag_with_quantized_embeddings.ipynb b/cookbook/rag_with_quantized_embeddings.ipynb
@@ -36,16 +36,16 @@
     "from bs4 import BeautifulSoup as Soup\n",
     "from langchain.retrievers.multi_vector import MultiVectorRetriever\n",
     "from langchain.storage import InMemoryByteStore, LocalFileStore\n",
-    "\n",
-    "# For our example, we'll load docs from the web\n",
-    "from langchain.text_splitter import RecursiveCharacterTextSplitter  # noqa\n",
     "from langchain_community.document_loaders.recursive_url_loader import (\n",
     "    RecursiveUrlLoader,\n",
     ")\n",
     "\n",
     "# noqa\n",
     "from langchain_community.vectorstores import Chroma\n",
     "\n",
+    "# For our example, we'll load docs from the web\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter  # noqa\n",
+    "\n",
     "DOCSTORE_DIR = \".\"\n",
     "DOCSTORE_ID_KEY = \"doc_id\""
    ]

diff --git a/cookbook/sales_agent_with_context.ipynb b/cookbook/sales_agent_with_context.ipynb
@@ -51,11 +51,11 @@
     "from langchain.chains.base import Chain\n",
     "from langchain.prompts import PromptTemplate\n",
     "from langchain.prompts.base import StringPromptTemplate\n",
-    "from langchain.text_splitter import CharacterTextSplitter\n",
     "from langchain_community.llms import BaseLLM\n",
     "from langchain_community.vectorstores import Chroma\n",
     "from langchain_core.agents import AgentAction, AgentFinish\n",
     "from langchain_openai import ChatOpenAI, OpenAI, OpenAIEmbeddings\n",
+    "from langchain_text_splitters import CharacterTextSplitter\n",
     "from pydantic import BaseModel, Field"
    ]
   },

diff --git a/cookbook/together_ai.ipynb b/cookbook/together_ai.ipynb
@@ -39,7 +39,7 @@
     "data = loader.load()\n",
     "\n",
     "# Split\n",
-    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
     "\n",
     "text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=0)\n",
     "all_splits = text_splitter.split_documents(data)\n",

diff --git a/cookbook/twitter-the-algorithm-analysis-deeplake.ipynb b/cookbook/twitter-the-algorithm-analysis-deeplake.ipynb
@@ -2610,7 +2610,7 @@
     }
    ],
    "source": [
-    "from langchain.text_splitter import CharacterTextSplitter\n",
+    "from langchain_text_splitters import CharacterTextSplitter\n",
     "\n",
     "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
     "texts = text_splitter.split_documents(docs)"

diff --git a/docs/docs/get_started/quickstart.mdx b/docs/docs/get_started/quickstart.mdx
@@ -281,7 +281,7 @@ Then we can build our index:
 
 ```python
 from langchain_community.vectorstores import FAISS
-from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 
 
 text_splitter = RecursiveCharacterTextSplitter()
@@ -531,7 +531,7 @@ from langchain_openai import ChatOpenAI
 from langchain_community.document_loaders import WebBaseLoader
 from langchain_openai import OpenAIEmbeddings
 from langchain_community.vectorstores import FAISS
-from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain.tools.retriever import create_retriever_tool
 from langchain_community.tools.tavily_search import TavilySearchResults
 from langchain_openai import ChatOpenAI

diff --git a/docs/docs/guides/privacy/presidio_data_anonymization/qa_privacy_protection.ipynb b/docs/docs/guides/privacy/presidio_data_anonymization/qa_privacy_protection.ipynb
@@ -643,9 +643,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
     "from langchain_community.vectorstores import FAISS\n",
     "from langchain_openai import OpenAIEmbeddings\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
     "\n",
     "# 2. Load the data: In our case data's already loaded\n",
     "# 3. Anonymize the data before indexing\n",

diff --git a/docs/docs/integrations/callbacks/confident.ipynb b/docs/docs/integrations/callbacks/confident.ipynb
@@ -215,10 +215,10 @@
    "source": [
     "import requests\n",
     "from langchain.chains import RetrievalQA\n",
-    "from langchain.text_splitter import CharacterTextSplitter\n",
     "from langchain_community.document_loaders import TextLoader\n",
     "from langchain_community.vectorstores import Chroma\n",
     "from langchain_openai import OpenAI, OpenAIEmbeddings\n",
+    "from langchain_text_splitters import CharacterTextSplitter\n",
     "\n",
     "text_file_url = \"https://raw.githubusercontent.com/hwchase17/chat-your-data/master/state_of_the_union.txt\"\n",
     "\n",

diff --git a/docs/docs/integrations/document_loaders/psychic.ipynb b/docs/docs/integrations/document_loaders/psychic.ipynb
@@ -78,9 +78,9 @@
    "outputs": [],
    "source": [
     "from langchain.chains import RetrievalQAWithSourcesChain\n",
-    "from langchain.text_splitter import CharacterTextSplitter\n",
     "from langchain_community.vectorstores import Chroma\n",
-    "from langchain_openai import OpenAI, OpenAIEmbeddings"
+    "from langchain_openai import OpenAI, OpenAIEmbeddings\n",
+    "from langchain_text_splitters import CharacterTextSplitter"
    ]
   },
   {

diff --git a/docs/docs/integrations/document_loaders/source_code.ipynb b/docs/docs/integrations/document_loaders/source_code.ipynb
@@ -62,9 +62,9 @@
     "warnings.filterwarnings(\"ignore\")\n",
     "from pprint import pprint\n",
     "\n",
-    "from langchain.text_splitter import Language\n",
     "from langchain_community.document_loaders.generic import GenericLoader\n",
-    "from langchain_community.document_loaders.parsers import LanguageParser"
+    "from langchain_community.document_loaders.parsers import LanguageParser\n",
+    "from langchain_text_splitters import Language"
    ]
   },
   {
@@ -323,7 +323,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.text_splitter import (\n",
+    "from langchain_text_splitters import (\n",
     "    Language,\n",
     "    RecursiveCharacterTextSplitter,\n",
     ")"
@@ -426,6 +426,7 @@
   },
   {
    "cell_type": "markdown",
+   "id": "7fb27b941602401d91542211134fc71a",
    "metadata": {},
    "source": [
     "## Adding Languages using Tree-sitter Template\n",

diff --git a/docs/docs/integrations/document_loaders/youtube_audio.ipynb b/docs/docs/integrations/document_loaders/youtube_audio.ipynb
@@ -168,9 +168,9 @@
    "outputs": [],
    "source": [
     "from langchain.chains import RetrievalQA\n",
-    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
     "from langchain_community.vectorstores import FAISS\n",
-    "from langchain_openai import ChatOpenAI, OpenAIEmbeddings"
+    "from langchain_openai import ChatOpenAI, OpenAIEmbeddings\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter"
    ]
   },
   {

diff --git a/docs/docs/integrations/llms/llm_caching.ipynb b/docs/docs/integrations/llms/llm_caching.ipynb
@@ -1463,7 +1463,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.text_splitter import CharacterTextSplitter\n",
+    "from langchain_text_splitters import CharacterTextSplitter\n",
     "\n",
     "text_splitter = CharacterTextSplitter()"
    ]

diff --git a/docs/docs/integrations/llms/manifest.ipynb b/docs/docs/integrations/llms/manifest.ipynb
@@ -82,7 +82,7 @@
     "# Map reduce example\n",
     "from langchain.chains.mapreduce import MapReduceChain\n",
     "from langchain.prompts import PromptTemplate\n",
-    "from langchain.text_splitter import CharacterTextSplitter\n",
+    "from langchain_text_splitters import CharacterTextSplitter\n",
     "\n",
     "_prompt = \"\"\"Write a concise summary of the following:\n",
     "\n",

diff --git a/docs/docs/integrations/platforms/openai.mdx b/docs/docs/integrations/platforms/openai.mdx
@@ -68,7 +68,7 @@ for OpenAI LLMs.
 
 You can also use it to count tokens when splitting documents with 
 ```python
-from langchain.text_splitter import CharacterTextSplitter
+from langchain_text_splitters import CharacterTextSplitter
 CharacterTextSplitter.from_tiktoken_encoder(...)
 ```
 For a more detailed walkthrough of this, see [this notebook](/docs/modules/data_connection/document_transformers/split_by_token#tiktoken)

diff --git a/docs/docs/integrations/providers/elasticsearch.mdx b/docs/docs/integrations/providers/elasticsearch.mdx
@@ -34,7 +34,7 @@ The vector store is a simple wrapper around Elasticsearch. It provides a simple
 from langchain_elasticsearch import ElasticsearchStore
 
 from langchain_community.document_loaders import TextLoader
-from langchain.text_splitter import CharacterTextSplitter
+from langchain_text_splitters import CharacterTextSplitter
 
 loader = TextLoader("./state_of_the_union.txt")
 documents = loader.load()

diff --git a/docs/docs/integrations/providers/ragatouille.ipynb b/docs/docs/integrations/providers/ragatouille.ipynb
@@ -87,9 +87,9 @@
    "outputs": [],
    "source": [
     "import requests\n",
-    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
     "from langchain_community.vectorstores import FAISS\n",
     "from langchain_openai import OpenAIEmbeddings\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
     "\n",
     "\n",
     "def get_wikipedia_page(title: str):\n",

diff --git a/docs/docs/integrations/providers/spacy.mdx b/docs/docs/integrations/providers/spacy.mdx
@@ -16,7 +16,7 @@ pip install spacy
 See a [usage example](/docs/modules/data_connection/document_transformers/split_by_token#spacy).
 
 ```python
-from langchain.text_splitter import SpacyTextSplitter
+from langchain_text_splitters import SpacyTextSplitter
 ```
 
 ## Text Embedding Models

diff --git a/docs/docs/integrations/retrievers/activeloop.ipynb b/docs/docs/integrations/retrievers/activeloop.ipynb
@@ -192,7 +192,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
     "\n",
     "chunk_size = 4096\n",
     "docs_new = []\n",

diff --git a/docs/docs/integrations/retrievers/cohere-reranker.ipynb b/docs/docs/integrations/retrievers/cohere-reranker.ipynb
@@ -301,10 +301,10 @@
     }
    ],
    "source": [
-    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
     "from langchain_community.document_loaders import TextLoader\n",
     "from langchain_community.embeddings import CohereEmbeddings\n",
     "from langchain_community.vectorstores import FAISS\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
     "\n",
     "documents = TextLoader(\"../../modules/state_of_the_union.txt\").load()\n",
     "text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)\n",

diff --git a/docs/docs/integrations/retrievers/flashrank-reranker.ipynb b/docs/docs/integrations/retrievers/flashrank-reranker.ipynb
@@ -288,10 +288,10 @@
     }
    ],
    "source": [
-    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
     "from langchain_community.document_loaders import TextLoader\n",
     "from langchain_community.vectorstores import FAISS\n",
     "from langchain_openai import OpenAIEmbeddings\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
     "\n",
     "documents = TextLoader(\n",
     "    \"../../modules/state_of_the_union.txt\",\n",

diff --git a/docs/docs/integrations/retrievers/jaguar.ipynb b/docs/docs/integrations/retrievers/jaguar.ipynb
@@ -52,10 +52,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.text_splitter import CharacterTextSplitter\n",
     "from langchain_community.document_loaders import TextLoader\n",
     "from langchain_community.vectorstores.jaguar import Jaguar\n",
     "from langchain_openai import OpenAIEmbeddings\n",
+    "from langchain_text_splitters import CharacterTextSplitter\n",
     "\n",
     "\"\"\" \n",
     "Load a text file into a set of documents \n",