diff --git a/docs/docs/community/integrations/vector_stores.md b/docs/docs/community/integrations/vector_stores.md index 7e01906bde0ca..967fcb90eb87a 100644 --- a/docs/docs/community/integrations/vector_stores.md +++ b/docs/docs/community/integrations/vector_stores.md @@ -32,7 +32,7 @@ as the storage backend for `VectorStoreIndex`. - Pinecone (`PineconeVectorStore`). [Installation/Quickstart](https://docs.pinecone.io/docs/quickstart). - Qdrant (`QdrantVectorStore`) [Installation](https://qdrant.tech/documentation/install/) [Python Client](https://qdrant.tech/documentation/install/#python-client) - LanceDB (`LanceDBVectorStore`) [Installation/Quickstart](https://lancedb.github.io/lancedb/basic/) -- Redis (`RedisVectorStore`). [Installation](https://redis.io/docs/getting-started/installation/). +- Redis (`RedisVectorStore`). [Installation](https://redis.io/docs/latest/operate/oss_and_stack/install/install-stack/). - Supabase (`SupabaseVectorStore`). [Quickstart](https://supabase.github.io/vecs/api/). - TiDB (`TiDBVectorStore`). [Quickstart](../../examples/vector_stores/TiDBVector.ipynb). [Installation](https://tidb.cloud/ai). [Python Client](https://github.com/pingcap/tidb-vector-python). - TimeScale (`TimescaleVectorStore`). [Installation](https://github.com/timescale/python-vector). diff --git a/docs/docs/examples/ingestion/ingestion_gdrive.ipynb b/docs/docs/examples/ingestion/ingestion_gdrive.ipynb index 9d60b8fb67870..57c244353480c 100644 --- a/docs/docs/examples/ingestion/ingestion_gdrive.ipynb +++ b/docs/docs/examples/ingestion/ingestion_gdrive.ipynb @@ -100,10 +100,107 @@ " IngestionPipeline,\n", " IngestionCache,\n", ")\n", - "from llama_index.core.ingestion.cache import RedisCache\n", + "from llama_index.storage.kvstore.redis import RedisKVStore as RedisCache\n", "from llama_index.storage.docstore.redis import RedisDocumentStore\n", "from llama_index.core.node_parser import SentenceSplitter\n", - "from llama_index.vector_stores.redis import RedisVectorStore" + "from llama_index.vector_stores.redis import RedisVectorStore\n", + "\n", + "from redisvl.schema import IndexSchema" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "baf744be", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ac74203675564f14b73882a6ae270d18", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "model.safetensors: 0%| | 0.00/133M [00:00=4.0.2 in /home/loganm/.cache/pypoetry/virtualenvs/llama-index-4a-wkI5X-py3.11/lib/python3.11/site-packages (from redis) (4.0.3)\n", - "\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n" - ] - } - ], - "source": [ - "!pip install redis" - ] - }, { "cell_type": "code", "execution_count": null, @@ -75,7 +54,8 @@ "source": [ "import os\n", "\n", - "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"" + "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"" ] }, { @@ -102,16 +82,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/loganm/.cache/pypoetry/virtualenvs/llama-index-4a-wkI5X-py3.11/lib/python3.11/site-packages/deeplake/util/check_latest_version.py:32: UserWarning: A newer version of deeplake (3.8.9) is available. It's recommended that you update to the latest version using `pip install -U deeplake`.\n", - " warnings.warn(\n" - ] - } - ], + "outputs": [], "source": [ "from llama_index.core import SimpleDirectoryReader\n", "\n", @@ -144,13 +115,46 @@ " IngestionPipeline,\n", " IngestionCache,\n", ")\n", - "from llama_index.core.ingestion.cache import RedisCache\n", + "from llama_index.storage.kvstore.redis import RedisKVStore as RedisCache\n", "from llama_index.storage.docstore.redis import RedisDocumentStore\n", "from llama_index.core.node_parser import SentenceSplitter\n", "from llama_index.vector_stores.redis import RedisVectorStore\n", "\n", + "from redisvl.schema import IndexSchema\n", + "\n", + "\n", "embed_model = HuggingFaceEmbedding(model_name=\"BAAI/bge-small-en-v1.5\")\n", "\n", + "custom_schema = IndexSchema.from_dict(\n", + " {\n", + " \"index\": {\"name\": \"redis_vector_store\", \"prefix\": \"doc\"},\n", + " # customize fields that are indexed\n", + " \"fields\": [\n", + " # required fields for llamaindex\n", + " {\"type\": \"tag\", \"name\": \"id\"},\n", + " {\"type\": \"tag\", \"name\": \"doc_id\"},\n", + " {\"type\": \"text\", \"name\": \"text\"},\n", + " # custom vector field for bge-small-en-v1.5 embeddings\n", + " {\n", + " \"type\": \"vector\",\n", + " \"name\": \"vector\",\n", + " \"attrs\": {\n", + " \"dims\": 384,\n", + " \"algorithm\": \"hnsw\",\n", + " \"distance_metric\": \"cosine\",\n", + " },\n", + " },\n", + " ],\n", + " }\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "pipeline = IngestionPipeline(\n", " transformations=[\n", " SentenceSplitter(),\n", @@ -160,8 +164,7 @@ " \"localhost\", 6379, namespace=\"document_store\"\n", " ),\n", " vector_store=RedisVectorStore(\n", - " index_name=\"redis_vector_store\",\n", - " index_prefix=\"vectore_store\",\n", + " schema=custom_schema,\n", " redis_url=\"redis://localhost:6379\",\n", " ),\n", " cache=IngestionCache(\n", @@ -221,7 +224,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "I see two documents: \"test2.txt\" and \"test1.txt\".\n" + "I see two documents.\n" ] } ], @@ -261,6 +264,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "13:32:07 redisvl.index.index INFO Index already exists, not overwriting.\n", "Ingested 2 Nodes\n" ] } @@ -284,7 +288,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "I see three documents: test3.txt, test1.txt, and test2.txt.\n", + "You see three documents: test3.txt, test1.txt, and test2.txt.\n", "This is a test file: three!\n", "This is a NEW test file: one!\n", "This is a test file: two!\n" diff --git a/docs/docs/examples/vector_stores/RedisIndexDemo.ipynb b/docs/docs/examples/vector_stores/RedisIndexDemo.ipynb index c3bd2d256844a..acde748797bb4 100644 --- a/docs/docs/examples/vector_stores/RedisIndexDemo.ipynb +++ b/docs/docs/examples/vector_stores/RedisIndexDemo.ipynb @@ -43,17 +43,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install llama-index-vector-stores-redis" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c71a947d", - "metadata": {}, - "outputs": [], - "source": [ - "!pip install llama-index" + "%pip install -U llama-index llama-index-vector-stores-redis llama-index-embeddings-cohere llama-index-embeddings-openai" ] }, { @@ -64,24 +54,19 @@ "outputs": [], "source": [ "import os\n", + "import getpass\n", "import sys\n", "import logging\n", "import textwrap\n", - "\n", "import warnings\n", "\n", "warnings.filterwarnings(\"ignore\")\n", "\n", - "# stop huggingface warnings\n", - "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", - "\n", "# Uncomment to see debug logs\n", - "# logging.basicConfig(stream=sys.stdout, level=logging.INFO)\n", - "# logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))\n", + "logging.basicConfig(stream=sys.stdout, level=logging.INFO)\n", "\n", - "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, Document\n", - "from llama_index.vector_stores.redis import RedisVectorStore\n", - "from IPython.display import Markdown, display" + "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n", + "from llama_index.vector_stores.redis import RedisVectorStore" ] }, { @@ -92,7 +77,8 @@ "source": [ "### Start Redis\n", "\n", - "The easiest way to start Redis as a vector database is using the [redis-stack](https://hub.docker.com/r/redis/redis-stack) docker image.\n", + "The easiest way to start Redis is using the [Redis Stack](https://hub.docker.com/r/redis/redis-stack) docker image or\n", + "quickly signing up for a [FREE Redis Cloud](https://redis.com/try-free) instance.\n", "\n", "To follow every step of this tutorial, launch the image as follows:\n", "\n", @@ -120,9 +106,8 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", - "\n", - "os.environ[\"OPENAI_API_KEY\"] = \"sk-\"" + "oai_api_key = getpass.getpass(\"OpenAI API Key:\")\n", + "os.environ[\"OPENAI_API_KEY\"] = oai_api_key" ] }, { @@ -139,7 +124,25 @@ "execution_count": null, "id": "304ad9d8", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--2024-04-10 19:35:33-- https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt\n", + "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 2606:50c0:8003::154, 2606:50c0:8000::154, 2606:50c0:8002::154, ...\n", + "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|2606:50c0:8003::154|:443... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 75042 (73K) [text/plain]\n", + "Saving to: ‘data/paul_graham/paul_graham_essay.txt’\n", + "\n", + "data/paul_graham/pa 100%[===================>] 73.28K --.-KB/s in 0.03s \n", + "\n", + "2024-04-10 19:35:33 (2.15 MB/s) - ‘data/paul_graham/paul_graham_essay.txt’ saved [75042/75042]\n", + "\n" + ] + } + ], "source": [ "!mkdir -p 'data/paul_graham/'\n", "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt' -O 'data/paul_graham/paul_graham_essay.txt'" @@ -165,7 +168,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Document ID: faa23c94-ac9e-4763-92ba-e0f87bf38195 Document Hash: 77ae91ab542f3abb308c4d7c77c9bc4c9ad0ccd63144802b7cbe7e1bb3a4094e\n" + "Document ID: 7056f7ba-3513-4ef4-9792-2bd28040aaed Document Filename: paul_graham_essay.txt\n" ] } ], @@ -174,399 +177,321 @@ "documents = SimpleDirectoryReader(\"./data/paul_graham\").load_data()\n", "print(\n", " \"Document ID:\",\n", - " documents[0].doc_id,\n", - " \"Document Hash:\",\n", - " documents[0].doc_hash,\n", + " documents[0].id_,\n", + " \"Document Filename:\",\n", + " documents[0].metadata[\"file_name\"],\n", ")" ] }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "ce335c9a", - "metadata": {}, - "source": [ - "You can process your files individually using [SimpleDirectoryReader](/examples/data_connectors/simple_directory_reader.ipynb):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1f67cb2a", - "metadata": {}, - "outputs": [], - "source": [ - "loader = SimpleDirectoryReader(\"./data/paul_graham\")\n", - "documents = loader.load_data()\n", - "for file in loader.input_files:\n", - " print(file)\n", - " # Here is where you would do any preprocessing" - ] - }, { "attachments": {}, "cell_type": "markdown", "id": "dd270925", "metadata": {}, "source": [ - "### Initialize the Redis Vector Store\n", - "\n", - "Now we have our documents read in, we can initialize the Redis Vector Store. This will allow us to store our vectors in Redis and create an index.\n", + "### Initialize the default Redis Vector Store\n", "\n", - "Below you can see the docstring for `RedisVectorStore`." + "Now we have our documents prepared, we can initialize the Redis Vector Store with **default** settings. This will allow us to store our vectors in Redis and create an index for real-time search." ] }, { "cell_type": "code", "execution_count": null, - "id": "f6d7f6c6-805b-4a2a-a9d3-a7c8c9de37ac", + "id": "ba1558b3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Initialize RedisVectorStore.\n", - "\n", - " For index arguments that can be passed to RediSearch, see\n", - " https://redis.io/docs/stack/search/reference/vectors/\n", - "\n", - " The index arguments will depend on the index type chosen. There\n", - " are two available index types\n", - " - FLAT: a flat index that uses brute force search\n", - " - HNSW: a hierarchical navigable small world graph index\n", - "\n", - " Args:\n", - " index_name (str): Name of the index.\n", - " index_prefix (str): Prefix for the index. Defaults to \"llama_index\".\n", - " The actual prefix used by Redis will be\n", - " \"{index_prefix}{prefix_ending}\".\n", - " prefix_ending (str): Prefix ending for the index. Be careful when\n", - " changing this: https://github.com/jerryjliu/llama_index/pull/6665.\n", - " Defaults to \"/vector\".\n", - " index_args (Dict[str, Any]): Arguments for the index. Defaults to None.\n", - " metadata_fields (List[str]): List of metadata fields to store in the index\n", - " (only supports TAG fields).\n", - " redis_url (str): URL for the redis instance.\n", - " Defaults to \"redis://localhost:6379\".\n", - " overwrite (bool): Whether to overwrite the index if it already exists.\n", - " Defaults to False.\n", - " kwargs (Any): Additional arguments to pass to the redis client.\n", - "\n", - " Raises:\n", - " ValueError: If redis-py is not installed\n", - " ValueError: If RediSearch is not installed\n", - "\n", - " Examples:\n", - " >>> from llama_index.vector_stores.redis import RedisVectorStore\n", - " >>> # Create a RedisVectorStore\n", - " >>> vector_store = RedisVectorStore(\n", - " >>> index_name=\"my_index\",\n", - " >>> index_prefix=\"llama_index\",\n", - " >>> index_args={\"algorithm\": \"HNSW\", \"m\": 16, \"ef_construction\": 200,\n", - " \"distance_metric\": \"cosine\"},\n", - " >>> redis_url=\"redis://localhost:6379/\",\n", - " >>> overwrite=True)\n", - "\n", - " \n" + "19:39:17 llama_index.vector_stores.redis.base INFO Using default RedisVectorStore schema.\n", + "19:39:19 httpx INFO HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n", + "19:39:19 llama_index.vector_stores.redis.base INFO Added 22 documents to index llama_index\n" ] } ], - "source": [ - "print(RedisVectorStore.__init__.__doc__)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ba1558b3", - "metadata": {}, - "outputs": [], "source": [ "from llama_index.core import StorageContext\n", + "from redis import Redis\n", "\n", - "vector_store = RedisVectorStore(\n", - " index_name=\"pg_essays\",\n", - " index_prefix=\"llama\",\n", - " redis_url=\"redis://localhost:6379\", # Default\n", - " overwrite=True,\n", - ")\n", + "# create a Redis client connection\n", + "redis_client = Redis.from_url(\"redis://localhost:6379\")\n", + "\n", + "# create the vector store wrapper\n", + "vector_store = RedisVectorStore(redis_client=redis_client, overwrite=True)\n", + "\n", + "# load storage context\n", "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n", + "\n", + "# build and load index from documents and storage context\n", "index = VectorStoreIndex.from_documents(\n", " documents, storage_context=storage_context\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "b481dc47", - "metadata": {}, - "source": [ - "With logging on, it prints out the following:\n", - "\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "d3f2c1e1", - "metadata": {}, - "source": [ - "```bash\n", - "INFO:llama_index.vector_stores.redis:Creating index pg_essays\n", - "Creating index pg_essays\n", - "INFO:llama_index.vector_stores.redis:Added 15 documents to index pg_essays\n", - "Added 15 documents to index pg_essays\n", - "INFO:llama_index.vector_stores.redis:Saving index to disk in background\n", - "```" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "e02256f9", - "metadata": {}, - "source": [ - "Now you can browse these index in redis-cli and read/write it as Redis hash. It looks like this:" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "83bb439b", - "metadata": {}, - "source": [ - "```bash\n", - "$ redis-cli\n", - "127.0.0.1:6379> keys *\n", - " 1) \"llama/vector_0f125320-f5cf-40c2-8462-aefc7dbff490\"\n", - " 2) \"llama/vector_bd667698-4311-4a67-bb8b-0397b03ec794\"\n", - "127.0.0.1:6379> HGETALL \"llama/vector_bd667698-4311-4a67-bb8b-0397b03ec794\"\n", - "...\n", - "```" + ")\n", + "# index = VectorStoreIndex.from_vector_store(vector_store=vector_store)" ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "405d445f", + "id": "dc00b3fb", "metadata": {}, "source": [ - "### Handle duplicated index\n", + "### Query the default vector store\n", + "\n", + "Now that we have our data stored in the index, we can ask questions against the index.\n", + "\n", + "The index will use the data as the knowledge base for an LLM. The default setting for as_query_engine() utilizes OpenAI embeddings and GPT as the language model. Therefore, an OpenAI key is required unless you opt for a customized or local language model.\n", "\n", - "Regardless of whether overwrite=True is used in RedisVectorStore(), the process of generating the index and storing data in Redis still takes time. Currently, it is necessary to implement your own logic to manage duplicate indexes. One possible approach is to set a flag in Redis to indicate the readiness of the index. If the flag is set, you can bypass the index generation step and directly load the index from Redis." + "Below we will test searches against out index and then full RAG with an LLM." ] }, { "cell_type": "code", "execution_count": null, - "id": "218e612a", + "id": "c50a593f", "metadata": {}, "outputs": [], "source": [ - "import redis\n", - "r = redis.Redis()\n", - "index_name = \"pg_essays\"\n", - "r.set(f\"added:{index_name}\", \"true\")\n", - "\n", - "# Later in code\n", - "if r.get(f\"added:{index_name}\"):\n", - " # Skip to deploy your index, restore it. Please see \"Restore index from Redis\" section below. " - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "04304299-fc3e-40a0-8600-f50c3292767e", - "metadata": {}, - "source": [ - "### Query the data\n", - "Now that we have our document stored in the index, we can ask questions against the index. The index will use the data stored in itself as the knowledge base for ChatGPT. The default setting for as_query_engine() utilizes OpenAI embeddings and ChatGPT as the language model. Therefore, an OpenAI key is required unless you opt for a customized or local language model." + "query_engine = index.as_query_engine()\n", + "retriever = index.as_retriever()" ] }, { "cell_type": "code", "execution_count": null, - "id": "35369eda", + "id": "e3f0daf7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " The author learned that it is possible to publish essays online, and that working on things that\n", - "are not prestigious can be a sign that one is on the right track. They also learned that impure\n", - "motives can lead ambitious people astray, and that it is possible to make connections with people\n", - "through cleverly planned events. Finally, the author learned that they could find love through a\n", - "chance meeting at a party.\n" + "19:39:22 httpx INFO HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n", + "19:39:22 llama_index.vector_stores.redis.base INFO Querying index llama_index with filters *\n", + "19:39:22 llama_index.vector_stores.redis.base INFO Found 2 results for query with id ['llama_index/vector_adb6b7ce-49bb-4961-8506-37082c02a389', 'llama_index/vector_e39be1fe-32d0-456e-b211-4efabd191108']\n", + "Node ID: adb6b7ce-49bb-4961-8506-37082c02a389\n", + "Text: What I Worked On February 2021 Before college the two main\n", + "things I worked on, outside of school, were writing and programming. I\n", + "didn't write essays. I wrote what beginning writers were supposed to\n", + "write then, and probably still are: short stories. My stories were\n", + "awful. They had hardly any plot, just characters with strong feelings,\n", + "which I ...\n", + "Score: 0.820\n", + "\n", + "Node ID: e39be1fe-32d0-456e-b211-4efabd191108\n", + "Text: Except for a few officially anointed thinkers who went to the\n", + "right parties in New York, the only people allowed to publish essays\n", + "were specialists writing about their specialties. There were so many\n", + "essays that had never been written, because there had been no way to\n", + "publish them. Now they could be, and I was going to write them. [12]\n", + "I've wor...\n", + "Score: 0.819\n", + "\n" ] } ], "source": [ - "query_engine = index.as_query_engine()\n", - "response = query_engine.query(\"What did the author learn?\")\n", - "print(textwrap.fill(str(response), 100))" + "result_nodes = retriever.retrieve(\"What did the author learn?\")\n", + "for node in result_nodes:\n", + " print(node)" ] }, { "cell_type": "code", "execution_count": null, - "id": "99212d33", + "id": "e13d7726", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " A hard moment for the author was when he realized that he had been working on things that weren't\n", - "prestigious. He had been drawn to these types of work despite their lack of prestige, and he was\n", - "worried that his ambition was leading him astray. He was also concerned that people would give him a\n", - "\"glassy eye\" when he explained what he was writing.\n" + "19:39:25 httpx INFO HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n", + "19:39:25 llama_index.vector_stores.redis.base INFO Querying index llama_index with filters *\n", + "19:39:25 llama_index.vector_stores.redis.base INFO Found 2 results for query with id ['llama_index/vector_adb6b7ce-49bb-4961-8506-37082c02a389', 'llama_index/vector_e39be1fe-32d0-456e-b211-4efabd191108']\n", + "19:39:27 httpx INFO HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n", + "The author learned that working on things that weren't prestigious often led to valuable discoveries\n", + "and indicated the right kind of motives. Despite the lack of initial prestige, pursuing such work\n", + "could be a sign of genuine potential and appropriate motivations, steering clear of the common\n", + "pitfall of being driven solely by the desire to impress others.\n" ] } ], "source": [ - "response = query_engine.query(\"What was a hard moment for the author?\")\n", + "response = query_engine.query(\"What did the author learn?\")\n", "print(textwrap.fill(str(response), 100))" ] }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "4d7bc976", - "metadata": {}, - "source": [ - "### Saving and Loading\n", - "\n", - "Redis allows the user to perform backups in the background or synchronously. With Llamaindex, the ``RedisVectorStore.persist()`` function can be used to trigger such a backup." - ] - }, { "cell_type": "code", "execution_count": null, - "id": "09836567", + "id": "4b99b79b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "redis redisinsight\n" + "19:39:27 httpx INFO HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n", + "19:39:27 llama_index.vector_stores.redis.base INFO Querying index llama_index with filters *\n", + "19:39:27 llama_index.vector_stores.redis.base INFO Found 2 results for query with id ['llama_index/vector_adb6b7ce-49bb-4961-8506-37082c02a389', 'llama_index/vector_e39be1fe-32d0-456e-b211-4efabd191108']\n", + "Node ID: adb6b7ce-49bb-4961-8506-37082c02a389\n", + "Text: What I Worked On February 2021 Before college the two main\n", + "things I worked on, outside of school, were writing and programming. I\n", + "didn't write essays. I wrote what beginning writers were supposed to\n", + "write then, and probably still are: short stories. My stories were\n", + "awful. They had hardly any plot, just characters with strong feelings,\n", + "which I ...\n", + "Score: 0.802\n", + "\n", + "Node ID: e39be1fe-32d0-456e-b211-4efabd191108\n", + "Text: Except for a few officially anointed thinkers who went to the\n", + "right parties in New York, the only people allowed to publish essays\n", + "were specialists writing about their specialties. There were so many\n", + "essays that had never been written, because there had been no way to\n", + "publish them. Now they could be, and I was going to write them. [12]\n", + "I've wor...\n", + "Score: 0.799\n", + "\n" ] } ], "source": [ - "!docker exec -it redis-vecdb ls /data" + "result_nodes = retriever.retrieve(\"What was a hard moment for the author?\")\n", + "for node in result_nodes:\n", + " print(node)" ] }, { "cell_type": "code", "execution_count": null, - "id": "93ef500b", + "id": "c0838ee1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "19:39:29 httpx INFO HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n", + "19:39:29 llama_index.vector_stores.redis.base INFO Querying index llama_index with filters *\n", + "19:39:29 llama_index.vector_stores.redis.base INFO Found 2 results for query with id ['llama_index/vector_adb6b7ce-49bb-4961-8506-37082c02a389', 'llama_index/vector_e39be1fe-32d0-456e-b211-4efabd191108']\n", + "19:39:31 httpx INFO HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n", + "A hard moment for the author was when one of his programs on the IBM 1401 mainframe didn't\n", + "terminate, leading to a technical error and an uncomfortable situation with the data center manager.\n" + ] + } + ], "source": [ - "# RedisVectorStore's persist method doesn't use the persist_path argument\n", - "vector_store.persist(persist_path=\"\")" + "response = query_engine.query(\"What was a hard moment for the author?\")\n", + "print(textwrap.fill(str(response), 100))" ] }, { "cell_type": "code", "execution_count": null, - "id": "ed5ab256", + "id": "ba33eb01", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "dump.rdb redis redisinsight\n" + "19:39:34 llama_index.vector_stores.redis.base INFO Deleting index llama_index\n" ] } ], "source": [ - "!docker exec -it redis-vecdb ls /data" + "index.vector_store.delete_index()" ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "8c8849ba", + "id": "831452c8", "metadata": {}, "source": [ - "### Restore index from Redis" + "### Use a custom index schema\n", + "\n", + "In most use cases, you need the ability to customize the underling index configuration\n", + "and specification. For example, this is handy in order to define specific metadata filters you wish to enable.\n", + "\n", + "With Redis, this is as simple as defining an index schema object\n", + "(from file or dict) and passing it through to the vector store client wrapper.\n", + "\n", + "For this example, we will:\n", + "1. switch the embedding model to [Cohere](cohereai.com)\n", + "2. add an additional metadata field for the document `updated_at` timestamp\n", + "3. index the existing `file_name` metadata field" ] }, { "cell_type": "code", "execution_count": null, - "id": "95817a85", + "id": "2022e92a", "metadata": {}, "outputs": [], "source": [ - "vector_store = RedisVectorStore(\n", - " index_name=\"pg_essays\",\n", - " index_prefix=\"llama\",\n", - " redis_url=\"redis://localhost:6379\",\n", - " overwrite=True,\n", - ")\n", - "index = VectorStoreIndex.from_vector_store(vector_store=vector_store)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "5b12c9f2", - "metadata": {}, - "source": [ - "Now you can reuse your index as discussed above." + "from llama_index.core.settings import Settings\n", + "from llama_index.embeddings.cohere import CohereEmbedding\n", + "\n", + "# set up Cohere Key\n", + "co_api_key = getpass.getpass(\"Cohere API Key:\")\n", + "os.environ[\"CO_API_KEY\"] = co_api_key\n", + "\n", + "# set llamaindex to use Cohere embeddings\n", + "Settings.embed_model = CohereEmbedding()" ] }, { "cell_type": "code", "execution_count": null, - "id": "437dc580", + "id": "c07e9747", "metadata": {}, "outputs": [], "source": [ - "pgQuery = index.as_query_engine()\n", - "pgQuery.query(\"What is the meaning of life?\")\n", - "# or\n", - "pgRetriever = index.as_retriever()\n", - "pgRetriever.retrieve(\"What is the meaning of life?\")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "e43b185f", - "metadata": {}, - "source": [ - "Learn more about [query_engine](/module_guides/deploying/query_engine/index.md) and [retrievers](/module_guides/querying/retriever/index.md)." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "52b975a7", - "metadata": {}, - "source": [ - "### Deleting documents or index completely\n", - "\n", - "Sometimes it may be useful to delete documents or the entire index. This can be done using the `delete` and `delete_index` methods." + "from redisvl.schema import IndexSchema\n", + "\n", + "\n", + "custom_schema = IndexSchema.from_dict(\n", + " {\n", + " # customize basic index specs\n", + " \"index\": {\n", + " \"name\": \"paul_graham\",\n", + " \"prefix\": \"essay\",\n", + " \"key_separator\": \":\",\n", + " },\n", + " # customize fields that are indexed\n", + " \"fields\": [\n", + " # required fields for llamaindex\n", + " {\"type\": \"tag\", \"name\": \"id\"},\n", + " {\"type\": \"tag\", \"name\": \"doc_id\"},\n", + " {\"type\": \"text\", \"name\": \"text\"},\n", + " # custom metadata fields\n", + " {\"type\": \"numeric\", \"name\": \"updated_at\"},\n", + " {\"type\": \"tag\", \"name\": \"file_name\"},\n", + " # custom vector field definition for cohere embeddings\n", + " {\n", + " \"type\": \"vector\",\n", + " \"name\": \"vector\",\n", + " \"attrs\": {\n", + " \"dims\": 1024,\n", + " \"algorithm\": \"hnsw\",\n", + " \"distance_metric\": \"cosine\",\n", + " },\n", + " },\n", + " ],\n", + " }\n", + ")" ] }, { "cell_type": "code", "execution_count": null, - "id": "6fe322f7", + "id": "22184dd0", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'faa23c94-ac9e-4763-92ba-e0f87bf38195'" + "IndexInfo(name='paul_graham', prefix='essay', key_separator=':', storage_type=)" ] }, "execution_count": null, @@ -575,224 +500,369 @@ } ], "source": [ - "document_id = documents[0].doc_id\n", - "document_id" + "custom_schema.index" ] }, { "cell_type": "code", "execution_count": null, - "id": "ae4fb2b0", + "id": "2bf50ab5", "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of documents 20\n" - ] + "data": { + "text/plain": [ + "{'id': TagField(name='id', type='tag', path=None, attrs=TagFieldAttributes(sortable=False, separator=',', case_sensitive=False, withsuffixtrie=False)),\n", + " 'doc_id': TagField(name='doc_id', type='tag', path=None, attrs=TagFieldAttributes(sortable=False, separator=',', case_sensitive=False, withsuffixtrie=False)),\n", + " 'text': TextField(name='text', type='text', path=None, attrs=TextFieldAttributes(sortable=False, weight=1, no_stem=False, withsuffixtrie=False, phonetic_matcher=None)),\n", + " 'updated_at': NumericField(name='updated_at', type='numeric', path=None, attrs=NumericFieldAttributes(sortable=False)),\n", + " 'file_name': TagField(name='file_name', type='tag', path=None, attrs=TagFieldAttributes(sortable=False, separator=',', case_sensitive=False, withsuffixtrie=False)),\n", + " 'vector': HNSWVectorField(name='vector', type='vector', path=None, attrs=HNSWVectorFieldAttributes(dims=1024, algorithm=, datatype=, distance_metric=, initial_cap=None, m=16, ef_construction=200, ef_runtime=10, epsilon=0.01))}" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "redis_client = vector_store.client\n", - "print(\"Number of documents\", len(redis_client.keys()))" + "custom_schema.fields" + ] + }, + { + "cell_type": "markdown", + "id": "b05ebd97", + "metadata": {}, + "source": [ + "Learn more about [schema and index design](https://redisvl.com) with redis." ] }, { "cell_type": "code", "execution_count": null, - "id": "0ce45788", + "id": "61b01276", "metadata": {}, "outputs": [], "source": [ - "vector_store.delete(document_id)" + "from datetime import datetime\n", + "\n", + "\n", + "def date_to_timestamp(date_string: str) -> int:\n", + " date_format: str = \"%Y-%m-%d\"\n", + " return int(datetime.strptime(date_string, date_format).timestamp())\n", + "\n", + "\n", + "# iterate through documents and add new field\n", + "for document in documents:\n", + " document.metadata[\"updated_at\"] = date_to_timestamp(\n", + " document.metadata[\"last_modified_date\"]\n", + " )" ] }, { "cell_type": "code", "execution_count": null, - "id": "4a1ac683", + "id": "e871823e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Number of documents 10\n" + "19:40:05 httpx INFO HTTP Request: POST https://api.cohere.ai/v1/embed \"HTTP/1.1 200 OK\"\n", + "19:40:06 httpx INFO HTTP Request: POST https://api.cohere.ai/v1/embed \"HTTP/1.1 200 OK\"\n", + "19:40:06 httpx INFO HTTP Request: POST https://api.cohere.ai/v1/embed \"HTTP/1.1 200 OK\"\n", + "19:40:06 llama_index.vector_stores.redis.base INFO Added 22 documents to index paul_graham\n" ] } ], "source": [ - "print(\"Number of documents\", len(redis_client.keys()))" + "vector_store = RedisVectorStore(\n", + " schema=custom_schema, # provide customized schema\n", + " redis_client=redis_client,\n", + " overwrite=True,\n", + ")\n", + "\n", + "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n", + "\n", + "# build and load index from documents and storage context\n", + "index = VectorStoreIndex.from_documents(\n", + " documents, storage_context=storage_context\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "3791a32c", + "metadata": {}, + "source": [ + "### Query the vector store and filter on metadata\n", + "Now that we have additional metadata indexed in Redis, let's try some queries with filters." ] }, { "cell_type": "code", "execution_count": null, - "id": "c380605a", + "id": "bb2c21ad", "metadata": {}, "outputs": [], "source": [ - "# now lets delete the index entirely (happens in the background, may take a second)\n", - "# this will delete all the documents and the index\n", - "vector_store.delete_index()" + "from llama_index.core.vector_stores import (\n", + " MetadataFilters,\n", + " MetadataFilter,\n", + " ExactMatchFilter,\n", + ")\n", + "\n", + "retriever = index.as_retriever(\n", + " similarity_top_k=3,\n", + " filters=MetadataFilters(\n", + " filters=[\n", + " ExactMatchFilter(key=\"file_name\", value=\"paul_graham_essay.txt\"),\n", + " MetadataFilter(\n", + " key=\"updated_at\",\n", + " value=date_to_timestamp(\"2023-01-01\"),\n", + " operator=\">=\",\n", + " ),\n", + " MetadataFilter(\n", + " key=\"text\",\n", + " value=\"learn\",\n", + " operator=\"text_match\",\n", + " ),\n", + " ],\n", + " condition=\"and\",\n", + " ),\n", + ")" ] }, { "cell_type": "code", "execution_count": null, - "id": "474ad4ee", + "id": "d136cfb3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Number of documents 0\n" + "19:40:22 httpx INFO HTTP Request: POST https://api.cohere.ai/v1/embed \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "19:40:22 llama_index.vector_stores.redis.base INFO Querying index paul_graham with filters ((@file_name:{paul_graham_essay\\.txt} @updated_at:[1672549200 +inf]) @text:(learn))\n", + "19:40:22 llama_index.vector_stores.redis.base INFO Found 3 results for query with id ['essay:0df3b734-ecdb-438e-8c90-f21a8c80f552', 'essay:01108c0d-140b-4dcc-b581-c38b7df9251e', 'essay:ced36463-ac36-46b0-b2d7-935c1b38b781']\n", + "Node ID: 0df3b734-ecdb-438e-8c90-f21a8c80f552\n", + "Text: All that seemed left for philosophy were edge cases that people\n", + "in other fields felt could safely be ignored. I couldn't have put\n", + "this into words when I was 18. All I knew at the time was that I kept\n", + "taking philosophy courses and they kept being boring. So I decided to\n", + "switch to AI. AI was in the air in the mid 1980s, but there were two\n", + "things...\n", + "Score: 0.410\n", + "\n", + "Node ID: 01108c0d-140b-4dcc-b581-c38b7df9251e\n", + "Text: It was not, in fact, simply a matter of teaching SHRDLU more\n", + "words. That whole way of doing AI, with explicit data structures\n", + "representing concepts, was not going to work. Its brokenness did, as\n", + "so often happens, generate a lot of opportunities to write papers\n", + "about various band-aids that could be applied to it, but it was never\n", + "going to get us ...\n", + "Score: 0.390\n", + "\n", + "Node ID: ced36463-ac36-46b0-b2d7-935c1b38b781\n", + "Text: Grad students could take classes in any department, and my\n", + "advisor, Tom Cheatham, was very easy going. If he even knew about the\n", + "strange classes I was taking, he never said anything. So now I was in\n", + "a PhD program in computer science, yet planning to be an artist, yet\n", + "also genuinely in love with Lisp hacking and working away at On Lisp.\n", + "In other...\n", + "Score: 0.389\n", + "\n" ] } ], "source": [ - "print(\"Number of documents\", len(redis_client.keys()))" + "result_nodes = retriever.retrieve(\"What did the author learn?\")\n", + "\n", + "for node in result_nodes:\n", + " print(node)" ] }, { "attachments": {}, "cell_type": "markdown", - "id": "61b67496", + "id": "8c8849ba", "metadata": {}, "source": [ - "### Working with Metadata\n", - "\n", - "RedisVectorStore supports adding metadata and then using it in your queries (for example, to limit the scope of documents retrieved). However, there are a couple of important caveats:\n", - "1. Currently, only [Tag fields](https://redis.io/docs/stack/search/reference/tags/) are supported, and only with exact match.\n", - "2. You must declare the metadata when creating the index (usually when initializing RedisVectorStore). If you do not do this, your queries will come back empty. There is no way to modify an existing index after it had already been created (this is a Redis limitation).\n", - "\n", - "Here's how to work with Metadata:\n", - "\n", - "\n", - "### When **creating** the index\n", - "\n", - "Make sure to declare the metadata when you **first** create the index:" + "### Restoring from an existing index in Redis\n", + "Restoring from an index requires a Redis connection client (or URL), `overwrite=False`, and passing in the same schema object used before. (This can be offloaded to a YAML file for convenience using `.to_yaml()`)" ] }, { "cell_type": "code", "execution_count": null, - "id": "9889ec79", + "id": "6792f189", "metadata": {}, "outputs": [], + "source": [ + "custom_schema.to_yaml(\"paul_graham.yaml\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "95817a85", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "19:40:28 redisvl.index.index INFO Index already exists, not overwriting.\n" + ] + } + ], "source": [ "vector_store = RedisVectorStore(\n", - " index_name=\"pg_essays_with_metadata\",\n", - " index_prefix=\"llama\",\n", - " redis_url=\"redis://localhost:6379\",\n", - " overwrite=True,\n", - " metadata_fields=[\"user_id\", \"favorite_color\"],\n", - ")" + " schema=IndexSchema.from_yaml(\"paul_graham.yaml\"),\n", + " redis_client=redis_client,\n", + ")\n", + "index = VectorStoreIndex.from_vector_store(vector_store=vector_store)" ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "f8d6dc21", + "id": "82ea32aa", "metadata": {}, "source": [ - "Note: the field names `text`, `doc_id`, `id` and the name of your vector field (`vector` by default) should **not** be used as metadata field names, as they are are reserved." + "**In the near future** -- we will implement a convenience method to load just using an index name:\n", + "```python\n", + "RedisVectorStore.from_existing_index(index_name=\"paul_graham\", redis_client=redis_client)\n", + "```" ] }, { "attachments": {}, "cell_type": "markdown", - "id": "429947d5", + "id": "52b975a7", "metadata": {}, "source": [ - "### When adding a document\n", + "### Deleting documents or index completely\n", "\n", - "Add your metadata under the `metadata` key. You can add metadata to documents you load in just by looping over them:" + "Sometimes it may be useful to delete documents or the entire index. This can be done using the `delete` and `delete_index` methods." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6fe322f7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'7056f7ba-3513-4ef4-9792-2bd28040aaed'" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "document_id = documents[0].doc_id\n", + "document_id" ] }, { "cell_type": "code", "execution_count": null, - "id": "89781b7d", + "id": "0ce45788", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Document ID: 6a5aa8dd-2771-454b-befc-bcfc311d2008 Document Hash: 77ae91ab542f3abb308c4d7c77c9bc4c9ad0ccd63144802b7cbe7e1bb3a4094e Metadata: {'user_id': '12345', 'favorite_color': 'blue'}\n" + "Number of documents before deleting 22\n", + "19:40:32 llama_index.vector_stores.redis.base INFO Deleted 22 documents from index paul_graham\n", + "Number of documents after deleting 0\n" ] } ], "source": [ - "# load your documents normally, then add your metadata\n", - "documents = SimpleDirectoryReader(\"../data/paul_graham\").load_data()\n", - "\n", - "for document in documents:\n", - " document.metadata = {\"user_id\": \"12345\", \"favorite_color\": \"blue\"}\n", - "\n", - "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n", - "index = VectorStoreIndex.from_documents(\n", - " documents, storage_context=storage_context\n", - ")\n", - "\n", - "# load documents\n", - "print(\n", - " \"Document ID:\",\n", - " documents[0].doc_id,\n", - " \"Document Hash:\",\n", - " documents[0].doc_hash,\n", - " \"Metadata:\",\n", - " documents[0].metadata,\n", - ")" + "print(\"Number of documents before deleting\", redis_client.dbsize())\n", + "vector_store.delete(document_id)\n", + "print(\"Number of documents after deleting\", redis_client.dbsize())" ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "42b24e76", + "id": "442e8acf", "metadata": {}, "source": [ - "### When querying the index\n", - "\n", - "To filter by your metadata fields, include one or more of your metadata keys, like so:" + "However, the Redis index still exists (with no associated documents) for continuous upsert." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12eda458", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vector_store.index_exists()" ] }, { "cell_type": "code", "execution_count": null, - "id": "0b01f346", + "id": "c380605a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " The author learned that it was possible to publish anything online, and that working on things that\n", - "weren't prestigious could lead to discovering something real. They also learned that impure motives\n", - "were a big danger for the ambitious, and that it was possible for programs not to terminate.\n", - "Finally, they learned that computers were expensive in those days, and that they could write\n", - "programs on the IBM 1401.\n" + "19:40:37 llama_index.vector_stores.redis.base INFO Deleting index paul_graham\n" ] } ], "source": [ - "from llama_index.core.vector_stores import MetadataFilters, ExactMatchFilter\n", - "\n", - "query_engine = index.as_query_engine(\n", - " similarity_top_k=3,\n", - " filters=MetadataFilters(\n", - " filters=[\n", - " ExactMatchFilter(key=\"user_id\", value=\"12345\"),\n", - " ExactMatchFilter(key=\"favorite_color\", value=\"blue\"),\n", - " ]\n", - " ),\n", - ")\n", - "\n", - "response = query_engine.query(\"What did the author learn?\")\n", - "print(textwrap.fill(str(response), 100))" + "# now lets delete the index entirely\n", + "# this will delete all the documents and the index\n", + "vector_store.delete_index()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "474ad4ee", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of documents after deleting 0\n" + ] + } + ], + "source": [ + "print(\"Number of documents after deleting\", redis_client.dbsize())" ] }, { @@ -801,29 +871,37 @@ "id": "07514f85", "metadata": {}, "source": [ - "## Troubleshooting\n", + "### Troubleshooting\n", "\n", - "In case you run into issues retrieving your documents from the index, you might get a message similar to this.\n", - "```\n", - "No docs found on index 'pg_essays' with prefix 'llama' and filters '(@user_id:{12345} & @favorite_color:{blue})'.\n", - "* Did you originally create the index with a different prefix?\n", - "* Did you index your metadata fields when you created the index?\n", - "```\n", + "If you get an empty query result, there a couple of issues to check:\n", + "\n", + "#### Schema\n", + "\n", + "Unlike other vector stores, Redis expects users to explicitly define the schema for the index. This is for a few reasons:\n", + "1. Redis is used for many use cases, including real-time vector search, but also for standard document storage/retrieval, caching, messaging, pub/sub, session mangement, and more. Not all attributes on records need to be indexed for search. This is partially an efficiency thing, and partially an attempt to minimize user foot guns.\n", + "2. All index schemas, when using Redis & LlamaIndex, must include the following fields `id`, `doc_id`, `text`, and `vector`, at a minimum.\n", + "\n", + "Instantiate your `RedisVectorStore` with the default schema (assumes OpenAI embeddings), or with a custom schema (see above).\n", "\n", - "If you get this error, there a couple of gotchas to be aware of when working with Redis:\n", "#### Prefix issues\n", "\n", - "If you first create your index with a specific `prefix` but later change that prefix in your code, your query will come back empty. Redis saves the prefix your originally created your index with and expects it to be consistent.\n", + "Redis expects all records to have a key prefix that segments the keyspace into \"partitions\"\n", + "for potentially different applications, use cases, and clients.\n", + "\n", + "Make sure that the chosen `prefix`, as part of the index schema, is consistent across your code (tied to a specific index).\n", "\n", "To see what prefix your index was created with, you can run `FT.INFO ` in the Redis CLI and look under `index_definition` => `prefixes`.\n", "\n", + "#### Data vs Index\n", + "Redis treats the records in the dataset and the index as different entities. This allows you more flexibility in performing updates, upserts, and index schema migrations.\n", + "\n", + "If you have an existing index and want to make sure it's dropped, you can run `FT.DROPINDEX ` in the Redis CLI. Note that this will *not* drop your actual data unless you pass `DD`\n", + "\n", "#### Empty queries when using metadata\n", "\n", "If you add metadata to the index *after* it has already been created and then try to query over that metadata, your queries will come back empty.\n", "\n", - "Redis indexes fields upon index creation only (similar to how it indexes the prefixes, above).\n", - "\n", - "If you have an existing index and want to make sure it's dropped, you can run `FT.DROPINDEX ` in the Redis CLI. Note that this will *not* drop your actual data." + "Redis indexes fields upon index creation only (similar to how it indexes the prefixes, above)." ] } ], diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/.gitignore b/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/.gitignore index 990c18de22908..9bed808d0b1bb 100644 --- a/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/.gitignore +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/.gitignore @@ -32,6 +32,7 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +poetry.lock # PyInstaller # Usually these files are written by a python script from a template diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/llama_index/vector_stores/redis/base.py b/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/llama_index/vector_stores/redis/base.py index 373cdee19248e..0fcf379ccf791 100644 --- a/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/llama_index/vector_stores/redis/base.py +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/llama_index/vector_stores/redis/base.py @@ -18,6 +18,7 @@ from llama_index.core.vector_stores.types import ( BasePydanticVectorStore, MetadataFilters, + MetadataFilter, VectorStoreQuery, VectorStoreQueryResult, ) @@ -25,125 +26,196 @@ metadata_dict_to_node, node_to_metadata_dict, ) -from llama_index.vector_stores.redis.utils import ( - TokenEscaper, - array_to_buffer, - check_redis_modules_exist, - convert_bytes, - get_redis_query, +from llama_index.vector_stores.redis.schema import ( + NODE_ID_FIELD_NAME, + NODE_CONTENT_FIELD_NAME, + DOC_ID_FIELD_NAME, + TEXT_FIELD_NAME, + VECTOR_FIELD_NAME, + RedisVectorStoreSchema, ) +from llama_index.vector_stores.redis.utils import REDIS_LLAMA_FIELD_SPEC -import redis -from redis import DataError -from redis.client import Redis as RedisType -from redis.commands.search.field import VectorField +from redis import Redis from redis.exceptions import RedisError from redis.exceptions import TimeoutError as RedisTimeoutError -_logger = logging.getLogger(__name__) +from redisvl.index import SearchIndex +from redisvl.schema import IndexSchema +from redisvl.query import VectorQuery, FilterQuery, CountQuery +from redisvl.query.filter import Tag, FilterExpression +from redisvl.schema.fields import BaseField +from redisvl.redis.utils import array_to_buffer + + +logger = logging.getLogger(__name__) class RedisVectorStore(BasePydanticVectorStore): + """RedisVectorStore. + + The RedisVectorStore takes a user-defined schema object and a Redis connection + client or URL string. The schema is optional, but useful for: + - Defining a custom index name, key prefix, and key separator. + - Defining *additional* metadata fields to use as query filters. + - Setting custom specifications on fields to improve search quality, e.g + which vector index algorithm to use. + + Other Notes: + - All embeddings and docs are stored in Redis. During query time, the index + uses Redis to query for the top k most similar nodes. + - Redis & LlamaIndex expect at least 4 *required* fields for any schema, default or custom, + `id`, `doc_id`, `text`, `vector`. + + Args: + schema (IndexSchema, optional): Redis index schema object. + redis_client (Redis, optional): Redis client connection. + redis_url (str, optional): Redis server URL. + Defaults to "redis://localhost:6379". + overwrite (bool, optional): Whether to overwrite the index if it already exists. + Defaults to False. + + Raises: + ValueError: If your Redis server does not have search or JSON enabled. + ValueError: If a Redis connection failed to be established. + ValueError: If an invalid schema is provided. + + Example: + from redisvl.schema import IndexSchema + from llama_index.vector_stores.redis import RedisVectorStore + + # Use default schema + rds = RedisVectorStore(redis_url="redis://localhost:6379") + + # Use custom schema from dict + schema = IndexSchema.from_dict({ + "index": {"name": "my-index", "prefix": "docs"}, + "fields": [ + {"name": "id", "type": "tag"}, + {"name": "doc_id", "type": "tag}, + {"name": "text", "type": "text"}, + {"name": "vector", "type": "vector", "attrs": {"dims": 1536, "algorithm": "flat"}} + ] + }) + vector_store = RedisVectorStore( + schema=schema, + redis_url="redis://localhost:6379" + ) + """ + stores_text = True stores_node = True flat_metadata = False - _tokenizer: Any = PrivateAttr() - _redis_client: Any = PrivateAttr() - _prefix: str = PrivateAttr() - _index_name: str = PrivateAttr() - _index_args: Dict[str, Any] = PrivateAttr() - _metadata_fields: List[str] = PrivateAttr() + _index: SearchIndex = PrivateAttr() _overwrite: bool = PrivateAttr() - _vector_field: str = PrivateAttr() - _vector_key: str = PrivateAttr() + _return_fields: List[str] = PrivateAttr() def __init__( self, - index_name: str, - index_prefix: str = "llama_index", - prefix_ending: str = "/vector", - index_args: Optional[Dict[str, Any]] = None, - metadata_fields: Optional[List[str]] = None, - redis_url: str = "redis://localhost:6379", + schema: Optional[IndexSchema] = None, + redis_client: Optional[Redis] = None, + redis_url: Optional[str] = None, overwrite: bool = False, + return_fields: Optional[List[str]] = None, **kwargs: Any, ) -> None: - """Initialize RedisVectorStore. - - For index arguments that can be passed to RediSearch, see - https://redis.io/docs/stack/search/reference/vectors/ + # check for indicators of old schema + self._flag_old_kwargs(**kwargs) + + # Setup schema + if not schema: + logger.info("Using default RedisVectorStore schema.") + schema = RedisVectorStoreSchema() + + self._validate_schema(schema) + self._return_fields = return_fields or [ + NODE_ID_FIELD_NAME, + DOC_ID_FIELD_NAME, + TEXT_FIELD_NAME, + NODE_CONTENT_FIELD_NAME, + ] + self._index = SearchIndex(schema=schema) + self._overwrite = overwrite - The index arguments will depend on the index type chosen. There - are two available index types - - FLAT: a flat index that uses brute force search - - HNSW: a hierarchical navigable small world graph index + # Establish redis connection + if redis_client: + self._index.set_client(redis_client) + elif redis_url: + self._index.connect(redis_url) + else: + raise ValueError( + "Failed to connect to Redis. Must provide a valid redis client or url" + ) - Args: - index_name (str): Name of the index. - index_prefix (str): Prefix for the index. Defaults to "llama_index". - The actual prefix used by Redis will be - "{index_prefix}{prefix_ending}". - prefix_ending (str): Prefix ending for the index. Be careful when - changing this: https://github.com/jerryjliu/llama_index/pull/6665. - Defaults to "/vector". - index_args (Dict[str, Any]): Arguments for the index. Defaults to None. - metadata_fields (List[str]): List of metadata fields to store in the index - (only supports TAG fields). - redis_url (str): URL for the redis instance. - Defaults to "redis://localhost:6379". - overwrite (bool): Whether to overwrite the index if it already exists. - Defaults to False. - kwargs (Any): Additional arguments to pass to the redis client. + # Create index + self.create_index() - Raises: - ValueError: If redis-py is not installed - ValueError: If RediSearch is not installed - - Examples: - `pip install llama-index-vector-stores-redis` - - ```python - from llama_index.core.vector_stores.redis import RedisVectorStore - - # Create a RedisVectorStore - vector_store = RedisVectorStore( - index_name="my_index", - index_prefix="llama_index", - index_args={ - "algorithm": "HNSW", - "m": 16, - "ef_construction": 200, - "distance_metric": "cosine" - }, - redis_url="redis://localhost:6379/", - overwrite=True - ) - ``` - """ - try: - # connect to redis from url - self._redis_client = redis.from_url(redis_url, **kwargs) - # check if redis has redisearch module installed - check_redis_modules_exist(self._redis_client) - except ValueError as e: - raise ValueError(f"Redis failed to connect: {e}") - - # index identifiers - self._prefix = index_prefix + prefix_ending - self._index_name = index_name - self._index_args = index_args if index_args is not None else {} - self._metadata_fields = metadata_fields if metadata_fields is not None else [] - self._overwrite = overwrite - self._vector_field = str(self._index_args.get("vector_field", "vector")) - self._vector_key = str(self._index_args.get("vector_key", "vector")) - self._tokenizer = TokenEscaper() super().__init__() + def _flag_old_kwargs(self, **kwargs): + old_kwargs = [ + "index_name", + "index_prefix", + "prefix_ending", + "index_args", + "metadata_fields", + ] + for kwarg in old_kwargs: + if kwarg in kwargs: + raise ValueError( + f"Deprecated kwarg, {kwarg}, found upon initialization. " + "RedisVectorStore now requires an IndexSchema object. " + "See the documentation for a complete example: https://docs.llamaindex.ai/en/stable/examples/vector_stores/RedisIndexDemo/" + ) + + def _validate_schema(self, schema: IndexSchema) -> str: + base_schema = RedisVectorStoreSchema() + for name, field in base_schema.fields.items(): + if (name not in schema.fields) or ( + not schema.fields[name].type == field.type + ): + raise ValueError( + f"Required field {name} must be present in the index " + f"and of type {schema.fields[name].type}" + ) + @property - def client(self) -> "RedisType": + def client(self) -> "Redis": """Return the redis client instance.""" - return self._redis_client + return self._index.client + + @property + def index_name(self) -> str: + """Return the name of the index based on the schema.""" + return self._index.name + + @property + def schema(self) -> IndexSchema: + """Return the index schema.""" + return self._index.schema + + def set_return_fields(self, return_fields: List[str]) -> None: + """Update the return fields for the query response.""" + self._return_fields = return_fields + + def index_exists(self) -> bool: + """Check whether the index exists in Redis. + + Returns: + bool: True or False. + """ + return self._index.exists() + + def create_index(self, overwrite: Optional[bool] = None) -> None: + """Create an index in Redis.""" + if overwrite is None: + overwrite = self._overwrite + # Create index honoring overwrite policy + if overwrite: + self._index.create(overwrite=True, drop=True) + else: + self._index.create() def add(self, nodes: List[BaseNode], **add_kwargs: Any) -> List[str]: """Add nodes to the index. @@ -157,41 +229,42 @@ def add(self, nodes: List[BaseNode], **add_kwargs: Any) -> List[str]: Raises: ValueError: If the index already exists and overwrite is False. """ - # check to see if empty document list was passed + # Check to see if empty document list was passed if len(nodes) == 0: return [] - # set vector dim for creation if index doesn't exist - self._index_args["dims"] = len(nodes[0].get_embedding()) - - if self._index_exists(): - if self._overwrite: - self.delete_index() - self._create_index() - else: - logging.info(f"Adding document to existing index {self._index_name}") - else: - self._create_index() + # Now check for the scenario where user is trying to index embeddings that don't align with schema + embedding_len = len(nodes[0].get_embedding()) + expected_dims = self._index.schema.fields[VECTOR_FIELD_NAME].attrs.dims + if expected_dims != embedding_len: + raise ValueError( + f"Attempting to index embeddings of dim {embedding_len} " + f"which doesn't match the index schema expectation of {expected_dims}. " + "Please review the Redis integration example to learn how to customize schema. " + "" + ) - ids = [] + data: List[Dict[str, Any]] = [] for node in nodes: - mapping = { - "id": node.node_id, - "doc_id": node.ref_doc_id, - "text": node.get_content(metadata_mode=MetadataMode.NONE), - self._vector_key: array_to_buffer(node.get_embedding()), + embedding = node.get_embedding() + record = { + NODE_ID_FIELD_NAME: node.node_id, + DOC_ID_FIELD_NAME: node.ref_doc_id, + TEXT_FIELD_NAME: node.get_content(metadata_mode=MetadataMode.NONE), + VECTOR_FIELD_NAME: array_to_buffer(embedding), } + # parse and append metadata additional_metadata = node_to_metadata_dict( node, remove_text=True, flat_metadata=self.flat_metadata ) - mapping.update(additional_metadata) + data.append({**record, **additional_metadata}) - ids.append(node.node_id) - key = "_".join([self._prefix, str(node.node_id)]) - self._redis_client.hset(key, mapping=mapping) # type: ignore - - _logger.info(f"Added {len(ids)} documents to index {self._index_name}") - return ids + # Load nodes to Redis + keys = self._index.load(data, id_field=NODE_ID_FIELD_NAME, **add_kwargs) + logger.info(f"Added {len(keys)} documents to index {self._index.name}") + return [ + key.strip(self._index.prefix + self._index.key_separator) for key in keys + ] def delete(self, ref_doc_id: str, **delete_kwargs: Any) -> None: """ @@ -201,29 +274,139 @@ def delete(self, ref_doc_id: str, **delete_kwargs: Any) -> None: ref_doc_id (str): The doc_id of the document to delete. """ - # use tokenizer to escape dashes in query - query_str = "@doc_id:{%s}" % self._tokenizer.escape(ref_doc_id) - # find all documents that match a doc_id - results = self._redis_client.ft(self._index_name).search(query_str) - if len(results.docs) == 0: - # don't raise an error but warn the user that document wasn't found - # could be a result of eviction policy - _logger.warning( - f"Document with doc_id {ref_doc_id} not found " - f"in index {self._index_name}" - ) - return - - for doc in results.docs: - self._redis_client.delete(doc.id) - _logger.info( - f"Deleted {len(results.docs)} documents from index {self._index_name}" + # build a filter to target specific docs by doc ID + doc_filter = Tag(DOC_ID_FIELD_NAME) == ref_doc_id + total = self._index.query(CountQuery(doc_filter)) + delete_query = FilterQuery( + return_fields=[NODE_ID_FIELD_NAME], + filter_expression=doc_filter, + num_results=total, + ) + # fetch docs to delete and flush them + docs_to_delete = self._index.search(delete_query.query, delete_query.params) + with self._index.client.pipeline(transaction=False) as pipe: + for doc in docs_to_delete.docs: + pipe.delete(doc.id) + res = pipe.execute() + + logger.info( + f"Deleted {len(docs_to_delete.docs)} documents from index {self._index.name}" ) def delete_index(self) -> None: """Delete the index and all documents.""" - _logger.info(f"Deleting index {self._index_name}") - self._redis_client.ft(self._index_name).dropindex(delete_documents=True) + logger.info(f"Deleting index {self._index.name}") + self._index.delete(drop=True) + + @staticmethod + def _to_redis_filter(field: BaseField, filter: MetadataFilter) -> FilterExpression: + """ + Translate a standard metadata filter to a Redis specific filter expression. + + Args: + field (BaseField): The field to be filtered on, must have a type attribute. + filter (MetadataFilter): The filter to apply, must have operator and value attributes. + + Returns: + FilterExpression: A Redis-specific filter expression constructed from the input. + + Raises: + ValueError: If the field type is unsupported or if the operator is not supported for the field type. + """ + # Check for unsupported field type + if field.type not in REDIS_LLAMA_FIELD_SPEC: + raise ValueError(f"Unsupported field type {field.type} for {field.name}") + + field_info = REDIS_LLAMA_FIELD_SPEC[field.type] + + # Check for unsupported operator + if filter.operator not in field_info["operators"]: + raise ValueError( + f"Filter operator {filter.operator} not supported for {field.name} of type {field.type}" + ) + + # Create field instance and apply the operator function + field_instance = field_info["class"](field.name) + return field_info["operators"][filter.operator](field_instance, filter.value) + + def _create_redis_filter_expression( + self, metadata_filters: MetadataFilters + ) -> FilterExpression: + """ + Generate a Redis Filter Expression as a combination of metadata filters. + + Args: + metadata_filters (MetadataFilters): List of metadata filters to use. + + Returns: + FilterExpression: A Redis filter expression. + """ + filter_expression = FilterExpression("*") + if metadata_filters: + if metadata_filters.filters: + for filter in metadata_filters.filters: + # Index must be created with the metadata field in the index schema + field = self._index.schema.fields.get(filter.key) + if not field: + logger.warning( + f"{filter.key} field was not included as part of the index schema, and thus cannot be used as a filter condition." + ) + continue + # Extract redis filter + redis_filter = self._to_redis_filter(field, filter) + # Combine with conditional + if metadata_filters.condition == "and": + filter_expression = filter_expression & redis_filter + else: + filter_expression = filter_expression | redis_filter + return filter_expression + + def _to_redis_query(self, query: VectorStoreQuery) -> VectorQuery: + """Creates a RedisQuery from a VectorStoreQuery.""" + filter_expression = self._create_redis_filter_expression(query.filters) + return_fields = self._return_fields.copy() + return VectorQuery( + vector=query.query_embedding, + vector_field_name=VECTOR_FIELD_NAME, + num_results=query.similarity_top_k, + filter_expression=filter_expression, + return_fields=return_fields, + ) + + def _extract_node_and_score(self, doc, redis_query: VectorQuery): + """Extracts a node and its score from a document.""" + try: + node = metadata_dict_to_node( + {NODE_CONTENT_FIELD_NAME: doc[NODE_CONTENT_FIELD_NAME]} + ) + node.text = doc[TEXT_FIELD_NAME] + except Exception: + # Handle legacy metadata format + node = TextNode( + text=doc[TEXT_FIELD_NAME], + id_=doc[NODE_ID_FIELD_NAME], + embedding=None, + relationships={ + NodeRelationship.SOURCE: RelatedNodeInfo( + node_id=doc[DOC_ID_FIELD_NAME] + ) + }, + ) + score = 1 - float(doc[redis_query.DISTANCE_ID]) + return node, score + + def _process_query_results( + self, results, redis_query: VectorQuery + ) -> VectorStoreQueryResult: + """Processes query results and returns a VectorStoreQueryResult.""" + ids, nodes, scores = [], [], [] + for doc in results: + node, score = self._extract_node_and_score(doc, redis_query) + ids.append(doc[NODE_ID_FIELD_NAME]) + nodes.append(node) + scores.append(score) + logger.info(f"Found {len(nodes)} results for query with id {ids}") + return VectorStoreQueryResult(nodes=nodes, ids=ids, similarities=scores) def query(self, query: VectorStoreQuery, **kwargs: Any) -> VectorStoreQueryResult: """Query the index. @@ -238,87 +421,36 @@ def query(self, query: VectorStoreQuery, **kwargs: Any) -> VectorStoreQueryResul ValueError: If query.query_embedding is None. redis.exceptions.RedisError: If there is an error querying the index. redis.exceptions.TimeoutError: If there is a timeout querying the index. - ValueError: If no documents are found when querying the index. """ - return_fields = [ - "id", - "doc_id", - "text", - self._vector_key, - "vector_score", - "_node_content", - ] - - filters = _to_redis_filters(query.filters) if query.filters is not None else "*" - - _logger.info(f"Using filters: {filters}") - - redis_query = get_redis_query( - return_fields=return_fields, - top_k=query.similarity_top_k, - vector_field=self._vector_field, - filters=filters, - ) - if not query.query_embedding: raise ValueError("Query embedding is required for querying.") - query_params = { - "vector": array_to_buffer(query.query_embedding), - } - _logger.info(f"Querying index {self._index_name}") + redis_query = self._to_redis_query(query) + logger.info( + f"Querying index {self._index.name} with filters {redis_query.get_filter()}" + ) try: - results = self._redis_client.ft(self._index_name).search( - redis_query, query_params=query_params # type: ignore - ) + results = self._index.query(redis_query) except RedisTimeoutError as e: - _logger.error(f"Query timed out on {self._index_name}: {e}") + logger.error(f"Query timed out on {self._index.name}: {e}") raise except RedisError as e: - _logger.error(f"Error querying {self._index_name}: {e}") + logger.error(f"Error querying {self._index.name}: {e}") raise - if len(results.docs) == 0: - raise ValueError( - f"No docs found on index '{self._index_name}' with " - f"prefix '{self._prefix}' and filters '{filters}'. " - "* Did you originally create the index with a different prefix? " - "* Did you index your metadata fields when you created the index?" - ) - - ids = [] - nodes = [] - scores = [] - for doc in results.docs: - try: - node = metadata_dict_to_node({"_node_content": doc._node_content}) - node.text = doc.text - except Exception: - # TODO: Legacy support for old metadata format - node = TextNode( - text=doc.text, - id_=doc.id, - embedding=None, - relationships={ - NodeRelationship.SOURCE: RelatedNodeInfo(node_id=doc.doc_id) - }, - ) - ids.append(doc.id.replace(self._prefix + "_", "")) - nodes.append(node) - scores.append(1 - float(doc.vector_score)) - _logger.info(f"Found {len(nodes)} results for query with id {ids}") - - return VectorStoreQueryResult(nodes=nodes, ids=ids, similarities=scores) + return self._process_query_results(results, redis_query) def persist( self, - persist_path: str, + persist_path: Optional[str] = None, fs: Optional[fsspec.AbstractFileSystem] = None, in_background: bool = True, ) -> None: """Persist the vector store to disk. + For Redis, more notes here: https://redis.io/docs/management/persistence/ + Args: persist_path (str): Path to persist the vector store to. (doesn't apply) in_background (bool, optional): Persist in background. Defaults to True. @@ -331,136 +463,12 @@ def persist( """ try: if in_background: - _logger.info("Saving index to disk in background") - self._redis_client.bgsave() + logger.info("Saving index to disk in background") + self._index.client.bgsave() else: - _logger.info("Saving index to disk") - self._redis_client.save() + logger.info("Saving index to disk") + self._index.client.save() except RedisError as e: - _logger.error(f"Error saving index to disk: {e}") + logger.error(f"Error saving index to disk: {e}") raise - - def _create_index(self) -> None: - # should never be called outside class and hence should not raise importerror - from redis.commands.search.field import TagField, TextField - from redis.commands.search.indexDefinition import IndexDefinition, IndexType - - # Create Index - default_fields = [ - TextField("text", weight=1.0), - TagField("doc_id", sortable=False), - TagField("id", sortable=False), - ] - # add vector field to list of index fields. Create lazily to allow user - # to specify index and search attributes in creation. - - fields = [ - *default_fields, - self._create_vector_field(self._vector_field, **self._index_args), - ] - - # add metadata fields to list of index fields or we won't be able to search them - for metadata_field in self._metadata_fields: - # TODO: allow addition of text fields as metadata - # TODO: make sure we're preventing overwriting other keys (e.g. text, - # doc_id, id, and other vector fields) - fields.append(TagField(metadata_field, sortable=False)) - - _logger.info(f"Creating index {self._index_name}") - self._redis_client.ft(self._index_name).create_index( - fields=fields, - definition=IndexDefinition( - prefix=[self._prefix], index_type=IndexType.HASH - ), # TODO support JSON - ) - - def _index_exists(self) -> bool: - # use FT._LIST to check if index exists - indices = convert_bytes(self._redis_client.execute_command("FT._LIST")) - return self._index_name in indices - - def _create_vector_field( - self, - name: str, - dims: int = 1536, - algorithm: str = "FLAT", - datatype: str = "FLOAT32", - distance_metric: str = "COSINE", - initial_cap: int = 20000, - block_size: int = 1000, - m: int = 16, - ef_construction: int = 200, - ef_runtime: int = 10, - epsilon: float = 0.8, - **kwargs: Any, - ) -> "VectorField": - """Create a RediSearch VectorField. - - Args: - name (str): The name of the field. - algorithm (str): The algorithm used to index the vector. - dims (int): The dimensionality of the vector. - datatype (str): The type of the vector. default: FLOAT32 - distance_metric (str): The distance metric used to compare vectors. - initial_cap (int): The initial capacity of the index. - block_size (int): The block size of the index. - m (int): The number of outgoing edges in the HNSW graph. - ef_construction (int): Number of maximum allowed potential outgoing edges - candidates for each node in the graph, - during the graph building. - ef_runtime (int): The umber of maximum top candidates to hold during the - KNN search - - Returns: - A RediSearch VectorField. - """ - try: - if algorithm.upper() == "HNSW": - return VectorField( - name, - "HNSW", - { - "TYPE": datatype.upper(), - "DIM": dims, - "DISTANCE_METRIC": distance_metric.upper(), - "INITIAL_CAP": initial_cap, - "M": m, - "EF_CONSTRUCTION": ef_construction, - "EF_RUNTIME": ef_runtime, - "EPSILON": epsilon, - }, - ) - else: - return VectorField( - name, - "FLAT", - { - "TYPE": datatype.upper(), - "DIM": dims, - "DISTANCE_METRIC": distance_metric.upper(), - "INITIAL_CAP": initial_cap, - "BLOCK_SIZE": block_size, - }, - ) - except DataError as e: - raise ValueError( - f"Failed to create Redis index vector field with error: {e}" - ) - - -# currently only supports exact tag match - {} denotes a tag -# must create the index with the correct metadata field before using a field as a -# filter, or it will return no results -def _to_redis_filters(metadata_filters: MetadataFilters) -> str: - tokenizer = TokenEscaper() - - filter_strings = [] - for filter in metadata_filters.legacy_filters(): - # adds quotes around the value to ensure that the filter is treated as an - # exact match - filter_string = f"@{filter.key}:{{{tokenizer.escape(str(filter.value))}}}" - filter_strings.append(filter_string) - - joined_filter_strings = " & ".join(filter_strings) - return f"({joined_filter_strings})" diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/llama_index/vector_stores/redis/schema.py b/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/llama_index/vector_stores/redis/schema.py new file mode 100644 index 0000000000000..8fe5f060799c4 --- /dev/null +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/llama_index/vector_stores/redis/schema.py @@ -0,0 +1,44 @@ +from typing import Any, Dict, List +from redisvl.schema import IndexSchema, IndexInfo, StorageType + +# required llama index fields +NODE_ID_FIELD_NAME: str = "id" +DOC_ID_FIELD_NAME: str = "doc_id" +TEXT_FIELD_NAME: str = "text" +NODE_CONTENT_FIELD_NAME: str = "_node_content" +VECTOR_FIELD_NAME: str = "vector" + + +class RedisIndexInfo(IndexInfo): + """The default Redis Vector Store Index Info.""" + + name: str = "llama_index" + """The unique name of the index.""" + prefix: str = "llama_index/vector" + """The prefix used for Redis keys associated with this index.""" + key_separator: str = "_" + """The separator character used in designing Redis keys.""" + storage_type: StorageType = StorageType.HASH + """The storage type used in Redis (e.g., 'hash' or 'json').""" + + +class RedisVectorStoreSchema(IndexSchema): + """The default Redis Vector Store Schema.""" + + def __init__(self, **data) -> None: + index = RedisIndexInfo() + fields: List[Dict[str, Any]] = [ + {"type": "tag", "name": NODE_ID_FIELD_NAME, "attrs": {"sortable": False}}, + {"type": "tag", "name": DOC_ID_FIELD_NAME, "attrs": {"sortable": False}}, + {"type": "text", "name": TEXT_FIELD_NAME, "attrs": {"weight": 1.0}}, + { + "type": "vector", + "name": VECTOR_FIELD_NAME, + "attrs": { + "dims": 1536, + "algorithm": "flat", + "distance_metric": "cosine", + }, + }, + ] + super().__init__(index=index.__dict__, fields=fields) diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/llama_index/vector_stores/redis/utils.py b/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/llama_index/vector_stores/redis/utils.py index 21b0d567d099e..8c0a815797b3b 100644 --- a/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/llama_index/vector_stores/redis/utils.py +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/llama_index/vector_stores/redis/utils.py @@ -1,105 +1,35 @@ -import logging -import re -from typing import Any, List, Optional, Pattern - -import numpy as np - -from redis.client import Redis as RedisType -from redis.commands.search.query import Query - -_logger = logging.getLogger(__name__) - - -class TokenEscaper: - """ - Escape punctuation within an input string. Taken from RedisOM Python. - """ - - # Characters that RediSearch requires us to escape during queries. - # Source: https://redis.io/docs/stack/search/reference/escaping/#the-rules-of-text-field-tokenization - DEFAULT_ESCAPED_CHARS = r"[,.<>{}\[\]\\\"\':;!@#$%^&*()\-+=~\/ ]" - - def __init__(self, escape_chars_re: Optional[Pattern] = None): - if escape_chars_re: - self.escaped_chars_re = escape_chars_re - else: - self.escaped_chars_re = re.compile(self.DEFAULT_ESCAPED_CHARS) - - def escape(self, value: str) -> str: - def escape_symbol(match: re.Match) -> str: - value = match.group(0) - return f"\\{value}" - - return self.escaped_chars_re.sub(escape_symbol, value) - - -# required modules -REDIS_REQUIRED_MODULES = [ - {"name": "search", "ver": 20400}, - {"name": "searchlight", "ver": 20400}, -] - - -def check_redis_modules_exist(client: "RedisType") -> None: - """Check if the correct Redis modules are installed.""" - installed_modules = client.module_list() - installed_modules = { - module[b"name"].decode("utf-8"): module for module in installed_modules - } - for module in REDIS_REQUIRED_MODULES: - if module["name"] in installed_modules and int( - installed_modules[module["name"]][b"ver"] - ) >= int( - module["ver"] - ): # type: ignore[call-overload] - return - # otherwise raise error - error_message = ( - "You must add the RediSearch (>= 2.4) module from Redis Stack. " - "Please refer to Redis Stack docs: https://redis.io/docs/stack/" - ) - _logger.error(error_message) - raise ValueError(error_message) - - -def get_redis_query( - return_fields: List[str], - top_k: int = 20, - vector_field: str = "vector", - sort: bool = True, - filters: str = "*", -) -> "Query": - """Create a vector query for use with a SearchIndex. - - Args: - return_fields (t.List[str]): A list of fields to return in the query results - top_k (int, optional): The number of results to return. Defaults to 20. - vector_field (str, optional): The name of the vector field in the index. - Defaults to "vector". - sort (bool, optional): Whether to sort the results by score. Defaults to True. - filters (str, optional): string to filter the results by. Defaults to "*". - - """ - base_query = f"{filters}=>[KNN {top_k} @{vector_field} $vector AS vector_score]" - - query = Query(base_query).return_fields(*return_fields).dialect(2).paging(0, top_k) - - if sort: - query.sort_by("vector_score") - return query - - -def convert_bytes(data: Any) -> Any: - if isinstance(data, bytes): - return data.decode("ascii") - if isinstance(data, dict): - return dict(map(convert_bytes, data.items())) - if isinstance(data, list): - return list(map(convert_bytes, data)) - if isinstance(data, tuple): - return map(convert_bytes, data) - return data - - -def array_to_buffer(array: List[float], dtype: Any = np.float32) -> bytes: - return np.array(array).astype(dtype).tobytes() +from redisvl.query.filter import Tag, Num, Text + + +# Global constant defining field specifications +REDIS_LLAMA_FIELD_SPEC = { + "tag": { + "class": Tag, + "operators": { + "==": lambda f, v: f == v, + "!=": lambda f, v: f != v, + "in": lambda f, v: f == v, + "nin": lambda f, v: f != v, + "contains": lambda f, v: f == v, + }, + }, + "numeric": { + "class": Num, + "operators": { + "==": lambda f, v: f == v, + "!=": lambda f, v: f != v, + ">": lambda f, v: f > v, + "<": lambda f, v: f < v, + ">=": lambda f, v: f >= v, + "<=": lambda f, v: f <= v, + }, + }, + "text": { + "class": Text, + "operators": { + "==": lambda f, v: f == v, + "!=": lambda f, v: f != v, + "text_match": lambda f, v: f % v, + }, + }, +} diff --git a/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/pyproject.toml b/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/pyproject.toml index f358d8fe635b0..533fcf3fea862 100644 --- a/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/pyproject.toml +++ b/llama-index-integrations/vector_stores/llama-index-vector-stores-redis/pyproject.toml @@ -12,7 +12,7 @@ contains_example = false import_path = "llama_index.vector_stores.redis" [tool.llamahub.class_authors] -RedisVectorStore = "llama-index" +RedisVectorStore = "redis" [tool.mypy] disallow_untyped_defs = true @@ -21,18 +21,18 @@ ignore_missing_imports = true python_version = "3.8" [tool.poetry] -authors = ["Your Name "] +authors = ["Tyler Hutcherson "] description = "llama-index vector_stores redis integration" exclude = ["**/BUILD"] license = "MIT" name = "llama-index-vector-stores-redis" readme = "README.md" -version = "0.1.2" +version = "0.2.0" [tool.poetry.dependencies] python = ">=3.8.1,<4.0" llama-index-core = "^0.10.1" -redis = "^5.0.1" +redisvl = "^0.1.3" [tool.poetry.group.dev.dependencies] ipython = "8.10.0"