From bf34a95b816b85c18f283f961dad4e181bdbf745 Mon Sep 17 00:00:00 2001 From: Justin Cechmanek Date: Tue, 18 Feb 2025 14:39:19 -0800 Subject: [PATCH 1/7] renames notebook --- .../{03_float16_support.ipynb => 03_dtype_support.ipynb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename python-recipes/vector-search/{03_float16_support.ipynb => 03_dtype_support.ipynb} (100%) diff --git a/python-recipes/vector-search/03_float16_support.ipynb b/python-recipes/vector-search/03_dtype_support.ipynb similarity index 100% rename from python-recipes/vector-search/03_float16_support.ipynb rename to python-recipes/vector-search/03_dtype_support.ipynb From 653df69952f6dafc1e4c0d783d083a4d53f37958 Mon Sep 17 00:00:00 2001 From: Justin Cechmanek Date: Tue, 18 Feb 2025 14:39:57 -0800 Subject: [PATCH 2/7] adds example demonstrating integer vector datatype --- .../vector-search/03_dtype_support.ipynb | 516 +++++++++++++++--- 1 file changed, 436 insertions(+), 80 deletions(-) diff --git a/python-recipes/vector-search/03_dtype_support.ipynb b/python-recipes/vector-search/03_dtype_support.ipynb index 16445e5e..23cc3b6a 100644 --- a/python-recipes/vector-search/03_dtype_support.ipynb +++ b/python-recipes/vector-search/03_dtype_support.ipynb @@ -7,21 +7,65 @@ "![Redis](https://redis.io/wp-content/uploads/2024/04/Logotype.svg?auto=webp&quality=85,75&width=120)\n", "# Using smaller vector types\n", "\n", - "With the [Redis 7.4 release](https://redis.io/blog/announcing-redis-community-edition-and-redis-stack-74/) there is now support for bfloat16 and float16 data types in the vector store.\n", + "With the [Redis 7.4 release](https://redis.io/blog/announcing-redis-community-edition-and-redis-stack-74/) there is now support for bfloat16 and float16 data types in the vector store. And with the release of [RedisVL 0.4.0](https://github.com/redis/redis-vl-python/tree/0.4.0) we've added support for integer vector types int8 and uint8 as well.\n", "\n", - "This tutorial will walk through how you can convert data stored in an existing index from float32 vectors to float16.\n", "\n", - "## Version requirements\n", + "This tutorial will walk through how you can convert data stored in an existing index from the default float32 vectors to float16 or 8 bit integers.\n", + "\n", + "## Version requirements for float16 and bfloat16 datatypes\n", "\n", "- redisvl >= 0.3.4\n", "- redis >= 7.4.0\n", "\n", + "\n", + "## Version requirements for int8 and uint8 datatypes\n", + "\n", + "- redisvl >= 0.4.0\n", + "- redis >= 7.4.0\n", + "\n", + "\n", "## Let's Begin!\n", - "\"Open\n", + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prepare data\n", + "In these examples we will load a list of movie objects with the following attributes: title, rating, description, and genre.\n", + "\n", + "For the vector part of our vector search we will embed the description so that users can search for movies that best match what they're looking for.\n", "\n", + "If you are running this notebook locally, FYI you may not need to perform this step at all." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# NBVAL_SKIP\n", + "!git clone https://github.com/redis-developer/redis-ai-resources.git temp_repo\n", + "!mv temp_repo/python-recipes/vector-search/resources .\n", + "!rm -rf temp_repo" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ "## Packages" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's start with float16 and bfloat16 support" + ] + }, { "cell_type": "code", "execution_count": 1, @@ -32,7 +76,7 @@ "output_type": "stream", "text": [ "\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.0.1\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", "Note: you may need to restart the kernel to use updated packages.\n" ] @@ -66,17 +110,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n# NBVAL_SKIP\\n%%sh\\ncurl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg\\necho \"deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main\" | sudo tee /etc/apt/sources.list.d/redis.list\\nsudo apt-get update > /dev/null 2>&1\\nsudo apt-get install redis-stack-server > /dev/null 2>&1\\nredis-stack-server --daemonize yes\\n'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ + "'''\n", "# NBVAL_SKIP\n", "%%sh\n", "curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg\n", "echo \"deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main\" | sudo tee /etc/apt/sources.list.d/redis.list\n", "sudo apt-get update > /dev/null 2>&1\n", "sudo apt-get install redis-stack-server > /dev/null 2>&1\n", - "redis-stack-server --daemonize yes" + "redis-stack-server --daemonize yes\n", + "'''" ] }, { @@ -122,7 +179,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -139,7 +196,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -148,16 +205,15 @@ "True" ] }, - "execution_count": 2, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "# from redis import Redis\n", - "import redis\n", + "from redis import Redis\n", "\n", - "client = redis.Redis.from_url(REDIS_URL)\n", + "client = Redis.from_url(REDIS_URL)\n", "client.ping()" ] }, @@ -172,7 +228,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -192,9 +248,17 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "14:19:02 redisvl.index.index INFO Index already exists, overwriting.\n" + ] + } + ], "source": [ "from redisvl.schema import IndexSchema\n", "from redisvl.index import SearchIndex\n", @@ -256,17 +320,15 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/Users/robert.shelton/.pyenv/versions/3.11.9/lib/python3.11/site-packages/huggingface_hub/file_download.py:1142: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", - " warnings.warn(\n", - "/Users/robert.shelton/.pyenv/versions/3.11.9/lib/python3.11/site-packages/huggingface_hub/file_download.py:1142: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", - " warnings.warn(\n" + "/Users/justin.cechmanek/.pyenv/versions/redis-ai-res/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], @@ -274,24 +336,24 @@ "from redisvl.utils.vectorize import HFTextVectorizer\n", "\n", "\n", - "# load model for embedding our movie descriptions\n", - "hf = HFTextVectorizer(model=\"sentence-transformers/all-MiniLM-L6-v2\")\n", + "# load a model to embed our movie descriptions, specifying the dtype we want to use\n", + "hf = HFTextVectorizer(model=\"sentence-transformers/all-MiniLM-L6-v2\", dtype=\"float32\")\n", "\n", - "embeddings_32 = hf.embed_many([movie[\"description\"] for movie in movies], dtype=\"float32\", as_buffer=True)" + "embeddings_32 = hf.embed_many([movie[\"description\"] for movie in movies], as_buffer=True)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "b'\\x9bf|=\\x0e`\\n;\"\\x92\\xb7;<\\xcb~\\xbd\\xfad\\xce\\xbb\\xc3\\x16J=V\\xa7?=\\xedv\\x95\\xaa\\x1c=\\xfd\\xee\\x89<\\xbd\\xb0-<\\x82\\xb2\\x9f\\xbc[\\x0b\\xc3\\xbd\\x98NR=xl\\xf7\\xbcN>\\x17\\xbe#\\x12\\x05\\xb99u\\xbf<\\xb0\\xe0b\\xba\\xd3\\xa6\\xa8\\xbdx\\xdc\\xec\\xbcRc%=\\xe4\\xe7r\\xbb\\x1eOG=?(\\x85=o@\\xa2\\xbc2Z\\xd0\\xbdC%K\\xbd\\xb9\\xed\\x94\\xbcR\\xddH=\\x92&F<\\xc6*\\xec<\\x90\\xd8\\x8d\\xbd\\xcbZ\\x98<\\t\\xa3\\xa3=>g3\\xbd&\\xcd\\xbd\\xbd\\x95$\\xf7;\\xfd\\xf4z=\\xfc\\xb4\\x8c=\\x85\\x0e\\xc6\\xbdnI\\x90\\xbdJ\\x16\\xbd;s\\xe7\\x0c\\xbd 3\\xc9\\xbc\\x85\\xf8\\xbb\\xbc\\xbf&u\\xbb5\\x8f\\xca<\\x05\\x80J=\\x0f\\xaf*=\\x8bOU\\xbd\\xc8\\xf0\\x95\\xbc\\x1d\\x02\\x19=)\\xf4K<\\xcb\\xc2\\t=F\\x83\\xac=\\x9f\\xd7\\xb8\\xbd\\xf2\\xb5\\x9c\\xbdB\\x85\\x18=\\x96d&=-3\\xf8<\\xfa\\xf7\\x88<\\x16v\\xf2\\xbb-=[\\xbd\\xf7\\xac\\xee\\xbb5:A\\xbd\\xd9d\\x19\\xbdrd\\xf2\\xbb!\\xbax;\\xdc;O<\\xb61,\\xbc\\xed\\xae\\xae=^\\x00-\\xbc\\x1a\\x06\\xae\\xbda\\xd6\\x1a=\\xcc\\xbf\\xcd=\\x1f\\x150=\\xcf\\xf1\\x9d\\xbc\\xa9GK=\\xaa\\xb8 =\\xb4\\xf1I\\xbd\"e\\x9e\\xbbF\\x8b\\xf7:\\x94\\xf8\\x1c=\\xa9\\xba\\xde<\\xcco\\x16\\xbb\\xe6]p\\xbb\\xbb\\xd5<<\\xac\\x95\\xa3\\xb8\\xc29s<&4&\\x10\\x90\\xbbvt\\xb9\\xbb\\x00\\xc9\\xb9\\xbb\\xfehk=\\x9a\\r\\xad<3f\\xa8\\xbd\\xbd]\\xcc=\\x15\\xe0 \\xbe\\xc74/\\xbd{f\\xf7\\xbcQ\\x9av=\\x11\\x0cq<,\\xda\\x1c\\xbd\\x01\\t\\x8b<\\xf0n\\xa6\\xbc\\xe4t\\x86<\\x82\\x87\\x19=v\\xae\\xe4\\xbc4m^\\xbc\\nV\\x0e\\xbd\\x81\\xb0\\xe3\\xbc\\xd3FU;\\xaaG|\\xbdW\\xfb\\x8b\\xbd\\x7f\\x81*\\xbdy\\x83\\xf4={\\xb7\\x10;\\x15!\\x0e\\xbd\\xfa\\xd3\\xb4=\\x15&\\x15\\xbdM\\x86\\x83=m$:\\xbdv\\x1bF\\xbd\\xa2?\\x14\\xbe\\xc5\\x8f(\\xbd\\xe3O\\x89\\xbd\\x17\\xae\\xd4<\\xa3\\x12\\xc3=\\xaf\\x05O\\xbd\\x7f\\x8ep\\xbc!\\xb5\\xac\\xbc\\xc4\\x9ee\\xbd9\\x8es;[a\\xc1;\\xd2\\xfaB\\xbd\\xf9#\\xfe:\\x90\\xe6\\xf4=\\xb2\\x15*<~\\xf8\\x1b=\\x01\\xfcV\\xbd\\xcf\\xd1\\r=*\\xee\\x06=\\x18u\\xba\\xbd\\x02\\xa4\\xd6<\\xf8\\xeb\\xd9;\\xc49/=\\xa8\\xc2\\x85=u\\x0b\"=\\xe9i\\xef<4\\xe8c=\\xfa2\\x08\\xbe\\xd4\\x12;=,VW;\\x15\\xa4b<\\xb0\\x9d\\xb7<\\x95r;\\xbd{z\\x91\\xbcI\\x00<\\xbd\\x18\\x1a\\xa3<\\xf9J%\\xbc\\n\\xe7\\xbf\\xbbr\\x87\\x12=\\x97\\x1d\\x95=\\x83|\\xfd\\xbc\\xed\\xf1\\xd1\\xbd%z\\x84;\\xcb\\tu=c\\x8ai\\x17\\xbe\\x18\\x0b\\x05\\xb99u\\xbf<\\xb5\\xe3b\\xba\\xd5\\xa6\\xa8\\xbd~\\xdc\\xec\\xbcPc%=\\xc1\\xe7r\\xbb\\x19OG=>(\\x85=c@\\xa2\\xbc1Z\\xd0\\xbd>%K\\xbd\\xba\\xed\\x94\\xbc\\\\\\xddH=\\xa6&F<\\xd2*\\xec<\\x8f\\xd8\\x8d\\xbd\\xb8Z\\x98<\\r\\xa3\\xa3=*g3\\xbd#\\xcd\\xbd\\xbd\\xde$\\xf7;\\xfd\\xf4z=\\xfc\\xb4\\x8c=\\x8b\\x0e\\xc6\\xbdfI\\x90\\xbdP\\x16\\xbd;x\\xe7\\x0c\\xbd\\x0e3\\xc9\\xbcj\\xf8\\xbb\\xbc\\xba&u\\xbb4\\x8f\\xca<\\x01\\x80J=\\x14\\xaf*=\\x84OU\\xbd\\xd1\\xf0\\x95\\xbc\\x1c\\x02\\x19=*\\xf4K<\\xca\\xc2\\t=B\\x83\\xac=\\x9a\\xd7\\xb8\\xbd\\xf1\\xb5\\x9c\\xbd>\\x85\\x18=\\xa4d&=\\x1f3\\xf8<\\xd8\\xf7\\x88<5v\\xf2\\xbb)=[\\xbd@\\xac\\xee\\xbb5:A\\xbd\\xd9d\\x19\\xbd/d\\xf2\\xbb4\\xbax;\\xeb;O<\\xe21,\\xbc\\xee\\xae\\xae=}\\x00-\\xbc\\x1e\\x06\\xae\\xbdo\\xd6\\x1a=\\xc4\\xbf\\xcd=\\x1b\\x150=\\xd6\\xf1\\x9d\\xbc\\xb6GK=\\xb0\\xb8 =\\xae\\xf1I\\xbd7e\\x9e\\xbb\\x96\\x8b\\xf7:\\x89\\xf8\\x1c=\\x97\\xba\\xde<\\x16p\\x16\\xbb\\xf2]p\\xbb\\xbc\\xd5<\\xbd~\\x1bF\\xbd\\xa2?\\x14\\xbe\\xc8\\x8f(\\xbd\\xe3O\\x89\\xbd\\x18\\xae\\xd4<\\xa6\\x12\\xc3=\\xb8\\x05O\\xbd\\x9e\\x8ep\\xbc\\x18\\xb5\\xac\\xbc\\xc9\\x9ee\\xbdV\\x8es;\\x07a\\xc1;\\xd2\\xfaB\\xbd\\xaa\"\\xfe:\\x92\\xe6\\xf4=\\xa4\\x15*<\\x91\\xf8\\x1b=\\x03\\xfcV\\xbd\\xdf\\xd1\\r=2\\xee\\x06=\\x17u\\xba\\xbd\\xff\\xa3\\xd6<\\x1c\\xec\\xd9;\\xba9/=\\xa9\\xc2\\x85=v\\x0b\"=\\xe3i\\xef<-\\xe8c=\\xfa2\\x08\\xbe\\xca\\x12;=\\xc0UW;Q\\xa4b<\\xd5\\x9d\\xb7<\\x90r;\\xbdUz\\x91\\xbcX\\x00<\\xbd\\r\\x1a\\xa3<\\xbfJ%\\xbc]\\xe7\\xbf\\xbb\\x84\\x87\\x12=\\x95\\x1d\\x95=||\\xfd\\xbc\\xf3\\xf1\\xd1\\xbd1z\\x84;\\xc7\\tu={\\x8ai\\xaa\\x1c=\\xfd\\xee\\x89<\\xbd\\xb0-<\\x82\\xb2\\x9f\\xbc[\\x0b\\xc3\\xbd\\x98NR=xl\\xf7\\xbcN>\\x17\\xbe#\\x12\\x05\\xb99u\\xbf<\\xb0\\xe0b\\xba\\xd3\\xa6\\xa8\\xbdx\\xdc\\xec\\xbcRc%=\\xe4\\xe7r\\xbb\\x1eOG=?(\\x85=o@\\xa2\\xbc2Z\\xd0\\xbdC%K\\xbd\\xb9\\xed\\x94\\xbcR\\xddH=\\x92&F<\\xc6*\\xec<\\x90\\xd8\\x8d\\xbd\\xcbZ\\x98<\\t\\xa3\\xa3=>g3\\xbd&\\xcd\\xbd\\xbd\\x95$\\xf7;\\xfd\\xf4z=\\xfc\\xb4\\x8c=\\x85\\x0e\\xc6\\xbdnI\\x90\\xbdJ\\x16\\xbd;s\\xe7\\x0c\\xbd 3\\xc9\\xbc\\x85\\xf8\\xbb\\xbc\\xbf&u\\xbb5\\x8f\\xca<\\x05\\x80J=\\x0f\\xaf*=\\x8bOU\\xbd\\xc8\\xf0\\x95\\xbc\\x1d\\x02\\x19=)\\xf4K<\\xcb\\xc2\\t=F\\x83\\xac=\\x9f\\xd7\\xb8\\xbd\\xf2\\xb5\\x9c\\xbdB\\x85\\x18=\\x96d&=-3\\xf8<\\xfa\\xf7\\x88<\\x16v\\xf2\\xbb-=[\\xbd\\xf7\\xac\\xee\\xbb5:A\\xbd\\xd9d\\x19\\xbdrd\\xf2\\xbb!\\xbax;\\xdc;O<\\xb61,\\xbc\\xed\\xae\\xae=^\\x00-\\xbc\\x1a\\x06\\xae\\xbda\\xd6\\x1a=\\xcc\\xbf\\xcd=\\x1f\\x150=\\xcf\\xf1\\x9d\\xbc\\xa9GK=\\xaa\\xb8 =\\xb4\\xf1I\\xbd\"e\\x9e\\xbbF\\x8b\\xf7:\\x94\\xf8\\x1c=\\xa9\\xba\\xde<\\xcco\\x16\\xbb\\xe6]p\\xbb\\xbb\\xd5<<\\xac\\x95\\xa3\\xb8\\xc29s<&4&\\x10\\x90\\xbbvt\\xb9\\xbb\\x00\\xc9\\xb9\\xbb\\xfehk=\\x9a\\r\\xad<3f\\xa8\\xbd\\xbd]\\xcc=\\x15\\xe0 \\xbe\\xc74/\\xbd{f\\xf7\\xbcQ\\x9av=\\x11\\x0cq<,\\xda\\x1c\\xbd\\x01\\t\\x8b<\\xf0n\\xa6\\xbc\\xe4t\\x86<\\x82\\x87\\x19=v\\xae\\xe4\\xbc4m^\\xbc\\nV\\x0e\\xbd\\x81\\xb0\\xe3\\xbc\\xd3FU;\\xaaG|\\xbdW\\xfb\\x8b\\xbd\\x7f\\x81*\\xbdy\\x83\\xf4={\\xb7\\x10;\\x15!\\x0e\\xbd\\xfa\\xd3\\xb4=\\x15&\\x15\\xbdM\\x86\\x83=m$:\\xbdv\\x1bF\\xbd\\xa2?\\x14\\xbe\\xc5\\x8f(\\xbd\\xe3O\\x89\\xbd\\x17\\xae\\xd4<\\xa3\\x12\\xc3=\\xaf\\x05O\\xbd\\x7f\\x8ep\\xbc!\\xb5\\xac\\xbc\\xc4\\x9ee\\xbd9\\x8es;[a\\xc1;\\xd2\\xfaB\\xbd\\xf9#\\xfe:\\x90\\xe6\\xf4=\\xb2\\x15*<~\\xf8\\x1b=\\x01\\xfcV\\xbd\\xcf\\xd1\\r=*\\xee\\x06=\\x18u\\xba\\xbd\\x02\\xa4\\xd6<\\xf8\\xeb\\xd9;\\xc49/=\\xa8\\xc2\\x85=u\\x0b\"=\\xe9i\\xef<4\\xe8c=\\xfa2\\x08\\xbe\\xd4\\x12;=,VW;\\x15\\xa4b<\\xb0\\x9d\\xb7<\\x95r;\\xbd{z\\x91\\xbcI\\x00<\\xbd\\x18\\x1a\\xa3<\\xf9J%\\xbc\\n\\xe7\\xbf\\xbbr\\x87\\x12=\\x97\\x1d\\x95=\\x83|\\xfd\\xbc\\xed\\xf1\\xd1\\xbd%z\\x84;\\xcb\\tu=c\\x8ai\\x17\\xbe\\x18\\x0b\\x05\\xb99u\\xbf<\\xb5\\xe3b\\xba\\xd5\\xa6\\xa8\\xbd~\\xdc\\xec\\xbcPc%=\\xc1\\xe7r\\xbb\\x19OG=>(\\x85=c@\\xa2\\xbc1Z\\xd0\\xbd>%K\\xbd\\xba\\xed\\x94\\xbc\\\\\\xddH=\\xa6&F<\\xd2*\\xec<\\x8f\\xd8\\x8d\\xbd\\xb8Z\\x98<\\r\\xa3\\xa3=*g3\\xbd#\\xcd\\xbd\\xbd\\xde$\\xf7;\\xfd\\xf4z=\\xfc\\xb4\\x8c=\\x8b\\x0e\\xc6\\xbdfI\\x90\\xbdP\\x16\\xbd;x\\xe7\\x0c\\xbd\\x0e3\\xc9\\xbcj\\xf8\\xbb\\xbc\\xba&u\\xbb4\\x8f\\xca<\\x01\\x80J=\\x14\\xaf*=\\x84OU\\xbd\\xd1\\xf0\\x95\\xbc\\x1c\\x02\\x19=*\\xf4K<\\xca\\xc2\\t=B\\x83\\xac=\\x9a\\xd7\\xb8\\xbd\\xf1\\xb5\\x9c\\xbd>\\x85\\x18=\\xa4d&=\\x1f3\\xf8<\\xd8\\xf7\\x88<5v\\xf2\\xbb)=[\\xbd@\\xac\\xee\\xbb5:A\\xbd\\xd9d\\x19\\xbd/d\\xf2\\xbb4\\xbax;\\xeb;O<\\xe21,\\xbc\\xee\\xae\\xae=}\\x00-\\xbc\\x1e\\x06\\xae\\xbdo\\xd6\\x1a=\\xc4\\xbf\\xcd=\\x1b\\x150=\\xd6\\xf1\\x9d\\xbc\\xb6GK=\\xb0\\xb8 =\\xae\\xf1I\\xbd7e\\x9e\\xbb\\x96\\x8b\\xf7:\\x89\\xf8\\x1c=\\x97\\xba\\xde<\\x16p\\x16\\xbb\\xf2]p\\xbb\\xbc\\xd5<\\xbd~\\x1bF\\xbd\\xa2?\\x14\\xbe\\xc8\\x8f(\\xbd\\xe3O\\x89\\xbd\\x18\\xae\\xd4<\\xa6\\x12\\xc3=\\xb8\\x05O\\xbd\\x9e\\x8ep\\xbc\\x18\\xb5\\xac\\xbc\\xc9\\x9ee\\xbdV\\x8es;\\x07a\\xc1;\\xd2\\xfaB\\xbd\\xaa\"\\xfe:\\x92\\xe6\\xf4=\\xa4\\x15*<\\x91\\xf8\\x1b=\\x03\\xfcV\\xbd\\xdf\\xd1\\r=2\\xee\\x06=\\x17u\\xba\\xbd\\xff\\xa3\\xd6<\\x1c\\xec\\xd9;\\xba9/=\\xa9\\xc2\\x85=v\\x0b\"=\\xe3i\\xef<-\\xe8c=\\xfa2\\x08\\xbe\\xca\\x12;=\\xc0UW;Q\\xa4b<\\xd5\\x9d\\xb7<\\x90r;\\xbdUz\\x91\\xbcX\\x00<\\xbd\\r\\x1a\\xa3<\\xbfJ%\\xbc]\\xe7\\xbf\\xbb\\x84\\x87\\x12=\\x95\\x1d\\x95=||\\xfd\\xbc\\xf3\\xf1\\xd1\\xbd1z\\x84;\\xc7\\tu={\\x8ai=0.4.0'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert redisvl.__version__ >= '0.4.0'" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "int_hf = HFTextVectorizer(model=\"sentence-transformers/all-MiniLM-L6-v2\", dtype='int8')\n", + "\n", + "\n", + "embeddings_int8 = int_hf.embed_many([movie[\"description\"] for movie in movies], as_buffer=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00'" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "embeddings_int8[0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "What happened? Why is the vector all zeros?\n", + "\n", + "While Redis supports integer data types, many embedding models scale their vector length to 1.0, which means each value is less than 1.0 - typically much less than 1.0, and so are rounded down when using int8\n", + "\n", + "\n", + "You have two options if you want to use integers\n", + "1. use an embedding model that is not normalized\n", + "2. scale the vectors up yourself before converting them to integers\n", + "\n", + "The large majority of models are normalized, so rather than hunt around for an elusive one that isn't we'll show you how to easily scale up any model" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: scikit-learn in /Users/justin.cechmanek/.pyenv/versions/3.11.9/envs/redis-ai-res/lib/python3.11/site-packages (1.6.1)\n", + "Requirement already satisfied: numpy>=1.19.5 in /Users/justin.cechmanek/.pyenv/versions/3.11.9/envs/redis-ai-res/lib/python3.11/site-packages (from scikit-learn) (1.26.4)\n", + "Requirement already satisfied: scipy>=1.6.0 in /Users/justin.cechmanek/.pyenv/versions/3.11.9/envs/redis-ai-res/lib/python3.11/site-packages (from scikit-learn) (1.15.1)\n", + "Requirement already satisfied: joblib>=1.2.0 in /Users/justin.cechmanek/.pyenv/versions/3.11.9/envs/redis-ai-res/lib/python3.11/site-packages (from scikit-learn) (1.4.2)\n", + "Requirement already satisfied: threadpoolctl>=3.1.0 in /Users/justin.cechmanek/.pyenv/versions/3.11.9/envs/redis-ai-res/lib/python3.11/site-packages (from scikit-learn) (3.5.0)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.0.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython -m pip install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!pip install scikit-learn" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "# use any embedding model, normalized or not\n", + "# this model isn't normalized, but most values are still between -5.0 and +5.0\n", + "# for int8 we want to use the full range of -128 to +127\n", + "\n", + "integer_hf = HFTextVectorizer(model=\"BAAI/bge-base-en-v1.5\", dtype='int8')\n", + "\n", + "embedding = integer_hf.embed('this string will be converted to an integer embedding')\n", + "\n", + "from sklearn.preprocessing import minmax_scale\n", + "from redisvl.redis.utils import array_to_buffer\n", + "\n", + "scaled_embedding = minmax_scale(embedding, feature_range=(-128, 127))\n", + "\n", + "scaled_byte_embedding = array_to_buffer(scaled_embedding, dtype='int8')" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "b'\"\\x1e&-BSQ\\x0e+\\x1c6/\\n@(NH\\x17A3\\x1c8\\x1e/<.7\\x02\\x1248-9%8\\x12\\x13\\x1f3)\\x15$A;\\x00\\x16\\xf801\\x1d\\xf8@\\x150G\\\\@%&g\\'\\x1d\\x11TL\\x1a/\\x11\\x136?\\x1b.\\x08@\\x1c?.*1+9\\x19!D$(@\\x1d3\\x14:5)\\x1b*+MG)\\x03\\x19(\\x14T;#(Z\\x1aR\\x1c\\xf57$?(\\x0cA)#$\\x10\\x05:\\x13T\"/*\\x194\\x1c7+\\x1a5:S \\x0f.\")-\\x13-5/17\\x0c/#\\x15\\x1a\\x1e\\x7fD\\x1d<\\x12\\x18\\xf7(\"\\'I\\x11\\x17_\\x04.^\\n4*51\\x10\\x1a\\x1f.))+\\x14\\x07-&:R\\x15,)\\x10\\x0f\\x0c\\x15ET,5;,/%-]\"\\x1c\\x17)@\\x0b\\x03/\\x18\\x1d\\x0b\\x1e*DF6?[\\x159F?\\x1f*E-?\"\\x1d;!\\x80A\\x05*1!;\\x12,8\\x15!\\x1c23\\x1e3/(3/\\x123Z b\\x1f\\x15/-\\'\\x16U\\x0f%\\x194\\x1212\\r.+\\x15I),(,D#05+)A\\x10\\r\\x13\\x1c\\x15\\x17\\x0cB(/\\x0f502=\\xfc.B$A\\x1b2;*F893)9/>4+:<<5\\xf3\\x15=A\\n9\\x0b\\x1fN0$%\\x159+\\x16\\x1b>9\\x1c#A;*(&9B 8 O5$)B_%&\\x13$$\\x08#-0\\xff\\x0e$Q+0A \\x1b&B\\x101&+.A\"\\x0e\\x19,#5\"-\\xf6E\\x1d2\\x1f20+#;\\x0f@\\x17;N\\r\\x1a\\x00K9O-,#/\"9\\x19\\x04(5A9\\x0c\\x07F8\\x12*)#$9A]SH(\\x1bB&\\'\\x072,2\\x14\\x0b\\x121\\xfa\\x19:;C\"%)>\\x12#,:\\x1c5->DR? \\x19N\\'\\x19\\x033\\'\\xfc\\xf81$\\x12dH! /\\x1c\\x14,\\x1d+[B!;\\x16C2L5M B\\x12\\x1744%C\\x13S#[\\x0c3D5,6,$\\x16\\x18-2 9\\x19E\\x1a9Z#\\x044W9/\\'0(\\'&1\"\\'0\\x18W/&\\x13\\x12\\x1e\\xf8\\x02$\\x1e46\\x0b\\r5 \\x1fJL9FD\\x16(D1[-\\x0f\\xf3-W)\\x0c=#A;\\x1b4$\\x1c &h\\x0cV=\\x1a56!0\\x073\\x0b\\x11:82\\x1245$C\\x10N2\\x1c\\xe8M0.2(C,U#7L'" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scaled_byte_embedding" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "from here we can use the same process as before to convert our existing embeddings to our new desired datatype appropriately scaled.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.preprocessing import minmax_scale\n", + "def scale_and_replace_vectors(index, pattern, old_dtype, new_dtype, scale_range=None):\n", + " cursor = \"0\"\n", + "\n", + " while cursor != 0:\n", + " items_to_convert = []\n", + " # Fetch a batch of records\n", + " cursor, keys = index.client.scan(cursor=cursor, match=pattern)\n", + "\n", + " # Use a Redis pipeline to make this more scalable\n", + " with index.client.pipeline(transaction=False) as pipe:\n", + " if index.storage_type.value == \"hash\":\n", + " for key in keys:\n", + " pipe.hgetall(key)\n", + " if index.storage_type.value == \"json\":\n", + " for key in keys:\n", + " pipe.json().get(key)\n", + "\n", + " items_to_convert.extend(pipe.execute())\n", + "\n", + " if items_to_convert:\n", + "\n", + " print(f\"Converting vectors for {len(items_to_convert)} records\")\n", + " new_vecs = [np.frombuffer(item[b'vector'], dtype=old_dtype) for item in items_to_convert]\n", + " if scale_range:\n", + " new_vecs = minmax_scale(new_vecs, feature_range=scale_range)\n", + " new_vecs = new_vecs.astype(new_dtype).tobytes()\n", + " updated_data = [{**item, 'vector': new_vecs[i]} for i, item in enumerate(items_to_convert)]\n", + "\n", + " # write back data\n", + " new_keys = index.load(updated_data, keys=keys)\n", + "\n", + " return new_keys\n" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Converting vectors for 1 records\n", + "Converting vectors for 1 records\n", + "Converting vectors for 1 records\n", + "Converting vectors for 1 records\n", + "Converting vectors for 1 records\n", + "Converting vectors for 1 records\n", + "Converting vectors for 1 records\n", + "Converting vectors for 1 records\n", + "Converting vectors for 1 records\n", + "Converting vectors for 1 records\n", + "Converting vectors for 1 records\n", + "Converting vectors for 2 records\n", + "Converting vectors for 1 records\n", + "Converting vectors for 1 records\n", + "Converting vectors for 1 records\n", + "Converting vectors for 1 records\n", + "Converting vectors for 1 records\n", + "Converting vectors for 1 records\n", + "Converting vectors for 1 records\n" + ] + } + ], + "source": [ + "pattern = \"movies:*\" # prefix of data to convert\n", + "storage_type = \"hash\"\n", + "updated_keys = scale_and_replace_vectors(index, pattern, \"float32\", \"int8\", (-128, 127))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Update the schema by removing the old vector field\n", + "index.schema.remove_field(\"vector\")\n", + "\n", + "# Add updated vector field including the new datatype here\n", + "index.schema.add_field({\n", + " \"name\": \"vector\",\n", + " \"type\": \"vector\",\n", + " \"attrs\": {\n", + " \"dims\": 384,\n", + " \"distance_metric\": \"cosine\",\n", + " \"algorithm\": \"hnsw\",\n", + " \"datatype\": \"int8\" # as simple as updating this field\n", + " }\n", + "})\n", + "\n", + "# Update the index schema by dropping the old and updating with the new -- will NOT delete data\n", + "index.create(overwrite=True, drop=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "index.info()[\"num_docs\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client.memory_usage(updated_keys[0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# some important notes\n", + "When dealing with integer data types in search indices be aware of overflow and underflow. Depending on the math library you are using trying to converting 256 to an unsigned 8 bit integer may either throw an error, or wrap around and return -1. Numpy versions before 2.0 wrap around, while later versions will raise an `OverflowError`.\n", + "\n", + "When doing vector similarity search in Redis the choice of distance metric also matters, as Inner Product (IP) and Euclidian Distance (L2), will not return scaled values, but cosine (COSINE) will always be scaled regardless of the vector values, because that's how angles work." + ] + }, { "cell_type": "code", "execution_count": null, @@ -659,7 +1015,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "redis-ai-res", "language": "python", "name": "python3" }, From c2bd141dc8e4851f6516efe1bcb0eb0a0644d63e Mon Sep 17 00:00:00 2001 From: Justin Cechmanek Date: Tue, 18 Feb 2025 14:54:58 -0800 Subject: [PATCH 3/7] removes version check until 0.4.0 is released --- python-recipes/vector-search/03_dtype_support.ipynb | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/python-recipes/vector-search/03_dtype_support.ipynb b/python-recipes/vector-search/03_dtype_support.ipynb index 23cc3b6a..655d7c6d 100644 --- a/python-recipes/vector-search/03_dtype_support.ipynb +++ b/python-recipes/vector-search/03_dtype_support.ipynb @@ -110,7 +110,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -125,15 +125,13 @@ } ], "source": [ - "'''\n", "# NBVAL_SKIP\n", "%%sh\n", "curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg\n", "echo \"deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main\" | sudo tee /etc/apt/sources.list.d/redis.list\n", "sudo apt-get update > /dev/null 2>&1\n", "sudo apt-get install redis-stack-server > /dev/null 2>&1\n", - "redis-stack-server --daemonize yes\n", - "'''" + "redis-stack-server --daemonize yes\n" ] }, { @@ -738,7 +736,7 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install 'redisvl>=0.4.0'" + "#!pip install 'redisvl>=0.4.0'" ] }, { @@ -747,7 +745,7 @@ "metadata": {}, "outputs": [], "source": [ - "assert redisvl.__version__ >= '0.4.0'" + "#assert redisvl.__version__ >= '0.4.0'" ] }, { From 1f9c66d1b08ed94c6b76aa59aa5b79ca228413e1 Mon Sep 17 00:00:00 2001 From: Justin Cechmanek Date: Tue, 18 Feb 2025 15:03:18 -0800 Subject: [PATCH 4/7] updates notebook links in README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3cbe48a5..8a4634ce 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Need quickstarts to begin your Redis AI journey? **Start here.** | [/vector-search/00_redispy.ipynb](/python-recipes/vector-search/00_redispy.ipynb) | Vector search with Redis python client | | [/vector-search/01_redisvl.ipynb](/python-recipes/vector-search/01_redisvl.ipynb) | Vector search with Redis Vector Library | | [/vector-search/02_hybrid_search.ipynb](/python-recipes/vector-search/02_hybrid_search.ipynb) | Hybrid search techniques with Redis (BM25 + Vector) | -| [/vector-search/03_float16_support.ipynb](/python-recipes/vector-search/03_float16_support.ipynb) | Shows how to convert a float32 index to use float16 | +| [/vector-search/03_dtype_support.ipynb](/python-recipes/vector-search/03_dtype_support.ipynb) | Shows how to convert a float32 index to float16 or integer dataypes| ### Retrieval Augmented Generation (RAG) @@ -107,6 +107,7 @@ Routing is a simple and effective way of preventing misuses with your AI applica | --- | --- | | [/recommendation-systems/00_content_filtering.ipynb](python-recipes/recommendation-systems/00_content_filtering.ipynb) | Intro content filtering example with redisvl | | [/recommendation-systems/01_collaborative_filtering.ipynb](python-recipes/recommendation-systems/01_collaborative_filtering.ipynb) | Intro collaborative filtering example with redisvl | +| [/recommendation-systems/02_two_towers.ipynb](python-recipes/recommendation-systems/02_two_towers.ipynb) | Intro deep learning two tower example with redisvl | ### Feature Store | Recipe | Description | From 37ed7ab5d9f327b7c71ca63c1102621b202915fb Mon Sep 17 00:00:00 2001 From: Justin Cechmanek Date: Thu, 6 Mar 2025 16:08:08 -0800 Subject: [PATCH 5/7] shows how to convert index to use int8 vectors --- .../vector-search/03_dtype_support.ipynb | 290 ++++++++++-------- 1 file changed, 157 insertions(+), 133 deletions(-) diff --git a/python-recipes/vector-search/03_dtype_support.ipynb b/python-recipes/vector-search/03_dtype_support.ipynb index d2836920..c566cb1a 100644 --- a/python-recipes/vector-search/03_dtype_support.ipynb +++ b/python-recipes/vector-search/03_dtype_support.ipynb @@ -21,7 +21,7 @@ "## Version requirements for int8 and uint8 datatypes\n", "\n", "- redisvl >= 0.4.0\n", - "- redis >= 7.4.0\n", + "- redis >= 7.9.226\n", "\n", "\n", "## Let's Begin!\n", @@ -44,7 +44,18 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n# NBVAL_SKIP\\n!git clone https://github.com/redis-developer/redis-ai-resources.git temp_repo\\n!mv temp_repo/python-recipes/vector-search/resources .\\n!rm -rf temp_repo\\n'" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# NBVAL_SKIP\n", "!git clone https://github.com/redis-developer/redis-ai-resources.git temp_repo\n", @@ -68,7 +79,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -88,7 +99,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -119,7 +130,7 @@ "'\\n# NBVAL_SKIP\\n%%sh\\ncurl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg\\necho \"deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main\" | sudo tee /etc/apt/sources.list.d/redis.list\\nsudo apt-get update > /dev/null 2>&1\\nsudo apt-get install redis-stack-server > /dev/null 2>&1\\nredis-stack-server --daemonize yes\\n'" ] }, - "execution_count": 3, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -131,7 +142,7 @@ "echo \"deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main\" | sudo tee /etc/apt/sources.list.d/redis.list\n", "sudo apt-get update > /dev/null 2>&1\n", "sudo apt-get install redis-stack-server > /dev/null 2>&1\n", - "redis-stack-server --daemonize yes\n" + "redis-stack-server --daemonize yes" ] }, { @@ -177,7 +188,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -194,7 +205,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -203,7 +214,7 @@ "True" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -226,7 +237,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -246,14 +257,14 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "14:19:02 redisvl.index.index INFO Index already exists, overwriting.\n" + "16:04:57 redisvl.index.index INFO Index already exists, overwriting.\n" ] } ], @@ -318,7 +329,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -333,7 +344,6 @@ "source": [ "from redisvl.utils.vectorize import HFTextVectorizer\n", "\n", - "\n", "# load a model to embed our movie descriptions, specifying the dtype we want to use\n", "hf = HFTextVectorizer(model=\"sentence-transformers/all-MiniLM-L6-v2\", dtype=\"float32\")\n", "\n", @@ -342,7 +352,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -351,7 +361,7 @@ "b'\\x8df|=*a\\n;-\\x92\\xb7;3\\xcb~\\xbd`e\\xce\\xbb\\xca\\x16J==\\xa7?=\\xefv\\x95\\x17\\xbe\\x18\\x0b\\x05\\xb99u\\xbf<\\xb5\\xe3b\\xba\\xd5\\xa6\\xa8\\xbd~\\xdc\\xec\\xbcPc%=\\xc1\\xe7r\\xbb\\x19OG=>(\\x85=c@\\xa2\\xbc1Z\\xd0\\xbd>%K\\xbd\\xba\\xed\\x94\\xbc\\\\\\xddH=\\xa6&F<\\xd2*\\xec<\\x8f\\xd8\\x8d\\xbd\\xb8Z\\x98<\\r\\xa3\\xa3=*g3\\xbd#\\xcd\\xbd\\xbd\\xde$\\xf7;\\xfd\\xf4z=\\xfc\\xb4\\x8c=\\x8b\\x0e\\xc6\\xbdfI\\x90\\xbdP\\x16\\xbd;x\\xe7\\x0c\\xbd\\x0e3\\xc9\\xbcj\\xf8\\xbb\\xbc\\xba&u\\xbb4\\x8f\\xca<\\x01\\x80J=\\x14\\xaf*=\\x84OU\\xbd\\xd1\\xf0\\x95\\xbc\\x1c\\x02\\x19=*\\xf4K<\\xca\\xc2\\t=B\\x83\\xac=\\x9a\\xd7\\xb8\\xbd\\xf1\\xb5\\x9c\\xbd>\\x85\\x18=\\xa4d&=\\x1f3\\xf8<\\xd8\\xf7\\x88<5v\\xf2\\xbb)=[\\xbd@\\xac\\xee\\xbb5:A\\xbd\\xd9d\\x19\\xbd/d\\xf2\\xbb4\\xbax;\\xeb;O<\\xe21,\\xbc\\xee\\xae\\xae=}\\x00-\\xbc\\x1e\\x06\\xae\\xbdo\\xd6\\x1a=\\xc4\\xbf\\xcd=\\x1b\\x150=\\xd6\\xf1\\x9d\\xbc\\xb6GK=\\xb0\\xb8 =\\xae\\xf1I\\xbd7e\\x9e\\xbb\\x96\\x8b\\xf7:\\x89\\xf8\\x1c=\\x97\\xba\\xde<\\x16p\\x16\\xbb\\xf2]p\\xbb\\xbc\\xd5<\\xbd~\\x1bF\\xbd\\xa2?\\x14\\xbe\\xc8\\x8f(\\xbd\\xe3O\\x89\\xbd\\x18\\xae\\xd4<\\xa6\\x12\\xc3=\\xb8\\x05O\\xbd\\x9e\\x8ep\\xbc\\x18\\xb5\\xac\\xbc\\xc9\\x9ee\\xbdV\\x8es;\\x07a\\xc1;\\xd2\\xfaB\\xbd\\xaa\"\\xfe:\\x92\\xe6\\xf4=\\xa4\\x15*<\\x91\\xf8\\x1b=\\x03\\xfcV\\xbd\\xdf\\xd1\\r=2\\xee\\x06=\\x17u\\xba\\xbd\\xff\\xa3\\xd6<\\x1c\\xec\\xd9;\\xba9/=\\xa9\\xc2\\x85=v\\x0b\"=\\xe3i\\xef<-\\xe8c=\\xfa2\\x08\\xbe\\xca\\x12;=\\xc0UW;Q\\xa4b<\\xd5\\x9d\\xb7<\\x90r;\\xbdUz\\x91\\xbcX\\x00<\\xbd\\r\\x1a\\xa3<\\xbfJ%\\xbc]\\xe7\\xbf\\xbb\\x84\\x87\\x12=\\x95\\x1d\\x95=||\\xfd\\xbc\\xf3\\xf1\\xd1\\xbd1z\\x84;\\xc7\\tu={\\x8ai\\x17\\xbe\\x18\\x0b\\x05\\xb99u\\xbf<\\xb5\\xe3b\\xba\\xd5\\xa6\\xa8\\xbd~\\xdc\\xec\\xbcPc%=\\xc1\\xe7r\\xbb\\x19OG=>(\\x85=c@\\xa2\\xbc1Z\\xd0\\xbd>%K\\xbd\\xba\\xed\\x94\\xbc\\\\\\xddH=\\xa6&F<\\xd2*\\xec<\\x8f\\xd8\\x8d\\xbd\\xb8Z\\x98<\\r\\xa3\\xa3=*g3\\xbd#\\xcd\\xbd\\xbd\\xde$\\xf7;\\xfd\\xf4z=\\xfc\\xb4\\x8c=\\x8b\\x0e\\xc6\\xbdfI\\x90\\xbdP\\x16\\xbd;x\\xe7\\x0c\\xbd\\x0e3\\xc9\\xbcj\\xf8\\xbb\\xbc\\xba&u\\xbb4\\x8f\\xca<\\x01\\x80J=\\x14\\xaf*=\\x84OU\\xbd\\xd1\\xf0\\x95\\xbc\\x1c\\x02\\x19=*\\xf4K<\\xca\\xc2\\t=B\\x83\\xac=\\x9a\\xd7\\xb8\\xbd\\xf1\\xb5\\x9c\\xbd>\\x85\\x18=\\xa4d&=\\x1f3\\xf8<\\xd8\\xf7\\x88<5v\\xf2\\xbb)=[\\xbd@\\xac\\xee\\xbb5:A\\xbd\\xd9d\\x19\\xbd/d\\xf2\\xbb4\\xbax;\\xeb;O<\\xe21,\\xbc\\xee\\xae\\xae=}\\x00-\\xbc\\x1e\\x06\\xae\\xbdo\\xd6\\x1a=\\xc4\\xbf\\xcd=\\x1b\\x150=\\xd6\\xf1\\x9d\\xbc\\xb6GK=\\xb0\\xb8 =\\xae\\xf1I\\xbd7e\\x9e\\xbb\\x96\\x8b\\xf7:\\x89\\xf8\\x1c=\\x97\\xba\\xde<\\x16p\\x16\\xbb\\xf2]p\\xbb\\xbc\\xd5<\\xbd~\\x1bF\\xbd\\xa2?\\x14\\xbe\\xc8\\x8f(\\xbd\\xe3O\\x89\\xbd\\x18\\xae\\xd4<\\xa6\\x12\\xc3=\\xb8\\x05O\\xbd\\x9e\\x8ep\\xbc\\x18\\xb5\\xac\\xbc\\xc9\\x9ee\\xbdV\\x8es;\\x07a\\xc1;\\xd2\\xfaB\\xbd\\xaa\"\\xfe:\\x92\\xe6\\xf4=\\xa4\\x15*<\\x91\\xf8\\x1b=\\x03\\xfcV\\xbd\\xdf\\xd1\\r=2\\xee\\x06=\\x17u\\xba\\xbd\\xff\\xa3\\xd6<\\x1c\\xec\\xd9;\\xba9/=\\xa9\\xc2\\x85=v\\x0b\"=\\xe3i\\xef<-\\xe8c=\\xfa2\\x08\\xbe\\xca\\x12;=\\xc0UW;Q\\xa4b<\\xd5\\x9d\\xb7<\\x90r;\\xbdUz\\x91\\xbcX\\x00<\\xbd\\r\\x1a\\xa3<\\xbfJ%\\xbc]\\xe7\\xbf\\xbb\\x84\\x87\\x12=\\x95\\x1d\\x95=||\\xfd\\xbc\\xf3\\xf1\\xd1\\xbd1z\\x84;\\xc7\\tu={\\x8ai=0.4.0'" + "# NBVAL_SKIP\n", + "%%sh\n", + "docker run -d --name redis -p 6379:6379 redis:8.0-M03" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ - "#assert redisvl.__version__ >= '0.4.0'" + "assert redisvl.__version__ >= '0.4.0'" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ "int_hf = HFTextVectorizer(model=\"sentence-transformers/all-MiniLM-L6-v2\", dtype='int8')\n", "\n", - "\n", "embeddings_int8 = int_hf.embed_many([movie[\"description\"] for movie in movies], as_buffer=True)" ] }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -771,7 +777,7 @@ "b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00'" ] }, - "execution_count": 24, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -786,7 +792,7 @@ "source": [ "What happened? Why is the vector all zeros?\n", "\n", - "While Redis supports integer data types, many embedding models scale their vector length to 1.0, which means each value is less than 1.0 - typically much less than 1.0, and so are rounded down when using int8\n", + "While Redis supports integer data types, many embedding models scale their vector length to 1.0, which means each value is less than 1.0 - typically much less than 1.0, and so are rounded down when using `int8`\n", "\n", "\n", "You have two options if you want to use integers\n", @@ -798,7 +804,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -812,7 +818,7 @@ "Requirement already satisfied: threadpoolctl>=3.1.0 in /Users/justin.cechmanek/.pyenv/versions/3.11.9/envs/redis-ai-res/lib/python3.11/site-packages (from scikit-learn) (3.5.0)\n", "\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.0.1\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython -m pip install --upgrade pip\u001b[0m\n" + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n" ] } ], @@ -822,7 +828,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -830,6 +836,8 @@ "# this model isn't normalized, but most values are still between -5.0 and +5.0\n", "# for int8 we want to use the full range of -128 to +127\n", "\n", + "from redisvl.redis.utils import array_to_buffer\n", + "\n", "integer_hf = HFTextVectorizer(model=\"BAAI/bge-base-en-v1.5\", dtype='int8')\n", "\n", "embedding = integer_hf.embed('this string will be converted to an integer embedding')\n", @@ -838,13 +846,14 @@ "from redisvl.redis.utils import array_to_buffer\n", "\n", "scaled_embedding = minmax_scale(embedding, feature_range=(-128, 127))\n", - "\n", + "#print(scaled_embedding)\n", + "#print('####')\n", "scaled_byte_embedding = array_to_buffer(scaled_embedding, dtype='int8')" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -853,7 +862,7 @@ "b'\"\\x1e&-BSQ\\x0e+\\x1c6/\\n@(NH\\x17A3\\x1c8\\x1e/<.7\\x02\\x1248-9%8\\x12\\x13\\x1f3)\\x15$A;\\x00\\x16\\xf801\\x1d\\xf8@\\x150G\\\\@%&g\\'\\x1d\\x11TL\\x1a/\\x11\\x136?\\x1b.\\x08@\\x1c?.*1+9\\x19!D$(@\\x1d3\\x14:5)\\x1b*+MG)\\x03\\x19(\\x14T;#(Z\\x1aR\\x1c\\xf57$?(\\x0cA)#$\\x10\\x05:\\x13T\"/*\\x194\\x1c7+\\x1a5:S \\x0f.\")-\\x13-5/17\\x0c/#\\x15\\x1a\\x1e\\x7fD\\x1d<\\x12\\x18\\xf7(\"\\'I\\x11\\x17_\\x04.^\\n4*51\\x10\\x1a\\x1f.))+\\x14\\x07-&:R\\x15,)\\x10\\x0f\\x0c\\x15ET,5;,/%-]\"\\x1c\\x17)@\\x0b\\x03/\\x18\\x1d\\x0b\\x1e*DF6?[\\x159F?\\x1f*E-?\"\\x1d;!\\x80A\\x05*1!;\\x12,8\\x15!\\x1c23\\x1e3/(3/\\x123Z b\\x1f\\x15/-\\'\\x16U\\x0f%\\x194\\x1212\\r.+\\x15I),(,D#05+)A\\x10\\r\\x13\\x1c\\x15\\x17\\x0cB(/\\x0f502=\\xfc.B$A\\x1b2;*F893)9/>4+:<<5\\xf3\\x15=A\\n9\\x0b\\x1fN0$%\\x159+\\x16\\x1b>9\\x1c#A;*(&9B 8 O5$)B_%&\\x13$$\\x08#-0\\xff\\x0e$Q+0A \\x1b&B\\x101&+.A\"\\x0e\\x19,#5\"-\\xf6E\\x1d2\\x1f20+#;\\x0f@\\x17;N\\r\\x1a\\x00K9O-,#/\"9\\x19\\x04(5A9\\x0c\\x07F8\\x12*)#$9A]SH(\\x1bB&\\'\\x072,2\\x14\\x0b\\x121\\xfa\\x19:;C\"%)>\\x12#,:\\x1c5->DR? \\x19N\\'\\x19\\x033\\'\\xfc\\xf81$\\x12dH! /\\x1c\\x14,\\x1d+[B!;\\x16C2L5M B\\x12\\x1744%C\\x13S#[\\x0c3D5,6,$\\x16\\x18-2 9\\x19E\\x1a9Z#\\x044W9/\\'0(\\'&1\"\\'0\\x18W/&\\x13\\x12\\x1e\\xf8\\x02$\\x1e46\\x0b\\r5 \\x1fJL9FD\\x16(D1[-\\x0f\\xf3-W)\\x0c=#A;\\x1b4$\\x1c &h\\x0cV=\\x1a56!0\\x073\\x0b\\x11:82\\x1245$C\\x10N2\\x1c\\xe8M0.2(C,U#7L'" ] }, - "execution_count": 31, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -866,12 +875,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "from here we can use the same process as before to convert our existing embeddings to our new desired datatype appropriately scaled.\n" + "From here we can use the same process as before to convert our existing embeddings to our new desired datatype appropriately scaled.\n" ] }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -881,10 +890,10 @@ "\n", " while cursor != 0:\n", " items_to_convert = []\n", - " # Fetch a batch of records\n", + " # fetch a batch of records\n", " cursor, keys = index.client.scan(cursor=cursor, match=pattern)\n", "\n", - " # Use a Redis pipeline to make this more scalable\n", + " # use a Redis pipeline to make this more scalable\n", " with index.client.pipeline(transaction=False) as pipe:\n", " if index.storage_type.value == \"hash\":\n", " for key in keys:\n", @@ -897,12 +906,12 @@ "\n", " if items_to_convert:\n", "\n", - " print(f\"Converting vectors for {len(items_to_convert)} records\")\n", - " new_vecs = [np.frombuffer(item[b'vector'], dtype=old_dtype) for item in items_to_convert]\n", + " old_vecs = [np.frombuffer(item[b'vector'], dtype=old_dtype) for item in items_to_convert]\n", + "\n", " if scale_range:\n", - " new_vecs = minmax_scale(new_vecs, feature_range=scale_range)\n", - " new_vecs = new_vecs.astype(new_dtype).tobytes()\n", - " updated_data = [{**item, 'vector': new_vecs[i]} for i, item in enumerate(items_to_convert)]\n", + " new_vecs = minmax_scale(old_vecs, feature_range=scale_range)\n", + " new_vecs = [vec.astype(new_dtype).tobytes() for vec in new_vecs]\n", + " updated_data = [{**item, b'vector': new_vecs[i]} for i, item in enumerate(items_to_convert)]\n", "\n", " # write back data\n", " new_keys = index.load(updated_data, keys=keys)\n", @@ -912,46 +921,28 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 31, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Converting vectors for 1 records\n", - "Converting vectors for 1 records\n", - "Converting vectors for 1 records\n", - "Converting vectors for 1 records\n", - "Converting vectors for 1 records\n", - "Converting vectors for 1 records\n", - "Converting vectors for 1 records\n", - "Converting vectors for 1 records\n", - "Converting vectors for 1 records\n", - "Converting vectors for 1 records\n", - "Converting vectors for 1 records\n", - "Converting vectors for 2 records\n", - "Converting vectors for 1 records\n", - "Converting vectors for 1 records\n", - "Converting vectors for 1 records\n", - "Converting vectors for 1 records\n", - "Converting vectors for 1 records\n", - "Converting vectors for 1 records\n", - "Converting vectors for 1 records\n" - ] - } - ], + "outputs": [], "source": [ "pattern = \"movies:*\" # prefix of data to convert\n", "storage_type = \"hash\"\n", - "updated_keys = scale_and_replace_vectors(index, pattern, \"float32\", \"int8\", (-128, 127))" + "updated_keys = scale_and_replace_vectors(index, pattern, \"float16\", \"int8\", (-128, 127))" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "16:05:09 redisvl.index.index INFO Index already exists, overwriting.\n" + ] + } + ], "source": [ "# Update the schema by removing the old vector field\n", "index.schema.remove_field(\"vector\")\n", @@ -974,18 +965,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "20" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "index.info()[\"num_docs\"]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "952" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "client.memory_usage(updated_keys[0])" ] @@ -994,7 +1007,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# some important notes\n", + "# Some important notes\n", "When dealing with integer data types in search indices be aware of overflow and underflow. Depending on the math library you are using trying to converting 256 to an unsigned 8 bit integer may either throw an error, or wrap around and return -1. Numpy versions before 2.0 wrap around, while later versions will raise an `OverflowError`.\n", "\n", "When doing vector similarity search in Redis the choice of distance metric also matters, as Inner Product (IP) and Euclidian Distance (L2), will not return scaled values, but cosine (COSINE) will always be scaled regardless of the vector values, because that's how angles work." @@ -1002,9 +1015,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# cleanup\n", "client.flushall()" From fb3eba602fd96fafff2c80982e0c3baa3c1221f4 Mon Sep 17 00:00:00 2001 From: Justin Cechmanek Date: Fri, 7 Mar 2025 14:43:24 -0800 Subject: [PATCH 6/7] updates redis docker image --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 17af1c43..fca2aa1e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -87,7 +87,7 @@ jobs: services: redis: - image: redis/redis-stack-server:latest + image: redis:8.0-M03 ports: - 6379:6379 From 4fec368294059697b83254f04dce8614c450a4f8 Mon Sep 17 00:00:00 2001 From: Justin Cechmanek Date: Fri, 7 Mar 2025 14:57:56 -0800 Subject: [PATCH 7/7] fixes execution count error --- python-recipes/vector-search/03_dtype_support.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-recipes/vector-search/03_dtype_support.ipynb b/python-recipes/vector-search/03_dtype_support.ipynb index c566cb1a..b19403e8 100644 --- a/python-recipes/vector-search/03_dtype_support.ipynb +++ b/python-recipes/vector-search/03_dtype_support.ipynb @@ -329,7 +329,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [ {