From 7e66769f4a3d6f43b46ae103b383db5b5cd9de29 Mon Sep 17 00:00:00 2001 From: Robert Shelton Date: Thu, 23 May 2024 08:36:25 -0400 Subject: [PATCH] WIP --- .github/workflows/test.yml | 9 +-- README.md | 2 +- python-recipes/RAG/00_intro_redispy.ipynb | 2 +- python-recipes/RAG/02_langchain.ipynb | 1 + python-recipes/RAG/03_llamaindex.ipynb | 1 + python-recipes/RAG/04_advanced_redisvl.ipynb | 62 +++---------------- .../semantic_caching_gemini.ipynb | 52 +++++++++++++--- test_requirements.txt | 3 + 8 files changed, 62 insertions(+), 70 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2901d22..8d2ff22 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,7 +5,7 @@ on: push: branches: - - RAAE-67/resource-updates + - feature/auto-testing - main jobs: @@ -46,12 +46,13 @@ jobs: if: matrix.connection == 'plain' && matrix.redis-stack-version == 'latest' env: OPENAI_API_KEY: ${{ secrets.OPENAI_KEY }} - # GCP_LOCATION: ${{ secrets.GCP_LOCATION }} - # GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }} + GCP_REGION: ${{ secrets.GCP_REGION }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} # COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} # AZURE_OPENAI_API_KEY: ${{secrets.AZURE_OPENAI_API_KEY}} # AZURE_OPENAI_ENDPOINT: ${{secrets.AZURE_OPENAI_ENDPOINT}} # AZURE_OPENAI_DEPLOYMENT_NAME: ${{secrets.AZURE_OPENAI_DEPLOYMENT_NAME}} # OPENAI_API_VERSION: ${{secrets.OPENAI_API_VERSION}} run: | - pytest --nbval python-examples/auto_test/01_redis-py.ipynb \ No newline at end of file + pytest --nbval-lax python-recipes/ \ No newline at end of file diff --git a/README.md b/README.md index 889f805..647cf8d 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Within [this repository](https://github.com/redis-developer/redis-nvidia-recsys) # Integrations/Tools - [⭐ RedisVL](https://github.com/redis/redis-vl-python) - a dedicated Python client lib for Redis as a Vector DB. -- [⭐ AWS Bedrock][https://redis.io/docs/latest/integrate/amazon-bedrock/] +- [⭐ AWS Bedrock](https://redis.io/docs/latest/integrate/amazon-bedrock/) - Streamlines GenAI deployment by offering foundational models as a unified API. - [⭐ LangChain Python](https://github.com/langchain-ai/langchain) - popular Python client lib for building LLM applications. powered by Redis. - [⭐ LangChain JS](https://github.com/langchain-ai/langchainjs) - popular JS client lib for building LLM applications. diff --git a/python-recipes/RAG/00_intro_redispy.ipynb b/python-recipes/RAG/00_intro_redispy.ipynb index 6c80a3e..21774ce 100644 --- a/python-recipes/RAG/00_intro_redispy.ipynb +++ b/python-recipes/RAG/00_intro_redispy.ipynb @@ -110,7 +110,7 @@ "\n", "Later in this tutorial, Redis will be used to store, index, and query vector\n", "embeddings created from PDF document chunks. **We need to make sure we have a Redis\n", - "instance available.**" + "instance available." ] }, { diff --git a/python-recipes/RAG/02_langchain.ipynb b/python-recipes/RAG/02_langchain.ipynb index fe0e860..a5d178f 100644 --- a/python-recipes/RAG/02_langchain.ipynb +++ b/python-recipes/RAG/02_langchain.ipynb @@ -113,6 +113,7 @@ "metadata": {}, "outputs": [], "source": [ + "# NBVAL_SKIP\n", "%%sh\n", "curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg\n", "echo \"deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main\" | sudo tee /etc/apt/sources.list.d/redis.list\n", diff --git a/python-recipes/RAG/03_llamaindex.ipynb b/python-recipes/RAG/03_llamaindex.ipynb index 5b5d4f9..5cb103f 100644 --- a/python-recipes/RAG/03_llamaindex.ipynb +++ b/python-recipes/RAG/03_llamaindex.ipynb @@ -89,6 +89,7 @@ } ], "source": [ + "# NBVAL_SKIP\n", "%pip install -U -q llama-index llama-index-vector-stores-redis llama-index-embeddings-cohere llama-index-embeddings-openai" ] }, diff --git a/python-recipes/RAG/04_advanced_redisvl.ipynb b/python-recipes/RAG/04_advanced_redisvl.ipynb index 0739eea..ffec3b1 100644 --- a/python-recipes/RAG/04_advanced_redisvl.ipynb +++ b/python-recipes/RAG/04_advanced_redisvl.ipynb @@ -73,14 +73,9 @@ } ], "source": [ - "# This clones the supporting git repository into a directory named 'temp_repo'.\n", - "!git clone https://github.com/redis-developer/financial-vss.git temp_repo\n", - "\n", - "# This command moves the 'resources' directory from 'temp_repo' to your current directory.\n", - "!mv temp_repo/resources .\n", - "!mv temp_repo/requirements.txt .\n", - "\n", - "# This deletes the 'temp_repo' directory, cleaning up the unwanted files.\n", + "# NBVAL_SKIP\n", + "!git clone https://github.com/redis-developer/redis-ai-resources.git temp_repo\n", + "!mv temp_repo/python-recipes/RAG/resources .\n", "!rm -rf temp_repo" ] }, @@ -101,20 +96,8 @@ }, "outputs": [], "source": [ - "!pip install -r requirements.txt" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "oTeBg4AvjdYs" - }, - "outputs": [], - "source": [ - "import warnings\n", - "\n", - "warnings.filterwarnings(\"ignore\")" + "# NBVAL_SKIP\n", + "!pip install -q redis redisvl pandas unstructured[pdf] sentence-transformers langchain langchain-community openai tqdm" ] }, { @@ -132,7 +115,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "#### Localized Redis Stack for Colab Environment\n", + "#### For Colab\n", "Use the shell script below to download, extract, and install [Redis Stack](https://redis.io/docs/getting-started/install-stack/) directly\n", "from the Redis package archive." ] @@ -152,6 +135,7 @@ } ], "source": [ + "# NBVAL_SKIP\n", "%%sh\n", "curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg\n", "echo \"deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main\" | sudo tee /etc/apt/sources.list.d/redis.list\n", @@ -164,7 +148,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "#### Running Redis for Alternative Environments\n", + "#### For Alternative Environments\n", "There are many ways to get the necessary redis-stack instance running\n", "1. On cloud, deploy a [FREE instance of Redis in the cloud](https://redis.com/try-free/). Or, if you have your\n", "own version of Redis Enterprise running, that works too!\n", @@ -172,15 +156,6 @@ "3. With docker: `docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest`" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -885,15 +860,6 @@ "pd.DataFrame(columns=[\"question\", \"answer\"], data=list(zip(questions, results)))" ] }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [], - "source": [ - "# TODO: note on difference" - ] - }, { "cell_type": "markdown", "metadata": { @@ -1451,18 +1417,6 @@ "source": [ "await index.client.flushall()" ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "oU9qZsAn1ujD" - }, - "source": [ - "Now that you have tried the easy-to-use RedisVL client, try your hand with LangChain -- the highest level of abstraction for using and integrating Redis as a vector database.\n", - "\n", - "\n", - "\"Open" - ] } ], "metadata": { diff --git a/python-recipes/semantic-cache/semantic_caching_gemini.ipynb b/python-recipes/semantic-cache/semantic_caching_gemini.ipynb index fe45ce6..cda1358 100644 --- a/python-recipes/semantic-cache/semantic_caching_gemini.ipynb +++ b/python-recipes/semantic-cache/semantic_caching_gemini.ipynb @@ -53,11 +53,18 @@ }, "outputs": [], "source": [ + "# NBVAL_SKIP\n", "!pip install redisvl>=0.2.0 unstructured[pdf]\n", "!pip install llama-parse llama-index-readers-file\n", - "!pip install langchain langchain-google-vertexai\n", - "\n", - "\n", + "!pip install langchain langchain-google-vertexai" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "import IPython\n", "\n", "app = IPython.Application.instance()\n", @@ -93,11 +100,24 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 1, "metadata": { "id": "-fR3mWQvdKxb" }, - "outputs": [], + "outputs": [ + { + "ename": "KeyError", + "evalue": "'GCP_PROJECT_ID'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[1], line 11\u001b[0m\n\u001b[1;32m 8\u001b[0m os\u001b[38;5;241m.\u001b[39menviron[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLLAMA_CLOUD_API_KEY\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m getpass(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLLAMA_CLOUD_API_KEY:\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 10\u001b[0m \u001b[38;5;66;03m# input your GCP project ID and region for Vertex AI\u001b[39;00m\n\u001b[0;32m---> 11\u001b[0m PROJECT_ID \u001b[38;5;241m=\u001b[39m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menviron\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mGCP_PROJECT_ID\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m \u001b[38;5;129;01mor\u001b[39;00m getpass(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGCP_PROJECT_ID:\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;66;03m#'central-beach-194106'\u001b[39;00m\n\u001b[1;32m 12\u001b[0m REGION \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39menviron[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGCP_REGION\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28minput\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGCP_REGION:\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;66;03m#'us-central1'\u001b[39;00m\n\u001b[1;32m 14\u001b[0m \u001b[38;5;66;03m# need this for running llama-index code in Jupyter Notebooks\u001b[39;00m\n", + "File \u001b[0;32m~/.pyenv/versions/3.9.12/lib/python3.9/os.py:679\u001b[0m, in \u001b[0;36m_Environ.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 676\u001b[0m value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_data[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mencodekey(key)]\n\u001b[1;32m 677\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n\u001b[1;32m 678\u001b[0m \u001b[38;5;66;03m# raise KeyError with the original key value\u001b[39;00m\n\u001b[0;32m--> 679\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(key) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 680\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdecodevalue(value)\n", + "\u001b[0;31mKeyError\u001b[0m: 'GCP_PROJECT_ID'" + ] + } + ], "source": [ "import os\n", "import nest_asyncio\n", @@ -109,8 +129,12 @@ " os.environ[\"LLAMA_CLOUD_API_KEY\"] = getpass(\"LLAMA_CLOUD_API_KEY:\")\n", "\n", "# input your GCP project ID and region for Vertex AI\n", - "PROJECT_ID = getpass(\"PROJECT_ID:\") #'central-beach-194106'\n", - "REGION = input(\"REGION:\") #'us-central1'\n", + "if \"GCP_PROJECT_ID\" not in os.environ:\n", + " PROJECT_ID = getpass(\"GCP_PROJECT_ID:\") #'central-beach-194106'\n", + " REGION = input(\"GCP_REGION:\") #'us-central1'\n", + "else:\n", + " PROJECT_ID = os.environ[\"GCP_PROJECT_ID\"]\n", + " REGION = os.environ[\"GCP_REGION\"]\n", "\n", "# need this for running llama-index code in Jupyter Notebooks\n", "nest_asyncio.apply()" @@ -152,7 +176,6 @@ "source": [ "import vertexai\n", "\n", - "# TODO handle env var\n", "vertexai.init(project=PROJECT_ID, location=REGION)" ] }, @@ -185,7 +208,6 @@ "from redisvl.utils.vectorize import VertexAITextVectorizer\n", "\n", "# Create vectorizer instance\n", - "# TODO - handle env var\n", "vectorizer = VertexAITextVectorizer(\n", " model = \"textembedding-gecko@003\",\n", " api_config = {\"project_id\": PROJECT_ID, \"location\": REGION}\n", @@ -268,6 +290,7 @@ } ], "source": [ + "# NBVAL_SKIP\n", "%%sh\n", "curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg\n", "echo \"deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main\" | sudo tee /etc/apt/sources.list.d/redis.list\n", @@ -1813,7 +1836,16 @@ "name": "python3" }, "language_info": { - "name": "python" + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" } }, "nbformat": 4, diff --git a/test_requirements.txt b/test_requirements.txt index d376b28..0f4fcc7 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -5,6 +5,9 @@ tiktoken langchain langchain-community llama-index +llama-index-vector-stores-redis +llama-index-embeddings-cohere +llama-index-embeddings-openai unstructured[pdf] sentence-transformers pandas