WIP

redis-developer · May 23, 2024 · 7e66769 · 7e66769
1 parent 61dce96
commit 7e66769
Show file tree

Hide file tree

Showing 8 changed files with 62 additions and 70 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -5,7 +5,7 @@ on:
 
   push:
     branches:
-      - RAAE-67/resource-updates
+      - feature/auto-testing
       - main
 
 jobs:
@@ -46,12 +46,13 @@ jobs:
       if: matrix.connection == 'plain' && matrix.redis-stack-version == 'latest'
       env:
         OPENAI_API_KEY: ${{ secrets.OPENAI_KEY }}
-      #   GCP_LOCATION: ${{ secrets.GCP_LOCATION }}
-      #   GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+        LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
+        GCP_REGION: ${{ secrets.GCP_REGION }}
+        GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
       #   COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
       #   AZURE_OPENAI_API_KEY: ${{secrets.AZURE_OPENAI_API_KEY}}
       #   AZURE_OPENAI_ENDPOINT: ${{secrets.AZURE_OPENAI_ENDPOINT}}
       #   AZURE_OPENAI_DEPLOYMENT_NAME: ${{secrets.AZURE_OPENAI_DEPLOYMENT_NAME}}
       #   OPENAI_API_VERSION: ${{secrets.OPENAI_API_VERSION}}
       run: |
-        pytest --nbval python-examples/auto_test/01_redis-py.ipynb
+        pytest --nbval-lax python-recipes/
diff --git a/README.md b/README.md
@@ -85,7 +85,7 @@ Within [this repository](https://github.com/redis-developer/redis-nvidia-recsys)
 
 # Integrations/Tools
 - [⭐ RedisVL](https://github.com/redis/redis-vl-python) - a dedicated Python client lib for Redis as a Vector DB.
-- [⭐ AWS Bedrock][https://redis.io/docs/latest/integrate/amazon-bedrock/]
+- [⭐ AWS Bedrock](https://redis.io/docs/latest/integrate/amazon-bedrock/) - Streamlines GenAI deployment by offering foundational models as a unified API.
 - [⭐ LangChain Python](https://github.com/langchain-ai/langchain) - popular Python client lib for building LLM applications.
 powered by Redis.
 - [⭐ LangChain JS](https://github.com/langchain-ai/langchainjs) - popular JS client lib for building LLM applications.

diff --git a/python-recipes/RAG/00_intro_redispy.ipynb b/python-recipes/RAG/00_intro_redispy.ipynb
@@ -110,7 +110,7 @@
         "\n",
         "Later in this tutorial, Redis will be used to store, index, and query vector\n",
         "embeddings created from PDF document chunks. **We need to make sure we have a Redis\n",
-        "instance available.**"
+        "instance available."
       ]
     },
     {

diff --git a/python-recipes/RAG/02_langchain.ipynb b/python-recipes/RAG/02_langchain.ipynb
@@ -113,6 +113,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
+        "# NBVAL_SKIP\n",
         "%%sh\n",
         "curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg\n",
         "echo \"deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main\" | sudo tee /etc/apt/sources.list.d/redis.list\n",

diff --git a/python-recipes/RAG/03_llamaindex.ipynb b/python-recipes/RAG/03_llamaindex.ipynb
@@ -89,6 +89,7 @@
         }
       ],
       "source": [
+        "# NBVAL_SKIP\n",
         "%pip install -U -q llama-index llama-index-vector-stores-redis llama-index-embeddings-cohere llama-index-embeddings-openai"
       ]
     },

diff --git a/python-recipes/RAG/04_advanced_redisvl.ipynb b/python-recipes/RAG/04_advanced_redisvl.ipynb
@@ -73,14 +73,9 @@
         }
       ],
       "source": [
-        "# This clones the supporting git repository into a directory named 'temp_repo'.\n",
-        "!git clone https://github.com/redis-developer/financial-vss.git temp_repo\n",
-        "\n",
-        "# This command moves the 'resources' directory from 'temp_repo' to your current directory.\n",
-        "!mv temp_repo/resources .\n",
-        "!mv temp_repo/requirements.txt .\n",
-        "\n",
-        "# This deletes the 'temp_repo' directory, cleaning up the unwanted files.\n",
+        "# NBVAL_SKIP\n",
+        "!git clone https://github.com/redis-developer/redis-ai-resources.git temp_repo\n",
+        "!mv temp_repo/python-recipes/RAG/resources .\n",
         "!rm -rf temp_repo"
       ]
     },
@@ -101,20 +96,8 @@
       },
       "outputs": [],
       "source": [
-        "!pip install -r requirements.txt"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 2,
-      "metadata": {
-        "id": "oTeBg4AvjdYs"
-      },
-      "outputs": [],
-      "source": [
-        "import warnings\n",
-        "\n",
-        "warnings.filterwarnings(\"ignore\")"
+        "# NBVAL_SKIP\n",
+        "!pip install -q redis redisvl pandas unstructured[pdf] sentence-transformers langchain langchain-community openai tqdm"
       ]
     },
     {
@@ -132,7 +115,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "#### Localized Redis Stack for Colab Environment\n",
+        "#### For Colab\n",
         "Use the shell script below to download, extract, and install [Redis Stack](https://redis.io/docs/getting-started/install-stack/) directly\n",
         "from the Redis package archive."
       ]
@@ -152,6 +135,7 @@
         }
       ],
       "source": [
+        "# NBVAL_SKIP\n",
         "%%sh\n",
         "curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg\n",
         "echo \"deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main\" | sudo tee /etc/apt/sources.list.d/redis.list\n",
@@ -164,23 +148,14 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "#### Running Redis for Alternative Environments\n",
+        "#### For Alternative Environments\n",
         "There are many ways to get the necessary redis-stack instance running\n",
         "1. On cloud, deploy a [FREE instance of Redis in the cloud](https://redis.com/try-free/). Or, if you have your\n",
         "own version of Redis Enterprise running, that works too!\n",
         "2. Per OS, [see the docs](https://redis.io/docs/latest/operate/oss_and_stack/install/install-stack/)\n",
         "3. With docker: `docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest`"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "!docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -885,15 +860,6 @@
         "pd.DataFrame(columns=[\"question\", \"answer\"], data=list(zip(questions, results)))"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": 33,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "# TODO: note on difference"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1451,18 +1417,6 @@
       "source": [
         "await index.client.flushall()"
       ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "oU9qZsAn1ujD"
-      },
-      "source": [
-        "Now that you have tried the easy-to-use RedisVL client, try your hand with LangChain -- the highest level of abstraction for using and integrating Redis as a vector database.\n",
-        "\n",
-        "\n",
-        "<a href=\"https://colab.research.google.com/github/redis-developer/financial-vss/blob/main/langchain-03.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
-      ]
     }
   ],
   "metadata": {

diff --git a/python-recipes/semantic-cache/semantic_caching_gemini.ipynb b/python-recipes/semantic-cache/semantic_caching_gemini.ipynb
@@ -53,11 +53,18 @@
       },
       "outputs": [],
       "source": [
+        "# NBVAL_SKIP\n",
         "!pip install redisvl>=0.2.0 unstructured[pdf]\n",
         "!pip install llama-parse llama-index-readers-file\n",
-        "!pip install langchain langchain-google-vertexai\n",
-        "\n",
-        "\n",
+        "!pip install langchain langchain-google-vertexai"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
         "import IPython\n",
         "\n",
         "app = IPython.Application.instance()\n",
@@ -93,11 +100,24 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 10,
+      "execution_count": 1,
       "metadata": {
         "id": "-fR3mWQvdKxb"
       },
-      "outputs": [],
+      "outputs": [
+        {
+          "ename": "KeyError",
+          "evalue": "'GCP_PROJECT_ID'",
+          "output_type": "error",
+          "traceback": [
+            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
+            "Cell \u001b[0;32mIn[1], line 11\u001b[0m\n\u001b[1;32m      8\u001b[0m     os\u001b[38;5;241m.\u001b[39menviron[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLLAMA_CLOUD_API_KEY\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m getpass(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLLAMA_CLOUD_API_KEY:\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     10\u001b[0m \u001b[38;5;66;03m# input your GCP project ID and region for Vertex AI\u001b[39;00m\n\u001b[0;32m---> 11\u001b[0m PROJECT_ID \u001b[38;5;241m=\u001b[39m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menviron\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mGCP_PROJECT_ID\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m \u001b[38;5;129;01mor\u001b[39;00m getpass(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGCP_PROJECT_ID:\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;66;03m#'central-beach-194106'\u001b[39;00m\n\u001b[1;32m     12\u001b[0m REGION \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39menviron[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGCP_REGION\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28minput\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGCP_REGION:\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;66;03m#'us-central1'\u001b[39;00m\n\u001b[1;32m     14\u001b[0m \u001b[38;5;66;03m# need this for running llama-index code in Jupyter Notebooks\u001b[39;00m\n",
+            "File \u001b[0;32m~/.pyenv/versions/3.9.12/lib/python3.9/os.py:679\u001b[0m, in \u001b[0;36m_Environ.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    676\u001b[0m     value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_data[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mencodekey(key)]\n\u001b[1;32m    677\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n\u001b[1;32m    678\u001b[0m     \u001b[38;5;66;03m# raise KeyError with the original key value\u001b[39;00m\n\u001b[0;32m--> 679\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(key) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    680\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdecodevalue(value)\n",
+            "\u001b[0;31mKeyError\u001b[0m: 'GCP_PROJECT_ID'"
+          ]
+        }
+      ],
       "source": [
         "import os\n",
         "import nest_asyncio\n",
@@ -109,8 +129,12 @@
         "    os.environ[\"LLAMA_CLOUD_API_KEY\"] = getpass(\"LLAMA_CLOUD_API_KEY:\")\n",
         "\n",
         "# input your GCP project ID and region for Vertex AI\n",
-        "PROJECT_ID = getpass(\"PROJECT_ID:\") #'central-beach-194106'\n",
-        "REGION = input(\"REGION:\") #'us-central1'\n",
+        "if \"GCP_PROJECT_ID\" not in os.environ:\n",
+        "    PROJECT_ID = getpass(\"GCP_PROJECT_ID:\") #'central-beach-194106'\n",
+        "    REGION = input(\"GCP_REGION:\") #'us-central1'\n",
+        "else:\n",
+        "    PROJECT_ID = os.environ[\"GCP_PROJECT_ID\"]\n",
+        "    REGION = os.environ[\"GCP_REGION\"]\n",
         "\n",
         "# need this for running llama-index code in Jupyter Notebooks\n",
         "nest_asyncio.apply()"
@@ -152,7 +176,6 @@
       "source": [
         "import vertexai\n",
         "\n",
-        "# TODO handle env var\n",
         "vertexai.init(project=PROJECT_ID, location=REGION)"
       ]
     },
@@ -185,7 +208,6 @@
         "from redisvl.utils.vectorize import VertexAITextVectorizer\n",
         "\n",
         "# Create vectorizer instance\n",
-        "# TODO - handle env var\n",
         "vectorizer = VertexAITextVectorizer(\n",
         "    model = \"textembedding-gecko@003\",\n",
         "    api_config = {\"project_id\": PROJECT_ID, \"location\": REGION}\n",
@@ -268,6 +290,7 @@
         }
       ],
       "source": [
+        "# NBVAL_SKIP\n",
         "%%sh\n",
         "curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg\n",
         "echo \"deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main\" | sudo tee /etc/apt/sources.list.d/redis.list\n",
@@ -1813,7 +1836,16 @@
       "name": "python3"
     },
     "language_info": {
-      "name": "python"
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.9.12"
     }
   },
   "nbformat": 4,

diff --git a/test_requirements.txt b/test_requirements.txt
@@ -5,6 +5,9 @@ tiktoken
 langchain
 langchain-community
 llama-index
+llama-index-vector-stores-redis
+llama-index-embeddings-cohere
+llama-index-embeddings-openai
 unstructured[pdf]
 sentence-transformers
 pandas