From d1e696860724300c20bae56a940c3afe7043757b Mon Sep 17 00:00:00 2001 From: Craig Chi Date: Fri, 16 Feb 2024 09:41:02 -0800 Subject: [PATCH] feat: add notebooks for integrations --- docs/chat_message_history.ipynb | 257 +++++++++++++------ docs/document_loader.ipynb | 432 +++++++++++++++++++------------- docs/vector_store.ipynb | 2 +- 3 files changed, 444 insertions(+), 247 deletions(-) diff --git a/docs/chat_message_history.ipynb b/docs/chat_message_history.ipynb index 8b1a4cf..8d01fc2 100644 --- a/docs/chat_message_history.ipynb +++ b/docs/chat_message_history.ipynb @@ -1,79 +1,184 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Google DATABASE\n", - "\n", - "[Google DATABASE](https://cloud.google.com/DATABASE).\n", - "\n", - "Save chat messages into `DATABASE`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Pre-reqs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "%pip install PACKAGE_NAME" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "from PACKAGE import LOADER" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Basic Usage" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.6" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "6-0_o3DxsFGi" + }, + "source": [ + "Google Database\n", + "\n", + "Use [Google Memorystore for Redis](https://cloud.google.com/memorystore/docs/redis/memorystore-for-redis-overview) to store chat message history for LangChain." + ] }, - "nbformat": 4, - "nbformat_minor": 4 + { + "cell_type": "markdown", + "metadata": { + "id": "dWakBoPnsFGj" + }, + "source": [ + "## Pre-reqs" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EudfLv_UsFGk" + }, + "source": [ + "### Setting Up a Memorystore for Redis Instance\n", + "\n", + "Before proceeding, an active Memorystore for Redis instance is needed to store chat message history:\n", + "\n", + "* Create a Memorystore for Redis Instance (>= 5.0): If an instance doesn't exist, follow the instructions at https://cloud.google.com/memorystore/docs/redis/create-instance-console to create a new one. Ensure the version is greater than or equal to 5.0.\n", + "* Obtain Endpoint: Note the endpoint associated with the instance." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "J5nxjYxHsFGk" + }, + "source": [ + "### Installing the LangChain Memorystore for Redis Module\n", + "\n", + "Interaction with the Memorystore for Redis instance from LangChain requires installing the necessary module:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [], + "id": "iLwVMVkYsFGk" + }, + "outputs": [], + "source": [ + "# Install Memorystore for Redis for LangChain module\n", + "%pip install langchain_google_memorystore_redis" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2L7kMu__sFGl" + }, + "source": [ + "## Basic Usage" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "A2fT1iEhsFGl" + }, + "source": [ + "### Initialize a MemorystoreChatMessageHistory\n", + "\n", + "Each chat message history object must have a unique session ID. If the session ID already has messages stored in Redis, they will can be retrieved." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YEDKWR6asFGl" + }, + "outputs": [], + "source": [ + "import redis\n", + "from langchain_google_memorystore_redis import MemorystoreChatMessageHistory\n", + "\n", + "# Connect to a Memorystore for Redis instance\n", + "redis_client = redis.from_url(\"redis://127.0.0.1:6379\")\n", + "\n", + "message_history = MemorystoreChatMessageHistory(redis_client, session_id='session1')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EmoJcTgosFGl" + }, + "source": [ + "### Add Messages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gB1PGe6wsFGm" + }, + "outputs": [], + "source": [ + "message_history.add_ai_message('Hey! I am AI!')\n", + "message_history.add_user_message('Hey! I am human!')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "02xxvmzTsFGm" + }, + "source": [ + "### Retrieve All Messages Stored in the Session" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BvS3UFsysFGm" + }, + "outputs": [], + "source": [ + "message_history.messages" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sFJdt3ubsFGo" + }, + "source": [ + "### Clear Messages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "H5I7K3MTsFGo" + }, + "outputs": [], + "source": [ + "message_history.clear()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + }, + "colab": { + "provenance": [] + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/docs/document_loader.ipynb b/docs/document_loader.ipynb index 5f48d1d..99343af 100644 --- a/docs/document_loader.ipynb +++ b/docs/document_loader.ipynb @@ -1,172 +1,264 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Google DATABASE\n", - "\n", - "[Google DATABASE](https://cloud.google.com/DATABASE).\n", - "\n", - "Load documents from `DATABASE`." - ] + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "6-0_o3DxsFGi" + }, + "source": [ + "Google Database\n", + "\n", + "Use [Google Memorystore for Redis](https://cloud.google.com/memorystore/docs/redis/memorystore-for-redis-overview) to store Documents for LangChain." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dWakBoPnsFGj" + }, + "source": [ + "## Pre-reqs" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EudfLv_UsFGk" + }, + "source": [ + "### Setting Up a Memorystore for Redis Instance\n", + "\n", + "Before proceeding, an active Memorystore for Redis instance is needed to store chat message history:\n", + "\n", + "* Create a Memorystore for Redis Instance (>= 5.0): If an instance doesn't exist, follow the instructions at https://cloud.google.com/memorystore/docs/redis/create-instance-console to create a new one. Ensure the version is greater than or equal to 5.0.\n", + "* Obtain Endpoint: Note the endpoint associated with the instance." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "J5nxjYxHsFGk" + }, + "source": [ + "### Installing the LangChain Memorystore for Redis Module\n", + "\n", + "Interaction with the Memorystore for Redis instance from LangChain requires installing the necessary module:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [], + "id": "iLwVMVkYsFGk" + }, + "outputs": [], + "source": [ + "# Install Memorystore for Redis for LangChain module\n", + "%pip install langchain_google_memorystore_redis" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2L7kMu__sFGl" + }, + "source": [ + "## Basic Usage" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "A2fT1iEhsFGl" + }, + "source": [ + "### Initialize a MemorystoreDocumentLoader\n", + "\n", + "Initialize a loader that loads all documents stored in the Memorystore for Redis instance with a specific prefix." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YEDKWR6asFGl" + }, + "outputs": [], + "source": [ + "import redis\n", + "from langchain_google_memorystore_redis import MemorystoreDocumentLoader\n", + "\n", + "# Connect to a Memorystore for Redis instance\n", + "redis_client = redis.from_url(\"redis://127.0.0.1:6379\")\n", + "prefix = \"doc:\"\n", + "\n", + "loader = MemorystoreDocumentLoader(\n", + " client=redis_client,\n", + " key_prefix=prefix,\n", + " content_fields=set([\"page_content\"]),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EmoJcTgosFGl" + }, + "source": [ + "### Load Documents\n", + "\n", + "Load all documents stored in the Memorystore for Redis instance at once." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gB1PGe6wsFGm" + }, + "outputs": [], + "source": [ + "documents = loader.load()" + ] + }, + { + "cell_type": "markdown", + "source": [ + "### Lazy Load Documents\n", + "\n", + "Load the document one-by-one with lazy_load generator." + ], + "metadata": { + "id": "Vbs8gIa24YvJ" + } + }, + { + "cell_type": "code", + "source": [ + "for document in loader.lazy_load():\n", + " # Do something\n", + " print(document)" + ], + "metadata": { + "id": "nPhpvLtA4kBM" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "02xxvmzTsFGm" + }, + "source": [ + "## Customize Document Page Content & Metadata\n", + "\n", + "When initializing a loader with more than 1 content field, the `page_content` of the loaded Documents will contain a JSON-encoded string with top level fields equal to the specified fields in `content_fields`.\n", + "\n", + "If the `metadata_fields` are specified, the `metadata` field of the loaded Documents will only have the top level fields equal to the specified `metadata_fields`. If any of the values of the metadata fields is stored as a JSON-encoded string, it will be decoded prior to being loaded to the metadata fields." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BvS3UFsysFGm" + }, + "outputs": [], + "source": [ + "loader = MemorystoreDocumentLoader(\n", + " client=redis_client,\n", + " key_prefix=prefix,\n", + " content_fields=set([\"content_field_1\", \"content_field_2\"]),\n", + " metadata_fields=set([\"title\", \"author\"]),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sFJdt3ubsFGo" + }, + "source": [ + "## Save Documents\n", + "\n", + "You can save a list of Documents into Memorystore for Redis instance like below. The Documents will be stored into randomly generated keys with the specified prefix of `key_prefix`. Alternatively, you can designate the suffixes of the keys by specifying `ids` in the `add_documents` method." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "H5I7K3MTsFGo" + }, + "outputs": [], + "source": [ + "from langchain_google_memorystore_redis import MemorystoreDocumentSaver\n", + "\n", + "# Connect to a Memorystore for Redis instance\n", + "redis_client = redis.from_url(\"redis://127.0.0.1:6379\")\n", + "prefix = \"doc:\"\n", + "\n", + "saver = MemorystoreDocumentSaver(\n", + " client=redis_client,\n", + " key_prefix=prefix,\n", + " content_field=\"page_content\",\n", + ")\n", + "saver.add_documents(documents)" + ] + }, + { + "cell_type": "markdown", + "source": [ + "### Clean up Saved Documents\n", + "\n", + "Delete all of keys with the specified prefix in the Memorystore for Redis instance. You can also specify the suffixes of the keys if you know.\n", + "\n" + ], + "metadata": { + "id": "FLVI7Kp7mhL-" + } + }, + { + "cell_type": "code", + "source": [ + "saver.delete()" + ], + "metadata": { + "id": "1ArfDYUGmrP3" + }, + "execution_count": null, + "outputs": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + }, + "colab": { + "provenance": [ + { + "file_id": "1kuFhDfyzOdzS1apxQ--1efXB1pJ79yVY", + "timestamp": 1708033015250 + } + ] + } }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Pre-reqs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "%pip install PACKAGE_NAME" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "from PACKAGE import LOADER" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Basic Usage" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Load from table" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "loader = LOADER()\n", - "\n", - "data = loader.load()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Load from query" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "loader = LOADER()\n", - "\n", - "data = loader.load()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Customize Document Page Content & Metadata" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "loader = LOADER()\n", - "\n", - "data = loader.load()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Customize Page Content Format" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Save Documents to table" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "saver = SAVER()\n", - "saver.add_documents(docs)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Customize Connection & Authentication" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from google.cloud.DATABASE import Client\n", - "\n", - "creds = \"\"\n", - "client = Client(creds=creds)\n", - "loader = LOADER(\n", - " client=client,\n", - ")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.6" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} \ No newline at end of file + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/docs/vector_store.ipynb b/docs/vector_store.ipynb index 564e338..df6a9b6 100644 --- a/docs/vector_store.ipynb +++ b/docs/vector_store.ipynb @@ -26,7 +26,7 @@ "\n", "Before proceeding, an active Memorystore for Redis instance is needed to store vectors:\n", "\n", - "* Create a Memorystore for Reids Instance (v7.2): If an instance doesn't exist, follow the instructions at https://cloud.google.com/memorystore/docs/redis/create-instance-console to create a new one. Ensure version 7.2 is selected.\n", + "* Create a Memorystore for Redis Instance (v7.2): If an instance doesn't exist, follow the instructions at https://cloud.google.com/memorystore/docs/redis/create-instance-console to create a new one. Ensure version 7.2 is selected.\n", "* Obtain Endpoint: Note the endpoint associated with the instance." ] },