From c0eadb787e538947c0dbcfdc425e9653f826b458 Mon Sep 17 00:00:00 2001 From: vmanghnani <50720827+vmanghnani@users.noreply.github.com> Date: Sat, 28 Sep 2024 18:15:14 -0700 Subject: [PATCH 01/11] Update vector_store.ipynb 1. Added Installation of additional dependencies "langchain-google-vertexai" it is needed for text embedding generation. 2. Set "GOOGLE_CLOUD_PROJECT" env variable for PROJECT_ID, it is needed for table creation / SQL execution in vector_store module. 3. Specified "id_column" value as "row_id", to match it with table creation in "init_vector_store_table" function, if "id_column" is not set then default value is "langchain_id" --- docs/vector_store.ipynb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/vector_store.ipynb b/docs/vector_store.ipynb index 05ad8acb..e3ceca73 100644 --- a/docs/vector_store.ipynb +++ b/docs/vector_store.ipynb @@ -52,7 +52,9 @@ } ], "source": [ - "%pip install --upgrade --quiet langchain-google-spanner" + "%pip install --upgrade --quiet langchain-google-spanner\n", + "# Install additional dependencies\n", + "%pip install langchain langchain-google-vertexai" ] }, { @@ -124,7 +126,8 @@ "PROJECT_ID = \"my-project-id\" # @param {type:\"string\"}\n", "\n", "# Set the project id\n", - "!gcloud config set project {PROJECT_ID}" + "!gcloud config set project {PROJECT_ID}\n", + "%env GOOGLE_CLOUD_PROJECT={PROJECT_ID}" ] }, { @@ -249,6 +252,7 @@ "1. `instance_id` - The name of the Spanner instance\n", "1. `database_id` - The name of the Spanner database\n", "1. `table_name` - The name of the table within the database to store the documents & their embeddings.\n", + "1. `row_id` - The name of the row identifier column of the table\n", "1. `embedding_service` - The Embeddings implementation which is used to generate the embeddings." ] }, @@ -263,6 +267,7 @@ " database_id=DATABASE,\n", " table_name=TABLE_NAME,\n", " ignore_metadata_columns=[],\n", + " id_column=\"row_id\",\n", " embedding_service=embeddings,\n", " metadata_json_column=\"metadata\",\n", ")" From 7e419591c5044cd1c9124fda834ab7f64392576b Mon Sep 17 00:00:00 2001 From: Vikram Manghnani <50720827+vmanghnani@users.noreply.github.com> Date: Tue, 1 Oct 2024 15:47:04 -0700 Subject: [PATCH 02/11] Update docs/vector_store.ipynb Co-authored-by: Averi Kitsch --- docs/vector_store.ipynb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/vector_store.ipynb b/docs/vector_store.ipynb index e3ceca73..93f490c8 100644 --- a/docs/vector_store.ipynb +++ b/docs/vector_store.ipynb @@ -52,9 +52,7 @@ } ], "source": [ - "%pip install --upgrade --quiet langchain-google-spanner\n", - "# Install additional dependencies\n", - "%pip install langchain langchain-google-vertexai" + "%pip install --upgrade --quiet langchain-google-spanner langchain-google-vertexai", ] }, { From c51079943b8f67d28a6a46976c6980251d340310 Mon Sep 17 00:00:00 2001 From: Vikram Manghnani <50720827+vmanghnani@users.noreply.github.com> Date: Tue, 1 Oct 2024 16:29:51 -0700 Subject: [PATCH 03/11] Update vector_store.ipynb --- docs/vector_store.ipynb | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/vector_store.ipynb b/docs/vector_store.ipynb index 93f490c8..a333d379 100644 --- a/docs/vector_store.ipynb +++ b/docs/vector_store.ipynb @@ -52,7 +52,7 @@ } ], "source": [ - "%pip install --upgrade --quiet langchain-google-spanner langchain-google-vertexai", + "%pip install --upgrade --quiet langchain-google-spanner langchain-google-vertexai" ] }, { @@ -195,14 +195,15 @@ " instance_id=INSTANCE,\n", " database_id=DATABASE,\n", " table_name=TABLE_NAME,\n", - " id_column=\"row_id\",\n", - " metadata_columns=[\n", - " TableColumn(name=\"metadata\", type=\"JSON\", is_null=True),\n", - " TableColumn(name=\"title\", type=\"STRING(MAX)\", is_null=False),\n", - " ],\n", - " secondary_indexes=[\n", - " SecondaryIndex(index_name=\"row_id_and_title\", columns=[\"row_id\", \"title\"])\n", - " ],\n", + " # Customize the table creation\n", + " #id_column=\"row_id\",\n", + " #metadata_columns=[\n", + " # TableColumn(name=\"metadata\", type=\"JSON\", is_null=True),\n", + " # TableColumn(name=\"title\", type=\"STRING(MAX)\", is_null=False),\n", + " #],\n", + " #secondary_indexes=[\n", + " # SecondaryIndex(index_name=\"row_id_and_title\", columns=[\"row_id\", \"title\"])\n", + " #],\n", ")" ] }, @@ -264,9 +265,11 @@ " instance_id=INSTANCE,\n", " database_id=DATABASE,\n", " table_name=TABLE_NAME,\n", - " ignore_metadata_columns=[],\n", - " id_column=\"row_id\",\n", " embedding_service=embeddings,\n", + " # Connect to a custom vector store table\n", + " # id_column=\"row_id\",\n", + " # content_column=\"content_column\",\n", + " # metadata_columns=[\"metadata\", \"title\"],\n", " metadata_json_column=\"metadata\",\n", ")" ] From 5e8989e914832ddffb9e59cdf89e89076b176541 Mon Sep 17 00:00:00 2001 From: Vikram Manghnani <50720827+vmanghnani@users.noreply.github.com> Date: Tue, 1 Oct 2024 16:31:32 -0700 Subject: [PATCH 04/11] Update vector_store.ipynb --- docs/vector_store.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/vector_store.ipynb b/docs/vector_store.ipynb index a333d379..993ee248 100644 --- a/docs/vector_store.ipynb +++ b/docs/vector_store.ipynb @@ -270,7 +270,7 @@ " # id_column=\"row_id\",\n", " # content_column=\"content_column\",\n", " # metadata_columns=[\"metadata\", \"title\"],\n", - " metadata_json_column=\"metadata\",\n", + " #metadata_json_column=\"metadata\",\n", ")" ] }, From aa632fbc9ac7174c6fd9fc8cd109d87e69f17f19 Mon Sep 17 00:00:00 2001 From: Vikram Manghnani <50720827+vmanghnani@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:06:24 -0700 Subject: [PATCH 05/11] Update vector_store.ipynb --- docs/vector_store.ipynb | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/vector_store.ipynb b/docs/vector_store.ipynb index 993ee248..e41a30e5 100644 --- a/docs/vector_store.ipynb +++ b/docs/vector_store.ipynb @@ -196,14 +196,14 @@ " database_id=DATABASE,\n", " table_name=TABLE_NAME,\n", " # Customize the table creation\n", - " #id_column=\"row_id\",\n", - " #metadata_columns=[\n", - " # TableColumn(name=\"metadata\", type=\"JSON\", is_null=True),\n", - " # TableColumn(name=\"title\", type=\"STRING(MAX)\", is_null=False),\n", - " #],\n", - " #secondary_indexes=[\n", - " # SecondaryIndex(index_name=\"row_id_and_title\", columns=[\"row_id\", \"title\"])\n", - " #],\n", + " id_column=\"row_id\",\n", + " metadata_columns=[\n", + " TableColumn(name=\"metadata\", type=\"JSON\", is_null=True),\n", + " TableColumn(name=\"title\", type=\"STRING(MAX)\", is_null=False),\n", + " ],\n", + " secondary_indexes=[\n", + " SecondaryIndex(index_name=\"row_id_and_title\", columns=[\"row_id\", \"title\"])\n", + " ],\n", ")" ] }, @@ -267,10 +267,10 @@ " table_name=TABLE_NAME,\n", " embedding_service=embeddings,\n", " # Connect to a custom vector store table\n", - " # id_column=\"row_id\",\n", - " # content_column=\"content_column\",\n", - " # metadata_columns=[\"metadata\", \"title\"],\n", - " #metadata_json_column=\"metadata\",\n", + " id_column=\"row_id\",\n", + " content_column=\"content_column\",\n", + " metadata_columns=[\"metadata\", \"title\"],\n", + " metadata_json_column=\"metadata\",\n", ")" ] }, From 0a8e8bebfbd0f126af81347307448a4b6cdcad7b Mon Sep 17 00:00:00 2001 From: Vikram Manghnani <50720827+vmanghnani@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:19:09 -0700 Subject: [PATCH 06/11] Update vector_store.ipynb --- docs/vector_store.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/vector_store.ipynb b/docs/vector_store.ipynb index e41a30e5..cafbab7f 100644 --- a/docs/vector_store.ipynb +++ b/docs/vector_store.ipynb @@ -268,7 +268,7 @@ " embedding_service=embeddings,\n", " # Connect to a custom vector store table\n", " id_column=\"row_id\",\n", - " content_column=\"content_column\",\n", + " content_column=\"content\",\n", " metadata_columns=[\"metadata\", \"title\"],\n", " metadata_json_column=\"metadata\",\n", ")" From 08d3fcc0ea78a7b761b5638036d03c2f7e192b0d Mon Sep 17 00:00:00 2001 From: Vikram Manghnani <50720827+vmanghnani@users.noreply.github.com> Date: Tue, 1 Oct 2024 18:07:34 -0700 Subject: [PATCH 07/11] Update vector_store.ipynb --- docs/vector_store.ipynb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/vector_store.ipynb b/docs/vector_store.ipynb index cafbab7f..189e25cc 100644 --- a/docs/vector_store.ipynb +++ b/docs/vector_store.ipynb @@ -266,11 +266,12 @@ " database_id=DATABASE,\n", " table_name=TABLE_NAME,\n", " embedding_service=embeddings,\n", + " ignore_metadata_columns=[],\", " # Connect to a custom vector store table\n", " id_column=\"row_id\",\n", - " content_column=\"content\",\n", - " metadata_columns=[\"metadata\", \"title\"],\n", - " metadata_json_column=\"metadata\",\n", + " #content_column=\"content\",\n", + " #metadata_columns=[\"metadata\", \"title\"],\n", + " #metadata_json_column=\"metadata\",\n", ")" ] }, From 9945523e56399f3d6a4c77b8a02ee888ea5a74e6 Mon Sep 17 00:00:00 2001 From: Vikram Manghnani <50720827+vmanghnani@users.noreply.github.com> Date: Tue, 1 Oct 2024 18:10:32 -0700 Subject: [PATCH 08/11] Update vector_store.ipynb --- docs/vector_store.ipynb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/vector_store.ipynb b/docs/vector_store.ipynb index 189e25cc..312225ff 100644 --- a/docs/vector_store.ipynb +++ b/docs/vector_store.ipynb @@ -266,11 +266,10 @@ " database_id=DATABASE,\n", " table_name=TABLE_NAME,\n", " embedding_service=embeddings,\n", - " ignore_metadata_columns=[],\", + " ignore_metadata_columns=[],\n", " # Connect to a custom vector store table\n", " id_column=\"row_id\",\n", " #content_column=\"content\",\n", - " #metadata_columns=[\"metadata\", \"title\"],\n", " #metadata_json_column=\"metadata\",\n", ")" ] From e23703d2e2972a33387b06ff518dd8915b8647c6 Mon Sep 17 00:00:00 2001 From: Vikram Manghnani <50720827+vmanghnani@users.noreply.github.com> Date: Tue, 1 Oct 2024 18:13:26 -0700 Subject: [PATCH 09/11] Update vector_store.ipynb --- docs/vector_store.ipynb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/vector_store.ipynb b/docs/vector_store.ipynb index 312225ff..685621b8 100644 --- a/docs/vector_store.ipynb +++ b/docs/vector_store.ipynb @@ -266,11 +266,10 @@ " database_id=DATABASE,\n", " table_name=TABLE_NAME,\n", " embedding_service=embeddings,\n", - " ignore_metadata_columns=[],\n", " # Connect to a custom vector store table\n", " id_column=\"row_id\",\n", " #content_column=\"content\",\n", - " #metadata_json_column=\"metadata\",\n", + " ignore_metadata_columns=[],\n", ")" ] }, From 716fdf3147eb11ed5b8b4efb4a8532c80296a6cd Mon Sep 17 00:00:00 2001 From: Averi Kitsch Date: Fri, 4 Oct 2024 13:09:45 -0700 Subject: [PATCH 10/11] Update vector_store.ipynb --- docs/vector_store.ipynb | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/vector_store.ipynb b/docs/vector_store.ipynb index 685621b8..146d4d8d 100644 --- a/docs/vector_store.ipynb +++ b/docs/vector_store.ipynb @@ -196,14 +196,15 @@ " database_id=DATABASE,\n", " table_name=TABLE_NAME,\n", " # Customize the table creation\n", - " id_column=\"row_id\",\n", - " metadata_columns=[\n", - " TableColumn(name=\"metadata\", type=\"JSON\", is_null=True),\n", - " TableColumn(name=\"title\", type=\"STRING(MAX)\", is_null=False),\n", - " ],\n", - " secondary_indexes=[\n", - " SecondaryIndex(index_name=\"row_id_and_title\", columns=[\"row_id\", \"title\"])\n", - " ],\n", + " # id_column=\"row_id\",\n", + " # content_column=\"content_column\",\n", + " # metadata_columns=[\n", + " # TableColumn(name=\"metadata\", type=\"JSON\", is_null=True),\n", + " # TableColumn(name=\"title\", type=\"STRING(MAX)\", is_null=False),\n", + " # ],\n", + " # secondary_indexes=[\n", + " # SecondaryIndex(index_name=\"row_id_and_title\", columns=[\"row_id\", \"title\"])\n", + " # ],\n", ")" ] }, @@ -251,7 +252,6 @@ "1. `instance_id` - The name of the Spanner instance\n", "1. `database_id` - The name of the Spanner database\n", "1. `table_name` - The name of the table within the database to store the documents & their embeddings.\n", - "1. `row_id` - The name of the row identifier column of the table\n", "1. `embedding_service` - The Embeddings implementation which is used to generate the embeddings." ] }, @@ -267,9 +267,9 @@ " table_name=TABLE_NAME,\n", " embedding_service=embeddings,\n", " # Connect to a custom vector store table\n", - " id_column=\"row_id\",\n", - " #content_column=\"content\",\n", - " ignore_metadata_columns=[],\n", + " # id_column=\"row_id\",\n", + " # content_column=\"content\",\n", + " # metadata_columns=["metadata", "title"],\n", ")" ] }, From 7afcefaef69e12613b249593b685a15412916cb6 Mon Sep 17 00:00:00 2001 From: Averi Kitsch Date: Fri, 4 Oct 2024 13:10:30 -0700 Subject: [PATCH 11/11] Update vector_store.ipynb --- docs/vector_store.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/vector_store.ipynb b/docs/vector_store.ipynb index 146d4d8d..b5948543 100644 --- a/docs/vector_store.ipynb +++ b/docs/vector_store.ipynb @@ -269,7 +269,7 @@ " # Connect to a custom vector store table\n", " # id_column=\"row_id\",\n", " # content_column=\"content\",\n", - " # metadata_columns=["metadata", "title"],\n", + " # metadata_columns=[\"metadata\", \"title\"],\n", ")" ] },