Python: Feature python vector stores preb

python/.coveragerc

@@ -1,20 +1,7 @@
 [run]
 source = semantic_kernel
 omit = 
-    semantic_kernel/connectors/memory/astradb/*
-    semantic_kernel/connectors/memory/azure_cognitive_search/*
-    semantic_kernel/connectors/memory/azure_cosmosdb/*
-    semantic_kernel/connectors/memory/azure_cosmosdb_no_sql/*
-    semantic_kernel/connectors/memory/chroma/chroma_memory_store.py
-    semantic_kernel/connectors/memory/milvus/*
-    semantic_kernel/connectors/memory/mongodb_atlas/mongodb_atlas_memory_store.py
-    semantic_kernel/connectors/memory/pinecone/pinecone_memory_store.py
-    semantic_kernel/connectors/memory/pinecone/utils.py
-    semantic_kernel/connectors/memory/postgres/postgres_memory_store.py
-    semantic_kernel/connectors/memory/qdrant/qdrant_memory_store.py
-    semantic_kernel/connectors/memory/redis/redis_memory_store.py
-    semantic_kernel/connectors/memory/usearch/*
-    semantic_kernel/connectors/memory/weaviate/weaviate_memory_store.py
+    semantic_kernel/connectors/memory_stores/*
     semantic_kernel/reliability/*
     semantic_kernel/memory/*
 

python/mypy.ini

@@ -17,50 +17,9 @@ disable_error_code = method-assign
 
 [mypy-semantic_kernel.memory.*]
 ignore_errors = true
-# TODO (eavanvalkenburg): remove this
-# https://github.com/microsoft/semantic-kernel/issues/6463
+# TODO (eavanvalkenburg): remove this when removing the memory stores
 
-[mypy-semantic_kernel.connectors.memory.astradb.*]
+[mypy-semantic_kernel.connectors.memory_stores.*]
 ignore_errors = true
+# TODO (eavanvalkenburg): remove this when removing the memory stores
 
-[mypy-semantic_kernel.connectors.memory.azure_ai_search.*]
-ignore_errors = false
-[mypy-semantic_kernel.connectors.memory.azure_cognitive_search.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.azure_cosmosdb.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.azure_cosmosdb_no_sql.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.chroma.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.milvus.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.mongodb_atlas.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.pinecone.pinecone_memory_store]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.postgres.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.qdrant.qdrant_vector_record_store.*]
-ignore_errors = true
-[mypy-semantic_kernel.connectors.memory.qdrant.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.redis.redis_vector_record_store.*]
-ignore_errors = true
-[mypy-semantic_kernel.connectors.memory.redis.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.usearch.*]
-ignore_errors = true
-
-[mypy-semantic_kernel.connectors.memory.weaviate.*]
-ignore_errors = true

python/pyproject.toml

@@ -123,7 +123,7 @@ pandas = [
     "pandas ~= 2.2"
 ]
 pinecone = [
-    "pinecone[asyncio, grpc] ~= 6.0"
+    "pinecone[asyncio, grpc] ~= 7.0"
 ]
 postgres = [
     "psycopg[binary,pool] ~= 3.2"
@@ -132,7 +132,7 @@ qdrant = [
     "qdrant-client ~= 1.9"
 ]
 redis = [
-    "redis[hiredis] >= 5,< 6",
+    "redis[hiredis] ~= 6.0",
     "types-redis ~= 4.6.0.20240425",
     "redisvl ~= 0.4"
 ]
@@ -215,6 +215,7 @@ select = [
 ignore = [
     "D100", #allow missing docstring in public module
     "D104", #allow missing docstring in public package
+    "D418", #allow docstring on overloaded function
     "TD003", #allow missing link to todo issue
     "FIX002" #allow todo
 ]

python/samples/concepts/caching/semantic_caching.py

@@ -8,36 +8,25 @@
 from uuid import uuid4
 
 from semantic_kernel import Kernel
-from semantic_kernel.connectors.ai.embedding_generator_base import EmbeddingGeneratorBase
 from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion, OpenAITextEmbedding
-from semantic_kernel.connectors.memory.in_memory.in_memory_store import InMemoryVectorStore
-from semantic_kernel.data import (
-    VectorizedSearchMixin,
-    VectorSearchOptions,
-    VectorStore,
-    VectorStoreRecordCollection,
-    VectorStoreRecordDataField,
-    VectorStoreRecordKeyField,
-    VectorStoreRecordVectorField,
-    vectorstoremodel,
-)
+from semantic_kernel.connectors.in_memory import InMemoryStore
+from semantic_kernel.data.vector import VectorStore, VectorStoreCollection, VectorStoreField, vectorstoremodel
 from semantic_kernel.filters import FilterTypes, FunctionInvocationContext, PromptRenderContext
 from semantic_kernel.functions import FunctionResult
 
 COLLECTION_NAME = "llm_responses"
 RECORD_ID_KEY = "cache_record_id"
 
 
-# Define a simple data model to store, the prompt, the result, and the prompt embedding.
-@vectorstoremodel
+# Define a simple data model to store, the prompt and the result
+# we annotate the prompt field as the vector field, the prompt itself will not be stored.
+# and if you use `include_vectors` in the search, it will return the vector, but not the prompt.
+@vectorstoremodel(collection_name=COLLECTION_NAME)
 @dataclass
 class CacheRecord:
-    prompt: Annotated[str, VectorStoreRecordDataField(embedding_property_name="prompt_embedding")]
-    result: Annotated[str, VectorStoreRecordDataField(is_full_text_searchable=True)]
-    prompt_embedding: Annotated[list[float], VectorStoreRecordVectorField(dimensions=1536)] = field(
-        default_factory=list
-    )
-    id: Annotated[str, VectorStoreRecordKeyField] = field(default_factory=lambda: str(uuid4()))
+    result: Annotated[str, VectorStoreField("data", is_full_text_indexed=True)]
+    prompt: Annotated[str | None, VectorStoreField("vector", dimensions=1536)] = None
+    id: Annotated[str, VectorStoreField("key")] = field(default_factory=lambda: str(uuid4()))
 
 
 # Define the filters, one for caching the results and one for using the cache.
@@ -46,16 +35,13 @@ class PromptCacheFilter:
 
     def __init__(
         self,
-        embedding_service: EmbeddingGeneratorBase,
         vector_store: VectorStore,
-        collection_name: str = COLLECTION_NAME,
         score_threshold: float = 0.2,
     ):
-        self.embedding_service = embedding_service
+        if vector_store.embedding_generator is None:
+            raise ValueError("The vector store must have an embedding generator.")
         self.vector_store = vector_store
-        self.collection: VectorStoreRecordCollection[str, CacheRecord] = vector_store.get_collection(
-            collection_name, data_model_type=CacheRecord
-        )
+        self.collection: VectorStoreCollection[str, CacheRecord] = vector_store.get_collection(record_type=CacheRecord)
         self.score_threshold = score_threshold
 
     async def on_prompt_render(
@@ -69,15 +55,10 @@ async def on_prompt_render(
         closer the match.
         """
         await next(context)
-        assert context.rendered_prompt  # nosec
-        prompt_embedding = await self.embedding_service.generate_raw_embeddings([context.rendered_prompt])
-        await self.collection.create_collection_if_not_exists()
-        assert isinstance(self.collection, VectorizedSearchMixin)  # nosec
-        results = await self.collection.vectorized_search(
-            vector=prompt_embedding[0], options=VectorSearchOptions(vector_field_name="prompt_embedding", top=1)
-        )
+        await self.collection.ensure_collection_exists()
+        results = await self.collection.search(context.rendered_prompt, vector_property_name="prompt", top=1)
         async for result in results.results:
-            if result.score < self.score_threshold:
+            if result.score and result.score < self.score_threshold:
                 context.function_result = FunctionResult(
                     function=context.function.metadata,
                     value=result.record.result,
@@ -92,13 +73,8 @@ async def on_function_invocation(
         await next(context)
         result = context.result
         if result and result.rendered_prompt and RECORD_ID_KEY not in result.metadata:
-            prompt_embedding = await self.embedding_service.generate_embeddings([result.rendered_prompt])
-            cache_record = CacheRecord(
-                prompt=result.rendered_prompt,
-                result=str(result),
-                prompt_embedding=prompt_embedding[0],
-            )
-            await self.collection.create_collection_if_not_exists()
+            cache_record = CacheRecord(prompt=result.rendered_prompt, result=str(result))
+            await self.collection.ensure_collection_exists()
             await self.collection.upsert(cache_record)
 
 
@@ -118,11 +94,10 @@ async def main():
     chat = OpenAIChatCompletion(service_id="default")
     embedding = OpenAITextEmbedding(service_id="embedder")
     kernel.add_service(chat)
-    kernel.add_service(embedding)
     # create the in-memory vector store
-    vector_store = InMemoryVectorStore()
+    vector_store = InMemoryStore(embedding_generator=embedding)
     # create the cache filter and add the filters to the kernel
-    cache = PromptCacheFilter(embedding_service=embedding, vector_store=vector_store)
+    cache = PromptCacheFilter(vector_store=vector_store)
     kernel.add_filter(FilterTypes.PROMPT_RENDERING, cache.on_prompt_render)
     kernel.add_filter(FilterTypes.FUNCTION_INVOCATION, cache.on_function_invocation)
 

python/samples/concepts/chat_history/store_chat_history_in_cosmosdb.py

@@ -7,17 +7,11 @@
 from samples.concepts.setup.chat_completion_services import Services, get_chat_completion_service_and_request_settings
 from semantic_kernel import Kernel
 from semantic_kernel.connectors.ai import FunctionChoiceBehavior
-from semantic_kernel.connectors.memory.azure_cosmos_db.azure_cosmos_db_no_sql_store import AzureCosmosDBNoSQLStore
+from semantic_kernel.connectors.azure_cosmos_db import CosmosNoSqlStore
 from semantic_kernel.contents import ChatHistory, ChatMessageContent
 from semantic_kernel.core_plugins.math_plugin import MathPlugin
 from semantic_kernel.core_plugins.time_plugin import TimePlugin
-from semantic_kernel.data import (
-    VectorStore,
-    VectorStoreRecordCollection,
-    VectorStoreRecordDataField,
-    VectorStoreRecordKeyField,
-    vectorstoremodel,
-)
+from semantic_kernel.data.vector import VectorStore, VectorStoreCollection, VectorStoreField, vectorstoremodel
 
 """
 This sample demonstrates how to build a conversational chatbot
@@ -39,9 +33,9 @@
 @vectorstoremodel
 @dataclass
 class ChatHistoryModel:
-    session_id: Annotated[str, VectorStoreRecordKeyField]
-    user_id: Annotated[str, VectorStoreRecordDataField(is_filterable=True)]
-    messages: Annotated[list[dict[str, str]], VectorStoreRecordDataField(is_filterable=True)]
+    session_id: Annotated[str, VectorStoreField("key")]
+    user_id: Annotated[str, VectorStoreField("data", is_indexed=True)]
+    messages: Annotated[list[dict[str, str]], VectorStoreField("data", is_indexed=True)]
 
 
 # 2. We then create a class that extends the ChatHistory class
@@ -55,7 +49,7 @@ class ChatHistoryInCosmosDB(ChatHistory):
     session_id: str
     user_id: str
     store: VectorStore
-    collection: VectorStoreRecordCollection[str, ChatHistoryModel] | None = None
+    collection: VectorStoreCollection[str, ChatHistoryModel] | None = None
 
     async def create_collection(self, collection_name: str) -> None:
         """Create a collection with the inbuild data model using the vector store.
@@ -64,9 +58,9 @@ async def create_collection(self, collection_name: str) -> None:
         """
         self.collection = self.store.get_collection(
             collection_name=collection_name,
-            data_model_type=ChatHistoryModel,
+            record_type=ChatHistoryModel,
         )
-        await self.collection.create_collection_if_not_exists()
+        await self.collection.ensure_collection_exists()
 
     async def store_messages(self) -> None:
         """Store the chat history in the Cosmos DB.
@@ -175,7 +169,7 @@ async def main() -> None:
 
     # First we enter the store context manager to connect.
     # The create_database flag will create the database if it does not exist.
-    async with AzureCosmosDBNoSQLStore(create_database=True) as store:
+    async with CosmosNoSqlStore(create_database=True) as store:
         # Then we create the chat history in CosmosDB.
         history = ChatHistoryInCosmosDB(store=store, session_id=session_id, user_id="user")
         # Finally we create the collection.
@@ -191,7 +185,7 @@ async def main() -> None:
         except Exception:
             print("Closing chat...")
         if delete_when_done and history.collection:
-            await history.collection.delete_collection()
+            await history.collection.ensure_collection_deleted()
 
 
 if __name__ == "__main__":

python/samples/concepts/memory/azure_ai_search_hotel_samples/1_interact_with_the_collection.py

@@ -0,0 +1,73 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import asyncio
+
+from samples.concepts.memory.azure_ai_search_hotel_samples.data_model import (
+    HotelSampleClass,
+    custom_index,
+    load_records,
+)
+from semantic_kernel.connectors.ai.open_ai import OpenAITextEmbedding
+from semantic_kernel.connectors.azure_ai_search import AzureAISearchCollection
+
+"""
+With the data model and records defined in data_model.py, this script will create an Azure AI Search collection,
+upsert the records, and then search the collection using vector and hybrid search.
+The script will print the first five records in the collection and the search results.
+The script will also delete the collection at the end.
+
+Note that we add the OpenAITextEmbedding to the collection, which is used to generate the vectors.
+To use the built-in embedding in Azure AI Search, remove this and add that definition to the custom_index.
+"""
+
+
+async def main(query: str):
+    records = load_records()
+    # Create the Azure AI Search collection
+    async with AzureAISearchCollection[str, HotelSampleClass](
+        record_type=HotelSampleClass, embedding_generator=OpenAITextEmbedding()
+    ) as collection:
+        # Check if the collection exists.
+        await collection.ensure_collection_exists(index=custom_index)
+        await collection.upsert(records)
+        # get the first five records to check the upsert worked.
+        results = await collection.get(order_by="HotelName", top=5)
+        print("Get first five records: ")
+        if results:
+            for result in results:
+                print(
+                    f"    {result.HotelId} (in {result.Address.City}, {result.Address.Country}): {result.Description}"
+                )
+
+        print("\n")
+        print("Search results using vector: ")
+        # Use search to search using the vector.
+        results = await collection.search(
+            query,
+            vector_property_name="DescriptionVector",
+        )
+        async for result in results.results:
+            print(
+                f"    {result.record.HotelId} (in {result.record.Address.City}, "
+                f"{result.record.Address.Country}): {result.record.Description} (score: {result.score})"
+            )
+        print("\n")
+        print("Search results using hybrid: ")
+        # Use hybrid search to search using the vector.
+        results = await collection.hybrid_search(
+            query,
+            vector_property_name="DescriptionVector",
+            additional_property_name="Description",
+        )
+        async for result in results.results:
+            print(
+                f"    {result.record.HotelId} (in {result.record.Address.City}, "
+                f"{result.record.Address.Country}): {result.record.Description} (score: {result.score})"
+            )
+
+        await collection.ensure_collection_deleted()
+
+
+if __name__ == "__main__":
+    query = "swimming pool and good internet connection"
+    asyncio.run(main(query=query))