New Semantic Memory Storage based on Azure Cognitive Search with cust…

…om embeddings (#1174) * Add new Memory Storage based on Azure Cognitive Search with custom embeddings * Delete previous Azure Search memory component based on old semantic search/Lucene Add Azure Cognitive Search vector connector (#1103) * Azure Cognitive Search integration with Python SK
microsoft · Jul 6, 2023 · b7d5012 · b7d5012
1 parent 91ccbc5
commit b7d5012
Show file tree

Hide file tree

Showing 25 changed files with 1,513 additions and 653 deletions.
diff --git a/.github/workflows/python-integration-tests.yml b/.github/workflows/python-integration-tests.yml
@@ -38,13 +38,13 @@ jobs:
           export HNSWLIB_NO_NATIVE=1
           python -m pip install --upgrade pip setuptools wheel
           python -m pip install poetry pytest
-          cd python && poetry install --with hugging_face --with chromadb --with weaviate
+          cd python && poetry install --with hugging_face --with chromadb --with weaviate --with azure_cognitive_search
       - name: Install dependencies with hnswlib native enabled
         if: matrix.os != 'macos-latest' || matrix.python-version != '3.11'
         run: |
           python -m pip install --upgrade pip setuptools wheel
           python -m pip install poetry pytest
-          cd python && poetry install --with hugging_face --with chromadb --with weaviate
+          cd python && poetry install --with hugging_face --with chromadb --with weaviate --with azure_cognitive_search
       - name: Run Integration Tests
         shell: bash
         env: # Set Azure credentials secret as an input

diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props
@@ -7,7 +7,7 @@
   <ItemGroup>
     <PackageVersion Include="Azure.AI.OpenAI" Version="1.0.0-beta.5" />
     <PackageVersion Include="Azure.Identity" Version="1.9.0" />
-    <PackageVersion Include="Azure.Search.Documents" Version="11.5.0-beta.2" />
+    <PackageVersion Include="Azure.Search.Documents" Version="11.5.0-alpha.20230706.1" />
     <PackageVersion Include="Microsoft.Bcl.HashCode" Version="[1.1.0, )" />
     <PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="6.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Http" Version="6.0.0" />

diff --git a/dotnet/SK-dotnet.sln b/dotnet/SK-dotnet.sln
@@ -90,7 +90,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Extensions.UnitTests", "src
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Planning.SequentialPlanner", "src\Extensions\Planning.SequentialPlanner\Planning.SequentialPlanner.csproj", "{A350933D-F9D5-4AD3-8C4F-B856B5020297}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.AzureCognitiveSearch", "src\Connectors\Connectors.Memory.AzureCognitiveSearch\Connectors.Memory.AzureCognitiveSearch.csproj", "{EC3BB6D1-2FB2-4702-84C6-F791DE533ED4}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.AzureSearch", "src\Connectors\Connectors.Memory.AzureSearch\Connectors.Memory.AzureSearch.csproj", "{EC3BB6D1-2FB2-4702-84C6-F791DE533ED4}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.Pinecone", "src\Connectors\Connectors.Memory.Pinecone\Connectors.Memory.Pinecone.csproj", "{4D226C2F-AE9F-4EFB-AF2D-45C8FE5CB34E}"
 EndProject

diff --git a/dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs b/dotnet/samples/KernelSyntaxExamples/Example14_SemanticMemory.cs
@@ -4,6 +4,7 @@
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Connectors.Memory.AzureSearch;
 using Microsoft.SemanticKernel.Memory;
 using RepoUtils;
 
@@ -35,7 +36,8 @@ public static async Task RunAsync()
 
         var kernelWithACS = Kernel.Builder
             .WithLogger(ConsoleLogger.Log)
-            .WithAzureCognitiveSearchMemory(Env.Var("ACS_ENDPOINT"), Env.Var("ACS_API_KEY"))
+            .WithOpenAITextEmbeddingGenerationService("text-embedding-ada-002", Env.Var("OPENAI_API_KEY"))
+            .WithMemoryStorage(new AzureSearchMemoryStore(Env.Var("ACS_ENDPOINT"), Env.Var("ACS_API_KEY")))
             .Build();
 
         await RunExampleAsync(kernelWithACS);
@@ -55,7 +57,7 @@ public static async Task RunAsync()
 
         var kernelWithCustomDb = Kernel.Builder
             .WithLogger(ConsoleLogger.Log)
-            .WithOpenAITextEmbeddingGenerationService("ada", "text-embedding-ada-002", Env.Var("OPENAI_API_KEY"))
+            .WithOpenAITextEmbeddingGenerationService("text-embedding-ada-002", Env.Var("OPENAI_API_KEY"))
             .WithMemoryStorage(new VolatileMemoryStore())
             .Build();
 
@@ -113,6 +115,7 @@ await foreach (MemoryQueryResult memory in memories)
             Console.WriteLine($"Result {++i}:");
             Console.WriteLine("  URL:     : " + memory.Metadata.Id);
             Console.WriteLine("  Title    : " + memory.Metadata.Description);
+            Console.WriteLine("  Relevance: " + memory.Relevance);
             Console.WriteLine();
         }
 
@@ -136,11 +139,11 @@ private static async Task StoreMemoryAsync(IKernel kernel)
         {
             await kernel.Memory.SaveReferenceAsync(
                 collection: MemoryCollectionName,
-                description: entry.Value,
-                text: entry.Value,
+                externalSourceName: "GitHub",
                 externalId: entry.Key,
-                externalSourceName: "GitHub"
-            );
+                description: entry.Value,
+                text: entry.Value);
+
             Console.Write($" #{++i} saved.");
         }
 

diff --git a/dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj b/dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj
@@ -28,7 +28,7 @@
   <ItemGroup>
     <ProjectReference Include="..\..\src\Connectors\Connectors.AI.OpenAI\Connectors.AI.OpenAI.csproj" />
     <ProjectReference Include="..\..\src\Connectors\Connectors.AI.HuggingFace\Connectors.AI.HuggingFace.csproj" />
-    <ProjectReference Include="..\..\src\Connectors\Connectors.Memory.AzureCognitiveSearch\Connectors.Memory.AzureCognitiveSearch.csproj" />
+    <ProjectReference Include="..\..\src\Connectors\Connectors.Memory.AzureSearch\Connectors.Memory.AzureSearch.csproj" />
     <ProjectReference Include="..\..\src\Connectors\Connectors.Memory.Chroma\Connectors.Memory.Chroma.csproj" />
     <ProjectReference Include="..\..\src\Connectors\Connectors.Memory.Postgres\Connectors.Memory.Postgres.csproj" />
     <ProjectReference Include="..\..\src\Connectors\Connectors.Memory.Weaviate\Connectors.Memory.Weaviate.csproj" />
@@ -37,6 +37,7 @@
     <ProjectReference Include="..\..\src\Extensions\Planning.SequentialPlanner\Planning.SequentialPlanner.csproj" />
     <ProjectReference Include="..\..\src\Extensions\Planning.StepwisePlanner\Planning.StepwisePlanner.csproj" />
     <ProjectReference Include="..\..\src\Connectors\Connectors.Memory.Pinecone\Connectors.Memory.Pinecone.csproj" />
+    <ProjectReference Include="..\..\src\SemanticKernel.Abstractions\SemanticKernel.Abstractions.csproj" />
     <ProjectReference Include="..\..\src\Skills\Skills.Core\Skills.Core.csproj" />
     <ProjectReference Include="..\..\src\Skills\Skills.OpenAPI\Skills.OpenAPI.csproj" />
     <ProjectReference Include="..\..\src\Skills\Skills.Grpc\Skills.Grpc.csproj" />