-
Notifications
You must be signed in to change notification settings - Fork 233
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Start adding support for Mongodb Atlas database (#311)
It works with cluster from M10 upward (does not work still with shared cluster). This code contains also the necessary code to automatically create the Vector Indexes needed to execute queries. --------- Co-authored-by: Devis Lucato <devis@microsoft.com>
- Loading branch information
1 parent
ae03364
commit 87e12b1
Showing
31 changed files
with
2,062 additions
and
157 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| FROM mongodb/atlas | ||
|
|
||
| COPY startatlas6.sh /usr/startatlas6.sh | ||
|
|
||
| CMD /usr/startatlas6.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| #!/bin/bash | ||
|
|
||
| # The name of the deployment to search for | ||
|
|
||
| # Run the command and save the output | ||
| OUTPUT=$(atlas deployments list) | ||
|
|
||
| echo "Output: " | ||
| echo $OUTPUT | ||
|
|
||
| # count line of output | ||
| LINE=$(echo "$OUTPUT" | wc -l) | ||
| echo "Count line of output: $LINE " | ||
|
|
||
| if [ $LINE -lt 2 ]; then | ||
| echo "No deployment found. create a new one" | ||
| atlas deployments setup local --mdbVersion 6.0 --bindIpAll --username root --password root --type local --force | ||
| else | ||
| echo "Deployment found. Start it" | ||
| atlas deployments start local | ||
| fi | ||
|
|
||
| function pause_atlas() { | ||
| atlas deployments pause local | ||
| } | ||
| # This will call the 'on_exit' function when the container exits | ||
| trap pause_atlas EXIT | ||
|
|
||
| tail -f /dev/null |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| FROM mongodb/atlas | ||
|
|
||
| COPY startatlas.sh /usr/startatlas.sh | ||
|
|
||
| CMD /usr/startatlas.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| #!/bin/bash | ||
|
|
||
| # The name of the deployment to search for | ||
|
|
||
| # Run the command and save the output | ||
| OUTPUT=$(atlas deployments list) | ||
|
|
||
| echo "Output: " | ||
| echo $OUTPUT | ||
|
|
||
| # count line of output | ||
| LINE=$(echo "$OUTPUT" | wc -l) | ||
| echo "Count line of output: $LINE " | ||
|
|
||
| if [ $LINE -lt 2 ]; then | ||
| echo "No deployment found. create a new one" | ||
| atlas deployments setup local --mdbVersion 7.0 --bindIpAll --username root --password root --type local --force | ||
| else | ||
| echo "Deployment found. Start it" | ||
| atlas deployments start local | ||
| fi | ||
|
|
||
| function pause_atlas() { | ||
| atlas deployments pause local | ||
| } | ||
| # This will call the 'on_exit' function when the container exits | ||
| trap pause_atlas EXIT | ||
|
|
||
| tail -f /dev/null |
143 changes: 143 additions & 0 deletions
143
extensions/MongoDbAtlas/MongoDbAtlas.FunctionalTests/DefaultTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,143 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using FunctionalTests.DefaultTestCases; | ||
| using Microsoft.KernelMemory; | ||
| using Microsoft.KernelMemory.MongoDbAtlas; | ||
| using Microsoft.TestHelpers; | ||
|
|
||
| namespace MongoDbAtlas.FunctionalTests; | ||
|
|
||
| public class DefaultTestsSingleCollection : DefaultTests | ||
| { | ||
| public DefaultTestsSingleCollection(IConfiguration cfg, ITestOutputHelper output) | ||
| : base(cfg, output, multiCollection: false) | ||
| { | ||
| } | ||
| } | ||
|
|
||
| public class DefaultTestsMultipleCollections : DefaultTests | ||
| { | ||
| public DefaultTestsMultipleCollections(IConfiguration cfg, ITestOutputHelper output) | ||
| : base(cfg, output, multiCollection: true) | ||
| { | ||
| } | ||
| } | ||
|
|
||
| public abstract class DefaultTests : BaseFunctionalTestCase | ||
| { | ||
| private readonly MemoryServerless _memory; | ||
|
|
||
| protected DefaultTests(IConfiguration cfg, ITestOutputHelper output, bool multiCollection) : base(cfg, output) | ||
| { | ||
| Assert.False(string.IsNullOrEmpty(this.OpenAiConfig.APIKey), "OpenAI API Key is empty"); | ||
|
|
||
| if (multiCollection) | ||
| { | ||
| // this._config = this.MongoDbAtlasConfig; | ||
| this.MongoDbAtlasConfig | ||
| .WithSingleCollectionForVectorSearch(false) | ||
| // Need to wait for atlas to grab the data from the collection and index. | ||
| .WithAfterIndexCallback(async () => await Task.Delay(2000)); | ||
|
|
||
| this.MongoDbAtlasConfig.DatabaseName += "multicoll"; | ||
| } | ||
| else | ||
| { | ||
| this.MongoDbAtlasConfig | ||
| .WithSingleCollectionForVectorSearch(true) | ||
| //Need to wait for atlas to grab the data from the collection and index. | ||
| .WithAfterIndexCallback(async () => await Task.Delay(2000)); | ||
| } | ||
|
|
||
| // Clear all content in any collection before running the test. | ||
| var ash = new MongoDbAtlasSearchHelper(this.MongoDbAtlasConfig.ConnectionString, this.MongoDbAtlasConfig.DatabaseName); | ||
| if (this.MongoDbAtlasConfig.UseSingleCollectionForVectorSearch) | ||
| { | ||
| //delete everything for every collection | ||
| ash.DropAllDocumentsFromCollectionsAsync().Wait(); | ||
| } | ||
| else | ||
| { | ||
| //drop the entire db to be sure we can start with a clean test. | ||
| ash.DropDatabaseAsync().Wait(); | ||
| } | ||
|
|
||
| this._memory = new KernelMemoryBuilder() | ||
| .WithSearchClientConfig(new SearchClientConfig { EmptyAnswer = NotFound }) | ||
| .WithOpenAI(this.OpenAiConfig) | ||
| // .WithAzureOpenAITextGeneration(this.AzureOpenAITextConfiguration) | ||
| // .WithAzureOpenAITextEmbeddingGeneration(this.AzureOpenAIEmbeddingConfiguration) | ||
| .WithMongoDbAtlasMemoryDb(this.MongoDbAtlasConfig) | ||
| .Build<MemoryServerless>(); | ||
| } | ||
|
|
||
| [Fact] | ||
| [Trait("Category", "MongoDbAtlas")] | ||
| public async Task ItSupportsASingleFilter() | ||
| { | ||
| await FilteringTest.ItSupportsASingleFilter(this._memory, this.Log); | ||
| } | ||
|
|
||
| [Fact] | ||
| [Trait("Category", "MongoDbAtlas")] | ||
| public async Task ItSupportsMultipleFilters() | ||
| { | ||
| await FilteringTest.ItSupportsMultipleFilters(this._memory, this.Log); | ||
| } | ||
|
|
||
| [Fact] | ||
| [Trait("Category", "MongoDbAtlas")] | ||
| public async Task ItIgnoresEmptyFilters() | ||
| { | ||
| await FilteringTest.ItIgnoresEmptyFilters(this._memory, this.Log, true); | ||
| } | ||
|
|
||
| [Fact] | ||
| [Trait("Category", "MongoDbAtlas")] | ||
| public async Task ItListsIndexes() | ||
| { | ||
| await IndexListTest.ItListsIndexes(this._memory, this.Log); | ||
| } | ||
|
|
||
| [Fact] | ||
| [Trait("Category", "MongoDbAtlas")] | ||
| public async Task ItNormalizesIndexNames() | ||
| { | ||
| await IndexListTest.ItNormalizesIndexNames(this._memory, this.Log); | ||
| } | ||
|
|
||
| [Fact] | ||
| [Trait("Category", "MongoDbAtlas")] | ||
| public async Task ItUsesDefaultIndexName() | ||
| { | ||
| await IndexListTest.ItUsesDefaultIndexName(this._memory, this.Log, "default4tests"); | ||
| } | ||
|
|
||
| [Fact] | ||
| [Trait("Category", "MongoDbAtlas")] | ||
| public async Task ItDeletesIndexes() | ||
| { | ||
| await IndexDeletionTest.ItDeletesIndexes(this._memory, this.Log); | ||
| } | ||
|
|
||
| [Fact] | ||
| [Trait("Category", "MongoDbAtlas")] | ||
| public async Task ItHandlesMissingIndexesConsistently() | ||
| { | ||
| await MissingIndexTest.ItHandlesMissingIndexesConsistently(this._memory, this.Log); | ||
| } | ||
|
|
||
| [Fact] | ||
| [Trait("Category", "MongoDbAtlas")] | ||
| public async Task ItUploadsPDFDocsAndDeletes() | ||
| { | ||
| await DocumentUploadTest.ItUploadsPDFDocsAndDeletes(this._memory, this.Log); | ||
| } | ||
|
|
||
| [Fact] | ||
| [Trait("Category", "MongoDbAtlas")] | ||
| public async Task ItSupportsTags() | ||
| { | ||
| await DocumentUploadTest.ItSupportsTags(this._memory, this.Log); | ||
| } | ||
| } |
42 changes: 42 additions & 0 deletions
42
extensions/MongoDbAtlas/MongoDbAtlas.FunctionalTests/MongoDbAtlas.FunctionalTests.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk.Web"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFramework>net6.0</TargetFramework> | ||
| <RollForward>LatestMajor</RollForward> | ||
| <IsTestProject>true</IsTestProject> | ||
| <ImplicitUsings>enable</ImplicitUsings> | ||
| <Nullable>enable</Nullable> | ||
| <IsPackable>false</IsPackable> | ||
| <GenerateDocumentationFile>false</GenerateDocumentationFile> | ||
| <UserSecretsId>5ee045b0-aea3-4f08-8d31-32d1a6f8fed0</UserSecretsId> | ||
| <NoWarn>$(NoWarn);CA2007;CA1303;IDE0005;IDE1006;IDE0130;CA1711</NoWarn> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\..\..\service\tests\Core.FunctionalTests\Core.FunctionalTests.csproj" /> | ||
| <ProjectReference Include="..\..\..\service\tests\TestHelpers\TestHelpers.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Microsoft.Extensions.DependencyInjection" /> | ||
| <PackageReference Include="Microsoft.NET.Test.Sdk" /> | ||
| <PackageReference Include="xunit" /> | ||
| <PackageReference Include="xunit.abstractions" /> | ||
| <PackageReference Include="Xunit.DependencyInjection" /> | ||
| <PackageReference Include="xunit.runner.visualstudio"> | ||
| <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> | ||
| <PrivateAssets>all</PrivateAssets> | ||
| </PackageReference> | ||
| <PackageReference Include="coverlet.collector"> | ||
| <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> | ||
| <PrivateAssets>all</PrivateAssets> | ||
| </PackageReference> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <Content Update="file1-NASA-news.pdf"> | ||
| <CopyToOutputDirectory>Always</CopyToOutputDirectory> | ||
| </Content> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
21 changes: 21 additions & 0 deletions
21
extensions/MongoDbAtlas/MongoDbAtlas.FunctionalTests/Startup.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| /* IMPORTANT: the Startup class must be at the root of the namespace and | ||
| * the namespace must match exactly (required by Xunit.DependencyInjection) */ | ||
|
|
||
| namespace MongoDbAtlas.FunctionalTests; | ||
|
|
||
| public class Startup | ||
| { | ||
| public void ConfigureHost(IHostBuilder hostBuilder) | ||
| { | ||
| var config = new ConfigurationBuilder() | ||
| .AddJsonFile("appsettings.json") | ||
| .AddJsonFile("appsettings.development.json", optional: true) | ||
| .AddUserSecrets<Startup>() | ||
| .AddEnvironmentVariables() | ||
| .Build(); | ||
|
|
||
| hostBuilder.ConfigureHostConfiguration(builder => builder.AddConfiguration(config)); | ||
| } | ||
| } |
Oops, something went wrong.