Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
// Copyright (c) Microsoft. All rights reserved.

using System.Text.Json;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.HuggingFace;
using Microsoft.SemanticKernel.Connectors.Sqlite;
using Microsoft.SemanticKernel.Memory;
using Microsoft.SemanticKernel.Embeddings;

#pragma warning disable CS8602 // Dereference of a possibly null reference.

Expand Down Expand Up @@ -31,14 +32,29 @@ public async Task RunInferenceApiEmbeddingCustomHttpHandlerAsync()
})
);

var sqliteMemory = await SqliteMemoryStore.ConnectAsync("./../../../Sqlite.sqlite");
var sqliteCollection = new SqliteVectorStoreRecordCollection<string, Record>(
"Data Source=./../../../Sqlite.sqlite",
name: "Test",
new() { EmbeddingGenerator = hf.AsEmbeddingGenerator() });

var skMemory = new MemoryBuilder()
.WithTextEmbeddingGeneration(hf)
.WithMemoryStore(sqliteMemory)
.Build();
await sqliteCollection.UpsertAsync(new Record
{
Id = "1",
Text = "THIS IS A SAMPLE",
Embedding = "An embedding will be generated from this text"
});
}

public class Record
{
[VectorStoreRecordKey]
public string Id { get; set; }

[VectorStoreRecordData]
public string Text { get; set; }

await skMemory.SaveInformationAsync("Test", "THIS IS A SAMPLE", "sample", "TEXT");
[VectorStoreRecordVector(Dimensions: 768)]
public string Embedding { get; set; }
}

private sealed class CustomHttpClientHandler : HttpClientHandler
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,10 @@
using System.Text;
using System.Text.Json;
using Azure;
using Azure.Identity;
using Azure.Search.Documents.Indexes;
using Memory.VectorStoreFixtures;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.AzureAISearch;
using Microsoft.SemanticKernel.Connectors.AzureOpenAI;
using Microsoft.SemanticKernel.Memory;

namespace Memory;
Expand All @@ -26,41 +24,24 @@ namespace Memory;
/// dotnet user-secrets set "AzureAISearch:Endpoint" "https://myazureaisearchinstance.search.windows.net"
/// dotnet user-secrets set "AzureAISearch:ApiKey" "samplesecret"
/// </remarks>
[Obsolete("The IMemoryStore abstraction is being obsoleted")]
public class VectorStore_ConsumeFromMemoryStore_AzureAISearch(ITestOutputHelper output, VectorStoreQdrantContainerFixture qdrantFixture) : BaseTest(output), IClassFixture<VectorStoreQdrantContainerFixture>
{
private const int VectorSize = 1536;
private const string MemoryStoreCollectionName = "memorystorecollection";
private readonly static JsonSerializerOptions s_consoleFormatting = new() { WriteIndented = true };

[Fact]
public async Task ConsumeExampleAsync()
{
// Setup the supporting infra and embedding generation.
await qdrantFixture.ManualInitializeAsync();
var textEmbeddingService = new AzureOpenAITextEmbeddingGenerationService(
TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
new AzureCliCredential());

// Construct a legacy MemoryStore.
var memoryStore = new AzureAISearchMemoryStore(
TestConfiguration.AzureAISearch.Endpoint,
TestConfiguration.AzureAISearch.ApiKey);

// Construct a VectorStore.
var vectorStore = new AzureAISearchVectorStore(new SearchIndexClient(
new Uri(TestConfiguration.AzureAISearch.Endpoint),
new AzureKeyCredential(TestConfiguration.AzureAISearch.ApiKey)));

// Build a collection with sample data using the MemoryStore abstraction.
await VectorStore_ConsumeFromMemoryStore_Common.CreateCollectionAndAddSampleDataAsync(
memoryStore,
MemoryStoreCollectionName,
textEmbeddingService);

// Connect to the same collection using the VectorStore abstraction.
var collection = vectorStore.GetCollection<string, VectorStoreRecord>(MemoryStoreCollectionName);
// Use the VectorStore abstraction to connect to an existing collection which was previously created via the IMemoryStore abstraction
var collection = vectorStore.GetCollection<string, VectorStoreRecord>("memorystorecollection");
await collection.CreateCollectionIfNotExistsAsync();

// Show that the data can be read using the VectorStore abstraction.
Expand All @@ -77,7 +58,7 @@ await VectorStore_ConsumeFromMemoryStore_Common.CreateCollectionAndAddSampleData

/// <summary>
/// A data model with Vector Store attributes that matches the storage representation of
/// <see cref="MemoryRecord"/> objects as created by <see cref="AzureAISearchMemoryStore"/>.
/// <see cref="MemoryRecord"/> objects as created by <c>AzureAISearchMemoryStore</c>.
/// </summary>
private sealed class VectorStoreRecord
{
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
// Copyright (c) Microsoft. All rights reserved.

using System.Text.Json;
using Azure.Identity;
using Memory.VectorStoreFixtures;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.AzureOpenAI;
using Microsoft.SemanticKernel.Connectors.Qdrant;
using Microsoft.SemanticKernel.Memory;
using Qdrant.Client;
Expand All @@ -23,37 +21,22 @@ namespace Memory;
/// To run this sample, you need a local instance of Docker running, since the associated fixture
/// will try and start a Qdrant container in the local docker instance to run against.
/// </remarks>
[Obsolete("The IMemoryStore abstraction is being obsoleted")]
public class VectorStore_ConsumeFromMemoryStore_Qdrant(ITestOutputHelper output, VectorStoreQdrantContainerFixture qdrantFixture) : BaseTest(output), IClassFixture<VectorStoreQdrantContainerFixture>
{
private const int VectorSize = 1536;
private const string MemoryStoreCollectionName = "memorystorecollection";
private readonly static JsonSerializerOptions s_consoleFormatting = new() { WriteIndented = true };

[Fact]
public async Task ConsumeExampleAsync()
{
// Setup the supporting infra and embedding generation.
await qdrantFixture.ManualInitializeAsync();
var textEmbeddingService = new AzureOpenAITextEmbeddingGenerationService(
TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
new AzureCliCredential());

// Construct a legacy MemoryStore.
var memoryStore = new QdrantMemoryStore("http://localhost:6333", VectorSize);

// Construct a VectorStore.
var vectorStore = new QdrantVectorStore(new QdrantClient("localhost"));

// Build a collection with sample data using the MemoryStore abstraction.
await VectorStore_ConsumeFromMemoryStore_Common.CreateCollectionAndAddSampleDataAsync(
memoryStore,
MemoryStoreCollectionName,
textEmbeddingService);

// Connect to the same collection using the VectorStore abstraction.
var collection = vectorStore.GetCollection<Guid, VectorStoreRecord>(MemoryStoreCollectionName);
// Use the VectorStore abstraction to connect to an existing collection which was previously created via the IMemoryStore abstraction
var collection = vectorStore.GetCollection<Guid, VectorStoreRecord>("memorystorecollection");
await collection.CreateCollectionIfNotExistsAsync();

// Show that the data can be read using the VectorStore abstraction.
Expand All @@ -68,7 +51,7 @@ await VectorStore_ConsumeFromMemoryStore_Common.CreateCollectionAndAddSampleData

/// <summary>
/// A data model with Vector Store attributes that matches the storage representation of
/// <see cref="MemoryRecord"/> objects as created by <see cref="QdrantMemoryStore"/>.
/// <see cref="MemoryRecord"/> objects as created by <c>QdrantMemoryStore</c>.
/// </summary>
private sealed class VectorStoreRecord
{
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.

using Azure.Identity;
using Memory.VectorStoreFixtures;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.AzureOpenAI;
using Microsoft.SemanticKernel.Connectors.Redis;
using Microsoft.SemanticKernel.Memory;
using StackExchange.Redis;
Expand All @@ -22,7 +20,6 @@ namespace Memory;
/// To run this sample, you need a local instance of Docker running, since the associated fixture
/// will try and start a Redis container in the local docker instance to run against.
/// </remarks>
[Obsolete("The IMemoryStore abstraction is being obsoleted")]
public class VectorStore_ConsumeFromMemoryStore_Redis(ITestOutputHelper output, VectorStoreRedisContainerFixture redisFixture) : BaseTest(output), IClassFixture<VectorStoreRedisContainerFixture>
{
private const int VectorSize = 1536;
Expand All @@ -33,26 +30,13 @@ public async Task ConsumeExampleAsync()
{
// Setup the supporting infra and embedding generation.
await redisFixture.ManualInitializeAsync();
var textEmbeddingService = new AzureOpenAITextEmbeddingGenerationService(
TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
new AzureCliCredential());

// Construct a legacy MemoryStore.
var memoryStore = new RedisMemoryStore("localhost:6379", VectorSize);

// Construct a VectorStore and indicate that we want to use hashes
// since the legacy memory store uses hashes to store memory records.
// Use the VectorStore abstraction to connect to an existing collection which was previously created via the IMemoryStore abstraction.
// Note that we use HashSet since the legacy memory store uses hashes to store memory records.
var vectorStore = new RedisVectorStore(
ConnectionMultiplexer.Connect("localhost:6379").GetDatabase(),
new() { StorageType = RedisStorageType.HashSet });

// Build a collection with sample data using the MemoryStore abstraction.
await VectorStore_ConsumeFromMemoryStore_Common.CreateCollectionAndAddSampleDataAsync(
memoryStore,
MemoryStoreCollectionName,
textEmbeddingService);

// Connect to the same collection using the VectorStore abstraction.
var collection = vectorStore.GetCollection<string, VectorStoreRecord>(MemoryStoreCollectionName);
await collection.CreateCollectionIfNotExistsAsync();
Expand All @@ -69,7 +53,7 @@ await VectorStore_ConsumeFromMemoryStore_Common.CreateCollectionAndAddSampleData

/// <summary>
/// A data model with Vector Store attributes that matches the storage representation of
/// <see cref="MemoryRecord"/> objects as created by <see cref="RedisMemoryStore"/>.
/// <see cref="MemoryRecord"/> objects as created by <c>RedisMemoryStore</c>.
/// </summary>
private sealed class VectorStoreRecord
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,16 +55,16 @@ public async Task ExampleWithDIAsync(string databaseType)
if (databaseType == "Redis")
{
await redisFixture.ManualInitializeAsync();
kernelBuilder.AddRedisVectorStore("localhost:6379");
kernelBuilder.Services.AddRedisVectorStore("localhost:6379");
}
else if (databaseType == "Qdrant")
{
await qdrantFixture.ManualInitializeAsync();
kernelBuilder.AddQdrantVectorStore("localhost");
kernelBuilder.Services.AddQdrantVectorStore("localhost");
}
else if (databaseType == "InMemory")
{
kernelBuilder.AddInMemoryVectorStore();
kernelBuilder.Services.AddInMemoryVectorStore();
}

// Register the DataIngestor with the DI container.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.

#if DISABLED
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@westey-m I've converted some other samples which show how to connect via MEVD to an IMemoryStore-created collection, but this one is slightly different since it shows an actual data migration... We should decide what we want to do with it (but can probably do it later?).


using System.Text.Json;
using Azure.Identity;
using Memory.VectorStoreFixtures;
Expand Down Expand Up @@ -189,3 +191,5 @@ All methods to upsert or get records use strongly typed model classes. The prope
key: "33333333-3333-3333-3333-333333333333");
}
}

#endif
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public async Task ExampleWithDIAsync()
credential: new AzureCliCredential());

// Register the Azure AI Search VectorStore.
kernelBuilder.AddAzureAISearchVectorStore(
kernelBuilder.Services.AddAzureAISearchVectorStore(
new Uri(TestConfiguration.AzureAISearch.Endpoint),
new AzureKeyCredential(TestConfiguration.AzureAISearch.ApiKey));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public async Task ExampleWithDIAsync()
credential: new AzureCliCredential());

// Register the InMemory VectorStore.
kernelBuilder.AddInMemoryVectorStore();
kernelBuilder.Services.AddInMemoryVectorStore();

// Register the test output helper common processor with the DI container.
kernelBuilder.Services.AddSingleton<ITestOutputHelper>(this.Output);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public async Task ExampleWithDIAsync()

// Initialize the Qdrant docker container via the fixtures and register the Qdrant VectorStore.
await qdrantFixture.ManualInitializeAsync();
kernelBuilder.AddQdrantVectorStore("localhost");
kernelBuilder.Services.AddQdrantVectorStore("localhost");

// Register the test output helper common processor with the DI container.
kernelBuilder.Services.AddSingleton<ITestOutputHelper>(this.Output);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public async Task ExampleWithDIAsync(RedisStorageType redisStorageType)

// Initialize the Redis docker container via the fixtures and register the Redis VectorStore with the preferred storage type.
await redisFixture.ManualInitializeAsync();
kernelBuilder.AddRedisVectorStore("localhost:6379", new() { StorageType = redisStorageType });
kernelBuilder.Services.AddRedisVectorStore("localhost:6379", new() { StorageType = redisStorageType });

// Register the test output helper common processor with the DI container.
kernelBuilder.Services.AddSingleton<ITestOutputHelper>(this.Output);
Expand Down
Loading
Loading