Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

.Net: Implementation of store using Azure SQL/SQL Server with vector search. #6142

Merged
merged 1 commit into from
May 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions dotnet/Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@
<PackageVersion Include="NRedisStack" Version="0.12.0" />
<PackageVersion Include="Milvus.Client" Version="2.3.0-preview.1" />
<PackageVersion Include="Testcontainers.Milvus" Version="3.8.0" />
<PackageVersion Include="Microsoft.Data.SqlClient" Version="5.2.0" />
<!-- Symbols -->
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
<!-- Toolset -->
Expand Down
9 changes: 9 additions & 0 deletions dotnet/SK-dotnet.sln
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Concepts", "samples\Concept
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FunctionInvocationApproval", "samples\Demos\FunctionInvocationApproval\FunctionInvocationApproval.csproj", "{6B56D8EE-9991-43E3-90B2-B8F5C5CE77C2}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.Memory.SqlServer", "src\Connectors\Connectors.Memory.SqlServer\Connectors.Memory.SqlServer.csproj", "{24B8041B-92C6-4BB3-A699-C593AF5A870F}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodeInterpreterPlugin", "samples\Demos\CodeInterpreterPlugin\CodeInterpreterPlugin.csproj", "{3ED53702-0E53-473A-A0F4-645DB33541C2}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TimePlugin", "samples\Demos\TimePlugin\TimePlugin.csproj", "{F312FCE1-12D7-4DEF-BC29-2FF6618509F3}"
Expand Down Expand Up @@ -734,6 +736,12 @@ Global
{6B56D8EE-9991-43E3-90B2-B8F5C5CE77C2}.Publish|Any CPU.Build.0 = Debug|Any CPU
{6B56D8EE-9991-43E3-90B2-B8F5C5CE77C2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6B56D8EE-9991-43E3-90B2-B8F5C5CE77C2}.Release|Any CPU.Build.0 = Release|Any CPU
{24B8041B-92C6-4BB3-A699-C593AF5A870F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{24B8041B-92C6-4BB3-A699-C593AF5A870F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{24B8041B-92C6-4BB3-A699-C593AF5A870F}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
{24B8041B-92C6-4BB3-A699-C593AF5A870F}.Publish|Any CPU.Build.0 = Debug|Any CPU
{24B8041B-92C6-4BB3-A699-C593AF5A870F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{24B8041B-92C6-4BB3-A699-C593AF5A870F}.Release|Any CPU.Build.0 = Release|Any CPU
{3ED53702-0E53-473A-A0F4-645DB33541C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3ED53702-0E53-473A-A0F4-645DB33541C2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3ED53702-0E53-473A-A0F4-645DB33541C2}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
Expand Down Expand Up @@ -847,6 +855,7 @@ Global
{6EF9663D-976C-4A27-B8D3-8B1E63BA3BF2} = {5D4C0700-BBB5-418F-A7B2-F392B9A18263}
{925B1185-8B58-4E2D-95C9-4CA0BA9364E5} = {FA3720F1-C99A-49B2-9577-A940257098BF}
{6B56D8EE-9991-43E3-90B2-B8F5C5CE77C2} = {5D4C0700-BBB5-418F-A7B2-F392B9A18263}
{24B8041B-92C6-4BB3-A699-C593AF5A870F} = {24503383-A8C4-4255-9998-28D70FE8E99A}
{3ED53702-0E53-473A-A0F4-645DB33541C2} = {5D4C0700-BBB5-418F-A7B2-F392B9A18263}
{F312FCE1-12D7-4DEF-BC29-2FF6618509F3} = {5D4C0700-BBB5-418F-A7B2-F392B9A18263}
EndGlobalSection
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
// Copyright (c) Microsoft. All rights reserved.

using System.Diagnostics.CodeAnalysis;

// This assembly is currently experimental.
[assembly: Experimental("SKEXP0020")]
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<!-- THIS PROPERTY GROUP MUST COME FIRST -->
<AssemblyName>Microsoft.SemanticKernel.Connectors.SqlServer</AssemblyName>
<RootNamespace>$(AssemblyName)</RootNamespace>
<TargetFramework>netstandard2.0</TargetFramework>
<VersionSuffix>alpha</VersionSuffix>
</PropertyGroup>

<!-- IMPORT NUGET PACKAGE SHARED PROPERTIES -->
<Import Project="$(RepoRoot)/dotnet/nuget/nuget-package.props" />
<Import Project="$(RepoRoot)/dotnet/src/InternalUtilities/src/InternalUtilities.props" />

<PropertyGroup>
<!-- NuGet Package Settings -->
<Title>Semantic Kernel - SQL Server Connector</Title>
<Description>SQL Server connector for Semantic Kernel plugins and semantic memory</Description>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.Data.SqlClient" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\..\SemanticKernel.Core\SemanticKernel.Core.csproj" />
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
// Copyright (c) Microsoft. All rights reserved.

using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;

namespace Microsoft.SemanticKernel.Connectors.SqlServer;

/// <summary>
/// Interface for client managing SQL Server or Azure SQL database operations.
/// </summary>
internal interface ISqlServerClient
{
/// <summary>
/// Create a table.
/// </summary>
/// <param name="tableName">The name assigned to a table of entries.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
Task CreateTableAsync(string tableName, CancellationToken cancellationToken = default);

/// <summary>
/// Get all tables.
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>A group of tables.</returns>
IAsyncEnumerable<string> GetTablesAsync(CancellationToken cancellationToken = default);

/// <summary>
/// Check if a table exists.
/// </summary>
/// <param name="tableName">The name assigned to a table of entries.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
Task<bool> DoesTableExistsAsync(string tableName, CancellationToken cancellationToken = default);

/// <summary>
/// Delete a table.
/// </summary>
/// <param name="tableName">The name assigned to a table of entries.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
Task DeleteTableAsync(string tableName, CancellationToken cancellationToken = default);

/// <summary>
/// Upsert entry into a table.
/// </summary>
/// <param name="tableName">The name assigned to a table of entries.</param>
/// <param name="key">The key of the entry to upsert.</param>
/// <param name="metadata">The metadata of the entry.</param>
/// <param name="embedding">The embedding of the entry.</param>
/// <param name="timestamp">The timestamp of the entry.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
Task UpsertAsync(string tableName, string key, string metadata, ReadOnlyMemory<float> embedding, DateTimeOffset? timestamp, CancellationToken cancellationToken = default);

/// <summary>
/// Read multiple entries by their keys.
/// </summary>
/// <param name="tableName">The name assigned to a table of entries.</param>
/// <param name="keys">The keys of the entries to read.</param>
/// <param name="withEmbeddings">If true, the embeddings will be returned in the entries.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>An asynchronous stream of <see cref="SqlServerMemoryEntry"/> objects that match the given keys.</returns>
IAsyncEnumerable<SqlServerMemoryEntry> ReadBatchAsync(string tableName, IEnumerable<string> keys, bool withEmbeddings = false, CancellationToken cancellationToken = default);

/// <summary>
/// Delete multiple entries by their key.
/// </summary>
/// <param name="tableName">The name assigned to a table of entries.</param>
/// <param name="keys">The keys of the entries to delete.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
Task DeleteBatchAsync(string tableName, IEnumerable<string> keys, CancellationToken cancellationToken = default);

/// <summary>
/// Gets the nearest matches to the embedding.
/// </summary>
/// <param name="tableName">The name assigned to a table of entries.</param>
/// <param name="embedding">The embedding to compare the table's embeddings with.</param>
/// <param name="limit">The maximum number of similarity results to return.</param>
/// <param name="minRelevanceScore">The minimum relevance threshold for returned results.</param>
/// <param name="withEmbeddings">If true, the embeddings will be returned in the entries.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>An asynchronous stream of <see cref="SqlServerMemoryEntry"/> objects that the nearest matches to the embedding.</returns>
IAsyncEnumerable<(SqlServerMemoryEntry, double)> GetNearestMatchesAsync(string tableName, ReadOnlyMemory<float> embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, CancellationToken cancellationToken = default);
}
Loading
Loading