-
Notifications
You must be signed in to change notification settings - Fork 639
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Semantic Memory Integration into Copilot Chat (#152)
# BREAKING CHANGE ### Motivation and Context <!-- Thank you for your contribution to the copilot-chat repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> Integration of chat-copilot with https://github.com/microsoft/semantic-memory. With this change, chat-copilot no longer utilizes the https://github.com/microsoft/semantic-kernel connector ### Description <!-- Describe your changes, the overall approach, the underlying design. These notes will help understanding how your code works. Thanks! --> - New "pipeline" service - Memory retrieval updated - DocumentImportController removed - DocumentMemorySkill removed - SematicChatMeorySkill removed - Core OCR moved to memory-service - Orphaned options removed - **Support for Postgres and Chroma storage removed (for now)** - Workflows and deployment scripts updated Examples:       ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [x] The code builds clean without any errors or warnings - [x] The PR follows the [Contribution Guidelines](https://github.com/microsoft/copilot-chat/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/copilot-chat/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone 😄 --------- Co-authored-by: Tao Chen <taochen@microsoft.com> Co-authored-by: Tao Chen <TaoChenOSU@users.noreply.github.com> Co-authored-by: Teresa Hoang <125500434+teresaqhoang@users.noreply.github.com>
- Loading branch information
4 people
committed
Sep 21, 2023
1 parent
c9e585d
commit 2244ab2
Showing
104 changed files
with
4,972 additions
and
2,621 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| name: copilot-build-memorypipeline | ||
|
|
||
| on: | ||
| pull_request: | ||
| branches: ["main"] | ||
| paths: | ||
| - "memorypipeline/**" | ||
| workflow_call: | ||
| outputs: | ||
| artifact: | ||
| description: "The name of the uploaded memory pipeline artifact." | ||
| value: ${{jobs.memory-pipeline.outputs.artifact}} | ||
|
|
||
| permissions: | ||
| contents: read | ||
|
|
||
| jobs: | ||
| memory-pipeline: | ||
| runs-on: windows-latest | ||
|
|
||
| env: | ||
| NUGET_CERT_REVOCATION_MODE: offline | ||
|
|
||
| outputs: | ||
| artifact: ${{steps.artifactoutput.outputs.artifactname}} | ||
|
|
||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| with: | ||
| clean: true | ||
| fetch-depth: 0 | ||
|
|
||
| - name: Install GitVersion | ||
| uses: gittools/actions/gitversion/setup@v0 | ||
| with: | ||
| versionSpec: "5.x" | ||
|
|
||
| - name: Determine version | ||
| id: gitversion | ||
| uses: gittools/actions/gitversion/execute@v0 | ||
|
|
||
| - name: Set version tag | ||
| id: versiontag | ||
| run: | | ||
| $VERSION_TAG = "${{ steps.gitversion.outputs.Major }}." | ||
| $VERSION_TAG += "${{ steps.gitversion.outputs.Minor }}." | ||
| $VERSION_TAG += "${{ steps.gitversion.outputs.CommitsSinceVersionSource }}" | ||
| echo $VERSION_TAG | ||
| Write-Output "versiontag=$VERSION_TAG" >> $env:GITHUB_OUTPUT | ||
| - name: Set .Net Core version | ||
| uses: actions/setup-dotnet@v1 | ||
| with: | ||
| dotnet-version: 6.0.x | ||
|
|
||
| - name: Package Copilot Chat Memory Pipeline | ||
| run: | | ||
| scripts\deploy\package-memorypipeline.ps1 -Configuration Release -DotnetFramework net6.0 -TargetRuntime win-x64 -OutputDirectory ${{ github.workspace }}\scripts\deploy -Version ${{ steps.versiontag.outputs.versiontag }} -InformationalVersion "Built from commit ${{ steps.gitversion.outputs.ShortSha }} on $(Get-Date -Format "yyyy-MM-dd")" | ||
| - name: Upload package to artifacts | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: copilotchat-memorypipeline-${{ steps.versiontag.outputs.versiontag }} | ||
| path: ${{ github.workspace }}\scripts\deploy\out\memorypipeline.zip | ||
|
|
||
| - name: "Set outputs" | ||
| id: artifactoutput | ||
| run: Write-Output "artifactname=copilotchat-memorypipeline-${{ steps.versiontag.outputs.versiontag }}" >> $env:GITHUB_OUTPUT |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| name: copilot-deploy-memorypipeline | ||
|
|
||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| ARTIFACT_NAME: | ||
| required: true | ||
| type: string | ||
| ENVIRONMENT: | ||
| required: true | ||
| type: string | ||
| DEPLOYMENT_NAME: | ||
| required: true | ||
| type: string | ||
| secrets: | ||
| AZURE_CLIENT_ID: | ||
| required: true | ||
| AZURE_TENANT_ID: | ||
| required: true | ||
| AZURE_SUBSCRIPTION_ID: | ||
| required: true | ||
|
|
||
| permissions: | ||
| contents: read | ||
|
|
||
| jobs: | ||
| memorypipeline: | ||
| environment: ${{inputs.ENVIRONMENT}} | ||
| permissions: | ||
| id-token: write | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| include: | ||
| - { dotnet: "6.0", configuration: Release, os: ubuntu-latest } | ||
|
|
||
| runs-on: ${{ matrix.os }} | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| with: | ||
| clean: true | ||
|
|
||
| - uses: actions/download-artifact@v3 | ||
| with: | ||
| name: ${{inputs.ARTIFACT_NAME}} | ||
| path: "${{ github.workspace }}/${{inputs.ARTIFACT_NAME}}" | ||
|
|
||
| - name: "Display downloaded content" | ||
| run: ls -R | ||
| working-directory: "${{ github.workspace }}/${{inputs.ARTIFACT_NAME}}" | ||
|
|
||
| - name: Azure login | ||
| uses: azure/login@v1 | ||
| with: | ||
| client-id: ${{secrets.AZURE_CLIENT_ID}} | ||
| tenant-id: ${{secrets.AZURE_TENANT_ID}} | ||
| subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}} | ||
| enable-AzPSSession: false | ||
|
|
||
| - name: Get app name | ||
| run: | | ||
| WEB_APP_NAME=$(az deployment group show --name ${{inputs.DEPLOYMENT_NAME}} --resource-group ${{vars.CC_DEPLOYMENT_GROUP_NAME}} --output json | jq -r '.properties.outputs.memoryPipelineName.value') | ||
| echo "AZURE_WEBAPP_NAME=$WEB_APP_NAME" >> $GITHUB_ENV | ||
| - name: Enable Run From Package | ||
| uses: azure/CLI@v1 | ||
| with: | ||
| azcliversion: 2.30.0 | ||
| inlineScript: | | ||
| az webapp config appsettings set --resource-group ${{vars.CC_DEPLOYMENT_GROUP_NAME}} --name ${{ env.AZURE_WEBAPP_NAME }} --settings WEBSITE_RUN_FROM_PACKAGE="1" -o none | ||
| - name: "Deploy" | ||
| uses: azure/webapps-deploy@v2 | ||
| with: | ||
| app-name: ${{ env.AZURE_WEBAPP_NAME }} | ||
| package: "${{ github.workspace }}/${{inputs.ARTIFACT_NAME}}/memorypipeline.zip" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk.Web"> | ||
|
|
||
| <PropertyGroup> | ||
| <RootNamespace>CopilotChat.MemoryPipeline</RootNamespace> | ||
| <TargetFramework>net6.0</TargetFramework> | ||
| <RollForward>LatestMajor</RollForward> | ||
| <ImplicitUsings>disable</ImplicitUsings> | ||
| <Nullable>enable</Nullable> | ||
| <UserSecretsId>5ee045b0-aea3-4f08-8d31-32d1a6f8fed0</UserSecretsId> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\shared\CopilotChatShared.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.21.0" /> | ||
| <PackageReference Include="Microsoft.SemanticKernel" Version="0.24.230918.1-preview" /> | ||
| <PackageReference Include="Microsoft.SemanticMemory.Core" Version="0.2.230919.2-preview" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System; | ||
| using CopilotChat.Shared; | ||
| using Microsoft.AspNetCore.Builder; | ||
| using Microsoft.AspNetCore.Http; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
| using Microsoft.Extensions.Logging; | ||
| using Microsoft.SemanticMemory; | ||
| using Microsoft.SemanticMemory.Diagnostics; | ||
|
|
||
| // ******************************************************** | ||
| // ************** SETUP *********************************** | ||
| // ******************************************************** | ||
|
|
||
| var builder = WebApplication.CreateBuilder(); | ||
|
|
||
| ISemanticMemoryClient memory = | ||
| new MemoryClientBuilder(builder.Services) | ||
| .FromAppSettings() | ||
| .WithoutSummarizeHandlers() | ||
| .WithCustomOcr(builder.Configuration) | ||
| .Build(); | ||
|
|
||
| builder.Services.AddSingleton(memory); | ||
|
|
||
| builder.Services.AddApplicationInsightsTelemetry(); | ||
|
|
||
| var app = builder.Build(); | ||
|
|
||
| DateTimeOffset start = DateTimeOffset.UtcNow; | ||
|
|
||
| // Simple ping endpoint | ||
| app.MapGet("/", () => | ||
| { | ||
| var uptime = DateTimeOffset.UtcNow.ToUnixTimeSeconds() - start.ToUnixTimeSeconds(); | ||
| var environment = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"); | ||
| var message = $"Memory pipeline is running. Uptime: {uptime} secs."; | ||
| if (!string.IsNullOrEmpty(environment)) | ||
| { | ||
| message += $" Environment: {environment}"; | ||
| } | ||
| return Results.Ok(message); | ||
| }); | ||
|
|
||
| // ******************************************************** | ||
| // ************** START *********************************** | ||
| // ******************************************************** | ||
|
|
||
| app.Logger.LogInformation( | ||
| "Starting Chat Copilot Memory pipeline service, .NET Env: {0}, Log Level: {1}", | ||
| Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"), | ||
| app.Logger.GetLogLevelName()); | ||
|
|
||
| app.Run(); |
Oops, something went wrong.