Merge branch 'main' into users/taochen/copilot_chat_deduplicate_docum…

…ent_memory

dotnet/src/Connectors/Connectors.AI.OpenAI/CustomClient/OpenAIClientBase.cs

@@ -27,7 +27,7 @@ namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.CustomClient;
 [SuppressMessage("Design", "CA1054:URI-like parameters should not be strings", Justification = "OpenAI users use strings")]
 public abstract class OpenAIClientBase : IDisposable
 {
-    protected readonly static HttpClientHandler DefaultHttpClientHandler = new() { CheckCertificateRevocationList = true };
+    protected static readonly HttpClientHandler DefaultHttpClientHandler = new() { CheckCertificateRevocationList = true };
 
     /// <summary>
     /// Logger
@@ -53,7 +53,7 @@ internal OpenAIClientBase(HttpClient? httpClient = null, ILogger? logger = null)
             this.HTTPClient = httpClient;
         }
 
-        this.HTTPClient.DefaultRequestHeaders.Add("User-Agent", HTTPUseragent);
+        this.HTTPClient.DefaultRequestHeaders.Add("User-Agent", HTTPUserAgent);
     }
 
     /// <summary>
@@ -186,7 +186,7 @@ protected virtual void Dispose(bool disposing)
     #region private ================================================================================
 
     // HTTP user agent sent to remote endpoints
-    private const string HTTPUseragent = "Microsoft Semantic Kernel";
+    private const string HTTPUserAgent = "Microsoft-Semantic-Kernel";
 
     // Set to true to dispose of HttpClient when disposing. If HttpClient was passed in, then the caller can manage.
     private readonly bool _disposeHttpClient = false;
@@ -198,12 +198,9 @@ private async Task<T> ExecutePostRequestAsync<T>(string url, string requestBody,
         try
         {
             using HttpContent content = new StringContent(requestBody, Encoding.UTF8, "application/json");
-            HttpResponseMessage response = await this.HTTPClient.PostAsync(url, content, cancellationToken).ConfigureAwait(false);
 
-            if (response == null)
-            {
-                throw new AIException(AIException.ErrorCodes.NoResponse);
-            }
+            HttpResponseMessage response = await this.HTTPClient.PostAsync(url, content, cancellationToken).ConfigureAwait(false)
+                ?? throw new AIException(AIException.ErrorCodes.NoResponse);
 
             this.Log.LogTrace("HTTP response: {0} {1}", (int)response.StatusCode, response.StatusCode.ToString("G"));
 

dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/AzureCognitiveSearchMemory.cs

@@ -106,12 +106,22 @@ public AzureCognitiveSearchMemory(string endpoint, TokenCredential credentials)
     {
         collection = NormalizeIndexName(collection);
 
-        var client = await this.GetSearchClientAsync(collection, cancellationToken).ConfigureAwait(false);
+        var client = this.GetSearchClient(collection);
 
-        Response<AzureCognitiveSearchRecord>? result = await client.GetDocumentAsync<AzureCognitiveSearchRecord>(
-            EncodeId(key), cancellationToken: cancellationToken).ConfigureAwait(false);
+        Response<AzureCognitiveSearchRecord>? result;
+        try
+        {
+            result = await client
+                .GetDocumentAsync<AzureCognitiveSearchRecord>(EncodeId(key), cancellationToken: cancellationToken)
+                .ConfigureAwait(false);
+        }
+        catch (RequestFailedException e) when (e.Status == 404)
+        {
+            // Index not found, no data to return
+            return null;
+        }
 
-        if (result == null || result.Value == null)
+        if (result?.Value == null)
         {
             throw new AzureCognitiveSearchMemoryException("Memory read returned null");
         }
@@ -130,25 +140,37 @@ public AzureCognitiveSearchMemory(string endpoint, TokenCredential credentials)
     {
         collection = NormalizeIndexName(collection);
 
-        var client = await this.GetSearchClientAsync(collection, cancellationToken).ConfigureAwait(false);
+        var client = this.GetSearchClient(collection);
 
+        // TODO: use vectors
         var options = new SearchOptions
         {
             QueryType = SearchQueryType.Semantic,
             SemanticConfigurationName = "default",
-            QueryLanguage = "en-us", // TODO: this shouldn't be required
+            QueryLanguage = "en-us",
             Size = limit,
         };
 
-        Response<SearchResults<AzureCognitiveSearchRecord>>? searchResult = await client
-            .SearchAsync<AzureCognitiveSearchRecord>(query, options, cancellationToken: cancellationToken)
-            .ConfigureAwait(false);
+        Response<SearchResults<AzureCognitiveSearchRecord>>? searchResult = null;
+        try
+        {
+            searchResult = await client
+                .SearchAsync<AzureCognitiveSearchRecord>(query, options, cancellationToken: cancellationToken)
+                .ConfigureAwait(false);
+        }
+        catch (RequestFailedException e) when (e.Status == 404)
+        {
+            // Index not found, no data to return
+        }
 
-        await foreach (SearchResult<AzureCognitiveSearchRecord>? doc in searchResult.Value.GetResultsAsync())
+        if (searchResult != null)
         {
-            if (doc.RerankerScore < minRelevanceScore) { break; }
+            await foreach (SearchResult<AzureCognitiveSearchRecord>? doc in searchResult.Value.GetResultsAsync())
+            {
+                if (doc.RerankerScore < minRelevanceScore) { break; }
 
-            yield return new MemoryQueryResult(ToMemoryRecordMetadata(doc.Document), doc.RerankerScore ?? 1, null);
+                yield return new MemoryQueryResult(ToMemoryRecordMetadata(doc.Document), doc.RerankerScore ?? 1, null);
+            }
         }
     }
 
@@ -159,9 +181,15 @@ public async Task RemoveAsync(string collection, string key, CancellationToken c
 
         var records = new List<AzureCognitiveSearchRecord> { new() { Id = EncodeId(key) } };
 
-        var client = await this.GetSearchClientAsync(collection, cancellationToken).ConfigureAwait(false);
-
-        await client.DeleteDocumentsAsync(records, cancellationToken: cancellationToken).ConfigureAwait(false);
+        var client = this.GetSearchClient(collection);
+        try
+        {
+            await client.DeleteDocumentsAsync(records, cancellationToken: cancellationToken).ConfigureAwait(false);
+        }
+        catch (RequestFailedException e) when (e.Status == 404)
+        {
+            // Index not found, no data to delete
+        }
     }
 
     /// <inheritdoc />
@@ -192,30 +220,12 @@ await foreach (var index in indexes)
 
     /// <summary>
     /// Get a search client for the index specified.
+    /// Note: the index might not exist, but we avoid checking everytime and the extra latency.
     /// </summary>
     /// <param name="indexName">Index name</param>
-    /// <param name="cancellationToken">Task cancellation token</param>
     /// <returns>Search client ready to read/write</returns>
-    private async Task<SearchClient> GetSearchClientAsync(
-        string indexName,
-        CancellationToken cancellationToken = default)
+    private SearchClient GetSearchClient(string indexName)
     {
-        Response<SearchIndex>? existingIndex = null;
-        try
-        {
-            // Search the index
-            existingIndex = await this._adminClient.GetIndexAsync(indexName, cancellationToken).ConfigureAwait(false);
-        }
-        catch (RequestFailedException e) when (e.Status == 404)
-        {
-        }
-
-        // Create the index if it doesn't exist
-        if (existingIndex == null || existingIndex.Value == null)
-        {
-            await this.CreateIndexAsync(indexName, cancellationToken).ConfigureAwait(false);
-        }
-
         // Search an available client from the local cache
         if (!this._clientsByIndex.TryGetValue(indexName, out SearchClient client))
         {
@@ -243,6 +253,7 @@ await foreach (var index in indexes)
             {
                 Configurations =
                 {
+                    // TODO: replace with vector search
                     new SemanticConfiguration("default", new PrioritizedFields
                     {
                         TitleField = new SemanticField { FieldName = "Description" },
@@ -264,12 +275,23 @@ await foreach (var index in indexes)
         AzureCognitiveSearchRecord record,
         CancellationToken cancellationToken = default)
     {
-        var client = await this.GetSearchClientAsync(indexName, cancellationToken).ConfigureAwait(false);
+        var client = this.GetSearchClient(indexName);
+
+        Task<Response<IndexDocumentsResult>> UpsertCode() => client
+            .MergeOrUploadDocumentsAsync(new List<AzureCognitiveSearchRecord> { record },
+                new IndexDocumentsOptions { ThrowOnAnyError = true },
+                cancellationToken);
 
-        Response<IndexDocumentsResult>? result = await client.MergeOrUploadDocumentsAsync(
-            new List<AzureCognitiveSearchRecord> { record },
-            new IndexDocumentsOptions { ThrowOnAnyError = true },
-            cancellationToken).ConfigureAwait(false);
+        Response<IndexDocumentsResult>? result;
+        try
+        {
+            result = await UpsertCode().ConfigureAwait(false);
+        }
+        catch (RequestFailedException e) when (e.Status == 404)
+        {
+            await this.CreateIndexAsync(indexName, cancellationToken).ConfigureAwait(false);
+            result = await UpsertCode().ConfigureAwait(false);
+        }
 
         if (result == null || result.Value.Results.Count == 0)
         {

dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/AzureCognitiveSearchRecord.cs

@@ -5,23 +5,51 @@
 
 namespace Microsoft.SemanticKernel.Connectors.Memory.AzureCognitiveSearch;
 
+/// <summary>
+/// Azure Cognitive Search record and index definition.
+/// Note: once defined, index cannot be modified.
+/// </summary>
 public class AzureCognitiveSearchRecord
 {
-    [SimpleField(IsKey = true, IsFilterable = true)]
+    /// <summary>
+    /// Record Id.
+    /// The record is not filterable to save quota, also SK uses only semantic search.
+    /// </summary>
+    [SimpleField(IsKey = true, IsFilterable = false)]
     public string Id { get; set; } = string.Empty;
 
-    [SearchableField(AnalyzerName = LexicalAnalyzerName.Values.EnMicrosoft)]
+    /// <summary>
+    /// Content is stored here.
+    /// </summary>
+    [SearchableField(AnalyzerName = LexicalAnalyzerName.Values.StandardLucene)]
     public string? Text { get; set; } = string.Empty;
 
-    [SearchableField(AnalyzerName = LexicalAnalyzerName.Values.EnMicrosoft)]
+    /// <summary>
+    /// Optional description of the content, e.g. a title. This can be useful when
+    /// indexing external data without pulling in the entire content.
+    /// </summary>
+    [SearchableField(AnalyzerName = LexicalAnalyzerName.Values.StandardLucene)]
     public string? Description { get; set; } = string.Empty;
 
-    [SearchableField(AnalyzerName = LexicalAnalyzerName.Values.EnMicrosoft)]
+    /// <summary>
+    /// Additional metadata. Currently this is a string, where you could store serialized data as JSON.
+    /// In future the design might change to allow storing named values and leverage filters.
+    /// </summary>
+    [SearchableField(AnalyzerName = LexicalAnalyzerName.Values.StandardLucene)]
     public string? AdditionalMetadata { get; set; } = string.Empty;
 
+    /// <summary>
+    /// Name of the external source, in cases where the content and the Id are
+    /// referenced to external information.
+    /// </summary>
     [SimpleField(IsFilterable = false)]
     public string ExternalSourceName { get; set; } = string.Empty;
 
+    /// <summary>
+    /// Whether the record references external information.
+    /// </summary>
     [SimpleField(IsFilterable = false)]
     public bool IsReference { get; set; } = false;
+
+    // TODO: add one more field with the vector, float array, mark it as searchable
 }

dotnet/src/SemanticKernel/Connectors/HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs

@@ -18,7 +18,7 @@ namespace Microsoft.SemanticKernel.Connectors.HuggingFace.TextCompletion;
 /// </summary>
 public sealed class HuggingFaceTextCompletion : ITextCompletion, IDisposable
 {
-    private const string HttpUserAgent = "Microsoft Semantic Kernel";
+    private const string HttpUserAgent = "Microsoft-Semantic-Kernel";
     private const string HuggingFaceApiEndpoint = "https://api-inference.huggingface.co/models";
 
     private readonly string _model;

dotnet/src/Skills/Skills.MsGraph/Connectors/Client/MsGraphClientLoggingHandler.cs

@@ -63,7 +63,7 @@ private void LogHttpMessage(HttpHeaders headers, Uri uri, string prefix)
     {
         if (this._logger.IsEnabled(LogLevel.Debug))
         {
-            StringBuilder message = new StringBuilder();
+            StringBuilder message = new();
             message.AppendLine($"{prefix} {uri}");
             foreach (string headerName in this._headerNamesToLog)
             {

dotnet/src/Skills/Skills.MsGraph/Connectors/MicrosoftToDoConnector.cs

@@ -77,13 +77,18 @@ public async Task<IEnumerable<TaskManagementTaskList>> GetTaskListsAsync(Cancell
     }
 
     /// <inheritdoc/>
-    public async Task<IEnumerable<TaskManagementTask>> GetTasksAsync(string listId, CancellationToken cancellationToken = default)
+    public async Task<IEnumerable<TaskManagementTask>> GetTasksAsync(string listId, bool includeCompleted, CancellationToken cancellationToken = default)
     {
         Ensure.NotNullOrWhitespace(listId, nameof(listId));
 
+        string filterValue = string.Empty;
+        if (!includeCompleted)
+        {
+            filterValue = "status ne 'completed'";
+        }
         ITodoTaskListTasksCollectionPage tasksPage = await this._graphServiceClient.Me
             .Todo.Lists[listId]
-            .Tasks.Request().GetAsync(cancellationToken).ConfigureAwait(false);
+            .Tasks.Request().Filter(filterValue).GetAsync(cancellationToken).ConfigureAwait(false);
 
         List<TodoTask> tasks = tasksPage.ToList();
 

dotnet/src/Skills/Skills.MsGraph/Connectors/OneDriveConnector.cs

@@ -39,18 +39,6 @@ public async Task<Stream> GetFileContentStreamAsync(string filePath, Cancellatio
             .Request().GetAsync(cancellationToken).ConfigureAwait(false);
     }
 
-    /// <exception cref="NotImplementedException">This method is not yet supported for <see cref="OneDriveConnector"/>.</exception>
-    public Task<Stream> GetWriteableFileStreamAsync(string filePath, CancellationToken cancellationToken = default)
-    {
-        throw new NotImplementedException();
-    }
-
-    /// <exception cref="NotImplementedException">This method is not yet supported for <see cref="OneDriveConnector"/>.</exception>
-    public Task<Stream> CreateFileAsync(string filePath, CancellationToken cancellationToken = default)
-    {
-        throw new NotImplementedException();
-    }
-
     public async Task<bool> FileExistsAsync(string filePath, CancellationToken cancellationToken = default)
     {
         Ensure.NotNullOrWhitespace(filePath, nameof(filePath));

dotnet/src/Skills/Skills.MsGraph/ITaskManagementConnector.cs

@@ -46,7 +46,8 @@ public interface ITaskManagementConnector
     /// Get the all tasks in a task list.
     /// </summary>
     /// <param name="listId">ID of the list from which to get the tasks.</param>
+    /// <param name="includeCompleted">Whether to include completed tasks.</param>
     /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
     /// <returns>All of the tasks in the specified task list.</returns>
-    Task<IEnumerable<TaskManagementTask>> GetTasksAsync(string listId, CancellationToken cancellationToken = default);
+    Task<IEnumerable<TaskManagementTask>> GetTasksAsync(string listId, bool includeCompleted, CancellationToken cancellationToken = default);
 }