diff --git a/dotnet/typeagent/README.md b/dotnet/typeagent/README.md index 9ddb8b4e7..e600564bd 100644 --- a/dotnet/typeagent/README.md +++ b/dotnet/typeagent/README.md @@ -4,6 +4,7 @@ This **sample code** is in early stage experimental developement with **frequent Working towards .NET versions of the following **TypeAgent** packages: * [KnowPro](../../ts/packages/knowPro/README.md) +* [Memory](../../ts/packages/memory/README.md) * [AIClient](../../ts/packages/aiclient/README.md) * [TypeAgent Common Libs](../../ts/packages/typeagent/README.md) @@ -13,11 +14,19 @@ TypeAgent.NET also incorporates [TypeChat.NET](https://github.com/microsoft/type KnowPro.NET will implement Structured RAG in C# for .NET platforms. This work is in curently in progress It will improve on the Typescript implementation in the following ways: +* Storage and indexing using Storage providers * Fully asynchronous * Operators are/will be reworked for more efficient async operation * Asynchronous storage providers with improved Sql schemas * Larger index sizes +Libraries: +* [KnowPro](./src/knowpro): KnowPro Core +* [KnowProStorage](./src/knowproStorage): KnowPro Storage Providers +* [Conversation Memory](./src/conversationMemory): Implementations of memory types using KnowPro.NET +* [Vector](./src/vector) +* [AIClient](./src/aiclient) + ## Trademarks This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft diff --git a/dotnet/typeagent/examples/knowProConsole/Includes.cs b/dotnet/typeagent/examples/knowProConsole/Includes.cs index a5609d72d..c47fa9c0b 100644 --- a/dotnet/typeagent/examples/knowProConsole/Includes.cs +++ b/dotnet/typeagent/examples/knowProConsole/Includes.cs @@ -18,6 +18,7 @@ global using TypeAgent.AIClient; global using TypeAgent.KnowPro; global using TypeAgent.KnowPro.KnowledgeExtractor; +global using TypeAgent.KnowPro.Answer; global using TypeAgent.KnowPro.Storage.Local; global using TypeAgent.KnowPro.Storage.Sqlite; global using TypeAgent.ConversationMemory; diff --git a/dotnet/typeagent/examples/knowProConsole/TestCommands.cs b/dotnet/typeagent/examples/knowProConsole/TestCommands.cs index b9979c9a7..6d0e7f3be 100644 --- a/dotnet/typeagent/examples/knowProConsole/TestCommands.cs +++ b/dotnet/typeagent/examples/knowProConsole/TestCommands.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using TypeAgent.KnowPro.Answer; using TypeAgent.KnowPro.Lang; namespace KnowProConsole; @@ -25,6 +26,7 @@ public IList GetCommands() SearchLangDef(), KnowledgeDef(), BuildIndexDef(), + AnswerDef() ]; } @@ -351,7 +353,6 @@ await KnowProWriter.WriteConversationSearchResultsAsync( } } - private Command KnowledgeDef() { Command cmd = new("kpTestKnowledge") @@ -382,6 +383,45 @@ private async Task KnowledgeAsync(ParseResult args, CancellationToken cancellati } } + private Command AnswerDef() + { + Command cmd = new("kpTestAnswer") + { + Args.Arg("text") + }; + cmd.TreatUnmatchedTokensAsErrors = false; + cmd.SetAction(this.AnswerAsync); + return cmd; + } + + private async Task AnswerAsync(ParseResult args, CancellationToken cancellationToken) + { + IConversation conversation = EnsureConversation(); + + NamedArgs namedArgs = new NamedArgs(args); + AnswerContext context = new AnswerContext(); + + List entities = await conversation.SemanticRefs.SelectAsync( + (sr) => sr.KnowledgeType == KnowledgeType.Entity ? sr.AsEntity() : null, + cancellationToken + ); + entities = [.. entities.ToDistinct()]; + + List topics = await conversation.SemanticRefs.SelectAsync( + (sr) => sr.KnowledgeType == KnowledgeType.Topic ? sr.AsTopic() : null, + cancellationToken + ); + topics = [.. topics.ToDistinct()]; + + context.Entities = entities.Map((e) => new RelevantEntity { Entity = e }); + context.Topics = topics.Map((t) => new RelevantTopic { Topic = t }); + + List messages = await conversation.Messages.GetAllAsync(cancellationToken); + context.Messages = messages.Map((m) => new RelevantMessage(m)); + string prompt = context.ToPromptString(); + ConsoleWriter.WriteLine(prompt); + } + private IConversation EnsureConversation() { return (_kpContext.Conversation is not null) diff --git a/dotnet/typeagent/src/common/DateTimeExtensions.cs b/dotnet/typeagent/src/common/DateTimeExtensions.cs index 703e978da..9b94d7c43 100644 --- a/dotnet/typeagent/src/common/DateTimeExtensions.cs +++ b/dotnet/typeagent/src/common/DateTimeExtensions.cs @@ -7,6 +7,6 @@ public static class DateTimeExtensions { public static string ToISOString(this DateTimeOffset dt) { - return dt.ToString("o", System.Globalization.CultureInfo.InvariantCulture); + return dt.ToString("o"); } } diff --git a/dotnet/typeagent/src/common/EnumerationExtensions.cs b/dotnet/typeagent/src/common/EnumerationExtensions.cs index 401db42c5..6b64f14dd 100644 --- a/dotnet/typeagent/src/common/EnumerationExtensions.cs +++ b/dotnet/typeagent/src/common/EnumerationExtensions.cs @@ -87,4 +87,52 @@ public static List> GetTopK(this IEnumerable> items, int topNList.Add(items); return topNList.ByRankAndClear(); } + + public static async ValueTask> ToListAsync( + this IAsyncEnumerable source, + CancellationToken cancellationToken = default) + { + List list = []; + await foreach (var item in source.WithCancellation(cancellationToken)) + { + list.Add(item); + } + return list; + } + + public static async ValueTask> SelectAsync( + this IAsyncEnumerable source, + Func selector, + CancellationToken cancellationToken = default) + { + ArgumentVerify.ThrowIfNull(selector, nameof(selector)); + + List list = []; + await foreach (var item in source.WithCancellation(cancellationToken)) + { + TSelect? selected = selector(item); + if (selected is not null) + { + list.Add(selected); + } + } + return list; + } + + public static async Task> WhereAsync( + this IAsyncEnumerable source, + Func predicate, + CancellationToken cancellationToken = default) + { + ArgumentVerify.ThrowIfNull(predicate, nameof(predicate)); + var list = new List(); + await foreach (var item in source.WithCancellation(cancellationToken)) + { + if (predicate(item)) + { + list.Add(item); + } + } + return list; + } } diff --git a/dotnet/typeagent/src/common/IsoDateJsonConvertor.cs b/dotnet/typeagent/src/common/IsoDateJsonConvertor.cs new file mode 100644 index 000000000..21135c31a --- /dev/null +++ b/dotnet/typeagent/src/common/IsoDateJsonConvertor.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Globalization; + +namespace TypeAgent.Common; + +/// +/// Forces DateTimeOffset serialization/deserialization to a stable ISO 8601 (round‑trip) string ("o"). +/// Example: 2025-11-05T14:23:17.1234567+00:00 +/// +public class IsoDateJsonConverter : JsonConverter +{ + public override DateTimeOffset Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.String) + { + string? s = reader.GetString(); + if (!string.IsNullOrEmpty(s)) + { + if (DateTimeOffset.TryParse(s, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var value)) + { + return value; + } + } + throw new JsonException($"Invalid DateTimeOffset value: '{s}'."); + } + throw new JsonException("Invalid DateTimeOffset value"); + } + + public override void Write(Utf8JsonWriter writer, DateTimeOffset value, JsonSerializerOptions options) + { + writer.WriteStringValue(value.ToISOString()); + } +} diff --git a/dotnet/typeagent/src/common/OneOrMany.cs b/dotnet/typeagent/src/common/OneOrMany.cs new file mode 100644 index 000000000..ac75c5652 --- /dev/null +++ b/dotnet/typeagent/src/common/OneOrMany.cs @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace TypeAgent.Common; + +public abstract class OneOrManyItem +{ + [JsonIgnore] + public abstract bool IsSingle { get; } + + public static OneOrManyItem? Create(T? value) + { + return value is not null ? new SingleItem(value) : null; + } + + public static OneOrManyItem? Create(IList? value) + { + return value.IsNullOrEmpty() + ? null + : value.Count == 1 + ? new SingleItem(value[0]) + : new ListItem(value); + } +} + +[JsonConverter(typeof(OneOrManyJsonConverterFactory))] +public abstract class OneOrManyItem : OneOrManyItem +{ +} + +public class SingleItem : OneOrManyItem +{ + public SingleItem() { } + + public SingleItem(T value) + { + Value = value; + } + + [JsonIgnore] + public override bool IsSingle => true; + + public T Value { get; set; } + + public static implicit operator T(SingleItem item) + { + return item.Value; + } +} + +public class ListItem : OneOrManyItem +{ + public ListItem() + { + + } + + public ListItem(IList value) + { + Value = value; + } + + [JsonIgnore] + public override bool IsSingle => false; + + public IList Value { get; set; } +} + +public class OneOrManyJsonConverterFactory : JsonConverterFactory +{ + public override bool CanConvert(Type typeToConvert) => true; + + public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options) + { + var elementType = typeToConvert.GetGenericArguments()[0]; + var converterType = typeof(OneOrManyJsonConverter<>).MakeGenericType(elementType); + return (JsonConverter)Activator.CreateInstance(converterType)!; + } +} + +public class OneOrManyJsonConverter : JsonConverter> +{ + public override OneOrManyItem? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + { + return null; + } + + if (reader.TokenType == JsonTokenType.StartArray) + { + var list = JsonSerializer.Deserialize>(ref reader, options); + return new ListItem() + { + Value = list + }; + } + + T value = JsonSerializer.Deserialize(ref reader, options); + return new SingleItem + { + Value = value + }; + } + + public override void Write(Utf8JsonWriter writer, OneOrManyItem value, JsonSerializerOptions options) + { + if (value is ListItem list) + { + JsonSerializer.Serialize(writer, list.Value, options); + } + else if (value is SingleItem item) + { + JsonSerializer.Serialize(writer, item.Value, options); + } + } +} diff --git a/dotnet/typeagent/src/common/StringExtensions.cs b/dotnet/typeagent/src/common/StringExtensions.cs index fbaaf60e4..f0eeb00bc 100644 --- a/dotnet/typeagent/src/common/StringExtensions.cs +++ b/dotnet/typeagent/src/common/StringExtensions.cs @@ -6,7 +6,7 @@ namespace TypeAgent.Common; -public static partial class StringExtensions +public static partial class StringExtensions { /// /// Splits an enumerable of strings into chunks, each chunk containing up to maxChunkLength strings and diff --git a/dotnet/typeagent/src/knowpro/Answer/AnswerContexSchema.cs b/dotnet/typeagent/src/knowpro/Answer/AnswerContexSchema.cs new file mode 100644 index 000000000..d7f477908 --- /dev/null +++ b/dotnet/typeagent/src/knowpro/Answer/AnswerContexSchema.cs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace TypeAgent.KnowPro.Answer; + +public class RelevantKnowledge +{ + // Entity or entities who mentioned the knowledge + [JsonPropertyName("origin")] + public OneOrManyItem? Origin { get; set; } + + // Entity or entities who received or consumed this knowledge + [JsonPropertyName("audience")] + public OneOrManyItem? Audience { get; set; } + + // Time period during which this knowledge was gathered + [JsonPropertyName("timeRange")] + public DateRange? TimeRange { get; set; } +}; + +public class RelevantTopic : RelevantKnowledge +{ + [JsonPropertyName("knowledge")] + public string? Topic { get; set; } +} + +public class RelevantEntity : RelevantKnowledge +{ + [JsonPropertyName("knowledge")] + public ConcreteEntity? Entity { get; set; } +} + +public partial class RelevantMessage +{ + [JsonPropertyName("from")] + public OneOrManyItem? From { get; set; } + + [JsonPropertyName("to")] + public OneOrManyItem? To { get; set; } + + [JsonPropertyName("timestamp")] + public string? Timestamp { get; set; } + + [JsonPropertyName("messageText")] + public OneOrManyItem? MessageText { get; set; } +} + +public partial class AnswerContext +{ + // Relevant entities + // Use the 'name' and 'type' properties of entities to PRECISELY identify those that answer the user question. + public IList? Entities { get; set; } + + // Relevant topics + public IList Topics { get; set; } + + // Relevant messages + public IList? Messages { get; set; } +}; diff --git a/dotnet/typeagent/src/knowpro/Answer/AnswerContextSchemaImpl.cs b/dotnet/typeagent/src/knowpro/Answer/AnswerContextSchemaImpl.cs new file mode 100644 index 000000000..0f484471d --- /dev/null +++ b/dotnet/typeagent/src/knowpro/Answer/AnswerContextSchemaImpl.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace TypeAgent.KnowPro.Answer; + +public partial class RelevantMessage +{ + public RelevantMessage() { } + + public RelevantMessage(IMessage message) + { + ArgumentVerify.ThrowIfNull(message, nameof(message)); + + To = OneOrManyItem.Create(message.Metadata?.Dest); + From = OneOrManyItem.Create(message.Metadata?.Source); + Timestamp = message.Timestamp; + MessageText = OneOrManyItem.Create(message.TextChunks); + } +} + +public partial class AnswerContext +{ + public string ToPromptString() + { + var json = new StringBuilder(); + json.Append("{\n"); + + int propertyCount = 0; + if (!Entities.IsNullOrEmpty()) + { + propertyCount = AddPrompt(json, propertyCount, "entities", Entities); + } + if (!Topics.IsNullOrEmpty()) + { + propertyCount = AddPrompt(json, propertyCount, "topics", Topics); + } + if (!Messages.IsNullOrEmpty()) + { + propertyCount = AddPrompt(json, propertyCount, "messages", Messages); + } + json.Append("\n}"); + + return json.ToString(); + } + + private int AddPrompt(StringBuilder text, int propertyCount, string name, object value) + { + if (propertyCount > 0) + { + text.Append(",\n"); + } + var json = Serializer.ToJson(value); + text.Append($"{name}: {json}"); + return propertyCount + 1; + } +} diff --git a/dotnet/typeagent/src/knowpro/Answer/AnswerGenerator.cs b/dotnet/typeagent/src/knowpro/Answer/AnswerGenerator.cs new file mode 100644 index 000000000..52d649668 --- /dev/null +++ b/dotnet/typeagent/src/knowpro/Answer/AnswerGenerator.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + + +namespace TypeAgent.KnowPro.Answer; + +public class AnswerGenerator : IAnswerGenerator +{ + public AnswerGenerator(AnswerGeneratorSettings settings) + { + ArgumentVerify.ThrowIfNull(settings, nameof(settings)); + Settings = settings; + } + + public Task GenerateAsync(string question, AnswerContext context, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public AnswerGeneratorSettings Settings { get; } + + public Task CombinePartialAsync(string question, IList responses, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } +} diff --git a/dotnet/typeagent/src/knowpro/Answer/AnswerGeneratorSettings.cs b/dotnet/typeagent/src/knowpro/Answer/AnswerGeneratorSettings.cs new file mode 100644 index 000000000..5b461ed21 --- /dev/null +++ b/dotnet/typeagent/src/knowpro/Answer/AnswerGeneratorSettings.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace TypeAgent.KnowPro.Answer; + +/// +/// Settings for answer generation (C# translation of the TypeScript AnswerGeneratorSettings). +/// +public sealed class AnswerGeneratorSettings +{ + public AnswerGeneratorSettings(IChatModel model) + : this(model, model) + { + } + + public AnswerGeneratorSettings(IChatModel generatorModel, IChatModel combinerModel) + { + ArgumentVerify.ThrowIfNull(generatorModel, nameof(generatorModel)); + ArgumentVerify.ThrowIfNull(combinerModel, nameof(combinerModel)); + + GeneratorModel = generatorModel; + CombinerModel = combinerModel; + } + + /// + /// Model used to generate answers from context. + /// + public IChatModel GeneratorModel { get; } + + /// + /// Model used to combine multiple partial answers (e.g. rewriting / merging). + /// Defaults to if not explicitly supplied. + /// + public IChatModel CombinerModel { get; } + + /// + /// Maximum number of characters allowed in the context for any given call. + /// (Default mirrors TS: 4096 tokens * ~4 chars per token). + /// + public int MaxCharsInBudget { get; set; } = 4096 * 4; + + /// + /// When chunking, number of chunks processed in parallel. + /// + public int Concurrency { get; set; } = 2; + + /// + /// Stop processing early if an answer is already found using just knowledge chunks. + /// + public bool FastStop { get; set; } = true; + + /// + /// Additional instructions (prompt sections) prepended when invoking the model. + /// + public IList? ModelInstructions { get; set; } +} diff --git a/dotnet/typeagent/src/knowpro/Answer/AnswerResponseSchema.ts b/dotnet/typeagent/src/knowpro/Answer/AnswerResponseSchema.ts new file mode 100644 index 000000000..322e88c25 --- /dev/null +++ b/dotnet/typeagent/src/knowpro/Answer/AnswerResponseSchema.ts @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export type AnswerType = + | "NoAnswer" // If question cannot be accurately answered from [ANSWER CONTEXT] + | "Answered"; // Fully answer question + +export type AnswerResponse = { + // use "NoAnswer" if no highly relevant answer found in the [ANSWER CONTEXT] + type: AnswerType; + // the answer to display if [ANSWER CONTEXT] is highly relevant and can be used to answer the user's question + answer?: string | undefined; + // If NoAnswer, explain why.. + // particularly explain why you didn't use any supplied entities + whyNoAnswer?: string; +}; diff --git a/dotnet/typeagent/src/knowpro/Answer/AnwerResponseSchema.cs b/dotnet/typeagent/src/knowpro/Answer/AnwerResponseSchema.cs new file mode 100644 index 000000000..fc1e711a7 --- /dev/null +++ b/dotnet/typeagent/src/knowpro/Answer/AnwerResponseSchema.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; +using Microsoft.TypeChat.Schema; + +namespace TypeAgent.KnowPro.Answer; + +/// +/// Type of answer produced by the generator. +/// +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum AnswerType +{ + [Comment("If question cannot be accurately answered from [ANSWER CONTEXT]")] + NoAnswer, + + [Comment("Fully answer question")] + Answered, +} + +public sealed class AnswerResponse +{ + [JsonPropertyName("type")] + [Comment("use \"NoAnswer\" if no highly relevant answer found in the [ANSWER CONTEXT]")] + public AnswerType Type { get; set; } + + [JsonPropertyName("answer")] + [Comment("the answer to display if [ANSWER CONTEXT] is highly relevant and can be used to answer the user's question")] + public string? Answer { get; set; } + + [JsonPropertyName("whyNoAnswer")] + [Comment("If NoAnswer, explain why..")] + [Comment("particularly explain why you didn't use any supplied entities")] + public string? WhyNoAnswer { get; set; } +} diff --git a/dotnet/typeagent/src/knowpro/Answer/IAnswerGenerator.cs b/dotnet/typeagent/src/knowpro/Answer/IAnswerGenerator.cs new file mode 100644 index 000000000..7199a78b1 --- /dev/null +++ b/dotnet/typeagent/src/knowpro/Answer/IAnswerGenerator.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace TypeAgent.KnowPro.Answer; + +public interface IAnswerGenerator +{ + AnswerGeneratorSettings Settings { get; } + + Task GenerateAsync( + string question, + AnswerContext context, + CancellationToken cancellationToken = default + ); + + Task CombinePartialAsync( + string question, + IList responses, + CancellationToken cancellationToken = default + ); +} diff --git a/dotnet/typeagent/src/knowpro/IMessage.cs b/dotnet/typeagent/src/knowpro/IMessage.cs index 32d54e749..28825030b 100644 --- a/dotnet/typeagent/src/knowpro/IMessage.cs +++ b/dotnet/typeagent/src/knowpro/IMessage.cs @@ -3,7 +3,6 @@ namespace TypeAgent.KnowPro; - public interface IMessage : IKnowledgeSource { IList TextChunks { get; set; } @@ -31,3 +30,33 @@ public interface IMessageEx : IMessage string? SerializeExtraDataToJson(); void DeserializeExtraDataFromJson(string json); } + +public static class MessageExtensions +{ + /** + * Get the total number of a characters in a message. + * A message can contain multiple text chunks + * @param {IMessage} message + * @returns + */ + public static int GetCharCount(this IMessage message) + { + int total = 0; + int count = message.TextChunks.Count; + for (int i = 0; i < count; ++i) + { + total += message.TextChunks[i].Length; + } + return total; + } + + public static List ToMessageOrdinals(this IList scoredOrdinals) + { + return scoredOrdinals.Map((s) => s.MessageOrdinal); + } + + public static IEnumerable AsMessageOrdinals(this IEnumerable scoredOrdinals) + { + return scoredOrdinals.Select((s) => s.MessageOrdinal); + } +} diff --git a/dotnet/typeagent/src/knowpro/IMessageCollection.cs b/dotnet/typeagent/src/knowpro/IMessageCollection.cs index 9d508dfcc..e68796431 100644 --- a/dotnet/typeagent/src/knowpro/IMessageCollection.cs +++ b/dotnet/typeagent/src/knowpro/IMessageCollection.cs @@ -22,6 +22,22 @@ public interface IMessageCollection : IReadOnlyAsyncCollection public static class MessageCollectionExtensions { + public static ValueTask> GetAllAsync( + this IMessageCollection messages, + CancellationToken cancellationToken + ) + where TMessage : IMessage + { + return messages.ToListAsync(cancellationToken); + } + public static ValueTask> GetAllAsync( + this IMessageCollection messages, + CancellationToken cancellationToken + ) + { + return messages.ToListAsync(cancellationToken); + } + internal static async ValueTask GetCountInCharBudgetAsync( this IMessageCollection messages, IList messageOrdinals, diff --git a/dotnet/typeagent/src/knowpro/ISemanticRefCollection.cs b/dotnet/typeagent/src/knowpro/ISemanticRefCollection.cs index ed033a2c3..d84ce7f74 100644 --- a/dotnet/typeagent/src/knowpro/ISemanticRefCollection.cs +++ b/dotnet/typeagent/src/knowpro/ISemanticRefCollection.cs @@ -11,6 +11,10 @@ public interface ISemanticRefCollection : IAsyncCollection ValueTask GetKnowledgeTypeAsync(int ordinal, CancellationToken cancellation = default); ValueTask> GetKnowledgeTypeAsync(IList ordinal, CancellationToken cancellation = default); + // TODO + // Add methods to enumerate by knowledge Type, casting appropriately. + // More efficient than looping over all + event Action OnKnowledgeExtracted; void NotifyKnowledgeProgress(BatchProgress progress); @@ -18,6 +22,14 @@ public interface ISemanticRefCollection : IAsyncCollection public static class SemanticRefCollectionExtensions { + public static ValueTask> GetAllAsync( + this ISemanticRefCollection semanticRefs, + CancellationToken cancellationToken + ) + { + return semanticRefs.ToListAsync(cancellationToken); + } + // // These methods use IAsyncCollectionReader because then they also work // with Caches...see ConversationCache.cs @@ -68,7 +80,7 @@ public static async ValueTask>> GetDistinctEntities semanticRefMatches ).ConfigureAwait(false); - Dictionary> mergedEntities = MergedEntity.MergeScoredEntities(scoredEntities, false); + Dictionary> mergedEntities = MergedEntity.MergeScored(scoredEntities, false); IEnumerable> entitites = mergedEntities.Values.Select((v) => { return new Scored(v.Item.ToConcrete(), v.Score); diff --git a/dotnet/typeagent/src/knowpro/KnowledgeMerge.cs b/dotnet/typeagent/src/knowpro/KnowledgeMerge.cs index 31372b118..25298d86e 100644 --- a/dotnet/typeagent/src/knowpro/KnowledgeMerge.cs +++ b/dotnet/typeagent/src/knowpro/KnowledgeMerge.cs @@ -21,10 +21,7 @@ public void MergeMessageOrdinals(SemanticRef sr) internal class MergedEntity : MergedKnowledge { - public MergedEntity() - { - - } + public MergedEntity() { } public string Name { get; set; } @@ -58,14 +55,34 @@ public static bool Union(MergedEntity to, MergedEntity other) return false; } - public static Dictionary> MergeScoredEntities( - IEnumerable> scoredEntities, + public static IEnumerable Merge(IEnumerable entities) + { + Dictionary mergedEntities = []; + + foreach (var entity in entities) + { + MergedEntity mergedEntity = entity.ToMerged(); + if (mergedEntities.TryGetValue(mergedEntity.Name, out var existing)) + { + Union(existing, mergedEntity); + } + else + { + mergedEntities.Add(mergedEntity.Name, mergedEntity); + } + } + + return mergedEntities.Values.Select((m) => m.ToConcrete()); + } + + public static Dictionary> MergeScored( + IEnumerable> semanticRefs, bool mergeOrdinals ) { Dictionary> mergedEntities = []; - foreach (var scoredEntity in scoredEntities) + foreach (var scoredEntity in semanticRefs) { if (scoredEntity.Item.KnowledgeType != KnowledgeType.Entity) { @@ -107,7 +124,60 @@ bool mergeOrdinals internal class MergedTopic : MergedKnowledge { - public Topic Topic { get; set; } + public string Topic { get; set; } + + public Topic ToTopic() => new Topic(Topic); + + public static IEnumerable Merge(IEnumerable topics) + { + ArgumentVerify.ThrowIfNull(topics, nameof(topics)); + + Dictionary distinct = new Dictionary(); + foreach (Topic topic in topics) + { + distinct.TryAdd(topic.Text.ToLower(), topic); + } + return distinct.Values; + } + + public static Dictionary> Merge( + IEnumerable> semanticRefs, + bool mergeOrdinals + ) + { + Dictionary> mergedTopics = []; + + foreach (var scoredTopic in semanticRefs) + { + if (scoredTopic.Item.KnowledgeType != KnowledgeType.Topic) + { + continue; + } + + MergedTopic mergedTopic = scoredTopic.Item.AsTopic().ToMerged(); + Scored? target = null; + if (mergedTopics.TryGetValue(mergedTopic.Topic, out var existing)) + { + if (existing.Score < scoredTopic.Score) + { + existing.Score = scoredTopic.Score; + } + } + else + { + var newMerged = new Scored(mergedTopic, scoredTopic.Score); + mergedTopics.Add(mergedTopic.Topic, newMerged); + target = newMerged; + } + if (target is not null && mergeOrdinals) + { + target.Value.Item.MergeMessageOrdinals(scoredTopic); + } + } + + return mergedTopics; + } + } internal class MergedFacets : Multiset diff --git a/dotnet/typeagent/src/knowpro/Knowledge.cs b/dotnet/typeagent/src/knowpro/KnowledgeSchema.cs similarity index 100% rename from dotnet/typeagent/src/knowpro/Knowledge.cs rename to dotnet/typeagent/src/knowpro/KnowledgeSchema.cs diff --git a/dotnet/typeagent/src/knowpro/KnowledgeImpl.cs b/dotnet/typeagent/src/knowpro/KnowledgeSchemaImpl.cs similarity index 90% rename from dotnet/typeagent/src/knowpro/KnowledgeImpl.cs rename to dotnet/typeagent/src/knowpro/KnowledgeSchemaImpl.cs index 845a8c31f..061325129 100644 --- a/dotnet/typeagent/src/knowpro/KnowledgeImpl.cs +++ b/dotnet/typeagent/src/knowpro/KnowledgeSchemaImpl.cs @@ -17,6 +17,7 @@ public ConcreteEntity(string name, string type) this.Type = [type]; } + [JsonIgnore] public override KnowledgeType KnowledgeType => KnowledgeType.Entity; [JsonIgnore] @@ -90,6 +91,7 @@ public Action() IndirectObjectEntityName = NoneEntityName; } + [JsonIgnore] public override KnowledgeType KnowledgeType => KnowledgeType.Action; [JsonIgnore] @@ -155,8 +157,17 @@ public Topic(string text) Text = text; } + [JsonIgnore] public override KnowledgeType KnowledgeType => KnowledgeType.Topic; + internal MergedTopic ToMerged() + { + return new MergedTopic + { + Topic = this.Text + }; + } + public static implicit operator string(Topic topic) { return topic.Text; @@ -165,6 +176,7 @@ public static implicit operator string(Topic topic) public partial class Tag { + [JsonIgnore] public override KnowledgeType KnowledgeType => KnowledgeType.Tag; public static implicit operator string(Tag tag) @@ -175,6 +187,7 @@ public static implicit operator string(Tag tag) public partial class StructuredTag { + [JsonIgnore] public override KnowledgeType KnowledgeType => KnowledgeType.STag; } @@ -238,3 +251,16 @@ internal void MergeActionKnowledge() } } } + +public static class KnowledgeExtensions +{ + public static IEnumerable ToDistinct(this IEnumerable entities) + { + return MergedEntity.Merge(entities); + } + + public static IEnumerable ToDistinct(this IEnumerable topics) + { + return MergedTopic.Merge(topics); + } +} diff --git a/dotnet/typeagent/src/knowpro/MessageExtensions.cs b/dotnet/typeagent/src/knowpro/MessageExtensions.cs deleted file mode 100644 index e1bd5991c..000000000 --- a/dotnet/typeagent/src/knowpro/MessageExtensions.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace TypeAgent.KnowPro; - -public static class MessageExtensions -{ - /** - * Get the total number of a characters in a message. - * A message can contain multiple text chunks - * @param {IMessage} message - * @returns - */ - public static int GetCharCount(this IMessage message) - { - int total = 0; - int count = message.TextChunks.Count; - for (int i = 0; i < count; ++i) - { - total += message.TextChunks[i].Length; - } - return total; - } - - public static List ToMessageOrdinals(this IList scoredOrdinals) - { - return scoredOrdinals.Map((s) => s.MessageOrdinal); - } - - public static IEnumerable AsMessageOrdinals(this IEnumerable scoredOrdinals) - { - return scoredOrdinals.Select((s) => s.MessageOrdinal); - } -} diff --git a/dotnet/typeagent/src/knowpro/Serializer.cs b/dotnet/typeagent/src/knowpro/Serializer.cs index eabd551d7..f91cf36fa 100644 --- a/dotnet/typeagent/src/knowpro/Serializer.cs +++ b/dotnet/typeagent/src/knowpro/Serializer.cs @@ -10,20 +10,26 @@ public class Serializer static Serializer() { + var enumConvertor = new JsonStringEnumConverter(); + var dateConvertor = new IsoDateJsonConverter(); var facetConvertor = new FacetValueJsonConverter(); var actionParamConvertor = new ActionParamJsonConverter(); - var enumConvertor = new JsonStringEnumConverter(); - + var oneOrManyConvertor = new OneOrManyJsonConverter(); s_options = Json.DefaultOptions(); + s_options.Converters.Add(enumConvertor); + s_options.Converters.Add(dateConvertor); s_options.Converters.Add(facetConvertor); s_options.Converters.Add(actionParamConvertor); - s_options.Converters.Add(enumConvertor); + s_options.Converters.Add(oneOrManyConvertor); s_optionsIndent = Json.DefaultOptions(); + s_optionsIndent.Converters.Add(enumConvertor); + s_optionsIndent.Converters.Add(dateConvertor); s_optionsIndent.Converters.Add(facetConvertor); s_optionsIndent.Converters.Add(actionParamConvertor); - s_optionsIndent.Converters.Add(enumConvertor); + s_optionsIndent.Converters.Add(oneOrManyConvertor); + s_optionsIndent.WriteIndented = true; } diff --git a/dotnet/typeagent/src/knowpro/knowpro.csproj b/dotnet/typeagent/src/knowpro/knowpro.csproj index d2644c28c..89a1fba85 100644 --- a/dotnet/typeagent/src/knowpro/knowpro.csproj +++ b/dotnet/typeagent/src/knowpro/knowpro.csproj @@ -9,6 +9,7 @@ + @@ -16,6 +17,7 @@ + @@ -30,8 +32,4 @@ - - - -