From 72326813c0b0b193d663df754f444cbf291edba5 Mon Sep 17 00:00:00 2001 From: Nori Zhang Date: Wed, 30 Oct 2024 12:34:09 +0800 Subject: [PATCH 1/3] remove --- shell/agents/Microsoft.Azure.Agent/AzureAgent.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs index 4a8f3cf6..671aa6ea 100644 --- a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs +++ b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs @@ -129,13 +129,16 @@ public void OnUserAction(UserActionPayload actionPayload) { if (action is UserAction.Dislike) { - var dislike = (DislikePayload) actionPayload; isUserFeedback = true; + DislikePayload dislike = (DislikePayload)actionPayload; + allowIdsForCorrelation = dislike.ShareConversation; details = string.Format("{0} | {1}", dislike.ShortFeedback, dislike.LongFeedback); } else if (action is UserAction.Like) { isUserFeedback = true; + LikePayload like = (LikePayload)actionPayload; + allowIdsForCorrelation = like.ShareConversation; } Telemetry.Trace(AzTrace.UserAction(action.ToString(), _copilotResponse, details, isUserFeedback)); From dc5061dc33be6dd63f8b861d1cf1308532617a13 Mon Sep 17 00:00:00 2001 From: Nori Zhang Date: Wed, 30 Oct 2024 12:32:15 +0800 Subject: [PATCH 2/3] remove ids --- .../Microsoft.Azure.Agent/AzureAgent.cs | 10 +++++- .../agents/Microsoft.Azure.Agent/Telemetry.cs | 32 +++++++++++++++---- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs index 671aa6ea..08526ee1 100644 --- a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs +++ b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs @@ -124,6 +124,7 @@ public void Initialize(AgentConfig config) public void OnUserAction(UserActionPayload actionPayload) { // Send telemetry about the user action. bool isUserFeedback = false; + bool allowIdsForCorrelation = false; string details = null; UserAction action = actionPayload.Action; @@ -141,7 +142,14 @@ public void OnUserAction(UserActionPayload actionPayload) { allowIdsForCorrelation = like.ShareConversation; } - Telemetry.Trace(AzTrace.UserAction(action.ToString(), _copilotResponse, details, isUserFeedback)); + if (isUserFeedback) + { + Telemetry.Trace(AzTrace.Feedback(action.ToString(), _copilotResponse, details, allowIdsForCorrelation)); + } + else + { + Telemetry.Trace(AzTrace.UserAction(action.ToString(), _copilotResponse, details)); + } } public async Task RefreshChatAsync(IShell shell, bool force) diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs index 195e8919..66d3d1a4 100644 --- a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs +++ b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs @@ -86,20 +86,40 @@ internal static void Initialize() internal object Details { get; set; } internal static AzTrace UserAction( + string shellCommand, + CopilotResponse response, + object details) + { + if (Telemetry.Enabled) + { + return new() + { + TopicName = response.TopicName, + ShellCommand = shellCommand, + EventType = "UserAction", + Details = details + }; + } + + // Don't create an object when telemetry is disabled. + return null; + } + + internal static AzTrace Feedback( string shellCommand, CopilotResponse response, object details, - bool isFeedback = false) + bool allowIdsForCorrelation) { if (Telemetry.Enabled) { return new() { - QueryId = response.ReplyToId, + QueryId = allowIdsForCorrelation ? response.ReplyToId : null, TopicName = response.TopicName, - ConversationId = response.ConversationId, + ConversationId = allowIdsForCorrelation ? response.ConversationId : null, ShellCommand = shellCommand, - EventType = isFeedback ? "Feedback" : "UserAction", + EventType = "Feedback", Details = details }; } @@ -115,9 +135,7 @@ internal static AzTrace Chat(CopilotResponse response) return new() { EventType = "Chat", - QueryId = response.ReplyToId, - TopicName = response.TopicName, - ConversationId = response.ConversationId + TopicName = response.TopicName }; } From 85242238f593ef8a34913f373fc1d0e11affaaef Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Thu, 31 Oct 2024 11:46:06 -0700 Subject: [PATCH 3/3] Update code and remove ids from exception telemetry --- .../Microsoft.Azure.Agent/AzureAgent.cs | 39 +++++++++++-------- .../Microsoft.Azure.Agent/ChatSession.cs | 1 - .../agents/Microsoft.Azure.Agent/Telemetry.cs | 16 ++++---- 3 files changed, 30 insertions(+), 26 deletions(-) diff --git a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs index 08526ee1..7266b393 100644 --- a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs +++ b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs @@ -121,34 +121,39 @@ public void Initialize(AgentConfig config) public IEnumerable GetCommands() => [new ReplaceCommand(this)]; public bool CanAcceptFeedback(UserAction action) => Telemetry.Enabled; - public void OnUserAction(UserActionPayload actionPayload) { + + public void OnUserAction(UserActionPayload actionPayload) + { // Send telemetry about the user action. bool isUserFeedback = false; - bool allowIdsForCorrelation = false; + bool shareConversation = false; string details = null; - UserAction action = actionPayload.Action; + string action = actionPayload.Action.ToString(); - if (action is UserAction.Dislike) - { - isUserFeedback = true; - DislikePayload dislike = (DislikePayload)actionPayload; - allowIdsForCorrelation = dislike.ShareConversation; - details = string.Format("{0} | {1}", dislike.ShortFeedback, dislike.LongFeedback); - } - else if (action is UserAction.Like) + switch (actionPayload) { - isUserFeedback = true; - LikePayload like = (LikePayload)actionPayload; - allowIdsForCorrelation = like.ShareConversation; + case DislikePayload dislike: + isUserFeedback = true; + shareConversation = dislike.ShareConversation; + details = string.Format("{0} | {1}", dislike.ShortFeedback, dislike.LongFeedback); + break; + + case LikePayload like: + isUserFeedback = true; + shareConversation = like.ShareConversation; + break; + + default: + break; } if (isUserFeedback) { - Telemetry.Trace(AzTrace.Feedback(action.ToString(), _copilotResponse, details, allowIdsForCorrelation)); + Telemetry.Trace(AzTrace.Feedback(action, shareConversation, _copilotResponse, details)); } else { - Telemetry.Trace(AzTrace.UserAction(action.ToString(), _copilotResponse, details)); + Telemetry.Trace(AzTrace.UserAction(action, _copilotResponse, details)); } } @@ -423,7 +428,7 @@ private ResponseData ParseCLIHandlerResponse(IShell shell) { // The placeholder section is not in the format as we've instructed ... Log.Error("Placeholder section not in expected format:\n{0}", text); - Telemetry.Trace(AzTrace.Exception(_copilotResponse, "Placeholder section not in expected format.")); + Telemetry.Trace(AzTrace.Exception("Placeholder section not in expected format.")); } ReplaceKnownPlaceholders(data); diff --git a/shell/agents/Microsoft.Azure.Agent/ChatSession.cs b/shell/agents/Microsoft.Azure.Agent/ChatSession.cs index d27e3015..afe88f15 100644 --- a/shell/agents/Microsoft.Azure.Agent/ChatSession.cs +++ b/shell/agents/Microsoft.Azure.Agent/ChatSession.cs @@ -14,7 +14,6 @@ internal class ChatSession : IDisposable // TODO: production URL not yet working for some regions. // private const string ACCESS_URL = "https://copilotweb.production.portalrp.azure.com/api/access?api-version=2024-09-01"; private const string ACCESS_URL = "https://copilotweb.canary.production.portalrp.azure.com/api/access?api-version=2024-09-01"; - private const string DL_TOKEN_URL = "https://copilotweb.production.portalrp.azure.com/api/conversations/start?api-version=2024-11-15"; private const string CONVERSATION_URL = "https://directline.botframework.com/v3/directline/conversations"; diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs index 66d3d1a4..6eb31c82 100644 --- a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs +++ b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs @@ -94,9 +94,9 @@ internal static AzTrace UserAction( { return new() { - TopicName = response.TopicName, - ShellCommand = shellCommand, EventType = "UserAction", + ShellCommand = shellCommand, + TopicName = response.TopicName, Details = details }; } @@ -107,19 +107,19 @@ internal static AzTrace UserAction( internal static AzTrace Feedback( string shellCommand, + bool shareConversation, CopilotResponse response, - object details, - bool allowIdsForCorrelation) + object details) { if (Telemetry.Enabled) { return new() { - QueryId = allowIdsForCorrelation ? response.ReplyToId : null, - TopicName = response.TopicName, - ConversationId = allowIdsForCorrelation ? response.ConversationId : null, - ShellCommand = shellCommand, EventType = "Feedback", + ShellCommand = shellCommand, + TopicName = response.TopicName, + QueryId = shareConversation ? response.ReplyToId : null, + ConversationId = shareConversation ? response.ConversationId : null, Details = details }; }