From a49b75f4953376dade9243110fc0a24df236ab4d Mon Sep 17 00:00:00 2001
From: Haiping Chen <hchen@lessen.com>
Date: Fri, 14 Mar 2025 13:59:13 -0500
Subject: [PATCH 1/3] Add Twilio settings.
---
.../Conversations/ConversationHookBase.cs | 3 --
.../Conversations/IConversationHook.cs | 7 ---
.../BotSharp.Core.Realtime.csproj | 6 ++-
.../Services/RealtimeHub.cs | 8 +---
.../Controllers/TwilioStreamController.cs | 22 +--------
.../Controllers/TwilioVoiceController.cs | 48 ++++++++++++++-----
.../Interfaces/ITwilioCallStatusHook.cs | 4 +-
.../Models/AssistantMessage.cs | 2 +-
.../Functions/OutboundPhoneCallFn.cs | 47 ++++++++++++++----
.../Services/TwilioService.cs | 15 +++++-
.../Settings/TwilioSetting.cs | 28 +++++++++--
11 files changed, 125 insertions(+), 65 deletions(-)
diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs
index a84c8693a..da87646e8 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs
@@ -78,7 +78,4 @@ public virtual Task OnBreakpointUpdated(string conversationId, bool resetStates)
public virtual Task OnNotificationGenerated(RoleDialogModel message)
=> Task.CompletedTask;
-
- public virtual Task OnUserDisconnected(Conversation conversation)
- => Task.CompletedTask;
}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs
index f99078eac..c764f3910 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs
@@ -25,13 +25,6 @@ public interface IConversationHook
/// <returns></returns>
Task OnUserAgentConnectedInitially(Conversation conversation);
- /// <summary>
- /// Triggered when user disconnects with agent.
- /// </summary>
- /// <param name="conversation"></param>
- /// <returns></returns>
- Task OnUserDisconnected(Conversation conversation);
-
/// <summary>
/// Triggered once for every new conversation.
/// </summary>
diff --git a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj
index c004dc503..25218b089 100644
--- a/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj
+++ b/src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj
@@ -1,7 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
- <TargetFramework>net8.0</TargetFramework>
+ <TargetFramework>$(TargetFramework)</TargetFramework>
+ <LangVersion>$(LangVersion)</LangVersion>
+ <VersionPrefix>$(BotSharpVersion)</VersionPrefix>
+ <GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
+ <OutputPath>$(SolutionDir)packages</OutputPath>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
index 763022627..8adf63f00 100644
--- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
+++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
@@ -1,7 +1,3 @@
-using BotSharp.Abstraction.Utilities;
-using BotSharp.Core.Infrastructures;
-using Microsoft.AspNetCore.Cors.Infrastructure;
-
namespace BotSharp.Core.Realtime.Services;
public class RealtimeHub : IRealtimeHub
@@ -257,9 +253,7 @@ private async Task HandleUserDtmfReceived()
private async Task HandleUserDisconnected()
{
- var convService = _services.GetRequiredService<IConversationService>();
- var conversation = await convService.GetConversation(_conn.ConversationId);
- await HookEmitter.Emit<IConversationHook>(_services, x => x.OnUserDisconnected(conversation));
+
}
private async Task SendEventToUser(WebSocket webSocket, object message)
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs
index e69f79f62..fb35c25ce 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs
@@ -42,7 +42,7 @@ public async Task<TwiMLResult> InitiateStreamConversation(ConversationalVoiceReq
request.InitAudioFile != null)
{
response = new VoiceResponse();
- response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}"));
+ response.Play(new Uri(request.InitAudioFile));
return TwiML(response);
}
@@ -54,7 +54,7 @@ public async Task<TwiMLResult> InitiateStreamConversation(ConversationalVoiceReq
if (request.InitAudioFile != null)
{
- instruction.SpeechPaths.Add(request.InitAudioFile);
+ instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}");
}
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
@@ -82,24 +82,6 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
return TwiML(response);
}
- [ValidateRequest]
- [HttpPost("twilio/stream/status")]
- public async Task<ActionResult> StreamConversationStatus(ConversationalVoiceRequest request)
- {
- if (request.AnsweredBy == "machine_start" &&
- request.Direction == "outbound-api" &&
- request.InitAudioFile != null &&
- request.CallStatus == "completed")
- {
- // voicemail
- await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
- {
- await hook.OnVoicemailLeft(request.ConversationId);
- });
- }
- return Ok();
- }
-
private async Task<string> InitConversation(ConversationalVoiceRequest request)
{
var convService = _services.GetRequiredService<IConversationService>();
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
index 6507a1053..0193cae9b 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
@@ -1,13 +1,11 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Infrastructures;
-using BotSharp.Abstraction.Repositories;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using BotSharp.Plugin.Twilio.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
-using System.ComponentModel.DataAnnotations;
using Twilio.Http;
namespace BotSharp.Plugin.Twilio.Controllers;
@@ -382,21 +380,21 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
}
[ValidateRequest]
- [HttpPost("twilio/voice/init-call")]
- public TwiMLResult InitiateOutboundCall(VoiceRequest request, [Required][FromQuery] string conversationId)
+ [HttpPost("twilio/voice/init-outbound-call")]
+ public TwiMLResult InitiateOutboundCall(ConversationalVoiceRequest request)
{
var instruction = new ConversationalVoiceResponse
{
ActionOnEmptyResult = true,
- CallbackPath = $"twilio/voice/receive/1?conversation-id={conversationId}",
- SpeechPaths = new List<string>
- {
- $"twilio/voice/speeches/{conversationId}/intial.mp3"
- }
+ CallbackPath = $"twilio/voice/receive/1?conversation-id={request.ConversationId}",
};
- string tag = $"twilio:{Request.Form["AnsweredBy"]}";
- var db = _services.GetRequiredService<IBotSharpRepository>();
- db.AppendConversationTags(conversationId, new List<string> { tag });
+
+ if (request.InitAudioFile != null)
+ {
+ instruction.CallbackPath += $"&init-audio-file={request.InitAudioFile}";
+ instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}");
+ }
+
var twilio = _services.GetRequiredService<TwilioService>();
var response = twilio.ReturnNoninterruptedInstructions(instruction);
return TwiML(response);
@@ -415,6 +413,32 @@ public async Task<FileContentResult> GetSpeechFile([FromRoute] string conversati
return result;
}
+ [ValidateRequest]
+ [HttpPost("twilio/voice/status")]
+ public async Task<ActionResult> PhoneCallStatus(ConversationalVoiceRequest request)
+ {
+ if (request.CallStatus == "completed")
+ {
+ if (request.AnsweredBy == "machine_start" &&
+ request.Direction == "outbound-api" &&
+ request.InitAudioFile != null)
+ {
+ // voicemail
+ await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
+ {
+ await hook.OnVoicemailLeft(request);
+ });
+ }
+ else
+ {
+ // phone call completed
+ await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnUserDisconnected(request));
+ }
+ }
+
+ return Ok();
+ }
+
private Dictionary<string, string> ParseStates(List<string> states)
{
var result = new Dictionary<string, string>();
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs
index d35a2a353..904f16a12 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs
@@ -1,8 +1,10 @@
+using BotSharp.Plugin.Twilio.Models;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Interfaces;
public interface ITwilioCallStatusHook
{
- Task OnVoicemailLeft(string conversationId);
+ Task OnVoicemailLeft(ConversationalVoiceRequest request);
+ Task OnUserDisconnected(ConversationalVoiceRequest request);
}
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs
index 2587fed9d..547b09d93 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs
@@ -6,7 +6,7 @@ public class AssistantMessage
public bool HumanIntervationNeeded { get; set; }
public string Content { get; set; }
public string MessageId { get; set; }
- public string SpeechFileName { get; set; }
+ public string? SpeechFileName { get; set; }
public string Hints { get; set; }
}
}
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs
index cb707474f..488ad79b2 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs
@@ -1,8 +1,11 @@
using BotSharp.Abstraction.Files;
+using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
+using BotSharp.Plugin.Twilio.Interfaces;
+using BotSharp.Plugin.Twilio.Models;
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
using Twilio.Rest.Api.V2010.Account;
using Twilio.Types;
@@ -58,20 +61,48 @@ public async Task<bool> Execute(RoleDialogModel message)
var newConversationId = Guid.NewGuid().ToString();
states.SetState(StateConst.SUB_CONVERSATION_ID, newConversationId);
+ var processUrl = $"{_twilioSetting.CallbackHost}/twilio";
+ var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?conversation-id={newConversationId}";
+
// Generate initial assistant audio
- var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
- var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage);
- var fileName = $"intial.mp3";
- fileStorage.SaveSpeechFile(newConversationId, fileName, data);
+ string initAudioUrl = null;
+ if (!string.IsNullOrEmpty(args.InitialMessage))
+ {
+ var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
+ var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage);
+ initAudioUrl = "intial.mp3";
+ fileStorage.SaveSpeechFile(newConversationId, initAudioUrl, data);
+
+ statusUrl += $"&init-audio-file={initAudioUrl}";
+ }
+
+ // Set up process URL streaming or synchronous
+ if (_twilioSetting.StreamingEnabled)
+ {
+ processUrl += "/stream";
+ }
+ else
+ {
+ var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
+ await sessionManager.SetAssistantReplyAsync(newConversationId, 0, new AssistantMessage
+ {
+ Content = args.InitialMessage,
+ SpeechFileName = initAudioUrl
+ });
+
+ processUrl += "/voice/init-outbound-call";
+ }
+
+ processUrl += $"?conversation-id={newConversationId}&init-audio-file={initAudioUrl}";
// Make outbound call
var call = await CallResource.CreateAsync(
- url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation-id={newConversationId}&init-audio-file={fileName}"),
+ url: new Uri(processUrl),
to: new PhoneNumber(args.PhoneNumber),
from: new PhoneNumber(_twilioSetting.PhoneNumber),
- statusCallback: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream/status?conversation-id={newConversationId}&init-audio-file={fileName}"),
+ statusCallback: new Uri(statusUrl),
// https://www.twilio.com/docs/voice/answering-machine-detection
- machineDetection: "Enable");
+ machineDetection: _twilioSetting.MachineDetection);
var convService = _services.GetRequiredService<IConversationService>();
var routing = _services.GetRequiredService<IRoutingContext>();
@@ -80,7 +111,7 @@ public async Task<bool> Execute(RoleDialogModel message)
await ForkConversation(args, entryAgentId, originConversationId, newConversationId, call);
- message.Content = $"The generated phone message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}]";
+ message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, STREAMING: {_twilioSetting.StreamingEnabled}, RECORDING: {_twilioSetting.RecordingEnabled}]";
message.StopCompletion = true;
return true;
}
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
index b17f287a0..d64c34c26 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
@@ -109,7 +109,14 @@ public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceRespons
{
foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
{
- response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
+ if (speechPath.StartsWith(_settings.CallbackHost))
+ {
+ response.Play(new Uri(speechPath));
+ }
+ else
+ {
+ response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
+ }
}
}
var gather = new Gather()
@@ -193,9 +200,13 @@ public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversa
{
response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
}
+ else if (speechPath.StartsWith(_settings.CallbackHost))
+ {
+ response.Play(new Uri(speechPath));
+ }
else
{
- response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}"));
+ response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
}
}
}
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs
index 4c65481fa..5d34299fd 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs
@@ -2,14 +2,36 @@ namespace BotSharp.Plugin.Twilio.Settings;
public class TwilioSetting
{
- public string PhoneNumber { get; set; }
+ /// <summary>
+ /// Outbound phone number
+ /// </summary>
+ public string? PhoneNumber { get; set; }
+
+ /// <summary>
+ /// Enable streaming for outbound phone call
+ /// </summary>
+ public bool StreamingEnabled { get; set; } = false;
public string AccountSID { get; set; }
public string AuthToken { get; set; }
public string AppSID { get; set; }
public string ApiKeySID { get; set; }
public string ApiSecret { get; set; }
public string CallbackHost { get; set; }
- public string AgentId { get; set; }
- public string CsrAgentNumber { get; set; }
+
+ /// <summary>
+ /// Default Agent Id to handle inbound phone call
+ /// </summary>
+ public string? AgentId { get; set; }
+
+ /// <summary>
+ /// Human agent phone number if AI can't handle the call
+ /// </summary>
+ public string? CsrAgentNumber { get; set; }
+
public int MaxGatherAttempts { get; set; } = 4;
+
+ public string? MachineDetection { get; set; }
+
+ public bool RecordingEnabled { get; set; } = false;
+ public bool RecordingTranscribe { get; set; } = false;
}
From addb1b1bba1052a449e6177bf79e3695f9d927ea Mon Sep 17 00:00:00 2001
From: Haiping Chen <hchen@lessen.com>
Date: Fri, 14 Mar 2025 15:08:34 -0500
Subject: [PATCH 2/3] Outbound call phone_recording_url
---
.../Controllers/TwilioRecordController.cs | 44 +++++++++++++++++++
.../Controllers/TwilioStreamController.cs | 1 +
.../Controllers/TwilioVoiceController.cs | 18 +++++++-
.../Interfaces/ITwilioCallStatusHook.cs | 1 +
.../Models/ConversationalVoiceResponse.cs | 1 +
.../Functions/OutboundPhoneCallFn.cs | 5 ++-
.../Services/TwilioService.cs | 18 ++++----
.../Settings/TwilioSetting.cs | 1 -
8 files changed, 78 insertions(+), 11 deletions(-)
create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs
new file mode 100644
index 000000000..65b2a5adf
--- /dev/null
+++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs
@@ -0,0 +1,44 @@
+using BotSharp.Core.Infrastructures;
+using BotSharp.Plugin.Twilio.Interfaces;
+using BotSharp.Plugin.Twilio.Models;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+
+namespace BotSharp.Plugin.Twilio.Controllers;
+
+public class TwilioRecordController : TwilioController
+{
+ private readonly TwilioSetting _settings;
+ private readonly IServiceProvider _services;
+ private readonly ILogger _logger;
+
+ public TwilioRecordController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioRecordController> logger)
+ {
+ _settings = settings;
+ _services = services;
+ _logger = logger;
+ }
+
+ [ValidateRequest]
+ [HttpPost("twilio/record/status")]
+ public async Task<ActionResult> PhoneRecordingStatus(ConversationalVoiceRequest request)
+ {
+ if (request.RecordingStatus == "completed")
+ {
+ _logger.LogInformation($"Recording completed for {request.CallSid}, the record URL is {request.RecordingUrl}");
+
+ // Set the recording URL to the conversation state
+ var convService = _services.GetRequiredService<IConversationService>();
+ convService.SetConversationId(request.ConversationId, new List<MessageState>
+ {
+ new("phone_recording_url", request.RecordingUrl)
+ });
+ convService.SaveStates();
+
+ // recording completed
+ await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnRecordingCompleted(request));
+ }
+
+ return Ok();
+ }
+}
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs
index fb35c25ce..8fea9bb4c 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs
@@ -48,6 +48,7 @@ public async Task<TwiMLResult> InitiateStreamConversation(ConversationalVoiceReq
var instruction = new ConversationalVoiceResponse
{
+ ConversationId = request.ConversationId,
SpeechPaths = [],
ActionOnEmptyResult = true
};
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
index 0193cae9b..18225d092 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
@@ -52,6 +52,7 @@ public async Task<TwiMLResult> InitiateConversation(ConversationalVoiceRequest r
VoiceResponse response = null;
var instruction = new ConversationalVoiceResponse
{
+ ConversationId = request.ConversationId,
SpeechPaths = ["twilio/welcome.mp3"],
ActionOnEmptyResult = true
};
@@ -170,6 +171,7 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
var instruction = new ConversationalVoiceResponse
{
+ ConversationId = request.ConversationId,
SpeechPaths = new List<string>(),
CallbackPath = $"twilio/voice/receive/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}",
ActionOnEmptyResult = true
@@ -273,6 +275,7 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
var instruction = new ConversationalVoiceResponse
{
+ ConversationId = request.ConversationId,
SpeechPaths = speechPaths,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
@@ -312,6 +315,7 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
var instruction = new ConversationalVoiceResponse
{
+ ConversationId = request.ConversationId,
SpeechPaths = instructions,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
@@ -358,6 +362,7 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
var instruction = new ConversationalVoiceResponse
{
+ ConversationId = request.ConversationId,
SpeechPaths = [$"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"],
CallbackPath = $"twilio/voice/receive/{nextSeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}",
ActionOnEmptyResult = true,
@@ -383,8 +388,19 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
[HttpPost("twilio/voice/init-outbound-call")]
public TwiMLResult InitiateOutboundCall(ConversationalVoiceRequest request)
{
+ VoiceResponse response = default!;
+ if (request.AnsweredBy == "machine_start" &&
+ request.Direction == "outbound-api" &&
+ request.InitAudioFile != null)
+ {
+ response = new VoiceResponse();
+ response.Play(new Uri(request.InitAudioFile));
+ return TwiML(response);
+ }
+
var instruction = new ConversationalVoiceResponse
{
+ ConversationId = request.ConversationId,
ActionOnEmptyResult = true,
CallbackPath = $"twilio/voice/receive/1?conversation-id={request.ConversationId}",
};
@@ -396,7 +412,7 @@ public TwiMLResult InitiateOutboundCall(ConversationalVoiceRequest request)
}
var twilio = _services.GetRequiredService<TwilioService>();
- var response = twilio.ReturnNoninterruptedInstructions(instruction);
+ response = twilio.ReturnNoninterruptedInstructions(instruction);
return TwiML(response);
}
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs
index 904f16a12..c9ed47106 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs
@@ -7,4 +7,5 @@ public interface ITwilioCallStatusHook
{
Task OnVoicemailLeft(ConversationalVoiceRequest request);
Task OnUserDisconnected(ConversationalVoiceRequest request);
+ Task OnRecordingCompleted(ConversationalVoiceRequest request);
}
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs
index ea8072d60..7870f9615 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs
@@ -2,6 +2,7 @@ namespace BotSharp.Plugin.Twilio.Models;
public class ConversationalVoiceResponse
{
+ public string ConversationId { get; set; } = null!;
public List<string> SpeechPaths { get; set; } = [];
public string CallbackPath { get; set; }
public bool ActionOnEmptyResult { get; set; }
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs
index 488ad79b2..3bcd02602 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs
@@ -63,6 +63,7 @@ public async Task<bool> Execute(RoleDialogModel message)
var processUrl = $"{_twilioSetting.CallbackHost}/twilio";
var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?conversation-id={newConversationId}";
+ var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?conversation-id={newConversationId}";
// Generate initial assistant audio
string initAudioUrl = null;
@@ -102,7 +103,9 @@ public async Task<bool> Execute(RoleDialogModel message)
from: new PhoneNumber(_twilioSetting.PhoneNumber),
statusCallback: new Uri(statusUrl),
// https://www.twilio.com/docs/voice/answering-machine-detection
- machineDetection: _twilioSetting.MachineDetection);
+ machineDetection: _twilioSetting.MachineDetection,
+ record: _twilioSetting.RecordingEnabled,
+ recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?conversation-id={newConversationId}");
var convService = _services.GetRequiredService<IConversationService>();
var routing = _services.GetRequiredService<IRoutingContext>();
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
index d64c34c26..8ef9f8e6a 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
@@ -1,7 +1,6 @@
using BotSharp.Abstraction.Utilities;
using BotSharp.Plugin.Twilio.Models;
using Twilio.Jwt.AccessToken;
-using Twilio.TwiML.Messaging;
using Token = Twilio.Jwt.AccessToken.Token;
namespace BotSharp.Plugin.Twilio.Services;
@@ -101,13 +100,13 @@ public VoiceResponse ReturnInstructions(ConversationalVoiceResponse conversation
return response;
}
- public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse conversationalVoiceResponse)
+ public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse voiceResponse)
{
var response = new VoiceResponse();
- response.Pause(2);
- if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
+
+ if (voiceResponse.SpeechPaths != null && voiceResponse.SpeechPaths.Any())
{
- foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
+ foreach (var speechPath in voiceResponse.SpeechPaths)
{
if (speechPath.StartsWith(_settings.CallbackHost))
{
@@ -119,6 +118,7 @@ public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceRespons
}
}
}
+
var gather = new Gather()
{
Input = new List<Gather.InputEnum>()
@@ -126,14 +126,15 @@ public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceRespons
Gather.InputEnum.Speech,
Gather.InputEnum.Dtmf
},
- Action = new Uri($"{_settings.CallbackHost}/{conversationalVoiceResponse.CallbackPath}"),
+ Action = new Uri($"{_settings.CallbackHost}/{voiceResponse.CallbackPath}"),
Enhanced = true,
SpeechModel = Gather.SpeechModelEnum.PhoneCall,
SpeechTimeout = "auto", // conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout.ToString() : "3",
- Timeout = conversationalVoiceResponse.Timeout > 0 ? conversationalVoiceResponse.Timeout : 3,
- ActionOnEmptyResult = conversationalVoiceResponse.ActionOnEmptyResult
+ Timeout = voiceResponse.Timeout > 0 ? voiceResponse.Timeout : 3,
+ ActionOnEmptyResult = voiceResponse.ActionOnEmptyResult,
};
response.Append(gather);
+
return response;
}
@@ -192,6 +193,7 @@ public VoiceResponse HoldOn(int interval, string message = null)
public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse)
{
var response = new VoiceResponse();
+
if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
{
foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs
index 5d34299fd..81629a9ed 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs
@@ -33,5 +33,4 @@ public class TwilioSetting
public string? MachineDetection { get; set; }
public bool RecordingEnabled { get; set; } = false;
- public bool RecordingTranscribe { get; set; } = false;
}
From ace2e906f04923e7d8da8c431db473b0b2100dc7 Mon Sep 17 00:00:00 2001
From: Haiping Chen <hchen@lessen.com>
Date: Fri, 14 Mar 2025 16:44:19 -0500
Subject: [PATCH 3/3] util-twilio-text_message
---
.../BotSharp.Plugin.Twilio.csproj | 3 ++
.../Functions/HangupPhoneCallFn.cs | 1 -
.../Functions/TextMessageFn.cs | 49 +++++++++++++++++++
.../OutboundPhoneCallHandlerUtilityHook.cs | 4 +-
.../Settings/TwilioSetting.cs | 2 +
.../functions/util-twilio-text_message.json | 18 +++++++
6 files changed, 75 insertions(+), 2 deletions(-)
create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TextMessageFn.cs
create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-text_message.json
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj
index 727143886..5b83d77d2 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj
+++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj
@@ -13,6 +13,9 @@
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-hangup_phone_call.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
+ <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-text_message.json">
+ <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+ </Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-outbound_phone_call.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs
index cd7553069..5a7eb3f39 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs
@@ -1,5 +1,4 @@
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
-using Microsoft.VisualBasic;
using Twilio.Rest.Api.V2010.Account;
using Task = System.Threading.Tasks.Task;
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TextMessageFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TextMessageFn.cs
new file mode 100644
index 000000000..c7c6fc888
--- /dev/null
+++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TextMessageFn.cs
@@ -0,0 +1,49 @@
+using BotSharp.Abstraction.Options;
+using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
+using Twilio.Rest.Api.V2010.Account;
+
+namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
+
+public class TextMessageFn : IFunctionCallback
+{
+ private readonly IServiceProvider _services;
+ private readonly ILogger _logger;
+ private readonly TwilioSetting _twilioSetting;
+
+ public string Name => "util-twilio-text_message";
+ public string Indication => "Sending text message";
+
+ public TextMessageFn(
+ IServiceProvider services,
+ ILogger<TextMessageFn> logger,
+ TwilioSetting twilioSetting)
+ {
+ _services = services;
+ _logger = logger;
+ _twilioSetting = twilioSetting;
+ }
+
+ public async Task<bool> Execute(RoleDialogModel message)
+ {
+ var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs, BotSharpOptions.defaultJsonOptions);
+
+ // Send the message
+ var twilioMessage = MessageResource.Create(
+ to: args.PhoneNumber,
+ from: _twilioSetting.MessagingShortCode,
+ body: args.InitialMessage
+ );
+
+ if (twilioMessage.Status == MessageResource.StatusEnum.Queued)
+ {
+ message.Content = $"Queued message to {args.PhoneNumber}: {args.InitialMessage} [MESSAGING SID: {twilioMessage.Sid}]";
+ message.StopCompletion = true;
+ }
+ else
+ {
+ message.Content = twilioMessage.ErrorMessage;
+ }
+
+ return true;
+ }
+}
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs
index 692610ffc..fa0b31798 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs
@@ -8,6 +8,7 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook
private static string PREFIX = "util-twilio-";
private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call";
private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call";
+ public static string TEXT_MESSAGE_FN = $"{PREFIX}text_message";
public void AddUtilities(List<AgentUtility> utilities)
{
@@ -17,7 +18,8 @@ public void AddUtilities(List<AgentUtility> utilities)
Functions =
[
new($"{OUTBOUND_PHONE_CALL_FN}"),
- new($"{HANGUP_PHONE_CALL_FN}")
+ new($"{HANGUP_PHONE_CALL_FN}"),
+ new($"{TEXT_MESSAGE_FN}")
],
Templates =
[
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs
index 81629a9ed..2de42ec8f 100644
--- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs
+++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs
@@ -18,6 +18,8 @@ public class TwilioSetting
public string ApiSecret { get; set; }
public string CallbackHost { get; set; }
+ public string? MessagingShortCode { get; set; }
+
/// <summary>
/// Default Agent Id to handle inbound phone call
/// </summary>
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-text_message.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-text_message.json
new file mode 100644
index 000000000..a4340410f
--- /dev/null
+++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-text_message.json
@@ -0,0 +1,18 @@
+{
+ "name": "util-twilio-text_message",
+ "description": "If the user wants to send SMS message to a phone.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "phone_number": {
+ "type": "string",
+ "description": "The phone number which will receive message. It needs to be a valid phone number starting with +1."
+ },
+ "initial_message": {
+ "type": "string",
+ "description": "The initial message which will be sent."
+ }
+ },
+ "required": [ "phone_number", "initial_message" ]
+ }
+}
\ No newline at end of file