From 6190c8eddd8fe74ff449351561619b3e2f7b75d7 Mon Sep 17 00:00:00 2001 From: aboutphilippe Date: Sun, 9 Mar 2025 01:10:00 +0100 Subject: [PATCH 01/11] advanced telephony example --- .../agent_twilio/src/agents/agent.py | 135 ++++++++++++---- .../src/functions/context_docs.py | 26 +++ ...livekit_room.py => livekit_create_room.py} | 4 +- .../src/functions/livekit_delete_room.py | 28 ++++ .../src/functions/livekit_outbound_trunk.py | 2 +- .../src/functions/livekit_send_data.py | 36 +++++ .../src/functions/livekit_token.py | 34 ++++ .../agent_twilio/src/functions/llm_chat.py | 52 ------ .../agent_twilio/src/functions/llm_logic.py | 48 ++++++ .../agent_twilio/src/functions/llm_talk.py | 64 ++++++++ .../src/functions/send_agent_event.py | 27 ++++ .../agent_twilio/src/services.py | 22 ++- .../agent_twilio/src/workflows/logic.py | 128 +++++++++++++++ .../livekit-trunk-setup/twilio_trunk.py | 1 - .../livekit_pipeline/pyproject.toml | 2 +- .../livekit_pipeline/src/client.py | 19 +++ .../livekit_pipeline/src/pipeline.py | 71 ++++++-- .../vapi/agent_vapi/src/agents/agent.py | 6 +- .../vapi/agent_vapi/src/services.py | 2 +- .../livekit_opentelemetry/pyproject.toml | 29 ++++ community/livekit_opentelemetry/src/client.py | 19 +++ .../src/otel_exporter.py | 48 ++++++ .../src/otel_provider.py | 140 ++++++++++++++++ .../livekit_opentelemetry/src/pipeline.py | 152 ++++++++++++++++++ 24 files changed, 991 insertions(+), 104 deletions(-) create mode 100644 agent_telephony/twilio_livekit/agent_twilio/src/functions/context_docs.py rename agent_telephony/twilio_livekit/agent_twilio/src/functions/{livekit_room.py => livekit_create_room.py} (87%) create mode 100644 agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_delete_room.py create mode 100644 agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_send_data.py create mode 100644 agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_token.py delete mode 100644 agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_chat.py create mode 100644 agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_logic.py create mode 100644 agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_talk.py create mode 100644 agent_telephony/twilio_livekit/agent_twilio/src/functions/send_agent_event.py create mode 100644 agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py create mode 100644 agent_telephony/twilio_livekit/livekit_pipeline/src/client.py create mode 100644 community/livekit_opentelemetry/pyproject.toml create mode 100644 community/livekit_opentelemetry/src/client.py create mode 100644 community/livekit_opentelemetry/src/otel_exporter.py create mode 100644 community/livekit_opentelemetry/src/otel_provider.py create mode 100644 community/livekit_opentelemetry/src/pipeline.py diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py b/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py index 87cdaccc..639b5c24 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py @@ -1,14 +1,32 @@ from datetime import timedelta +from typing import Any from pydantic import BaseModel -from restack_ai.agent import NonRetryableError, agent, agent_info, import_functions, log +from restack_ai.agent import ( + NonRetryableError, + RetryPolicy, + agent, + agent_info, + import_functions, + log, + uuid, +) + +from src.workflows.logic import LogicWorkflow, LogicWorkflowInput with import_functions(): from src.functions.livekit_call import LivekitCallInput, livekit_call + from src.functions.livekit_create_room import livekit_create_room + from src.functions.livekit_delete_room import livekit_delete_room from src.functions.livekit_dispatch import LivekitDispatchInput, livekit_dispatch from src.functions.livekit_outbound_trunk import livekit_outbound_trunk - from src.functions.livekit_room import livekit_room - from src.functions.llm_chat import LlmChatInput, Message, llm_chat + from src.functions.livekit_send_data import ( + LivekitSendDataInput, + SendDataResponse, + livekit_send_data, + ) + from src.functions.livekit_token import LivekitTokenInput, livekit_token + from src.functions.llm_talk import LlmTalkInput, Message, llm_talk class MessagesEvent(BaseModel): @@ -23,29 +41,59 @@ class CallInput(BaseModel): phone_number: str +class ContextEvent(BaseModel): + context: str + + +class PipelineMetricsEvent(BaseModel): + metrics: Any + latencies: str + + @agent.defn() class AgentTwilio: def __init__(self) -> None: self.end = False - self.messages: list[Message] = [] + self.messages = [] + self.context = "" self.room_id = "" @agent.event async def messages(self, messages_event: MessagesEvent) -> list[Message]: log.info(f"Received message: {messages_event.messages}") self.messages.extend(messages_event.messages) + try: - assistant_message = await agent.step( - function=llm_chat, - function_input=LlmChatInput(messages=self.messages), - start_to_close_timeout=timedelta(seconds=120), + await agent.child_start( + workflow=LogicWorkflow, + workflow_id=f"{uuid()}-logic", + workflow_input=LogicWorkflowInput( + messages=self.messages, + room_id=self.room_id, + context=str(self.context), + ), + ) + + fast_response = await agent.step( + function=llm_talk, + function_input=LlmTalkInput( + messages=self.messages[-3:], + context=str(self.context), + mode="default", + ), + start_to_close_timeout=timedelta(seconds=3), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=1), + maximum_attempts=1, + maximum_interval=timedelta(seconds=5), + ), ) + + self.messages.append(Message(role="assistant", content=fast_response)) + return self.messages except Exception as e: - error_message = f"Error during llm_chat: {e}" + error_message = f"Error during messages: {e}" raise NonRetryableError(error_message) from e - else: - self.messages.append(Message(role="assistant", content=str(assistant_message))) - return self.messages @agent.event async def call(self, call_input: CallInput) -> None: @@ -56,45 +104,70 @@ async def call(self, call_input: CallInput) -> None: run_id = agent_info().run_id try: sip_trunk_id = await agent.step(function=livekit_outbound_trunk) - except Exception as e: - error_message = f"Error during livekit_outbound_trunk: {e}" - raise NonRetryableError(error_message) from e - else: - try: - result =await agent.step( - function=livekit_call, - function_input=LivekitCallInput( + await agent.step( + function=livekit_call, + function_input=LivekitCallInput( sip_trunk_id=sip_trunk_id, phone_number=phone_number, room_id=self.room_id, agent_name=agent_name, agent_id=agent_id, - run_id=run_id, - ), - ) - except Exception as e: - error_message = f"Error during livekit_call: {e}" - raise NonRetryableError(error_message) from e - else: - return result + run_id=run_id, + ), + ) + except Exception as e: + error_message = f"Error during livekit_outbound_trunk: {e}" + raise NonRetryableError(error_message) from e + + @agent.event + async def say(self, say: str) -> SendDataResponse: + log.info("Received say") + return await agent.step( + function=livekit_send_data, function_input=LivekitSendDataInput(text=say) + ) @agent.event async def end(self, end: EndEvent) -> EndEvent: log.info("Received end") + await agent.step( + function=livekit_send_data, + function_input=LivekitSendDataInput( + room_id=self.room_id, text="Thank you for calling restack. Goodbye!" + ), + ) + await agent.step(function=livekit_delete_room) + self.end = True return end + @agent.event + async def context(self, context: ContextEvent) -> str: + log.info("Received context") + self.context = context.context + return self.context + + @agent.event + async def pipeline_metrics( + self, pipeline_metrics: PipelineMetricsEvent + ) -> PipelineMetricsEvent: + log.info("Received pipeline metrics", pipeline_metrics=pipeline_metrics) + return pipeline_metrics + @agent.run async def run(self) -> None: - room = await agent.step(function=livekit_room) - self.room_id = room.name try: + room = await agent.step(function=livekit_create_room) + self.room_id = room.name + await agent.step( + function=livekit_token, + function_input=LivekitTokenInput(room_id=self.room_id), + ) await agent.step( function=livekit_dispatch, function_input=LivekitDispatchInput(room_id=self.room_id), ) except Exception as e: - error_message = f"Error during livekit_dispatch: {e}" + error_message = f"Error during agent run: {e}" raise NonRetryableError(error_message) from e else: await agent.condition(lambda: self.end) diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/context_docs.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/context_docs.py new file mode 100644 index 00000000..64d7443f --- /dev/null +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/context_docs.py @@ -0,0 +1,26 @@ +import aiohttp +from restack_ai.function import NonRetryableError, function, log + + +async def fetch_content_from_url(url: str) -> str: + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + if response.status == 200: + return await response.text() + error_message = f"Failed to fetch content: {response.status}" + raise NonRetryableError(error_message) + + +@function.defn() +async def context_docs() -> str: + try: + docs_content = await fetch_content_from_url( + "https://docs.restack.io/llms-full.txt" + ) + log.info("Fetched content from URL", content=len(docs_content)) + + return docs_content + + except Exception as e: + error_message = f"context_docs function failed: {e}" + raise NonRetryableError(error_message) from e diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_room.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_create_room.py similarity index 87% rename from agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_room.py rename to agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_create_room.py index c11ca80f..34d7a79e 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_room.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_create_room.py @@ -6,7 +6,7 @@ @function.defn() -async def livekit_room() -> Room: +async def livekit_create_room() -> Room: try: lkapi = api.LiveKitAPI( url=os.getenv("LIVEKIT_API_URL"), @@ -27,7 +27,7 @@ async def livekit_room() -> Room: await lkapi.aclose() except Exception as e: - error_message = f"livekit_room function failed: {e}" + error_message = f"livekit_create_room function failed: {e}" raise NonRetryableError(error_message) from e else: diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_delete_room.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_delete_room.py new file mode 100644 index 00000000..dfda1dd6 --- /dev/null +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_delete_room.py @@ -0,0 +1,28 @@ +import os + +from livekit import api +from livekit.api import DeleteRoomRequest, DeleteRoomResponse +from restack_ai.function import NonRetryableError, function, function_info + + +@function.defn() +async def livekit_delete_room() -> DeleteRoomResponse: + try: + lkapi = api.LiveKitAPI( + url=os.getenv("LIVEKIT_API_URL"), + api_key=os.getenv("LIVEKIT_API_KEY"), + api_secret=os.getenv("LIVEKIT_API_SECRET"), + ) + + run_id = function_info().workflow_run_id + + deleted_room = await lkapi.room.delete_room(DeleteRoomRequest(room=run_id)) + + await lkapi.aclose() + + except Exception as e: + error_message = f"livekit_delete_room function failed: {e}" + raise NonRetryableError(error_message) from e + + else: + return deleted_room diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_outbound_trunk.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_outbound_trunk.py index ab0ca710..1d6e9544 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_outbound_trunk.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_outbound_trunk.py @@ -30,7 +30,7 @@ async def livekit_outbound_trunk() -> str: trunk = SIPOutboundTrunkInfo( name=run_id, - address=os.getenv("LIVEKIT_SIP_ADDRESS"), + address=os.getenv("TWILIO_TRUNK_TERMINATION_SIP_URL"), numbers=[os.getenv("TWILIO_PHONE_NUMBER")], auth_username=os.getenv("TWILIO_TRUNK_AUTH_USERNAME"), auth_password=os.getenv("TWILIO_TRUNK_AUTH_PASSWORD"), diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_send_data.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_send_data.py new file mode 100644 index 00000000..dd79e8b2 --- /dev/null +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_send_data.py @@ -0,0 +1,36 @@ +import os + +from livekit import api +from livekit.api import SendDataRequest, SendDataResponse +from pydantic import BaseModel +from restack_ai.function import NonRetryableError, function + + +class LivekitSendDataInput(BaseModel): + room_id: str + text: str + + +@function.defn() +async def livekit_send_data(function_input: LivekitSendDataInput) -> SendDataResponse: + try: + lkapi = api.LiveKitAPI( + url=os.getenv("LIVEKIT_API_URL"), + api_key=os.getenv("LIVEKIT_API_KEY"), + api_secret=os.getenv("LIVEKIT_API_SECRET"), + ) + + send_data_reponse = await lkapi.room.send_data( + SendDataRequest( + room=function_input.room_id, data=function_input.text.encode("utf-8") + ) + ) + + await lkapi.aclose() + + except Exception as e: + error_message = f"livekit_delete_room function failed: {e}" + raise NonRetryableError(error_message) from e + + else: + return send_data_reponse diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_token.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_token.py new file mode 100644 index 00000000..16be9570 --- /dev/null +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_token.py @@ -0,0 +1,34 @@ +import os + +from livekit import api +from pydantic import BaseModel +from restack_ai.function import NonRetryableError, function, log + + +class LivekitTokenInput(BaseModel): + room_id: str + + +@function.defn() +async def livekit_token(function_input: LivekitTokenInput) -> str: + try: + token = ( + api.AccessToken( + os.getenv("LIVEKIT_API_KEY"), os.getenv("LIVEKIT_API_SECRET") + ) + .with_identity("identity") + .with_name("dev_user") + .with_grants( + api.VideoGrants( + room_join=True, + room=function_input.room_id, + ) + ) + ) + log.info("Token generated", token=token.to_jwt()) + except Exception as e: + error_message = f"livekit_room function failed: {e}" + raise NonRetryableError(error_message) from e + + else: + return token.to_jwt() diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_chat.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_chat.py deleted file mode 100644 index ce82e2a4..00000000 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_chat.py +++ /dev/null @@ -1,52 +0,0 @@ -import os -from typing import TYPE_CHECKING, Literal - -from openai import OpenAI -from pydantic import BaseModel, Field -from restack_ai.function import NonRetryableError, function, stream_to_websocket - -from src.client import api_address - -if TYPE_CHECKING: - from openai.resources.chat.completions import ChatCompletionChunk, Stream - - -class Message(BaseModel): - role: Literal["system", "user", "assistant"] - content: str - - -class LlmChatInput(BaseModel): - system_content: str | None = None - model: str | None = None - messages: list[Message] = Field(default_factory=list) - stream: bool = True - - -@function.defn() -async def llm_chat(function_input: LlmChatInput) -> str: - try: - client = OpenAI( - base_url="https://ai.restack.io", api_key=os.environ.get("RESTACK_API_KEY") - ) - - if function_input.system_content: - # Insert the system message at the beginning - function_input.messages.insert( - 0, Message(role="system", content=function_input.system_content) - ) - - # Convert Message objects to dictionaries - messages_dicts = [message.model_dump() for message in function_input.messages] - # Get the streamed response from OpenAI API - response: Stream[ChatCompletionChunk] = client.chat.completions.create( - model=function_input.model or "gpt-4o-mini", - messages=messages_dicts, - stream=True, - ) - - return await stream_to_websocket(api_address=api_address, data=response) - - except Exception as e: - error_message = f"llm_chat function failed: {e}" - raise NonRetryableError(error_message) from e diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_logic.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_logic.py new file mode 100644 index 00000000..ad1db3b9 --- /dev/null +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_logic.py @@ -0,0 +1,48 @@ +import os +from typing import Literal + +from openai import OpenAI +from pydantic import BaseModel +from restack_ai.function import NonRetryableError, function + + +class LlmLogicResponse(BaseModel): + """Structured AI Decision Output for Intelligent Interruptions""" + + action: Literal["interrupt", "update_context", "end_call"] + reason: str + updated_context: str + + +class LlmLogicInput(BaseModel): + messages: list[dict] + documentation: str + + +@function.defn() +async def llm_logic(function_input: LlmLogicInput) -> LlmLogicResponse: + try: + client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY")) + + response = client.beta.chat.completions.parse( + model="gpt-4o", + messages=[ + { + "role": "system", + "content": ( + "Analyze the developer's questions and determine if an interruption is needed. " + "Use the Restack documentation for accurate answers. " + "Track what the developer has learned and update their belief state." + "End the call if a voice mail is detected." + f"Restack Documentation: {function_input.documentation}" + ), + }, + *function_input.messages, + ], + response_format=LlmLogicResponse, + ) + + return response.choices[0].message.parsed + + except Exception as e: + raise NonRetryableError(f"llm_slow failed: {e}") from e diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_talk.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_talk.py new file mode 100644 index 00000000..1b43e8da --- /dev/null +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_talk.py @@ -0,0 +1,64 @@ +import os +from typing import Literal + +from openai import OpenAI +from pydantic import BaseModel, Field +from restack_ai.function import NonRetryableError, function, stream_to_websocket + +from src.client import api_address + + +class Message(BaseModel): + role: str + content: str + + +class LlmTalkInput(BaseModel): + messages: list[Message] = Field(default_factory=list) + context: str | None = None # Updated context from Slow AI + mode: Literal["default", "interrupt"] + stream: bool = True + + +@function.defn() +async def llm_talk(function_input: LlmTalkInput) -> str: + """Fast AI generates responses while checking for memory updates.""" + try: + client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY")) + + common_prompt = ( + "Your are an AI assistant helping developers build with restack: the backend framework for accurate & reliable AI agents." + "Your interface with users will be voice. Be friendly, helpful and avoid usage of unpronouncable punctuation." + "Always try to bring back the conversation to restack if the user is talking about something else. " + "Current context: " + function_input.context + ) + + if function_input.mode == "default": + system_prompt = ( + common_prompt + + "If you don't know an answer, **do not make something up**. Instead, be friendly andacknowledge that " + "you will check for the correct response and let the user know. Keep your answer short in max 20 words" + ) + else: + system_prompt = ( + common_prompt + + "You are providing a short and precise update based on new information. " + "Do not re-explain everything, just deliver the most important update. Keep your answer short in max 20 words unless the user asks for more information." + ) + + function_input.messages.insert(0, Message(role="system", content=system_prompt)) + + messages_dicts = [msg.model_dump() for msg in function_input.messages] + + response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages=messages_dicts, + stream=function_input.stream, + ) + + if function_input.stream: + return await stream_to_websocket(api_address=api_address, data=response) + return response.choices[0].message.content + + except Exception as e: + raise NonRetryableError(f"llm_fast failed: {e}") from e diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/send_agent_event.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/send_agent_event.py new file mode 100644 index 00000000..9a4becc6 --- /dev/null +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/send_agent_event.py @@ -0,0 +1,27 @@ +from typing import Any + +from pydantic import BaseModel +from restack_ai.function import NonRetryableError, function + +from src.client import client + + +class SendAgentEventInput(BaseModel): + event_name: str + agent_id: str + run_id: str | None = None + event_input: dict[str, Any] | None = None + + +@function.defn() +async def send_agent_event(function_input: SendAgentEventInput) -> str: + try: + return await client.send_agent_event( + event_name=function_input.event_name, + agent_id=function_input.agent_id, + run_id=function_input.run_id, + event_input=function_input.event_input, + ) + + except Exception as e: + raise NonRetryableError(f"send_agent_event failed: {e}") from e diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/services.py b/agent_telephony/twilio_livekit/agent_twilio/src/services.py index af4bfa1a..496250b2 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/services.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/services.py @@ -7,22 +7,36 @@ from src.agents.agent import AgentTwilio from src.client import client +from src.functions.context_docs import context_docs from src.functions.livekit_call import livekit_call +from src.functions.livekit_create_room import livekit_create_room +from src.functions.livekit_delete_room import livekit_delete_room from src.functions.livekit_dispatch import livekit_dispatch from src.functions.livekit_outbound_trunk import livekit_outbound_trunk -from src.functions.livekit_room import livekit_room -from src.functions.llm_chat import llm_chat +from src.functions.livekit_send_data import livekit_send_data +from src.functions.livekit_token import livekit_token +from src.functions.llm_logic import llm_logic +from src.functions.llm_talk import llm_talk +from src.functions.send_agent_event import send_agent_event +from src.workflows.logic import LogicWorkflow async def main() -> None: await client.start_service( agents=[AgentTwilio], + workflows=[LogicWorkflow], functions=[ - llm_chat, + llm_talk, + llm_logic, livekit_dispatch, livekit_call, - livekit_room, + livekit_create_room, + livekit_delete_room, livekit_outbound_trunk, + livekit_token, + context_docs, + livekit_send_data, + send_agent_event, ], ) diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py b/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py new file mode 100644 index 00000000..c7f7e909 --- /dev/null +++ b/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py @@ -0,0 +1,128 @@ +from datetime import timedelta + +from pydantic import BaseModel +from restack_ai.workflow import ( + NonRetryableError, + import_functions, + log, + workflow, + workflow_info, +) + +with import_functions(): + from src.functions.context_docs import context_docs + from src.functions.livekit_send_data import LivekitSendDataInput, livekit_send_data + from src.functions.llm_logic import LlmLogicInput, LlmLogicResponse, llm_logic + from src.functions.llm_talk import LlmTalkInput, Message, llm_talk + from src.functions.send_agent_event import SendAgentEventInput, send_agent_event + + +class LogicWorkflowInput(BaseModel): + messages: list[Message] + room_id: str + context: str + + +class LogicWorkflowOutput(BaseModel): + result: str + + +@workflow.defn() +class LogicWorkflow: + @workflow.run + async def run(self, workflow_input: LogicWorkflowInput) -> str: + context = workflow_input.context + + parent_agent_id = workflow_info().parent.workflow_id + parent_agent_run_id = workflow_info().parent.run_id + + log.info("LogicWorkflow started") + try: + documentation = await workflow.step(function=context_docs) + + slow_response: LlmLogicResponse = await workflow.step( + function=llm_logic, + function_input=LlmLogicInput( + messages=[ + msg.model_dump() for msg in workflow_input.messages + ], # Convert messages to dict + documentation=documentation, + ), + start_to_close_timeout=timedelta(seconds=60), + ) + + log.info(f"Slow response: {slow_response}") + + context = slow_response.updated_context + + await workflow.step( + function=send_agent_event, + function_input=SendAgentEventInput( + event_name="context", + agent_id=parent_agent_id, + run_id=parent_agent_run_id, + event_input={"context": str(context)}, + ), + ) + + if slow_response.action == "interrupt": + interrupt_response = await workflow.step( + function=llm_talk, + function_input=LlmTalkInput( + messages=[Message(role="system", content=slow_response.reason)], + context=str(context), + mode="interrupt", + stream=False, + ), + start_to_close_timeout=timedelta(seconds=3), + ) + + await workflow.step( + function=livekit_send_data, + function_input=LivekitSendDataInput( + room_id=parent_agent_run_id, text=interrupt_response + ), + ) + + if slow_response.action == "end_call": + goodbye_message = await workflow.step( + function=llm_talk, + function_input=LlmTalkInput( + messages=[ + Message( + role="system", + content="Say goodbye to the user by providing a unique and short message based on context.", + ) + ], + context=str(context), + mode="interrupt", + stream=False, + ), + start_to_close_timeout=timedelta(seconds=3), + ) + + log.info(f"Goodbye message: {goodbye_message}") + + await workflow.step( + function=livekit_send_data, + function_input=LivekitSendDataInput( + room_id=parent_agent_run_id, text=goodbye_message + ), + ) + + await workflow.step( + function=send_agent_event, + function_input=SendAgentEventInput( + event_name="end", + agent_id=parent_agent_id, + run_id=parent_agent_run_id, + event_input={"end": True}, + ), + ) + + except Exception as e: + error_message = f"Error during welcome: {e}" + raise NonRetryableError(error_message) from e + else: + log.info("LogicWorkflow completed", context=str(context)) + return str(context) diff --git a/agent_telephony/twilio_livekit/livekit-trunk-setup/twilio_trunk.py b/agent_telephony/twilio_livekit/livekit-trunk-setup/twilio_trunk.py index aebea6f5..a078c046 100644 --- a/agent_telephony/twilio_livekit/livekit-trunk-setup/twilio_trunk.py +++ b/agent_telephony/twilio_livekit/livekit-trunk-setup/twilio_trunk.py @@ -104,7 +104,6 @@ def main(): else: logging.info(f"{trunk_name} already exists. Using the existing trunk.") - inbound_trunk_sid = create_inbound_trunk(phone_number, trunk_name) if inbound_trunk_sid: create_dispatch_rule(inbound_trunk_sid, trunk_name) diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/pyproject.toml b/agent_telephony/twilio_livekit/livekit_pipeline/pyproject.toml index 4ac54c4f..215f6be4 100644 --- a/agent_telephony/twilio_livekit/livekit_pipeline/pyproject.toml +++ b/agent_telephony/twilio_livekit/livekit_pipeline/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "livekit-plugins-silero>=0.7.4", "livekit-plugins-turn-detector>=0.4.2", "python-dotenv==1.0.1", - "restack-ai>=0.0.77", + "restack-ai>=0.0.80", ] [tool.hatch.build.targets.sdist] diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/src/client.py b/agent_telephony/twilio_livekit/livekit_pipeline/src/client.py new file mode 100644 index 00000000..885bf8ea --- /dev/null +++ b/agent_telephony/twilio_livekit/livekit_pipeline/src/client.py @@ -0,0 +1,19 @@ +import os + +from dotenv import load_dotenv +from restack_ai import Restack +from restack_ai.restack import CloudConnectionOptions + +# Load environment variables from a .env file +load_dotenv() + + +engine_id = os.getenv("RESTACK_ENGINE_ID") +address = os.getenv("RESTACK_ENGINE_ADDRESS") +api_key = os.getenv("RESTACK_ENGINE_API_KEY") +api_address = os.getenv("RESTACK_ENGINE_API_ADDRESS") + +connection_options = CloudConnectionOptions( + engine_id=engine_id, address=address, api_key=api_key, api_address=api_address +) +client = Restack(connection_options) diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py b/agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py index 85c28181..560ec96a 100644 --- a/agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py +++ b/agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py @@ -1,3 +1,4 @@ +import asyncio import json import logging import os @@ -13,11 +14,10 @@ ) from livekit.agents.pipeline import VoicePipelineAgent from livekit.plugins import deepgram, elevenlabs, openai, silero, turn_detector +from src.client import client -# Load environment variables from .env.local -load_dotenv(dotenv_path=".env.local") +load_dotenv(dotenv_path=".env") -# Setup basic logging configuration so that all logs are properly output. logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -35,7 +35,6 @@ def validate_envs() -> None: logger.warning("Environment variable %s (%s) is not set.", key, description) -# Validate environments at module load validate_envs() @@ -71,7 +70,6 @@ async def entrypoint(ctx: JobContext) -> None: agent_id = metadata_obj.get("agent_id") run_id = metadata_obj.get("run_id") - # Retrieve the Host from environment variables. engine_api_address = os.environ.get("RESTACK_ENGINE_API_ADDRESS") if not engine_api_address: agent_backend_host = "http://localhost:9233" @@ -88,7 +86,6 @@ async def entrypoint(ctx: JobContext) -> None: logger.info("Connecting to room: %s", ctx.room.name) await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY) - # Wait for the first participant to connect participant = await ctx.wait_for_participant() logger.info("Starting voice assistant for participant: %s", participant.identity) @@ -96,8 +93,6 @@ async def entrypoint(ctx: JobContext) -> None: vad=ctx.proc.userdata["vad"], stt=deepgram.STT(), llm=openai.LLM( - # model="gpt-4o-mini", - # api_key=os.environ.get("OPENAI_API_KEY"), api_key=f"{agent_id}-livekit", base_url=agent_url, ), @@ -114,11 +109,69 @@ async def entrypoint(ctx: JobContext) -> None: @agent.on("metrics_collected") def on_metrics_collected(agent_metrics: metrics.AgentMetrics) -> None: metrics.log_metrics(agent_metrics) + + async def send_metrics(agent_metrics): + try: + latencies = [] + if isinstance(agent_metrics, metrics.PipelineEOUMetrics): + total_latency = agent_metrics.end_of_utterance_delay + latencies.append(total_latency * 1000) + + elif isinstance(agent_metrics, metrics.PipelineLLMMetrics): + total_latency = agent_metrics.ttft + latencies.append(total_latency * 1000) + + elif isinstance(agent_metrics, metrics.PipelineTTSMetrics): + total_latency = agent_metrics.ttfb + latencies.append(total_latency * 1000) + + if latencies: + metrics_latencies = str(json.dumps({"latencies": latencies})) + logger.info(f"Sending pipeline metrics: {metrics_latencies!s}") + + await client.send_agent_event( + event_name="pipeline_metrics", + agent_id=agent_id.replace("local-", ""), + run_id=run_id, + event_input={ + "metrics": agent_metrics, + "latencies": metrics_latencies, + }, + ) + except Exception as e: + logger.error("Error sending pipeline metrics", error=e) + + asyncio.create_task(send_metrics(agent_metrics)) + usage_collector.collect(agent_metrics) - # Start the voice pipeline agent. + usage_collector.get_summary() + + async def say(text: str): + await agent.say(text) + + @ctx.room.on("data_received") + def on_data_received(data_packet) -> None: + logger.info(f"Received data: {data_packet}") + + byte_content = data_packet.data + if isinstance(byte_content, bytes): + text_data = byte_content.decode("utf-8") + logger.info(f"Text data: {text_data}") + + asyncio.create_task(say(text_data)) + + else: + logger.warning("Data is not in bytes format.") + agent.start(ctx.room, participant) + await asyncio.sleep(0.1) + + await agent.say( + "Welcome to restack, how can I help you today?", allow_interruptions=True + ) + if __name__ == "__main__": cli.run_app( diff --git a/agent_telephony/vapi/agent_vapi/src/agents/agent.py b/agent_telephony/vapi/agent_vapi/src/agents/agent.py index 0443c43d..914caf18 100644 --- a/agent_telephony/vapi/agent_vapi/src/agents/agent.py +++ b/agent_telephony/vapi/agent_vapi/src/agents/agent.py @@ -4,7 +4,7 @@ from restack_ai.agent import NonRetryableError, agent, import_functions, log with import_functions(): - from src.functions.llm_chat import LlmChatInput, Message, llm_chat + from src.functions.llm_fast import LlmChatInput, Message, llm_chat from src.functions.vapi_call import VapiCallInput, vapi_call @@ -42,7 +42,9 @@ async def messages(self, messages_event: MessagesEvent) -> list[Message]: error_message = f"llm_chat function failed: {e}" raise NonRetryableError(error_message) from e else: - self.messages.append(Message(role="assistant", content=str(assistant_message))) + self.messages.append( + Message(role="assistant", content=str(assistant_message)) + ) return self.messages @agent.event diff --git a/agent_telephony/vapi/agent_vapi/src/services.py b/agent_telephony/vapi/agent_vapi/src/services.py index 8ca0b99e..7be30eeb 100644 --- a/agent_telephony/vapi/agent_vapi/src/services.py +++ b/agent_telephony/vapi/agent_vapi/src/services.py @@ -7,7 +7,7 @@ from src.agents.agent import AgentVapi from src.client import client -from src.functions.llm_chat import llm_chat +from src.functions.llm_fast import llm_chat from src.functions.vapi_call import vapi_call diff --git a/community/livekit_opentelemetry/pyproject.toml b/community/livekit_opentelemetry/pyproject.toml new file mode 100644 index 00000000..64fe8f92 --- /dev/null +++ b/community/livekit_opentelemetry/pyproject.toml @@ -0,0 +1,29 @@ +[project] +name = "agent_telephony_livekit_pipeline" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "livekit-agents>=0.12.15", + "livekit-plugins-deepgram>=0.6.19", + "livekit-plugins-elevenlabs>=0.7.13", + "livekit-plugins-openai>=0.11.0", + "livekit-plugins-silero>=0.7.4", + "livekit-plugins-turn-detector>=0.4.2", + "opentelemetry-api>=1.30.0", + "opentelemetry-exporter-gcp-monitoring>=1.9.0a0", + "opentelemetry-sdk>=1.30.0", + "python-dotenv==1.0.1", + "restack-ai>=0.0.80", +] + +[tool.hatch.build.targets.sdist] +include = ["src"] + +[tool.hatch.build.targets.wheel] +include = ["src"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/community/livekit_opentelemetry/src/client.py b/community/livekit_opentelemetry/src/client.py new file mode 100644 index 00000000..885bf8ea --- /dev/null +++ b/community/livekit_opentelemetry/src/client.py @@ -0,0 +1,19 @@ +import os + +from dotenv import load_dotenv +from restack_ai import Restack +from restack_ai.restack import CloudConnectionOptions + +# Load environment variables from a .env file +load_dotenv() + + +engine_id = os.getenv("RESTACK_ENGINE_ID") +address = os.getenv("RESTACK_ENGINE_ADDRESS") +api_key = os.getenv("RESTACK_ENGINE_API_KEY") +api_address = os.getenv("RESTACK_ENGINE_API_ADDRESS") + +connection_options = CloudConnectionOptions( + engine_id=engine_id, address=address, api_key=api_key, api_address=api_address +) +client = Restack(connection_options) diff --git a/community/livekit_opentelemetry/src/otel_exporter.py b/community/livekit_opentelemetry/src/otel_exporter.py new file mode 100644 index 00000000..aaec554f --- /dev/null +++ b/community/livekit_opentelemetry/src/otel_exporter.py @@ -0,0 +1,48 @@ +# exporter_setup.py + +from opentelemetry.exporter.cloud_monitoring import CloudMonitoringMetricsExporter +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +import os +import atexit +import logging +from opentelemetry import metrics + +# Set up logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +class LoggingCloudMonitoringMetricsExporter(CloudMonitoringMetricsExporter): + def export(self, metrics, timeout_millis=None): + try: + result = super().export(metrics, timeout_millis=timeout_millis) + if result is None: # Assuming None indicates success + logging.info("Metrics successfully sent to Google Cloud Monitoring.") + else: + logging.error("Failed to send metrics to Google Cloud Monitoring.") + return result + except Exception as e: + logging.error(f"Exception during export: {e}") + return None + +def setup_google_cloud_exporter(): + """Set up the Google Cloud exporter.""" + try: + exporter = LoggingCloudMonitoringMetricsExporter( + project_id=os.environ.get("GOOGLE_CLOUD_PROJECT") + ) + logger.info("Google Cloud Monitoring exporter set up successfully.") + + # Set up a periodic exporting metric reader + reader = PeriodicExportingMetricReader(exporter, export_interval_millis=60000) + logger.info("Periodic exporting metric reader set up successfully.") + + # Ensure the exporter is flushed before the application exits + atexit.register(reader.shutdown) + + logger.info(f"Using Google Cloud project: {os.environ.get('GOOGLE_CLOUD_PROJECT')}") + return reader + except Exception as e: + logger.error(f"Failed to set up Google Cloud exporter: {e}") + return None \ No newline at end of file diff --git a/community/livekit_opentelemetry/src/otel_provider.py b/community/livekit_opentelemetry/src/otel_provider.py new file mode 100644 index 00000000..98d44f79 --- /dev/null +++ b/community/livekit_opentelemetry/src/otel_provider.py @@ -0,0 +1,140 @@ +# otel_setup.py + +from opentelemetry import metrics +from opentelemetry.sdk.metrics import MeterProvider +from livekit.agents.metrics.base import AgentMetrics, PipelineLLMMetrics, PipelineSTTMetrics, PipelineTTSMetrics, PipelineVADMetrics +import logging +from src.otel_exporter import setup_google_cloud_exporter +from opentelemetry.sdk.resources import Resource +# Set up logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Set up the meter provider with the exporter +reader = setup_google_cloud_exporter() +if reader: + metrics.set_meter_provider(MeterProvider( + metric_readers=[reader], + resource=Resource.create({ + "service.name": "livekit_metrics", + "service.namespace": "livekit_telemetry", + "service.instance.id": "livekit_pipeline", + }) + )) + logger.info("MeterProvider set up successfully with the MetricReader.") +else: + logger.error("Failed to set up the metric reader with the exporter.") + +# Create a meter +meter = metrics.get_meter(__name__) + +# Create ValueRecorder instruments for detailed metrics +ttft_recorder = meter.create_histogram( + name="llm_ttft", + description="Time to first token for LLM", + unit="ms", +) + +duration_recorder = meter.create_histogram( + name="llm_duration", + description="Duration of LLM processing", + unit="ms", +) + +completion_tokens_recorder = meter.create_histogram( + name="llm_completion_tokens", + description="Number of completion tokens", + unit="1", +) + +prompt_tokens_recorder = meter.create_histogram( + name="llm_prompt_tokens", + description="Number of prompt tokens", + unit="1", +) + +total_tokens_recorder = meter.create_histogram( + name="llm_total_tokens", + description="Total number of tokens", + unit="1", +) + +tokens_per_second_recorder = meter.create_histogram( + name="llm_tokens_per_second", + description="Tokens processed per second", + unit="tokens/s", +) + +stt_duration_recorder = meter.create_histogram( + name="stt_duration", + description="Duration of STT processing", + unit="ms", +) + +stt_audio_duration_recorder = meter.create_histogram( + name="stt_audio_duration", + description="Audio duration for STT", + unit="ms", +) + +tts_ttfb_recorder = meter.create_histogram( + name="tts_ttfb", + description="Time to first byte for TTS", + unit="ms", +) + +tts_duration_recorder = meter.create_histogram( + name="tts_duration", + description="Duration of TTS processing", + unit="ms", +) + +tts_audio_duration_recorder = meter.create_histogram( + name="tts_audio_duration", + description="Audio duration for TTS", + unit="ms", +) + +vad_idle_time_recorder = meter.create_histogram( + name="vad_idle_time", + description="Idle time for VAD", + unit="ms", +) + +vad_inference_duration_recorder = meter.create_histogram( + name="vad_inference_duration", + description="Total inference duration for VAD", + unit="ms", +) + +# Function to record metrics +def record_metrics(agent_metrics: AgentMetrics): + """Record detailed metrics based on the type of AgentMetrics.""" + if isinstance(agent_metrics, PipelineLLMMetrics): + ttft_recorder.record(agent_metrics.ttft * 1000, {"label": agent_metrics.label}) + duration_recorder.record(agent_metrics.duration * 1000, {"label": agent_metrics.label}) + completion_tokens_recorder.record(agent_metrics.completion_tokens, {"label": agent_metrics.label}) + prompt_tokens_recorder.record(agent_metrics.prompt_tokens, {"label": agent_metrics.label}) + total_tokens_recorder.record(agent_metrics.total_tokens, {"label": agent_metrics.label}) + tokens_per_second_recorder.record(agent_metrics.tokens_per_second, {"label": agent_metrics.label}) + elif isinstance(agent_metrics, PipelineSTTMetrics): + + stt_duration_recorder.record(agent_metrics.duration * 1000, {"label": agent_metrics.label}) + stt_audio_duration_recorder.record(agent_metrics.audio_duration * 1000, {"label": agent_metrics.label}) + elif isinstance(agent_metrics, PipelineTTSMetrics): + + tts_ttfb_recorder.record(agent_metrics.ttfb * 1000, {"label": agent_metrics.label}) + tts_duration_recorder.record(agent_metrics.duration * 1000, {"label": agent_metrics.label}) + tts_audio_duration_recorder.record(agent_metrics.audio_duration * 1000, {"label": agent_metrics.label}) + elif isinstance(agent_metrics, PipelineVADMetrics): + + vad_idle_time_recorder.record(agent_metrics.idle_time * 1000, {"label": agent_metrics.label}) + vad_inference_duration_recorder.record(agent_metrics.inference_duration_total * 1000, {"label": agent_metrics.label}) + +def calculate_total_latency(eou_metrics, llm_metrics, tts_metrics): + """Calculate the total latency.""" + return ( + eou_metrics.end_of_utterance_delay + + llm_metrics.ttft + + tts_metrics.ttfb + ) * 1000 # Convert to milliseconds \ No newline at end of file diff --git a/community/livekit_opentelemetry/src/pipeline.py b/community/livekit_opentelemetry/src/pipeline.py new file mode 100644 index 00000000..19111a7b --- /dev/null +++ b/community/livekit_opentelemetry/src/pipeline.py @@ -0,0 +1,152 @@ +import json +import logging +import os + +from dotenv import load_dotenv +from livekit.agents import ( + AutoSubscribe, + JobContext, + JobProcess, + WorkerOptions, + cli, + metrics, + +) +from livekit.agents.pipeline import VoicePipelineAgent +from livekit.plugins import deepgram, elevenlabs, openai, silero, turn_detector +from src.client import client +from src.otel_provider import record_metrics +from src.otel_exporter import setup_google_cloud_exporter + +# Set up the Google Cloud exporter +setup_google_cloud_exporter() + +# Load environment variables from .env +load_dotenv(dotenv_path=".env") + +# Setup basic logging configuration so that all logs are properly output. +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Initialize total variables at the beginning of the function or module +# These should be initialized at the module level if they need to persist across multiple calls + +total_llm_prompt_tokens = 0 +# Initialize other total variables similarly + +total_llm_completion_tokens = 0 +total_tts_characters_count = 0 +total_stt_audio_duration = 0 + +def validate_envs() -> None: + required_envs = { + "LIVEKIT_URL": "LiveKit server URL", + "LIVEKIT_API_KEY": "API Key for LiveKit", + "LIVEKIT_API_SECRET": "API Secret for LiveKit", + "DEEPGRAM_API_KEY": "API key for Deepgram (used for STT)", + "ELEVEN_API_KEY": "API key for ElevenLabs (used for TTS)", + } + for key, description in required_envs.items(): + if not os.environ.get(key): + logger.warning("Environment variable %s (%s) is not set.", key, description) + + +# Validate environments at module load +validate_envs() + + +def prewarm(proc: JobProcess) -> None: + logger.info("Prewarming: loading VAD model...") + proc.userdata["vad"] = silero.VAD.load() + logger.info("VAD model loaded successfully.") + + +async def entrypoint(ctx: JobContext) -> None: + metadata = ctx.job.metadata + + logger.info("job metadata: %s", metadata) + + if isinstance(metadata, str): + try: + metadata_obj = json.loads(metadata) + except json.JSONDecodeError: + try: + normalized = metadata.replace("'", '"') + metadata_obj = json.loads(normalized) + except json.JSONDecodeError as norm_error: + logger.warning( + "Normalization failed, using default values: %s", norm_error + ) + metadata_obj = {} + else: + metadata_obj = metadata + + logger.info("metadata_obj: %s", metadata_obj) + + agent_name = metadata_obj.get("agent_name") + agent_id = metadata_obj.get("agent_id") + run_id = metadata_obj.get("run_id") + + # Retrieve the Host from environment variables. + engine_api_address = os.environ.get("RESTACK_ENGINE_API_ADDRESS") + if not engine_api_address: + agent_backend_host = "http://localhost:9233" + elif not engine_api_address.startswith("https://"): + agent_backend_host = "https://" + engine_api_address + else: + agent_backend_host = engine_api_address + + logger.info("Using RESTACK_ENGINE_API_ADDRESS: %s", agent_backend_host) + + agent_url = f"{agent_backend_host}/stream/agents/{agent_name}/{agent_id}/{run_id}" + logger.info("Agent URL: %s", agent_url) + + logger.info("Connecting to room: %s", ctx.room.name) + await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY) + + # Wait for the first participant to connect + participant = await ctx.wait_for_participant() + logger.info("Starting voice assistant for participant: %s", participant.identity) + + agent = VoicePipelineAgent( + vad=ctx.proc.userdata["vad"], + stt=deepgram.STT(), + llm=openai.LLM( + # model="gpt-4o-mini", + # api_key=os.environ.get("OPENAI_API_KEY"), + api_key=f"{agent_id}-livekit", + base_url=agent_url, + ), + tts=elevenlabs.TTS(), + turn_detector=turn_detector.EOUModel(), + # minimum delay for endpointing, used when turn detector believes the user is done with their turn + # min_endpointing_delay=0.5, + # # maximum delay for endpointing, used when turn detector does not believe the user is done with their turn + # max_endpointing_delay=5.0, + ) + + + + usage_collector = metrics.UsageCollector() + + async def send_pipeline_metrics(summary: str) -> None: + logger.warning("Sending pipeline metrics") + + + @agent.on("metrics_collected") + def on_metrics_collected(agent_metrics: metrics.AgentMetrics) -> None: + record_metrics(agent_metrics) + + usage_collector.get_summary() + + agent.start(ctx.room, participant) + + +if __name__ == "__main__": + cli.run_app( + WorkerOptions( + entrypoint_fnc=entrypoint, + agent_name="AgentTelemetry", + prewarm_fnc=prewarm, + ) + ) From 124603263a87416631eed0ebb22d92e191380a94 Mon Sep 17 00:00:00 2001 From: aboutphilippe Date: Sun, 9 Mar 2025 01:43:54 +0100 Subject: [PATCH 02/11] bump restack_ai --- agent_apis/pyproject.toml | 2 +- agent_apis/requirements.txt | 2 +- agent_chat/pyproject.toml | 2 +- agent_chat/requirements.txt | 2 +- agent_humanloop/pyproject.toml | 2 +- agent_humanloop/requirements.txt | 2 +- agent_rag/pyproject.toml | 2 +- agent_rag/requirements.txt | 2 +- agent_stream/pyproject.toml | 2 +- agent_stream/requirements.txt | 2 +- agent_telephony/twilio_livekit/agent_twilio/pyproject.toml | 2 +- agent_telephony/twilio_livekit/livekit_pipeline/pyproject.toml | 2 +- agent_todo/pyproject.toml | 2 +- agent_todo/requirements.txt | 2 +- agent_tool/pyproject.toml | 2 +- agent_tool/requirements.txt | 2 +- agent_voice/livekit/agent/pyproject.toml | 2 +- audio_transcript/pyproject.toml | 2 +- audio_transcript/requirements.txt | 2 +- child_workflows/pyproject.toml | 2 +- child_workflows/requirements.txt | 2 +- community/bostondynamics_spot/pyproject.toml | 2 +- community/custom_llm_gemini/pyproject.toml | 2 +- .../pyproject.toml | 2 +- community/defense_quickstart_denoise/pyproject.toml | 2 +- .../defense_quickstart_news_scraper_summarizer/pyproject.toml | 2 +- community/e2b/pyproject.toml | 2 +- community/elevenlabs/pyproject.toml | 2 +- community/email_sender/pyproject.toml | 2 +- community/email_sender/requirements.txt | 2 +- community/fastapi_gemini_feedback/pyproject.toml | 2 +- community/flask_gemini/pyproject.toml | 2 +- community/flask_togetherai_llamaindex/pyproject.toml | 2 +- community/gemini/pyproject.toml | 2 +- community/livekit_opentelemetry/pyproject.toml | 2 +- community/llama_quickstart/pyproject.toml | 2 +- community/lmnt/pyproject.toml | 2 +- community/openai_greet/pyproject.toml | 2 +- community/openai_greet/requirements.txt | 2 +- community/re_act/pyproject.toml | 2 +- community/re_act/requirements.txt | 2 +- community/streamlit/pyproject.toml | 2 +- community/streamlit_fastapi_togetherai_llama/pyproject.toml | 2 +- community/stripe_ai/pyproject.toml | 2 +- community/weaviate_search/pyproject.toml | 2 +- encryption/pyproject.toml | 2 +- encryption/requirements.txt | 2 +- production_demo/pyproject.toml | 2 +- production_demo/requirements.txt | 2 +- 49 files changed, 49 insertions(+), 49 deletions(-) diff --git a/agent_apis/pyproject.toml b/agent_apis/pyproject.toml index 13b9e276..8ba62112 100644 --- a/agent_apis/pyproject.toml +++ b/agent_apis/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "python-dotenv==1.0.1", "openai>=1.61.0", "aiohttp>=3.11.12", - "restack-ai>=0.0.80",] + "restack-ai>=0.0.81",] [project.scripts] dev = "src.services:watch_services" diff --git a/agent_apis/requirements.txt b/agent_apis/requirements.txt index 9cd7e871..ec404a85 100644 --- a/agent_apis/requirements.txt +++ b/agent_apis/requirements.txt @@ -69,7 +69,7 @@ python-dotenv==1.0.1 # via # openai-greet (pyproject.toml) # restack-ai -restack-ai==0.0.80 +restack-ai==0.0.81 # via openai-greet (pyproject.toml) sniffio==1.3.1 # via diff --git a/agent_chat/pyproject.toml b/agent_chat/pyproject.toml index 20dc7676..a3a0257a 100644 --- a/agent_chat/pyproject.toml +++ b/agent_chat/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "watchfiles>=1.0.4", "python-dotenv==1.0.1", "openai>=1.61.0", - "restack-ai>=0.0.80",] + "restack-ai>=0.0.81",] [project.scripts] dev = "src.services:watch_services" diff --git a/agent_chat/requirements.txt b/agent_chat/requirements.txt index 99966269..31180607 100644 --- a/agent_chat/requirements.txt +++ b/agent_chat/requirements.txt @@ -67,7 +67,7 @@ python-dotenv==1.0.1 # via # agent-chat (pyproject.toml) # restack-ai -restack-ai==0.0.80 +restack-ai==0.0.81 # via agent-chat (pyproject.toml) sniffio==1.3.1 # via diff --git a/agent_humanloop/pyproject.toml b/agent_humanloop/pyproject.toml index 9e2598db..1226a39e 100644 --- a/agent_humanloop/pyproject.toml +++ b/agent_humanloop/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", ] diff --git a/agent_humanloop/requirements.txt b/agent_humanloop/requirements.txt index 8d673248..b6fc5f74 100644 --- a/agent_humanloop/requirements.txt +++ b/agent_humanloop/requirements.txt @@ -46,7 +46,7 @@ python-dotenv==1.0.1 # via # human-loop (pyproject.toml) # restack-ai -restack-ai==0.0.80 +restack-ai==0.0.81 # via human-loop (pyproject.toml) sniffio==1.3.1 # via anyio diff --git a/agent_rag/pyproject.toml b/agent_rag/pyproject.toml index 5f5d23f4..720c3a59 100644 --- a/agent_rag/pyproject.toml +++ b/agent_rag/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "watchfiles>=1.0.4", "requests==2.32.3", "python-dotenv==1.0.1", - "restack-ai>=0.0.80",] + "restack-ai>=0.0.81",] [project.scripts] dev = "src.services:watch_services" diff --git a/agent_rag/requirements.txt b/agent_rag/requirements.txt index 3559ac65..87d9d88a 100644 --- a/agent_rag/requirements.txt +++ b/agent_rag/requirements.txt @@ -73,7 +73,7 @@ python-dotenv==1.0.1 # restack-ai requests==2.32.3 # via agent-rag (pyproject.toml) -restack-ai==0.0.80 +restack-ai==0.0.81 # via agent-rag (pyproject.toml) sniffio==1.3.1 # via diff --git a/agent_stream/pyproject.toml b/agent_stream/pyproject.toml index 3095c240..81dc91b4 100644 --- a/agent_stream/pyproject.toml +++ b/agent_stream/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "python-dotenv==1.0.1", "openai>=1.61.0", "livekit-api>=0.8.2", - "restack-ai>=0.0.80", + "restack-ai>=0.0.81", ] [project.scripts] diff --git a/agent_stream/requirements.txt b/agent_stream/requirements.txt index a2825977..2379f729 100644 --- a/agent_stream/requirements.txt +++ b/agent_stream/requirements.txt @@ -78,7 +78,7 @@ python-dotenv==1.0.1 # via # agent-stream (pyproject.toml) # restack-ai -restack-ai==0.0.80 +restack-ai==0.0.81 # via agent-stream (pyproject.toml) sniffio==1.3.1 # via diff --git a/agent_telephony/twilio_livekit/agent_twilio/pyproject.toml b/agent_telephony/twilio_livekit/agent_twilio/pyproject.toml index 799e316c..836d4ca5 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/pyproject.toml +++ b/agent_telephony/twilio_livekit/agent_twilio/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "python-dotenv==1.0.1", "openai>=1.61.0", "livekit-api>=0.8.2", - "restack-ai>=0.0.80",] + "restack-ai>=0.0.81",] [project.scripts] dev = "src.services:watch_services" diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/pyproject.toml b/agent_telephony/twilio_livekit/livekit_pipeline/pyproject.toml index 215f6be4..9919aa89 100644 --- a/agent_telephony/twilio_livekit/livekit_pipeline/pyproject.toml +++ b/agent_telephony/twilio_livekit/livekit_pipeline/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "livekit-plugins-silero>=0.7.4", "livekit-plugins-turn-detector>=0.4.2", "python-dotenv==1.0.1", - "restack-ai>=0.0.80", + "restack-ai>=0.0.81", ] [tool.hatch.build.targets.sdist] diff --git a/agent_todo/pyproject.toml b/agent_todo/pyproject.toml index ceedc980..1a473533 100644 --- a/agent_todo/pyproject.toml +++ b/agent_todo/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "watchfiles>=1.0.4", "python-dotenv==1.0.1", "openai>=1.61.0", - "restack-ai>=0.0.80",] + "restack-ai>=0.0.81",] [project.scripts] dev = "src.services:watch_services" diff --git a/agent_todo/requirements.txt b/agent_todo/requirements.txt index fbe939fb..23eb1d39 100644 --- a/agent_todo/requirements.txt +++ b/agent_todo/requirements.txt @@ -67,7 +67,7 @@ python-dotenv==1.0.1 # via # quickstart (pyproject.toml) # restack-ai -restack-ai==0.0.80 +restack-ai==0.0.81 # via quickstart (pyproject.toml) sniffio==1.3.1 # via diff --git a/agent_tool/pyproject.toml b/agent_tool/pyproject.toml index b9b958f2..1e7108da 100644 --- a/agent_tool/pyproject.toml +++ b/agent_tool/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "watchfiles>=1.0.4", "requests==2.32.3", "python-dotenv==1.0.1", - "restack-ai>=0.0.80",] + "restack-ai>=0.0.81",] [project.scripts] dev = "src.services:watch_services" diff --git a/agent_tool/requirements.txt b/agent_tool/requirements.txt index 52fc17f0..24cbb7f3 100644 --- a/agent_tool/requirements.txt +++ b/agent_tool/requirements.txt @@ -73,7 +73,7 @@ python-dotenv==1.0.1 # restack-ai requests==2.32.3 # via agent-tool (pyproject.toml) -restack-ai==0.0.80 +restack-ai==0.0.81 # via agent-tool (pyproject.toml) sniffio==1.3.1 # via diff --git a/agent_voice/livekit/agent/pyproject.toml b/agent_voice/livekit/agent/pyproject.toml index dc7e9c63..179b9929 100644 --- a/agent_voice/livekit/agent/pyproject.toml +++ b/agent_voice/livekit/agent/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "python-dotenv==1.0.1", "openai>=1.61.0", "livekit-api>=0.8.2", - "restack-ai>=0.0.80",] + "restack-ai>=0.0.81",] [project.scripts] dev = "src.services:watch_services" diff --git a/audio_transcript/pyproject.toml b/audio_transcript/pyproject.toml index c6e799c9..5ddfa7d6 100644 --- a/audio_transcript/pyproject.toml +++ b/audio_transcript/pyproject.toml @@ -8,7 +8,7 @@ readme = "README.md" dependencies = [ "openai>=1.61.0", "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", ] diff --git a/audio_transcript/requirements.txt b/audio_transcript/requirements.txt index b044054d..b1caf28a 100644 --- a/audio_transcript/requirements.txt +++ b/audio_transcript/requirements.txt @@ -67,7 +67,7 @@ python-dotenv==1.0.1 # via # audio-transcript (pyproject.toml) # restack-ai -restack-ai==0.0.80 +restack-ai==0.0.81 # via audio-transcript (pyproject.toml) sniffio==1.3.1 # via diff --git a/child_workflows/pyproject.toml b/child_workflows/pyproject.toml index c21b2ac1..9831cbfb 100644 --- a/child_workflows/pyproject.toml +++ b/child_workflows/pyproject.toml @@ -9,7 +9,7 @@ dependencies = [ "pydantic>=2.10.6", "watchfiles>=1.0.4", "python-dotenv==1.0.1", - "restack-ai>=0.0.80",] + "restack-ai>=0.0.81",] [project.scripts] dev = "src.services:watch_services" diff --git a/child_workflows/requirements.txt b/child_workflows/requirements.txt index 1a6aa5a2..493161d9 100644 --- a/child_workflows/requirements.txt +++ b/child_workflows/requirements.txt @@ -46,7 +46,7 @@ python-dotenv==1.0.1 # via # child-workflows (pyproject.toml) # restack-ai -restack-ai==0.0.80 +restack-ai==0.0.81 # via child-workflows (pyproject.toml) sniffio==1.3.1 # via anyio diff --git a/community/bostondynamics_spot/pyproject.toml b/community/bostondynamics_spot/pyproject.toml index 6624eaaa..7462d96a 100644 --- a/community/bostondynamics_spot/pyproject.toml +++ b/community/bostondynamics_spot/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "openai>=1.61.0", diff --git a/community/custom_llm_gemini/pyproject.toml b/community/custom_llm_gemini/pyproject.toml index 8c98ac98..7340e0db 100644 --- a/community/custom_llm_gemini/pyproject.toml +++ b/community/custom_llm_gemini/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "flask==3.0.3", diff --git a/community/defense_quickstart_audio_transcription_translation/pyproject.toml b/community/defense_quickstart_audio_transcription_translation/pyproject.toml index 8ca5f02d..0fd1c5bd 100644 --- a/community/defense_quickstart_audio_transcription_translation/pyproject.toml +++ b/community/defense_quickstart_audio_transcription_translation/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "openai>=1.61.0", diff --git a/community/defense_quickstart_denoise/pyproject.toml b/community/defense_quickstart_denoise/pyproject.toml index 30fe432b..1136164c 100644 --- a/community/defense_quickstart_denoise/pyproject.toml +++ b/community/defense_quickstart_denoise/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "fastapi==0.115.4", diff --git a/community/defense_quickstart_news_scraper_summarizer/pyproject.toml b/community/defense_quickstart_news_scraper_summarizer/pyproject.toml index de308da8..ad0ffce6 100644 --- a/community/defense_quickstart_news_scraper_summarizer/pyproject.toml +++ b/community/defense_quickstart_news_scraper_summarizer/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "openai>=1.61.0", diff --git a/community/e2b/pyproject.toml b/community/e2b/pyproject.toml index 4e8c39f2..4a2687ac 100644 --- a/community/e2b/pyproject.toml +++ b/community/e2b/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "openai>=1.61.0", diff --git a/community/elevenlabs/pyproject.toml b/community/elevenlabs/pyproject.toml index fbcbf79e..708148a9 100644 --- a/community/elevenlabs/pyproject.toml +++ b/community/elevenlabs/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "openai>=1.61.0", diff --git a/community/email_sender/pyproject.toml b/community/email_sender/pyproject.toml index ef59129a..5ca5a899 100644 --- a/community/email_sender/pyproject.toml +++ b/community/email_sender/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "openai>=1.61.0", diff --git a/community/email_sender/requirements.txt b/community/email_sender/requirements.txt index 6727b40d..181bc2c7 100644 --- a/community/email_sender/requirements.txt +++ b/community/email_sender/requirements.txt @@ -69,7 +69,7 @@ python-dotenv==1.0.1 # restack-ai python-http-client==3.3.7 # via sendgrid -restack-ai==0.0.80 +restack-ai==0.0.81 # via email-sender (pyproject.toml) sendgrid==6.11.0 # via email-sender (pyproject.toml) diff --git a/community/fastapi_gemini_feedback/pyproject.toml b/community/fastapi_gemini_feedback/pyproject.toml index 5b8bffde..a95d0180 100644 --- a/community/fastapi_gemini_feedback/pyproject.toml +++ b/community/fastapi_gemini_feedback/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "google-generativeai==0.8.3", diff --git a/community/flask_gemini/pyproject.toml b/community/flask_gemini/pyproject.toml index 22a3c37b..f207c289 100644 --- a/community/flask_gemini/pyproject.toml +++ b/community/flask_gemini/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "flask[async]==3.0.3", diff --git a/community/flask_togetherai_llamaindex/pyproject.toml b/community/flask_togetherai_llamaindex/pyproject.toml index 8b7f7247..693c486a 100644 --- a/community/flask_togetherai_llamaindex/pyproject.toml +++ b/community/flask_togetherai_llamaindex/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "llama-index==0.11.22", diff --git a/community/gemini/pyproject.toml b/community/gemini/pyproject.toml index ab84c83e..74c1e7d8 100644 --- a/community/gemini/pyproject.toml +++ b/community/gemini/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "google-genai==0.5.0", diff --git a/community/livekit_opentelemetry/pyproject.toml b/community/livekit_opentelemetry/pyproject.toml index 64fe8f92..f601e2a7 100644 --- a/community/livekit_opentelemetry/pyproject.toml +++ b/community/livekit_opentelemetry/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "opentelemetry-exporter-gcp-monitoring>=1.9.0a0", "opentelemetry-sdk>=1.30.0", "python-dotenv==1.0.1", - "restack-ai>=0.0.80", + "restack-ai>=0.0.81", ] [tool.hatch.build.targets.sdist] diff --git a/community/llama_quickstart/pyproject.toml b/community/llama_quickstart/pyproject.toml index 3b5692f7..e4f6acbe 100644 --- a/community/llama_quickstart/pyproject.toml +++ b/community/llama_quickstart/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "fastapi==0.115.4", diff --git a/community/lmnt/pyproject.toml b/community/lmnt/pyproject.toml index 28bbc091..15a6aaa3 100644 --- a/community/lmnt/pyproject.toml +++ b/community/lmnt/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "lmnt==1.1.4", diff --git a/community/openai_greet/pyproject.toml b/community/openai_greet/pyproject.toml index da38dd09..e17a9c62 100644 --- a/community/openai_greet/pyproject.toml +++ b/community/openai_greet/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "openai>=1.61.0", diff --git a/community/openai_greet/requirements.txt b/community/openai_greet/requirements.txt index 4a0a0869..fff18c47 100644 --- a/community/openai_greet/requirements.txt +++ b/community/openai_greet/requirements.txt @@ -67,7 +67,7 @@ python-dotenv==1.0.1 # via # openai-greet (pyproject.toml) # restack-ai -restack-ai==0.0.80 +restack-ai==0.0.81 # via openai-greet (pyproject.toml) sniffio==1.3.1 # via diff --git a/community/re_act/pyproject.toml b/community/re_act/pyproject.toml index 009680e4..d931a48c 100644 --- a/community/re_act/pyproject.toml +++ b/community/re_act/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.13" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "openai>=1.61.0", diff --git a/community/re_act/requirements.txt b/community/re_act/requirements.txt index 6673a534..51e7ab02 100644 --- a/community/re_act/requirements.txt +++ b/community/re_act/requirements.txt @@ -69,7 +69,7 @@ python-dotenv==1.0.1 # restack-ai python-http-client==3.3.7 # via sendgrid -restack-ai==0.0.80 +restack-ai==0.0.81 # via re-act-example (pyproject.toml) sendgrid==6.11.0 # via re-act-example (pyproject.toml) diff --git a/community/streamlit/pyproject.toml b/community/streamlit/pyproject.toml index aa8f82c9..bcfdb9cc 100644 --- a/community/streamlit/pyproject.toml +++ b/community/streamlit/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "streamlit==1.39.0", diff --git a/community/streamlit_fastapi_togetherai_llama/pyproject.toml b/community/streamlit_fastapi_togetherai_llama/pyproject.toml index 5173716d..a697f098 100644 --- a/community/streamlit_fastapi_togetherai_llama/pyproject.toml +++ b/community/streamlit_fastapi_togetherai_llama/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "fastapi==0.115.4", diff --git a/community/stripe_ai/pyproject.toml b/community/stripe_ai/pyproject.toml index 06dad363..47bb4499 100644 --- a/community/stripe_ai/pyproject.toml +++ b/community/stripe_ai/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "pydantic>=2.10.3,<3", diff --git a/community/weaviate_search/pyproject.toml b/community/weaviate_search/pyproject.toml index 55a3159b..10d84d84 100644 --- a/community/weaviate_search/pyproject.toml +++ b/community/weaviate_search/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "requests==2.32.3", diff --git a/encryption/pyproject.toml b/encryption/pyproject.toml index 1b92de32..4f1fb2c4 100644 --- a/encryption/pyproject.toml +++ b/encryption/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "aiohttp>=3.11.10", - "restack-ai==0.0.80", + "restack-ai==0.0.81", ] [project.scripts] diff --git a/encryption/requirements.txt b/encryption/requirements.txt index 478f677e..411c02d2 100644 --- a/encryption/requirements.txt +++ b/encryption/requirements.txt @@ -40,7 +40,7 @@ pydantic-core==2.27.2 # via pydantic python-dotenv==1.0.1 # via restack-ai -restack-ai==0.0.80 +restack-ai==0.0.81 # via encryption (pyproject.toml) temporalio==1.10.0 # via restack-ai diff --git a/production_demo/pyproject.toml b/production_demo/pyproject.toml index 47d6a962..d3c3ca24 100644 --- a/production_demo/pyproject.toml +++ b/production_demo/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.10,<3.14" readme = "README.md" dependencies = [ "pydantic>=2.10.6", - "restack-ai==0.0.80", + "restack-ai==0.0.81", "watchfiles>=1.0.4", "python-dotenv==1.0.1", "openai>=1.61.0", diff --git a/production_demo/requirements.txt b/production_demo/requirements.txt index 0f029fe3..415447f0 100644 --- a/production_demo/requirements.txt +++ b/production_demo/requirements.txt @@ -67,7 +67,7 @@ python-dotenv==1.0.1 # via # production-demo (pyproject.toml) # restack-ai -restack-ai==0.0.80 +restack-ai==0.0.81 # via production-demo (pyproject.toml) sniffio==1.3.1 # via From da5bf12f3d447249b10a9fa654b40fd70ef08be2 Mon Sep 17 00:00:00 2001 From: aboutphilippe Date: Sun, 9 Mar 2025 01:46:27 +0100 Subject: [PATCH 03/11] cleanup --- .../twilio_livekit/agent_twilio/src/functions/llm_talk.py | 2 +- agent_telephony/vapi/agent_vapi/src/agents/agent.py | 2 +- agent_telephony/vapi/agent_vapi/src/services.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_talk.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_talk.py index 1b43e8da..4bc4e903 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_talk.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_talk.py @@ -61,4 +61,4 @@ async def llm_talk(function_input: LlmTalkInput) -> str: return response.choices[0].message.content except Exception as e: - raise NonRetryableError(f"llm_fast failed: {e}") from e + raise NonRetryableError(f"llm_talk failed: {e}") from e diff --git a/agent_telephony/vapi/agent_vapi/src/agents/agent.py b/agent_telephony/vapi/agent_vapi/src/agents/agent.py index 914caf18..f4643835 100644 --- a/agent_telephony/vapi/agent_vapi/src/agents/agent.py +++ b/agent_telephony/vapi/agent_vapi/src/agents/agent.py @@ -4,7 +4,7 @@ from restack_ai.agent import NonRetryableError, agent, import_functions, log with import_functions(): - from src.functions.llm_fast import LlmChatInput, Message, llm_chat + from src.functions.llm_chat import LlmChatInput, Message, llm_chat from src.functions.vapi_call import VapiCallInput, vapi_call diff --git a/agent_telephony/vapi/agent_vapi/src/services.py b/agent_telephony/vapi/agent_vapi/src/services.py index 7be30eeb..8ca0b99e 100644 --- a/agent_telephony/vapi/agent_vapi/src/services.py +++ b/agent_telephony/vapi/agent_vapi/src/services.py @@ -7,7 +7,7 @@ from src.agents.agent import AgentVapi from src.client import client -from src.functions.llm_fast import llm_chat +from src.functions.llm_chat import llm_chat from src.functions.vapi_call import vapi_call From 654af12035806488d6dbd602f0c1ba934cb16449 Mon Sep 17 00:00:00 2001 From: aboutphilippe Date: Sun, 9 Mar 2025 01:48:32 +0100 Subject: [PATCH 04/11] cleanup comment --- .../twilio_livekit/agent_twilio/src/workflows/logic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py b/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py index c7f7e909..d39aa299 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py @@ -45,7 +45,7 @@ async def run(self, workflow_input: LogicWorkflowInput) -> str: function_input=LlmLogicInput( messages=[ msg.model_dump() for msg in workflow_input.messages - ], # Convert messages to dict + ], documentation=documentation, ), start_to_close_timeout=timedelta(seconds=60), From eeef2721601e03837902cbd47d508c378abbfaff Mon Sep 17 00:00:00 2001 From: aboutphilippe Date: Sun, 9 Mar 2025 14:46:10 +0100 Subject: [PATCH 05/11] refactor ruff --- .../agent_twilio/src/workflows/logic.py | 4 +- .../livekit_pipeline/entrypoint.sh | 6 +- .../livekit_pipeline/src/client.py | 19 -- .../livekit_pipeline/src/env_check.py | 38 +++ .../livekit_pipeline/src/metrics.py | 115 +++++++++ .../livekit_pipeline/src/pipeline.py | 227 ++++-------------- .../livekit_pipeline/src/restack/client.py | 34 +++ .../livekit_pipeline/src/restack/utils.py | 53 ++++ .../livekit_pipeline/src/utils.py | 106 ++++++++ .../livekit_pipeline/src/worker.py | 139 +++++++++++ agent_telephony/twilio_livekit/readme.md | 6 +- pyproject.toml | 7 + 12 files changed, 546 insertions(+), 208 deletions(-) delete mode 100644 agent_telephony/twilio_livekit/livekit_pipeline/src/client.py create mode 100644 agent_telephony/twilio_livekit/livekit_pipeline/src/env_check.py create mode 100644 agent_telephony/twilio_livekit/livekit_pipeline/src/metrics.py create mode 100644 agent_telephony/twilio_livekit/livekit_pipeline/src/restack/client.py create mode 100644 agent_telephony/twilio_livekit/livekit_pipeline/src/restack/utils.py create mode 100644 agent_telephony/twilio_livekit/livekit_pipeline/src/utils.py create mode 100644 agent_telephony/twilio_livekit/livekit_pipeline/src/worker.py diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py b/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py index d39aa299..7af8c347 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py @@ -43,9 +43,7 @@ async def run(self, workflow_input: LogicWorkflowInput) -> str: slow_response: LlmLogicResponse = await workflow.step( function=llm_logic, function_input=LlmLogicInput( - messages=[ - msg.model_dump() for msg in workflow_input.messages - ], + messages=[msg.model_dump() for msg in workflow_input.messages], documentation=documentation, ), start_to_close_timeout=timedelta(seconds=60), diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/entrypoint.sh b/agent_telephony/twilio_livekit/livekit_pipeline/entrypoint.sh index cab9850f..b68f9255 100644 --- a/agent_telephony/twilio_livekit/livekit_pipeline/entrypoint.sh +++ b/agent_telephony/twilio_livekit/livekit_pipeline/entrypoint.sh @@ -2,7 +2,7 @@ set -e echo "Running download-files..." -uv run python src/pipeline.py download-files +uv run python src/worker.py download-files -echo "Starting pipeline..." -exec uv run python src/pipeline.py start \ No newline at end of file +echo "Starting livekit worker..." +exec uv run python src/worker.py start \ No newline at end of file diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/src/client.py b/agent_telephony/twilio_livekit/livekit_pipeline/src/client.py deleted file mode 100644 index 885bf8ea..00000000 --- a/agent_telephony/twilio_livekit/livekit_pipeline/src/client.py +++ /dev/null @@ -1,19 +0,0 @@ -import os - -from dotenv import load_dotenv -from restack_ai import Restack -from restack_ai.restack import CloudConnectionOptions - -# Load environment variables from a .env file -load_dotenv() - - -engine_id = os.getenv("RESTACK_ENGINE_ID") -address = os.getenv("RESTACK_ENGINE_ADDRESS") -api_key = os.getenv("RESTACK_ENGINE_API_KEY") -api_address = os.getenv("RESTACK_ENGINE_API_ADDRESS") - -connection_options = CloudConnectionOptions( - engine_id=engine_id, address=address, api_key=api_key, api_address=api_address -) -client = Restack(connection_options) diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/src/env_check.py b/agent_telephony/twilio_livekit/livekit_pipeline/src/env_check.py new file mode 100644 index 00000000..d711669d --- /dev/null +++ b/agent_telephony/twilio_livekit/livekit_pipeline/src/env_check.py @@ -0,0 +1,38 @@ +"""Env Check. + +This module checks that all required environment variables are set. +It is intended to be used during application startup to warn developers about missing configurations. +""" + +import os +from dotenv import load_dotenv +from src.utils import logger # Using the shared logger from utils + +REQUIRED_ENVS: dict[str, str] = { + "LIVEKIT_URL": "LiveKit server URL", + "LIVEKIT_API_KEY": "API Key for LiveKit", + "LIVEKIT_API_SECRET": "API Secret for LiveKit", + "DEEPGRAM_API_KEY": "API key for Deepgram (used for STT)", + "ELEVEN_API_KEY": "API key for ElevenLabs (used for TTS)", +} + + +# Load environment variables from the .env file. +load_dotenv(dotenv_path=".env") + +def check_env_vars() -> None: + """Check required environment variables and log warnings if any are missing.""" + try: + for key, description in REQUIRED_ENVS.items(): + if not os.environ.get(key): + logger.warning( + "Environment variable '%s' (%s) is not set.", + key, + description, + ) + logger.info("Environment variable check complete.") + except Exception as e: + logger.exception( + "Error during environment variable check: %s", e + ) + raise diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/src/metrics.py b/agent_telephony/twilio_livekit/livekit_pipeline/src/metrics.py new file mode 100644 index 00000000..82db468b --- /dev/null +++ b/agent_telephony/twilio_livekit/livekit_pipeline/src/metrics.py @@ -0,0 +1,115 @@ +"""Metrics. + +Provides functions for handling and sending metrics data from Livekit to Restack. +""" + +import asyncio +import json + +from livekit.agents import metrics +from livekit.agents.pipeline import VoicePipelineAgent +from src.restack.client import client +from src.utils import logger, track_task + + +async def send_metrics( + pipeline_metrics: metrics.AgentMetrics, + agent_id: str, + run_id: str, +) -> None: + """Send metrics data to restack asynchronously. + + Args: + pipeline_metrics (metrics.AgentMetrics): The metrics data to be sent. + agent_id (str): The identifier for the agent. + run_id (str): The current execution run identifier. + + """ + try: + latencies = [] + if isinstance( + pipeline_metrics, metrics.PipelineEOUMetrics + ): + total_latency = ( + pipeline_metrics.end_of_utterance_delay + ) + latencies.append(total_latency * 1000) + elif isinstance( + pipeline_metrics, metrics.PipelineLLMMetrics + ): + total_latency = pipeline_metrics.ttft + latencies.append(total_latency * 1000) + elif isinstance( + pipeline_metrics, metrics.PipelineTTSMetrics + ): + total_latency = pipeline_metrics.ttfb + latencies.append(total_latency * 1000) + if latencies: + metrics_latencies = str( + json.dumps({"latencies": latencies}) + ) + logger.info( + "Sending pipeline metrics: %s", metrics_latencies + ) + await client.send_agent_event( + event_name="pipeline_metrics", + agent_id=agent_id.replace("local-", ""), + run_id=run_id, + event_input={ + "metrics": pipeline_metrics, + "latencies": metrics_latencies, + }, + ) + except (TypeError, ValueError) as exc: + logger.exception("Error processing metrics data: %s", exc) + raise + except Exception as exc: + logger.exception( + "Unexpected error sending pipeline metrics: %s", exc + ) + raise + + +def setup_pipeline_metrics( + pipeline: VoicePipelineAgent, + agent_id: str, + run_id: str, + usage_collector: metrics.UsageCollector, +) -> None: + """Configure the pipeline to send metrics when they are collected. + + Attaches a callback to the pipeline that logs metrics data and sends it to restack. + + Args: + pipeline (VoicePipelineAgent): The pipeline instance. + agent_id (str): The identifier for the agent. + run_id (str): The current run identifier. + usage_collector (metrics.UsageCollector): Collector for aggregating usage metrics. + + """ + + @pipeline.on("metrics_collected") + def on_metrics_collected( + pipeline_metrics: metrics.AgentMetrics, + ) -> None: + try: + metrics.log_metrics(pipeline_metrics) + track_task( + asyncio.create_task( + send_metrics( + pipeline_metrics, agent_id, run_id + ) + ) + ) + usage_collector.collect(pipeline_metrics) + except (TypeError, ValueError) as exc: + logger.exception( + "Error processing collected metrics: %s", exc + ) + raise + except Exception as exc: + logger.exception( + "Unexpected error handling collected metrics: %s", + exc, + ) + raise diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py b/agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py index 560ec96a..c0c30e53 100644 --- a/agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py +++ b/agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py @@ -1,183 +1,50 @@ -import asyncio -import json -import logging -import os +"""Pipeline. -from dotenv import load_dotenv -from livekit.agents import ( - AutoSubscribe, - JobContext, - JobProcess, - WorkerOptions, - cli, - metrics, -) -from livekit.agents.pipeline import VoicePipelineAgent -from livekit.plugins import deepgram, elevenlabs, openai, silero, turn_detector -from src.client import client - -load_dotenv(dotenv_path=".env") - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -def validate_envs() -> None: - required_envs = { - "LIVEKIT_URL": "LiveKit server URL", - "LIVEKIT_API_KEY": "API Key for LiveKit", - "LIVEKIT_API_SECRET": "API Secret for LiveKit", - "DEEPGRAM_API_KEY": "API key for Deepgram (used for STT)", - "ELEVEN_API_KEY": "API key for ElevenLabs (used for TTS)", - } - for key, description in required_envs.items(): - if not os.environ.get(key): - logger.warning("Environment variable %s (%s) is not set.", key, description) - - -validate_envs() - - -def prewarm(proc: JobProcess) -> None: - logger.info("Prewarming: loading VAD model...") - proc.userdata["vad"] = silero.VAD.load() - logger.info("VAD model loaded successfully.") - - -async def entrypoint(ctx: JobContext) -> None: - metadata = ctx.job.metadata - - logger.info("job metadata: %s", metadata) - - if isinstance(metadata, str): - try: - metadata_obj = json.loads(metadata) - except json.JSONDecodeError: - try: - normalized = metadata.replace("'", '"') - metadata_obj = json.loads(normalized) - except json.JSONDecodeError as norm_error: - logger.warning( - "Normalization failed, using default values: %s", norm_error - ) - metadata_obj = {} - else: - metadata_obj = metadata - - logger.info("metadata_obj: %s", metadata_obj) - - agent_name = metadata_obj.get("agent_name") - agent_id = metadata_obj.get("agent_id") - run_id = metadata_obj.get("run_id") - - engine_api_address = os.environ.get("RESTACK_ENGINE_API_ADDRESS") - if not engine_api_address: - agent_backend_host = "http://localhost:9233" - elif not engine_api_address.startswith("https://"): - agent_backend_host = "https://" + engine_api_address - else: - agent_backend_host = engine_api_address - - logger.info("Using RESTACK_ENGINE_API_ADDRESS: %s", agent_backend_host) - - agent_url = f"{agent_backend_host}/stream/agents/{agent_name}/{agent_id}/{run_id}" - logger.info("Agent URL: %s", agent_url) - - logger.info("Connecting to room: %s", ctx.room.name) - await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY) - - participant = await ctx.wait_for_participant() - logger.info("Starting voice assistant for participant: %s", participant.identity) - - agent = VoicePipelineAgent( - vad=ctx.proc.userdata["vad"], - stt=deepgram.STT(), - llm=openai.LLM( - api_key=f"{agent_id}-livekit", - base_url=agent_url, - ), - tts=elevenlabs.TTS(), - turn_detector=turn_detector.EOUModel(), - # minimum delay for endpointing, used when turn detector believes the user is done with their turn - # min_endpointing_delay=0.5, - # # maximum delay for endpointing, used when turn detector does not believe the user is done with their turn - # max_endpointing_delay=5.0, - ) - - usage_collector = metrics.UsageCollector() +This module provides functions to create and configure LiveKit pipeline. +""" - @agent.on("metrics_collected") - def on_metrics_collected(agent_metrics: metrics.AgentMetrics) -> None: - metrics.log_metrics(agent_metrics) - - async def send_metrics(agent_metrics): - try: - latencies = [] - if isinstance(agent_metrics, metrics.PipelineEOUMetrics): - total_latency = agent_metrics.end_of_utterance_delay - latencies.append(total_latency * 1000) - - elif isinstance(agent_metrics, metrics.PipelineLLMMetrics): - total_latency = agent_metrics.ttft - latencies.append(total_latency * 1000) - - elif isinstance(agent_metrics, metrics.PipelineTTSMetrics): - total_latency = agent_metrics.ttfb - latencies.append(total_latency * 1000) - - if latencies: - metrics_latencies = str(json.dumps({"latencies": latencies})) - logger.info(f"Sending pipeline metrics: {metrics_latencies!s}") - - await client.send_agent_event( - event_name="pipeline_metrics", - agent_id=agent_id.replace("local-", ""), - run_id=run_id, - event_input={ - "metrics": agent_metrics, - "latencies": metrics_latencies, - }, - ) - except Exception as e: - logger.error("Error sending pipeline metrics", error=e) - - asyncio.create_task(send_metrics(agent_metrics)) - - usage_collector.collect(agent_metrics) - - usage_collector.get_summary() - - async def say(text: str): - await agent.say(text) - - @ctx.room.on("data_received") - def on_data_received(data_packet) -> None: - logger.info(f"Received data: {data_packet}") - - byte_content = data_packet.data - if isinstance(byte_content, bytes): - text_data = byte_content.decode("utf-8") - logger.info(f"Text data: {text_data}") - - asyncio.create_task(say(text_data)) - - else: - logger.warning("Data is not in bytes format.") - - agent.start(ctx.room, participant) - - await asyncio.sleep(0.1) - - await agent.say( - "Welcome to restack, how can I help you today?", allow_interruptions=True - ) - - -if __name__ == "__main__": - cli.run_app( - WorkerOptions( - entrypoint_fnc=entrypoint, - agent_name="AgentTwilio", - prewarm_fnc=prewarm, +from livekit.agents import JobContext +from livekit.agents.pipeline import VoicePipelineAgent +from livekit.plugins import ( + deepgram, + elevenlabs, + openai, + turn_detector, +) +from src.utils import logger + + +def create_livekit_pipeline( + ctx: JobContext, agent_id: str, agent_url: str +) -> VoicePipelineAgent: + """Create and configure a VoicePipelineAgent with the provided context, agent_id, and agent_url. + + Args: + ctx (JobContext): The job context containing user data and configuration. + agent_id (str): The identifier for the agent. + agent_url (str): The URL for the agent backend. + + Returns: + VoicePipelineAgent: A configured agent instance. + """ + try: + logger.info( + "Creating VoicePipelineAgent with agent_id: %s and agent_url: %s", + agent_id, + agent_url, + ) + return VoicePipelineAgent( + vad=ctx.proc.userdata["vad"], + stt=deepgram.STT(), + llm=openai.LLM( + api_key=f"{agent_id}-livekit", + base_url=agent_url, + ), + tts=elevenlabs.TTS(), + turn_detector=turn_detector.EOUModel(), + ) + except Exception as e: + logger.exception( + "Error creating VoicePipelineAgent: %s", e ) - ) + raise diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/src/restack/client.py b/agent_telephony/twilio_livekit/livekit_pipeline/src/restack/client.py new file mode 100644 index 00000000..dfb6329d --- /dev/null +++ b/agent_telephony/twilio_livekit/livekit_pipeline/src/restack/client.py @@ -0,0 +1,34 @@ +"""Client. + +Initializes and exposes the Restack client used for sending events from the LiveKit pipeline agent. +""" + +import os + +from dotenv import load_dotenv +from restack_ai import Restack +from restack_ai.restack import CloudConnectionOptions +from src.utils import logger + +# Load environment variables from the .env file. +load_dotenv() + +try: + engine_id = os.getenv("RESTACK_ENGINE_ID") + address = os.getenv("RESTACK_ENGINE_ADDRESS") + api_key = os.getenv("RESTACK_ENGINE_API_KEY") + api_address = os.getenv("RESTACK_ENGINE_API_ADDRESS") + + connection_options = CloudConnectionOptions( + engine_id=engine_id, + address=address, + api_key=api_key, + api_address=api_address, + ) + client = Restack(connection_options) + logger.info( + "Initialized Restack client with engine_id: %s", engine_id + ) +except Exception as e: + logger.exception("Error initializing Restack client: %s", e) + raise diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/src/restack/utils.py b/agent_telephony/twilio_livekit/livekit_pipeline/src/restack/utils.py new file mode 100644 index 00000000..848b01e8 --- /dev/null +++ b/agent_telephony/twilio_livekit/livekit_pipeline/src/restack/utils.py @@ -0,0 +1,53 @@ +"""Utils. + +Provides utility functions and a shared logger configuration for the LiveKit pipeline example project. +""" + +import os +from typing import Any + + +def extract_restack_agent_info( + metadata_obj: Any, +) -> tuple[str | None, str | None, str | None]: + """Extract agent-related information from the metadata object. + + Args: + metadata_obj (Any): The metadata object. + + Returns: + tuple[str | None, str | None, str | None]: A tuple of (agent_name, agent_id, run_id). + + """ + return ( + metadata_obj.get("agent_name"), + metadata_obj.get("agent_id"), + metadata_obj.get("run_id"), + ) + + +def get_restack_agent_url( + agent_name: str, agent_id: str, run_id: str +) -> str: + """Retrieve the agent base URL. + + Args: + agent_name (str): The name of the agent. + agent_id (str): The ID of the agent. + run_id (str): The run ID of the agent. + + Returns: + str: The agent base URL. + + """ + engine_api_address = os.environ.get( + "RESTACK_ENGINE_API_ADDRESS" + ) + if not engine_api_address: + hostname = "http://localhost:9233" + elif not engine_api_address.startswith("https://"): + hostname = "https://" + engine_api_address + else: + hostname = engine_api_address + + return f"{hostname}/stream/agents/{agent_name}/{agent_id}/{run_id}" diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/src/utils.py b/agent_telephony/twilio_livekit/livekit_pipeline/src/utils.py new file mode 100644 index 00000000..8c5701ea --- /dev/null +++ b/agent_telephony/twilio_livekit/livekit_pipeline/src/utils.py @@ -0,0 +1,106 @@ +"""Utils. + +Provides utility functions and a shared logger configuration for the LiveKit pipeline example project. +""" + +import asyncio +import json +import logging +import os +from typing import Any + +# Configure shared logger with a consistent format across modules. +logging.basicConfig( + level=logging.INFO, + format="[%(levelname)s] %(asctime)s - %(name)s - %(message)s", +) +logger = logging.getLogger("livekit_pipeline") + +pending_tasks: list[asyncio.Task] = [] + + +def track_task(task: asyncio.Task) -> None: + """Track an asyncio task by adding it to the global list `pending_tasks`. + + The task is removed automatically once completed. + + Args: + task (asyncio.Task): The task to track. + + """ + pending_tasks.append(task) + + def remove_task(t: asyncio.Task) -> None: + try: + pending_tasks.remove(t) + except ValueError: + logger.warning("Task not found in pending_tasks list") + + task.add_done_callback(remove_task) + + +def parse_metadata(metadata: Any) -> Any: + """Parse job metadata from a JSON string or return as-is if already parsed. + + Args: + metadata (Any): The metadata to parse. + + Returns: + Any: The parsed JSON object or the original metadata. + + """ + if isinstance(metadata, str): + try: + return json.loads(metadata) + except json.JSONDecodeError: + try: + normalized = metadata.replace("'", '"') + return json.loads(normalized) + except json.JSONDecodeError as norm_error: + logger.warning( + "Normalization failed, using default values: %s", + norm_error, + ) + return {} + except Exception: + logger.exception("Unexpected error parsing metadata") + return {} + return metadata + + +def extract_agent_info( + metadata_obj: Any, +) -> tuple[str | None, str | None, str | None]: + """Extract agent-related information from the metadata object. + + Args: + metadata_obj (Any): The metadata object. + + Returns: + tuple[str | None, str | None, str | None]: A tuple of (agent_name, agent_id, run_id). + + """ + return ( + metadata_obj.get("agent_name"), + metadata_obj.get("agent_id"), + metadata_obj.get("run_id"), + ) + + +def get_agent_backend_host() -> str: + """Retrieve the backend host URL from environment variables. + + Defaults to "http://localhost:9233" if not provided. + + Returns: + str: The backend host URL. + + """ + engine_api_address = os.environ.get( + "RESTACK_ENGINE_API_ADDRESS" + ) + if not engine_api_address: + return "http://localhost:9233" + if not engine_api_address.startswith("https://"): + return "https://" + engine_api_address + return engine_api_address diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/src/worker.py b/agent_telephony/twilio_livekit/livekit_pipeline/src/worker.py new file mode 100644 index 00000000..6a9e1e1a --- /dev/null +++ b/agent_telephony/twilio_livekit/livekit_pipeline/src/worker.py @@ -0,0 +1,139 @@ +"""Worker. + +This is the main entrypoint to run the LiveKit worker. +It orchestrates connection to a LiveKit room, agent creation, and +event handling for incoming data. +""" + +import asyncio +from typing import Any + +from livekit.agents import ( + AutoSubscribe, + JobContext, + JobProcess, + WorkerOptions, + cli, + metrics, +) +from livekit.plugins import silero +from src.env_check import check_env_vars +from src.metrics import setup_pipeline_metrics +from src.pipeline import create_livekit_pipeline +from src.restack.utils import ( + extract_restack_agent_info, + get_restack_agent_url, +) +from src.utils import ( + logger, + parse_metadata, + track_task, +) + + +def prewarm(proc: JobProcess) -> None: + """Prewarm the system by loading necessary models before the agent starts.""" + logger.info("Prewarming: loading VAD model...") + proc.userdata["vad"] = silero.VAD.load() + logger.info("VAD model loaded successfully.") + + +async def entrypoint(ctx: JobContext) -> None: + """Run main entrypoint for LiveKit. + + Processes job metadata, checks environment variables, sets up the agent and metrics, + and starts the data handling for the LiveKit room. + """ + # Check environment variables and log warnings if any are missing. + check_env_vars() + + # Extract metadata to match pipeline with Restack agent + + metadata = ctx.job.metadata + logger.info("Job metadata: %s", metadata) + metadata_obj = parse_metadata(metadata) + agent_name, agent_id, run_id = extract_restack_agent_info( + metadata_obj + ) + agent_url = get_restack_agent_url( + agent_name, agent_id, run_id + ) + + logger.info("Restack agent url: %s", agent_url) + + # Connect to LiveKit room + + logger.info("Connecting to room: %s", ctx.room.name) + + await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY) + + participant = await ctx.wait_for_participant() + + logger.info( + "Starting voice assistant for participant: %s", + participant.identity, + ) + + pipeline = create_livekit_pipeline(ctx, agent_id, agent_url) + + # Setup metrics + + usage_collector = metrics.UsageCollector() + + setup_pipeline_metrics( + pipeline, agent_id, run_id, usage_collector + ) + + usage_collector.get_summary() + + # Allow pipeline to speak when receiving data + + async def say(text: str) -> None: + """Send a text message to the pipeline's TTS system. + + Args: + text (str): The text to be spoken in the pipeline. + + """ + await pipeline.say(text) + + @ctx.room.on("data_received") + def on_data_received(data_packet: Any) -> None: + """Handle data received in the LiveKit room. + + Decode data and pass it to the pipeline for processing. + + Args: + data_packet (Any): The incoming data packet. + + """ + logger.info("Received data: %s", data_packet) + byte_content = data_packet.data + if isinstance(byte_content, bytes): + text_data = byte_content.decode("utf-8") + logger.info("Text data: %s", text_data) + track_task(asyncio.create_task(say(text_data))) + else: + logger.warning("Data is not in bytes format.") + + # Start pipeline and welcome user + + pipeline.start(ctx.room, participant) + + logger.info("Pipeline started", ctx.job.metadata) + + welcome_message = ( + "Welcome to restack, how can I help you today?" + ) + + await pipeline.say(welcome_message, allow_interruptions=True) + + +if __name__ == "__main__": + cli.run_app( + WorkerOptions( + entrypoint_fnc=entrypoint, + agent_name="AgentTwilio", + prewarm_fnc=prewarm, + ) + ) diff --git a/agent_telephony/twilio_livekit/readme.md b/agent_telephony/twilio_livekit/readme.md index 004d9d27..81b12992 100644 --- a/agent_telephony/twilio_livekit/readme.md +++ b/agent_telephony/twilio_livekit/readme.md @@ -66,7 +66,7 @@ pip install -e . python -c "from src.services import watch_services; watch_services()" ``` -## Start Livekit voice pipeline +## Start Livekit worker ### Start python shell @@ -88,14 +88,14 @@ If using uv: ```bash uv sync -uv run python src/pipeline.py dev +uv run python src/worker.py dev ``` If using pip: ```bash pip install -e . -python src/pipeline.py dev +python src/worker.py dev ``` ## Configure Your Environment Variables diff --git a/pyproject.toml b/pyproject.toml index ce6dcb17..08c30e66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,9 @@ dev = [ "ruff>=0.9.4", ] +[tool.ruff] +line-length = 66 + [tool.ruff.lint] extend-select = ["ALL"] ignore = ["ANN401", "E501", "D100", "D101", "D102", "D103", "D107", "TRY002", "D213", "D203", "COM812", "D104", "INP001"] @@ -16,3 +19,7 @@ ignore = ["ANN401", "E501", "D100", "D101", "D102", "D103", "D107", "TRY002", "D quote-style = "double" indent-style = "space" docstring-code-format = true +docstring-code-line-length = 66 + +[tool.ruff.lint.pydocstyle] +convention = "google" \ No newline at end of file From 0c1a0601d80525c79c7d158e222a4dd629133028 Mon Sep 17 00:00:00 2001 From: aboutphilippe Date: Sun, 9 Mar 2025 15:41:17 +0100 Subject: [PATCH 06/11] lint --- .../agent_twilio/src/agents/agent.py | 33 +++++++++++++++---- .../src/functions/context_docs.py | 11 +++---- .../src/functions/livekit_call.py | 5 ++- .../src/functions/livekit_create_room.py | 6 +++- .../src/functions/livekit_delete_room.py | 6 +++- .../src/functions/livekit_dispatch.py | 6 +++- .../src/functions/livekit_outbound_trunk.py | 7 +++- .../agent_twilio/src/functions/llm_logic.py | 2 +- .../agent_twilio/src/functions/llm_talk.py | 6 +++- .../agent_twilio/src/services.py | 4 ++- .../agent_twilio/src/workflows/logic.py | 22 ++++++++++--- 11 files changed, 83 insertions(+), 25 deletions(-) diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py b/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py index 639b5c24..c0e30c58 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py @@ -15,18 +15,37 @@ from src.workflows.logic import LogicWorkflow, LogicWorkflowInput with import_functions(): - from src.functions.livekit_call import LivekitCallInput, livekit_call - from src.functions.livekit_create_room import livekit_create_room - from src.functions.livekit_delete_room import livekit_delete_room - from src.functions.livekit_dispatch import LivekitDispatchInput, livekit_dispatch - from src.functions.livekit_outbound_trunk import livekit_outbound_trunk + from src.functions.livekit_call import ( + LivekitCallInput, + livekit_call, + ) + from src.functions.livekit_create_room import ( + livekit_create_room, + ) + from src.functions.livekit_delete_room import ( + livekit_delete_room, + ) + from src.functions.livekit_dispatch import ( + LivekitDispatchInput, + livekit_dispatch, + ) + from src.functions.livekit_outbound_trunk import ( + livekit_outbound_trunk, + ) from src.functions.livekit_send_data import ( LivekitSendDataInput, SendDataResponse, livekit_send_data, ) - from src.functions.livekit_token import LivekitTokenInput, livekit_token - from src.functions.llm_talk import LlmTalkInput, Message, llm_talk + from src.functions.livekit_token import ( + LivekitTokenInput, + livekit_token, + ) + from src.functions.llm_talk import ( + LlmTalkInput, + Message, + llm_talk, + ) class MessagesEvent(BaseModel): diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/context_docs.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/context_docs.py index 64d7443f..1a852f8d 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/context_docs.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/context_docs.py @@ -3,12 +3,11 @@ async def fetch_content_from_url(url: str) -> str: - async with aiohttp.ClientSession() as session: - async with session.get(url) as response: - if response.status == 200: - return await response.text() - error_message = f"Failed to fetch content: {response.status}" - raise NonRetryableError(error_message) + async with aiohttp.ClientSession() as session, session.get(url) as response: + if response.status == 200: + return await response.text() + error_message = f"Failed to fetch content: {response.status}" + raise NonRetryableError(error_message) @function.defn() diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_call.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_call.py index e9c5be82..c5a03d6e 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_call.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_call.py @@ -1,7 +1,10 @@ from dataclasses import dataclass from livekit import api -from livekit.protocol.sip import CreateSIPParticipantRequest, SIPParticipantInfo +from livekit.protocol.sip import ( + CreateSIPParticipantRequest, + SIPParticipantInfo, +) from restack_ai.function import NonRetryableError, function, log diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_create_room.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_create_room.py index 34d7a79e..59172346 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_create_room.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_create_room.py @@ -2,7 +2,11 @@ from livekit import api from livekit.api import CreateRoomRequest, Room -from restack_ai.function import NonRetryableError, function, function_info +from restack_ai.function import ( + NonRetryableError, + function, + function_info, +) @function.defn() diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_delete_room.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_delete_room.py index dfda1dd6..e6d4b7dd 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_delete_room.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_delete_room.py @@ -2,7 +2,11 @@ from livekit import api from livekit.api import DeleteRoomRequest, DeleteRoomResponse -from restack_ai.function import NonRetryableError, function, function_info +from restack_ai.function import ( + NonRetryableError, + function, + function_info, +) @function.defn() diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_dispatch.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_dispatch.py index fd587d83..0122775d 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_dispatch.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_dispatch.py @@ -3,7 +3,11 @@ from livekit import api from livekit.protocol.agent_dispatch import AgentDispatch -from restack_ai.function import NonRetryableError, function, function_info +from restack_ai.function import ( + NonRetryableError, + function, + function_info, +) @dataclass diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_outbound_trunk.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_outbound_trunk.py index 1d6e9544..5a5fe8fc 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_outbound_trunk.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_outbound_trunk.py @@ -6,7 +6,12 @@ ListSIPOutboundTrunkRequest, SIPOutboundTrunkInfo, ) -from restack_ai.function import NonRetryableError, function, function_info, log +from restack_ai.function import ( + NonRetryableError, + function, + function_info, + log, +) @function.defn() diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_logic.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_logic.py index ad1db3b9..e4191cde 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_logic.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_logic.py @@ -7,7 +7,7 @@ class LlmLogicResponse(BaseModel): - """Structured AI Decision Output for Intelligent Interruptions""" + """Structured AI decision output used to interrupt conversations.""" action: Literal["interrupt", "update_context", "end_call"] reason: str diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_talk.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_talk.py index 4bc4e903..a3a63207 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_talk.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_talk.py @@ -3,7 +3,11 @@ from openai import OpenAI from pydantic import BaseModel, Field -from restack_ai.function import NonRetryableError, function, stream_to_websocket +from restack_ai.function import ( + NonRetryableError, + function, + stream_to_websocket, +) from src.client import api_address diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/services.py b/agent_telephony/twilio_livekit/agent_twilio/src/services.py index 496250b2..2f80c895 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/services.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/services.py @@ -12,7 +12,9 @@ from src.functions.livekit_create_room import livekit_create_room from src.functions.livekit_delete_room import livekit_delete_room from src.functions.livekit_dispatch import livekit_dispatch -from src.functions.livekit_outbound_trunk import livekit_outbound_trunk +from src.functions.livekit_outbound_trunk import ( + livekit_outbound_trunk, +) from src.functions.livekit_send_data import livekit_send_data from src.functions.livekit_token import livekit_token from src.functions.llm_logic import llm_logic diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py b/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py index 7af8c347..05213a8a 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py @@ -11,10 +11,24 @@ with import_functions(): from src.functions.context_docs import context_docs - from src.functions.livekit_send_data import LivekitSendDataInput, livekit_send_data - from src.functions.llm_logic import LlmLogicInput, LlmLogicResponse, llm_logic - from src.functions.llm_talk import LlmTalkInput, Message, llm_talk - from src.functions.send_agent_event import SendAgentEventInput, send_agent_event + from src.functions.livekit_send_data import ( + LivekitSendDataInput, + livekit_send_data, + ) + from src.functions.llm_logic import ( + LlmLogicInput, + LlmLogicResponse, + llm_logic, + ) + from src.functions.llm_talk import ( + LlmTalkInput, + Message, + llm_talk, + ) + from src.functions.send_agent_event import ( + SendAgentEventInput, + send_agent_event, + ) class LogicWorkflowInput(BaseModel): From 41936d49c9071357aed78548ce26d64f863ba200 Mon Sep 17 00:00:00 2001 From: aboutphilippe Date: Sun, 9 Mar 2025 17:00:48 +0100 Subject: [PATCH 07/11] improve --- .../agent_twilio/src/agents/agent.py | 1 + .../agent_twilio/src/workflows/logic.py | 2 ++ .../livekit_pipeline/src/pipeline.py | 19 +++++++++++++++++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py b/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py index c0e30c58..9abd033e 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py @@ -154,6 +154,7 @@ async def end(self, end: EndEvent) -> EndEvent: room_id=self.room_id, text="Thank you for calling restack. Goodbye!" ), ) + await agent.sleep(1) await agent.step(function=livekit_delete_room) self.end = True diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py b/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py index 05213a8a..f7a4b8b9 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py @@ -122,6 +122,8 @@ async def run(self, workflow_input: LogicWorkflowInput) -> str: ), ) + await workflow.sleep(1) + await workflow.step( function=send_agent_event, function_input=SendAgentEventInput( diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py b/agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py index c0c30e53..713e3c61 100644 --- a/agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py +++ b/agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py @@ -35,12 +35,27 @@ def create_livekit_pipeline( ) return VoicePipelineAgent( vad=ctx.proc.userdata["vad"], - stt=deepgram.STT(), + stt=deepgram.STT( + model="nova-3-general", + ), llm=openai.LLM( api_key=f"{agent_id}-livekit", base_url=agent_url, ), - tts=elevenlabs.TTS(), + tts=elevenlabs.TTS( + voice=elevenlabs.tts.Voice( + id="UgBBYS2sOqTuMpoF3BR0", + name="Mark", + category="premade", + settings=elevenlabs.tts.VoiceSettings( + stability=0, + similarity_boost=0, + style=1, + speed=1.01, + use_speaker_boost=False + ), + ), + ), turn_detector=turn_detector.EOUModel(), ) except Exception as e: From ec2ea1c57bddc908c26722c74e7031567e190434 Mon Sep 17 00:00:00 2001 From: aboutphilippe Date: Sun, 9 Mar 2025 23:25:51 +0100 Subject: [PATCH 08/11] recording --- .../twilio_livekit/agent_twilio/.env.Example | 6 +- .../agent_twilio/event_agent.py | 13 +++- .../agent_twilio/schedule_agent.py | 4 +- .../agent_twilio/src/agents/agent.py | 65 ++++++++++++++++--- .../twilio_livekit/agent_twilio/src/client.py | 5 +- .../src/functions/context_docs.py | 13 +++- .../src/functions/livekit_call.py | 18 +++-- .../src/functions/livekit_create_room.py | 4 +- .../src/functions/livekit_delete_room.py | 8 ++- .../src/functions/livekit_dispatch.py | 14 +++- .../src/functions/livekit_outbound_trunk.py | 21 ++++-- .../src/functions/livekit_send_data.py | 11 +++- .../src/functions/livekit_start_recording.py | 63 ++++++++++++++++++ .../src/functions/livekit_token.py | 3 +- .../agent_twilio/src/functions/llm_logic.py | 4 +- .../agent_twilio/src/functions/llm_talk.py | 12 +++- .../src/functions/send_agent_event.py | 8 ++- .../agent_twilio/src/services.py | 13 +++- .../agent_twilio/src/workflows/logic.py | 30 +++++++-- .../livekit_pipeline/.env.example | 13 ++-- 20 files changed, 269 insertions(+), 59 deletions(-) create mode 100644 agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_start_recording.py diff --git a/agent_telephony/twilio_livekit/agent_twilio/.env.Example b/agent_telephony/twilio_livekit/agent_twilio/.env.Example index 4209859f..c02e4aa4 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/.env.Example +++ b/agent_telephony/twilio_livekit/agent_twilio/.env.Example @@ -1,18 +1,22 @@ RESTACK_API_KEY= +OPENAI_API_KEY= + LIVEKIT_API_KEY= LIVEKIT_API_SECRET= LIVEKIT_URL= -LIVEKIT_SIP_ADDRESS= TWILIO_PHONE_NUMBER= TWILIO_TRUNK_AUTH_USERNAME= TWILIO_TRUNK_AUTH_PASSWORD= +TWILIO_TRUNK_TERMINATION_SIP_URL= ELEVEN_API_KEY= DEEPGRAM_API_KEY= OPENAI_API_KEY= +GCP_CREDENTIALS= + # Restack Cloud (Optional) # RESTACK_ENGINE_ID= diff --git a/agent_telephony/twilio_livekit/agent_twilio/event_agent.py b/agent_telephony/twilio_livekit/agent_twilio/event_agent.py index 53002603..4ff92d98 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/event_agent.py +++ b/agent_telephony/twilio_livekit/agent_twilio/event_agent.py @@ -11,14 +11,23 @@ async def main(agent_id: str, run_id: str) -> None: agent_id=agent_id, run_id=run_id, event_name="call", - event_input={"messages": [{"role": "user", "content": "Tell me another joke"}]}, + event_input={ + "messages": [ + { + "role": "user", + "content": "What is Restack framework?", + } + ] + }, ) sys.exit(0) def run_event_workflow() -> None: - asyncio.run(main(agent_id="your-agent-id", run_id="your-run-id")) + asyncio.run( + main(agent_id="agent-id", run_id="run-id") + ) if __name__ == "__main__": diff --git a/agent_telephony/twilio_livekit/agent_twilio/schedule_agent.py b/agent_telephony/twilio_livekit/agent_twilio/schedule_agent.py index 20fdcb35..96858aea 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/schedule_agent.py +++ b/agent_telephony/twilio_livekit/agent_twilio/schedule_agent.py @@ -14,7 +14,9 @@ async def main() -> None: agent_name=AgentStream.__name__, agent_id=agent_id ) - await client.get_agent_result(agent_id=agent_id, run_id=run_id) + await client.get_agent_result( + agent_id=agent_id, run_id=run_id + ) sys.exit(0) diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py b/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py index 9abd033e..c001aa5b 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py @@ -37,6 +37,11 @@ SendDataResponse, livekit_send_data, ) + from src.functions.livekit_start_recording import ( + EgressInfo, + LivekitStartRecordingInput, + livekit_start_recording, + ) from src.functions.livekit_token import ( LivekitTokenInput, livekit_token, @@ -69,6 +74,13 @@ class PipelineMetricsEvent(BaseModel): latencies: str +class AgentTwilioOutput(BaseModel): + recording_url: str + livekit_room_id: str + messages: list[Message] + context: str + + @agent.defn() class AgentTwilio: def __init__(self) -> None: @@ -78,7 +90,9 @@ def __init__(self) -> None: self.room_id = "" @agent.event - async def messages(self, messages_event: MessagesEvent) -> list[Message]: + async def messages( + self, messages_event: MessagesEvent + ) -> list[Message]: log.info(f"Received message: {messages_event.messages}") self.messages.extend(messages_event.messages) @@ -108,7 +122,9 @@ async def messages(self, messages_event: MessagesEvent) -> list[Message]: ), ) - self.messages.append(Message(role="assistant", content=fast_response)) + self.messages.append( + Message(role="assistant", content=fast_response) + ) return self.messages except Exception as e: error_message = f"Error during messages: {e}" @@ -122,7 +138,9 @@ async def call(self, call_input: CallInput) -> None: agent_id = agent_info().workflow_id run_id = agent_info().run_id try: - sip_trunk_id = await agent.step(function=livekit_outbound_trunk) + sip_trunk_id = await agent.step( + function=livekit_outbound_trunk + ) await agent.step( function=livekit_call, function_input=LivekitCallInput( @@ -135,14 +153,17 @@ async def call(self, call_input: CallInput) -> None: ), ) except Exception as e: - error_message = f"Error during livekit_outbound_trunk: {e}" + error_message = ( + f"Error during livekit_outbound_trunk: {e}" + ) raise NonRetryableError(error_message) from e @agent.event async def say(self, say: str) -> SendDataResponse: log.info("Received say") return await agent.step( - function=livekit_send_data, function_input=LivekitSendDataInput(text=say) + function=livekit_send_data, + function_input=LivekitSendDataInput(text=say), ) @agent.event @@ -151,10 +172,11 @@ async def end(self, end: EndEvent) -> EndEvent: await agent.step( function=livekit_send_data, function_input=LivekitSendDataInput( - room_id=self.room_id, text="Thank you for calling restack. Goodbye!" + room_id=self.room_id, + text="Thank you for calling restack. Goodbye!", ), ) - await agent.sleep(1) + await agent.sleep(3) await agent.step(function=livekit_delete_room) self.end = True @@ -170,7 +192,10 @@ async def context(self, context: ContextEvent) -> str: async def pipeline_metrics( self, pipeline_metrics: PipelineMetricsEvent ) -> PipelineMetricsEvent: - log.info("Received pipeline metrics", pipeline_metrics=pipeline_metrics) + log.info( + "Received pipeline metrics", + pipeline_metrics=pipeline_metrics, + ) return pipeline_metrics @agent.run @@ -180,14 +205,34 @@ async def run(self) -> None: self.room_id = room.name await agent.step( function=livekit_token, - function_input=LivekitTokenInput(room_id=self.room_id), + function_input=LivekitTokenInput( + room_id=self.room_id + ), + ) + recording: EgressInfo = await agent.step( + function=livekit_start_recording, + function_input=LivekitStartRecordingInput( + room_id=self.room_id + ), ) await agent.step( function=livekit_dispatch, - function_input=LivekitDispatchInput(room_id=self.room_id), + function_input=LivekitDispatchInput( + room_id=self.room_id + ), ) + except Exception as e: error_message = f"Error during agent run: {e}" raise NonRetryableError(error_message) from e else: await agent.condition(lambda: self.end) + + recording_url = f"https://storage.googleapis.com/{recording.room_composite.file_outputs[0].gcp.bucket}/{recording.room_composite.file_outputs[0].filepath}" + + return AgentTwilioOutput( + recording_url=recording_url, + livekit_room_id=self.room_id, + messages=self.messages, + context=self.context, + ) diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/client.py b/agent_telephony/twilio_livekit/agent_twilio/src/client.py index 885bf8ea..2ff14fbb 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/client.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/client.py @@ -14,6 +14,9 @@ api_address = os.getenv("RESTACK_ENGINE_API_ADDRESS") connection_options = CloudConnectionOptions( - engine_id=engine_id, address=address, api_key=api_key, api_address=api_address + engine_id=engine_id, + address=address, + api_key=api_key, + api_address=api_address, ) client = Restack(connection_options) diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/context_docs.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/context_docs.py index 1a852f8d..aea0b5f3 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/context_docs.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/context_docs.py @@ -3,10 +3,15 @@ async def fetch_content_from_url(url: str) -> str: - async with aiohttp.ClientSession() as session, session.get(url) as response: + async with ( + aiohttp.ClientSession() as session, + session.get(url) as response, + ): if response.status == 200: return await response.text() - error_message = f"Failed to fetch content: {response.status}" + error_message = ( + f"Failed to fetch content: {response.status}" + ) raise NonRetryableError(error_message) @@ -16,7 +21,9 @@ async def context_docs() -> str: docs_content = await fetch_content_from_url( "https://docs.restack.io/llms-full.txt" ) - log.info("Fetched content from URL", content=len(docs_content)) + log.info( + "Fetched content from URL", content=len(docs_content) + ) return docs_content diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_call.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_call.py index c5a03d6e..d6c0f1dd 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_call.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_call.py @@ -19,7 +19,9 @@ class LivekitCallInput: @function.defn() -async def livekit_call(function_input: LivekitCallInput) -> SIPParticipantInfo: +async def livekit_call( + function_input: LivekitCallInput, +) -> SIPParticipantInfo: try: livekit_api = api.LiveKitAPI() @@ -32,13 +34,21 @@ async def livekit_call(function_input: LivekitCallInput) -> SIPParticipantInfo: play_dialtone=True, ) - log.info("livekit_call CreateSIPParticipantRequest: ", request=request) + log.info( + "livekit_call CreateSIPParticipantRequest: ", + request=request, + ) - participant = await livekit_api.sip.create_sip_participant(request) + participant = ( + await livekit_api.sip.create_sip_participant(request) + ) await livekit_api.aclose() - log.info("livekit_call SIPParticipantInfo:", participant=participant) + log.info( + "livekit_call SIPParticipantInfo:", + participant=participant, + ) except Exception as e: error_message = f"livekit_call function failed: {e}" raise NonRetryableError(error_message) from e diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_create_room.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_create_room.py index 59172346..7e8e1852 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_create_room.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_create_room.py @@ -31,7 +31,9 @@ async def livekit_create_room() -> Room: await lkapi.aclose() except Exception as e: - error_message = f"livekit_create_room function failed: {e}" + error_message = ( + f"livekit_create_room function failed: {e}" + ) raise NonRetryableError(error_message) from e else: diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_delete_room.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_delete_room.py index e6d4b7dd..647c72e8 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_delete_room.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_delete_room.py @@ -20,12 +20,16 @@ async def livekit_delete_room() -> DeleteRoomResponse: run_id = function_info().workflow_run_id - deleted_room = await lkapi.room.delete_room(DeleteRoomRequest(room=run_id)) + deleted_room = await lkapi.room.delete_room( + DeleteRoomRequest(room=run_id) + ) await lkapi.aclose() except Exception as e: - error_message = f"livekit_delete_room function failed: {e}" + error_message = ( + f"livekit_delete_room function failed: {e}" + ) raise NonRetryableError(error_message) from e else: diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_dispatch.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_dispatch.py index 0122775d..4a33606d 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_dispatch.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_dispatch.py @@ -16,7 +16,9 @@ class LivekitDispatchInput: @function.defn() -async def livekit_dispatch(function_input: LivekitDispatchInput) -> AgentDispatch: +async def livekit_dispatch( + function_input: LivekitDispatchInput, +) -> AgentDispatch: try: lkapi = api.LiveKitAPI( url=os.getenv("LIVEKIT_API_URL"), @@ -28,13 +30,19 @@ async def livekit_dispatch(function_input: LivekitDispatchInput) -> AgentDispatc agent_id = function_info().workflow_id run_id = function_info().workflow_run_id - metadata = {"agent_name": agent_name, "agent_id": agent_id, "run_id": run_id} + metadata = { + "agent_name": agent_name, + "agent_id": agent_id, + "run_id": run_id, + } room = function_input.room_id or run_id dispatch = await lkapi.agent_dispatch.create_dispatch( api.CreateAgentDispatchRequest( - agent_name=agent_name, room=room, metadata=str(metadata) + agent_name=agent_name, + room=room, + metadata=str(metadata), ) ) diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_outbound_trunk.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_outbound_trunk.py index 5a5fe8fc..3e788ecf 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_outbound_trunk.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_outbound_trunk.py @@ -20,8 +20,12 @@ async def livekit_outbound_trunk() -> str: livekit_api = api.LiveKitAPI() run_id = function_info().workflow_run_id - existing_trunk = await livekit_api.sip.list_sip_outbound_trunk( - list=ListSIPOutboundTrunkRequest(trunk_ids=[str(run_id)]) + existing_trunk = ( + await livekit_api.sip.list_sip_outbound_trunk( + list=ListSIPOutboundTrunkRequest( + trunk_ids=[str(run_id)] + ) + ) ) if existing_trunk.items and len(existing_trunk.items) > 0: @@ -43,14 +47,21 @@ async def livekit_outbound_trunk() -> str: request = CreateSIPOutboundTrunkRequest(trunk=trunk) - trunk = await livekit_api.sip.create_sip_outbound_trunk(request) + trunk = await livekit_api.sip.create_sip_outbound_trunk( + request + ) - log.info("livekit_outbound_trunk Successfully created, trunk: ", trunk=trunk) + log.info( + "livekit_outbound_trunk Successfully created, trunk: ", + trunk=trunk, + ) await livekit_api.aclose() except Exception as e: # Consider catching a more specific exception if possible - error_message = f"livekit_outbound_trunk function failed: {e}" + error_message = ( + f"livekit_outbound_trunk function failed: {e}" + ) raise NonRetryableError(error_message) from e else: diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_send_data.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_send_data.py index dd79e8b2..09dd66d2 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_send_data.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_send_data.py @@ -12,7 +12,9 @@ class LivekitSendDataInput(BaseModel): @function.defn() -async def livekit_send_data(function_input: LivekitSendDataInput) -> SendDataResponse: +async def livekit_send_data( + function_input: LivekitSendDataInput, +) -> SendDataResponse: try: lkapi = api.LiveKitAPI( url=os.getenv("LIVEKIT_API_URL"), @@ -22,14 +24,17 @@ async def livekit_send_data(function_input: LivekitSendDataInput) -> SendDataRes send_data_reponse = await lkapi.room.send_data( SendDataRequest( - room=function_input.room_id, data=function_input.text.encode("utf-8") + room=function_input.room_id, + data=function_input.text.encode("utf-8"), ) ) await lkapi.aclose() except Exception as e: - error_message = f"livekit_delete_room function failed: {e}" + error_message = ( + f"livekit_delete_room function failed: {e}" + ) raise NonRetryableError(error_message) from e else: diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_start_recording.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_start_recording.py new file mode 100644 index 00000000..5a6ecf36 --- /dev/null +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_start_recording.py @@ -0,0 +1,63 @@ +import os + +from livekit import api +from livekit.api import ( + EgressInfo, + EncodedFileType, + RoomCompositeEgressRequest, +) +from pydantic import BaseModel +from restack_ai.function import NonRetryableError, function, log + + +class LivekitStartRecordingInput(BaseModel): + room_id: str + + +@function.defn() +async def livekit_start_recording( + function_input: LivekitStartRecordingInput, +) -> EgressInfo: + try: + if os.getenv("GCP_CREDENTIALS") is None: + raise NonRetryableError( + message="GCP_CREDENTIALS is not set" + ) + + credentials = os.getenv("GCP_CREDENTIALS") + log.info("GCP_CREDENTIALS", credentials=credentials) + + lkapi = api.LiveKitAPI( + url=os.getenv("LIVEKIT_API_URL"), + api_key=os.getenv("LIVEKIT_API_KEY"), + api_secret=os.getenv("LIVEKIT_API_SECRET"), + ) + + recording = await lkapi.egress.start_room_composite_egress( + RoomCompositeEgressRequest( + room_name=function_input.room_id, + layout="grid", + audio_only=True, + file_outputs=[ + api.EncodedFileOutput( + file_type=EncodedFileType.MP4, + filepath=f"{function_input.room_id}-audio.mp4", + gcp=api.GCPUpload( + credentials=credentials, + bucket="livekit-local-recordings", + ), + ) + ], + ) + ) + + await lkapi.aclose() + + except Exception as e: + error_message = ( + f"livekit_start_recording function failed: {e}" + ) + raise NonRetryableError(error_message) from e + + else: + return recording diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_token.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_token.py index 16be9570..5d44e497 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_token.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/livekit_token.py @@ -14,7 +14,8 @@ async def livekit_token(function_input: LivekitTokenInput) -> str: try: token = ( api.AccessToken( - os.getenv("LIVEKIT_API_KEY"), os.getenv("LIVEKIT_API_SECRET") + os.getenv("LIVEKIT_API_KEY"), + os.getenv("LIVEKIT_API_SECRET"), ) .with_identity("identity") .with_name("dev_user") diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_logic.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_logic.py index e4191cde..6db3dfce 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_logic.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_logic.py @@ -20,7 +20,9 @@ class LlmLogicInput(BaseModel): @function.defn() -async def llm_logic(function_input: LlmLogicInput) -> LlmLogicResponse: +async def llm_logic( + function_input: LlmLogicInput, +) -> LlmLogicResponse: try: client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY")) diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_talk.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_talk.py index a3a63207..9bc1957a 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_talk.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_talk.py @@ -50,9 +50,13 @@ async def llm_talk(function_input: LlmTalkInput) -> str: "Do not re-explain everything, just deliver the most important update. Keep your answer short in max 20 words unless the user asks for more information." ) - function_input.messages.insert(0, Message(role="system", content=system_prompt)) + function_input.messages.insert( + 0, Message(role="system", content=system_prompt) + ) - messages_dicts = [msg.model_dump() for msg in function_input.messages] + messages_dicts = [ + msg.model_dump() for msg in function_input.messages + ] response = client.chat.completions.create( model="gpt-3.5-turbo", @@ -61,7 +65,9 @@ async def llm_talk(function_input: LlmTalkInput) -> str: ) if function_input.stream: - return await stream_to_websocket(api_address=api_address, data=response) + return await stream_to_websocket( + api_address=api_address, data=response + ) return response.choices[0].message.content except Exception as e: diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/send_agent_event.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/send_agent_event.py index 9a4becc6..33edcc34 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/send_agent_event.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/send_agent_event.py @@ -14,7 +14,9 @@ class SendAgentEventInput(BaseModel): @function.defn() -async def send_agent_event(function_input: SendAgentEventInput) -> str: +async def send_agent_event( + function_input: SendAgentEventInput, +) -> str: try: return await client.send_agent_event( event_name=function_input.event_name, @@ -24,4 +26,6 @@ async def send_agent_event(function_input: SendAgentEventInput) -> str: ) except Exception as e: - raise NonRetryableError(f"send_agent_event failed: {e}") from e + raise NonRetryableError( + f"send_agent_event failed: {e}" + ) from e diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/services.py b/agent_telephony/twilio_livekit/agent_twilio/src/services.py index 2f80c895..38c834c5 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/services.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/services.py @@ -16,6 +16,9 @@ livekit_outbound_trunk, ) from src.functions.livekit_send_data import livekit_send_data +from src.functions.livekit_start_recording import ( + livekit_start_recording, +) from src.functions.livekit_token import livekit_token from src.functions.llm_logic import llm_logic from src.functions.llm_talk import llm_talk @@ -39,6 +42,7 @@ async def main() -> None: context_docs, livekit_send_data, send_agent_event, + livekit_start_recording, ], ) @@ -47,12 +51,17 @@ def run_services() -> None: try: asyncio.run(main()) except KeyboardInterrupt: - logging.info("Service interrupted by user. Exiting gracefully.") + logging.info( + "Service interrupted by user. Exiting gracefully." + ) def watch_services() -> None: watch_path = Path.cwd() - logging.info("Watching %s and its subdirectories for changes...", watch_path) + logging.info( + "Watching %s and its subdirectories for changes...", + watch_path, + ) webbrowser.open("http://localhost:5233") run_process(watch_path, recursive=True, target=run_services) diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py b/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py index f7a4b8b9..9dd94dd5 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/workflows/logic.py @@ -44,7 +44,9 @@ class LogicWorkflowOutput(BaseModel): @workflow.defn() class LogicWorkflow: @workflow.run - async def run(self, workflow_input: LogicWorkflowInput) -> str: + async def run( + self, workflow_input: LogicWorkflowInput + ) -> str: context = workflow_input.context parent_agent_id = workflow_info().parent.workflow_id @@ -52,12 +54,17 @@ async def run(self, workflow_input: LogicWorkflowInput) -> str: log.info("LogicWorkflow started") try: - documentation = await workflow.step(function=context_docs) + documentation = await workflow.step( + function=context_docs + ) slow_response: LlmLogicResponse = await workflow.step( function=llm_logic, function_input=LlmLogicInput( - messages=[msg.model_dump() for msg in workflow_input.messages], + messages=[ + msg.model_dump() + for msg in workflow_input.messages + ], documentation=documentation, ), start_to_close_timeout=timedelta(seconds=60), @@ -81,7 +88,12 @@ async def run(self, workflow_input: LogicWorkflowInput) -> str: interrupt_response = await workflow.step( function=llm_talk, function_input=LlmTalkInput( - messages=[Message(role="system", content=slow_response.reason)], + messages=[ + Message( + role="system", + content=slow_response.reason, + ) + ], context=str(context), mode="interrupt", stream=False, @@ -92,7 +104,8 @@ async def run(self, workflow_input: LogicWorkflowInput) -> str: await workflow.step( function=livekit_send_data, function_input=LivekitSendDataInput( - room_id=parent_agent_run_id, text=interrupt_response + room_id=parent_agent_run_id, + text=interrupt_response, ), ) @@ -118,7 +131,8 @@ async def run(self, workflow_input: LogicWorkflowInput) -> str: await workflow.step( function=livekit_send_data, function_input=LivekitSendDataInput( - room_id=parent_agent_run_id, text=goodbye_message + room_id=parent_agent_run_id, + text=goodbye_message, ), ) @@ -138,5 +152,7 @@ async def run(self, workflow_input: LogicWorkflowInput) -> str: error_message = f"Error during welcome: {e}" raise NonRetryableError(error_message) from e else: - log.info("LogicWorkflow completed", context=str(context)) + log.info( + "LogicWorkflow completed", context=str(context) + ) return str(context) diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/.env.example b/agent_telephony/twilio_livekit/livekit_pipeline/.env.example index cb8a3df9..1a240a8c 100644 --- a/agent_telephony/twilio_livekit/livekit_pipeline/.env.example +++ b/agent_telephony/twilio_livekit/livekit_pipeline/.env.example @@ -1,7 +1,6 @@ -LIVEKIT_URL= -LIVEKIT_API_KEY= -LIVEKIT_API_SECRET= -OPENAI_API_KEY= -DEEPGRAM_API_KEY= -CARTESIA_API_KEY= -RESTACK_ENGINE_API_ADDRESS= \ No newline at end of file +LIVEKIT_URL= +LIVEKIT_API_KEY= +LIVEKIT_API_SECRET= + +DEEPGRAM_API_KEY= +ELEVEN_API_KEY= \ No newline at end of file From a7567b63329cc2755056520474679dcb1d10dfa7 Mon Sep 17 00:00:00 2001 From: aboutphilippe Date: Sun, 9 Mar 2025 23:44:13 +0100 Subject: [PATCH 09/11] allow direct phone number and output recording url --- .../agent_twilio/src/agents/agent.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py b/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py index c001aa5b..5bc10948 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/agents/agent.py @@ -51,6 +51,10 @@ Message, llm_talk, ) + from src.functions.send_agent_event import ( + SendAgentEventInput, + send_agent_event, + ) class MessagesEvent(BaseModel): @@ -73,6 +77,8 @@ class PipelineMetricsEvent(BaseModel): metrics: Any latencies: str +class AgentTwilioInput(BaseModel): + phone_number: str | None = None class AgentTwilioOutput(BaseModel): recording_url: str @@ -176,7 +182,7 @@ async def end(self, end: EndEvent) -> EndEvent: text="Thank you for calling restack. Goodbye!", ), ) - await agent.sleep(3) + await agent.sleep(8) await agent.step(function=livekit_delete_room) self.end = True @@ -199,7 +205,7 @@ async def pipeline_metrics( return pipeline_metrics @agent.run - async def run(self) -> None: + async def run(self, agent_input: AgentTwilioInput) -> AgentTwilioOutput: try: room = await agent.step(function=livekit_create_room) self.room_id = room.name @@ -222,6 +228,23 @@ async def run(self) -> None: ), ) + if agent_input.phone_number: + + agent_id = agent_info().workflow_id + run_id = agent_info().run_id + + await agent.step( + function=send_agent_event, + function_input=SendAgentEventInput( + event_name="call", + agent_id=agent_id, + run_id=run_id, + event_input={ + "phone_number": agent_input.phone_number, + }, + ), + ) + except Exception as e: error_message = f"Error during agent run: {e}" raise NonRetryableError(error_message) from e From ec8bab129f96c8927f289e2da90ebb575fb286ea Mon Sep 17 00:00:00 2001 From: aboutphilippe Date: Mon, 10 Mar 2025 16:25:16 +0100 Subject: [PATCH 10/11] fix voice mail --- .../agent_twilio/src/functions/llm_logic.py | 13 +++++++++++-- .../twilio_livekit/livekit_pipeline/src/metrics.py | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_logic.py b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_logic.py index 6db3dfce..39efc3c3 100644 --- a/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_logic.py +++ b/agent_telephony/twilio_livekit/agent_twilio/src/functions/llm_logic.py @@ -5,6 +5,9 @@ from pydantic import BaseModel from restack_ai.function import NonRetryableError, function +class Message(BaseModel): + role: str + content: str class LlmLogicResponse(BaseModel): """Structured AI decision output used to interrupt conversations.""" @@ -15,7 +18,7 @@ class LlmLogicResponse(BaseModel): class LlmLogicInput(BaseModel): - messages: list[dict] + messages: list[Message] documentation: str @@ -26,6 +29,12 @@ async def llm_logic( try: client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY")) + user_messages = [msg for msg in function_input.messages if msg.role == "user"] + if len(user_messages) == 1: + voice_mail_detection = "End the call if a voice mail is detected." + else: + voice_mail_detection = "" + response = client.beta.chat.completions.parse( model="gpt-4o", messages=[ @@ -35,7 +44,7 @@ async def llm_logic( "Analyze the developer's questions and determine if an interruption is needed. " "Use the Restack documentation for accurate answers. " "Track what the developer has learned and update their belief state." - "End the call if a voice mail is detected." + f"{voice_mail_detection}" f"Restack Documentation: {function_input.documentation}" ), }, diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/src/metrics.py b/agent_telephony/twilio_livekit/livekit_pipeline/src/metrics.py index 82db468b..79374943 100644 --- a/agent_telephony/twilio_livekit/livekit_pipeline/src/metrics.py +++ b/agent_telephony/twilio_livekit/livekit_pipeline/src/metrics.py @@ -53,7 +53,7 @@ async def send_metrics( ) await client.send_agent_event( event_name="pipeline_metrics", - agent_id=agent_id.replace("local-", ""), + agent_id=agent_id, run_id=run_id, event_input={ "metrics": pipeline_metrics, From 1729c6ea52d10852d2d23fb8f137e31583ecf770 Mon Sep 17 00:00:00 2001 From: aboutphilippe Date: Mon, 10 Mar 2025 16:28:04 +0100 Subject: [PATCH 11/11] reduce style --- agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py b/agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py index 713e3c61..be074732 100644 --- a/agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py +++ b/agent_telephony/twilio_livekit/livekit_pipeline/src/pipeline.py @@ -50,7 +50,7 @@ def create_livekit_pipeline( settings=elevenlabs.tts.VoiceSettings( stability=0, similarity_boost=0, - style=1, + style=0, speed=1.01, use_speaker_boost=False ),