fix handoff to Realtime model with existing session context

examples/bank-ivr/ivr_system_agent.py

@@ -78,7 +78,9 @@ async def collect_digits(
             result = await GetDtmfTask(
                 num_digits=num_digits,
                 ask_for_confirmation=confirmation,
-                chat_ctx=agent.chat_ctx.copy(exclude_instructions=True, exclude_function_call=True),
+                chat_ctx=agent.chat_ctx.copy(
+                    exclude_instructions=True, exclude_function_call=True, exclude_handoff=True
+                ),
                 extra_instructions=(
                     "You are gathering keypad digits from a bank customer. "
                     f"Prompt them with: {prompt}."

examples/dtmf/basic_dtmf_agent.py

@@ -67,7 +67,9 @@ async def ask_for_service_options(self, context: RunContext) -> str:
                 result = await GetDtmfTask(
                     num_digits=1,
                     chat_ctx=self.chat_ctx.copy(
-                        exclude_instructions=True, exclude_function_call=True
+                        exclude_instructions=True,
+                        exclude_function_call=True,
+                        exclude_handoff=True,
                     ),
                     extra_instructions=(
                         "Let the caller know they can choose one of three Horizon Wireless services: "
@@ -103,7 +105,9 @@ async def ask_for_phone_number(self, context: RunContext) -> str:
                 result = await GetDtmfTask(
                     num_digits=10,
                     chat_ctx=self.chat_ctx.copy(
-                        exclude_instructions=True, exclude_function_call=True
+                        exclude_instructions=True,
+                        exclude_function_call=True,
+                        exclude_handoff=True,
                     ),
                     ask_for_confirmation=True,
                     extra_instructions=(

examples/voice_agents/fast-preresponse.py

@@ -39,7 +39,9 @@ def __init__(self):
     async def on_user_turn_completed(self, turn_ctx: ChatContext, new_message: ChatMessage):
         # Create a short "silence filler" response to quickly acknowledge the user's input
         fast_llm_ctx = turn_ctx.copy(
-            exclude_instructions=True, exclude_function_call=True
+            exclude_instructions=True,
+            exclude_function_call=True,
+            exclude_handoff=True,
         ).truncate(max_items=3)
         fast_llm_ctx.items.insert(0, self._fast_llm_prompt)
         fast_llm_ctx.items.append(new_message)

examples/voice_agents/multi_agent.py

@@ -88,7 +88,7 @@ async def information_gathered(
         logger.info(
             "switching to the story agent with the provided user data: %s", context.userdata
         )
-        return story_agent, "Let's start the story!"
+        return story_agent
 
 
 class StoryAgent(Agent):

examples/voice_agents/restaurant_agent.py

@@ -123,7 +123,9 @@ async def on_enter(self) -> None:
         # add the previous agent's chat history to the current agent
         if isinstance(userdata.prev_agent, Agent):
             truncated_chat_ctx = userdata.prev_agent.chat_ctx.copy(
-                exclude_instructions=True, exclude_function_call=False
+                exclude_instructions=True,
+                exclude_function_call=False,
+                exclude_handoff=True,
             ).truncate(max_items=6)
             existing_ids = {item.id for item in chat_ctx.items}
             items_copy = [item for item in truncated_chat_ctx.items if item.id not in existing_ids]

livekit-agents/livekit/agents/beta/workflows/task_group.py

@@ -94,7 +94,10 @@ async def on_enter(self) -> None:
                 # when a task is done, the chat_ctx is going to be merged with the "caller" chat_ctx
                 # enabling summarization will result on only one ChatMessage added.
                 summarized_chat_ctx = await self.chat_ctx.copy(
-                    exclude_instructions=True
+                    exclude_instructions=True,
+                    exclude_handoff=True,
+                    exclude_empty_message=True,
+                    exclude_function_call=True,
                 )._summarize(llm_v=self.session.llm, keep_last_turns=0)
                 await self.update_chat_ctx(summarized_chat_ctx)
         except Exception as e:

livekit-agents/livekit/agents/beta/workflows/warm_transfer.py

@@ -131,7 +131,10 @@ def get_instructions(
         prev_convo = ""
         if chat_ctx:
             context_copy = chat_ctx.copy(
-                exclude_empty_message=True, exclude_instructions=True, exclude_function_call=True
+                exclude_empty_message=True,
+                exclude_instructions=True,
+                exclude_function_call=True,
+                exclude_handoff=True,
             )
             for msg in context_copy.items:
                 if msg.type != "message":

livekit-agents/livekit/agents/llm/chat_context.py

@@ -279,6 +279,7 @@ def copy(
         exclude_function_call: bool = False,
         exclude_instructions: bool = False,
         exclude_empty_message: bool = False,
+        exclude_handoff: bool = False,
         tools: NotGivenOr[Sequence[FunctionTool | RawFunctionTool | str | Any]] = NOT_GIVEN,
     ) -> ChatContext:
         items = []
@@ -318,6 +319,9 @@ def copy(
             if exclude_empty_message and item.type == "message" and not item.content:
                 continue
 
+            if exclude_handoff and item.type == "agent_handoff":
+                continue
+
             if (
                 is_given(tools)
                 and (item.type == "function_call" or item.type == "function_call_output")

livekit-plugins/livekit-plugins-aws/livekit/plugins/aws/experimental/realtime/realtime_model.py

@@ -1614,6 +1614,10 @@ async def update_chat_ctx(self, chat_ctx: llm.ChatContext) -> None:
         if self._chat_ctx_ready is None:
             self._chat_ctx_ready = asyncio.get_running_loop().create_future()
 
+        chat_ctx = chat_ctx.copy(
+            exclude_handoff=True, exclude_instructions=True, exclude_empty_message=True
+        )
+
         # Initial context setup (once)
         if not self._chat_ctx_ready.done():
             self._chat_ctx = chat_ctx.copy()

livekit-plugins/livekit-plugins-google/livekit/plugins/google/realtime/realtime_api.py

@@ -473,9 +473,12 @@ async def update_instructions(self, instructions: str) -> None:
             self._mark_restart_needed()
 
     async def update_chat_ctx(self, chat_ctx: llm.ChatContext) -> None:
+        chat_ctx = chat_ctx.copy(
+            exclude_handoff=True, exclude_instructions=True, exclude_empty_message=True
+        )
         async with self._session_lock:
             if not self._active_session:
-                self._chat_ctx = chat_ctx.copy()
+                self._chat_ctx = chat_ctx
                 return
 
         diff_ops = llm.utils.compute_chat_ctx_diff(self._chat_ctx, chat_ctx)
@@ -490,9 +493,9 @@ async def update_chat_ctx(self, chat_ctx: llm.ChatContext) -> None:
                 append_ctx.items.append(item)
 
         if append_ctx.items:
-            turns_dict, _ = append_ctx.copy(
-                exclude_function_call=True,
-            ).to_provider_format(format="google", inject_dummy_user_message=False)
+            turns_dict, _ = append_ctx.copy(exclude_function_call=True).to_provider_format(
+                format="google", inject_dummy_user_message=False
+            )
             # we are not generating, and do not need to inject
             turns = [types.Content.model_validate(turn) for turn in turns_dict]
             tool_results = get_tool_results_for_realtime(
@@ -507,7 +510,7 @@ async def update_chat_ctx(self, chat_ctx: llm.ChatContext) -> None:
 
         # since we don't have a view of the history on the server side, we'll assume
         # the current state is accurate. this isn't perfect because removals aren't done.
-        self._chat_ctx = chat_ctx.copy()
+        self._chat_ctx = chat_ctx
 
     async def update_tools(self, tools: list[llm.FunctionTool | llm.RawFunctionTool]) -> None:
         new_declarations: list[types.FunctionDeclaration] = to_fnc_ctx(
@@ -686,6 +689,9 @@ async def _main_task(self) -> None:
                         self._active_session = session
                         turns_dict, _ = self._chat_ctx.copy(
                             exclude_function_call=True,
+                            exclude_handoff=True,
+                            exclude_instructions=True,
+                            exclude_empty_message=True,
                         ).to_provider_format(format="google", inject_dummy_user_message=False)
                         if turns_dict:
                             turns = [types.Content.model_validate(turn) for turn in turns_dict]

livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py

@@ -724,6 +724,7 @@ async def _reconnect() -> None:
                 exclude_function_call=True,
                 exclude_instructions=True,
                 exclude_empty_message=True,
+                exclude_handoff=True,
             )
             old_chat_ctx = self._remote_chat_ctx
             self._remote_chat_ctx = llm.remote_chat_context.RemoteChatContext()
@@ -1103,6 +1104,7 @@ def update_options(
 
     async def update_chat_ctx(self, chat_ctx: llm.ChatContext) -> None:
         async with self._update_chat_ctx_lock:
+            chat_ctx = chat_ctx.copy(exclude_handoff=True, exclude_instructions=True)
             events = self._create_update_chat_ctx_events(chat_ctx)
             futs: list[asyncio.Future[None]] = []
 
@@ -1340,7 +1342,9 @@ def truncate(
             )
         elif utils.is_given(audio_transcript):
             # sync the forwarded text to the remote chat ctx
-            chat_ctx = self.chat_ctx.copy()
+            chat_ctx = self.chat_ctx.copy(
+                exclude_handoff=True,
+            )
             if (idx := chat_ctx.index_by_id(message_id)) is not None:
                 new_item = copy.copy(chat_ctx.items[idx])
                 assert new_item.type == "message"

livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model_beta.py

@@ -582,6 +582,7 @@ async def _reconnect() -> None:
                 exclude_function_call=True,
                 exclude_instructions=True,
                 exclude_empty_message=True,
+                exclude_handoff=True,
             )
             old_chat_ctx = self._remote_chat_ctx
             self._remote_chat_ctx = llm.remote_chat_context.RemoteChatContext()
@@ -1170,7 +1171,9 @@ def truncate(
             )
         elif utils.is_given(audio_transcript):
             # sync the forwarded text to the remote chat ctx
-            chat_ctx = self.chat_ctx.copy()
+            chat_ctx = self.chat_ctx.copy(
+                exclude_handoff=True,
+            )
             if (idx := chat_ctx.index_by_id(message_id)) is not None:
                 new_item = copy.copy(chat_ctx.items[idx])
                 assert new_item.type == "message"