+ Voice playback failed: {transcriptSpeech.error} The transcript is still available.
+
+ )}
{invocationMessagesLoading && invocationMessages.length > 0 && (
Refreshing transcript while keeping the current messages visible.
diff --git a/dashboard/src/v2/components/chat/__tests__/ChatPage.speech.test.tsx b/dashboard/src/v2/components/chat/__tests__/ChatPage.speech.test.tsx
index aded0517cf..d5e1d69013 100644
--- a/dashboard/src/v2/components/chat/__tests__/ChatPage.speech.test.tsx
+++ b/dashboard/src/v2/components/chat/__tests__/ChatPage.speech.test.tsx
@@ -16,7 +16,12 @@ const speechButtonMock = vi.hoisted(() => ({
}));
const synthesisMock = vi.hoisted(() => ({
- synthesizeSpeech: vi.fn<() => Promise
>(),
+ synthesizeSpeech: vi.fn<(
+ text: string,
+ projectId?: string | null,
+ voice?: string | null,
+ signal?: AbortSignal,
+ ) => Promise>(),
}));
const mocks = vi.hoisted(() => {
@@ -329,7 +334,12 @@ describe("ChatPage speech input", () => {
);
await waitFor(() => expect(synthesisMock.synthesizeSpeech).toHaveBeenCalledTimes(1));
- expect(synthesisMock.synthesizeSpeech).toHaveBeenCalledWith("Fresh agent reply", "p1");
+ expect(synthesisMock.synthesizeSpeech).toHaveBeenCalledWith(
+ "Fresh agent reply",
+ "p1",
+ undefined,
+ expect.any(AbortSignal),
+ );
});
it("replays the staged agent message only after its explicit replay control is clicked", async () => {
@@ -354,7 +364,12 @@ describe("ChatPage speech input", () => {
expect(synthesisMock.synthesizeSpeech).not.toHaveBeenCalled();
fireEvent.click(replay);
- await waitFor(() => expect(synthesisMock.synthesizeSpeech).toHaveBeenCalledWith("Replay this reply", "p1"));
+ await waitFor(() => expect(synthesisMock.synthesizeSpeech).toHaveBeenCalledWith(
+ "Replay this reply",
+ "p1",
+ undefined,
+ expect.any(AbortSignal),
+ ));
});
it("reports a 3D voice synthesis failure instead of silently discarding it", async () => {
@@ -383,6 +398,33 @@ describe("ChatPage speech input", () => {
);
});
+ it("reports thread replay failures through the existing composer status without hiding the transcript", async () => {
+ synthesisMock.synthesizeSpeech.mockRejectedValueOnce(new Error("Speech provider timed out."));
+ mocks.data = {
+ ...mocks.data,
+ chatMode: "threads",
+ messages: [{
+ id: "thread-replay-error",
+ threadId: "thread1",
+ direction: "connection_to_dashboard",
+ authorType: "connection",
+ authorConnectionId: "connection-1",
+ bodyMarkdown: "The transcript remains visible.",
+ deliveryStatus: "delivered",
+ createdAt: "2026-03-10T12:00:00.000Z",
+ metadata: null,
+ }],
+ };
+
+ renderChatPage();
+ fireEvent.click(await screen.findByRole("button", { name: "Replay message from Assistant" }));
+
+ expect(await screen.findByRole("alert")).toHaveTextContent(
+ "Voice playback failed: Speech provider timed out. The transcript is still available.",
+ );
+ expect(screen.getByText("The transcript remains visible.")).toBeInTheDocument();
+ });
+
it("auto-plays the first reply after sending in a brand-new empty 3D thread", async () => {
mocks.data = {
...mocks.data,
@@ -439,7 +481,12 @@ describe("ChatPage speech input", () => {
);
await waitFor(() => expect(synthesisMock.synthesizeSpeech).toHaveBeenCalledTimes(1));
- expect(synthesisMock.synthesizeSpeech).toHaveBeenCalledWith("First answer", "p1");
+ expect(synthesisMock.synthesizeSpeech).toHaveBeenCalledWith(
+ "First answer",
+ "p1",
+ undefined,
+ expect.any(AbortSignal),
+ );
});
it("keeps loaded thread and invocation transcripts silent until replay is requested", async () => {
@@ -496,5 +543,12 @@ describe("ChatPage speech input", () => {
expect(await screen.findByRole("button", { name: "Replay message from Assistant" })).toBeInTheDocument();
expect(synthesisMock.synthesizeSpeech).not.toHaveBeenCalled();
+
+ synthesisMock.synthesizeSpeech.mockRejectedValueOnce(new Error("Invocation replay failed."));
+ fireEvent.click(screen.getByRole("button", { name: "Replay message from Assistant" }));
+ expect(await screen.findByRole("alert")).toHaveTextContent(
+ "Voice playback failed: Invocation replay failed. The transcript is still available.",
+ );
+ expect(screen.getByText("Loaded invocation reply")).toBeInTheDocument();
});
});
diff --git a/dashboard/src/v2/hooks/__tests__/use-speech-playback.test.tsx b/dashboard/src/v2/hooks/__tests__/use-speech-playback.test.tsx
new file mode 100644
index 0000000000..a0e8f1a4ba
--- /dev/null
+++ b/dashboard/src/v2/hooks/__tests__/use-speech-playback.test.tsx
@@ -0,0 +1,301 @@
+/** @vitest-environment jsdom */
+import { act, cleanup, renderHook, waitFor } from "@testing-library/preact";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import { useSpeechPlayback } from "../use-speech-playback.js";
+
+const speechApiMock = vi.hoisted(() => ({
+ synthesizeSpeech: vi.fn(),
+}));
+
+vi.mock("../../lib/speech-api.js", () => ({
+ synthesizeSpeech: speechApiMock.synthesizeSpeech,
+}));
+
+vi.mock("../../lib/speech-playback.js", () => ({
+ speechTextFromMarkdown: (markdown: string) => markdown,
+ splitSpeechPlaybackText: (text: string) => text.split(" | ").filter(Boolean),
+}));
+
+interface Deferred {
+ promise: Promise;
+ reject: (error: unknown) => void;
+ resolve: (value: T) => void;
+}
+
+const deferred = (): Deferred => {
+ let resolve!: (value: T) => void;
+ let reject!: (error: unknown) => void;
+ const promise = new Promise((resolvePromise, rejectPromise) => {
+ resolve = resolvePromise;
+ reject = rejectPromise;
+ });
+ return { promise, reject, resolve };
+};
+
+class FakeAudio {
+ static instances: FakeAudio[] = [];
+
+ readonly onPause = vi.fn();
+ readonly onPlay = vi.fn(() => Promise.resolve());
+ onended: (() => void) | null = null;
+ onerror: (() => void) | null = null;
+
+ constructor(readonly src: string) {
+ FakeAudio.instances.push(this);
+ }
+
+ pause(): void {
+ this.onPause();
+ }
+
+ play(): Promise {
+ return this.onPlay();
+ }
+
+ end(): void {
+ this.onended?.();
+ }
+
+ fail(): void {
+ this.onerror?.();
+ }
+}
+
+const blobLabels = new WeakMap();
+const speechBlob = (label: string): Blob => {
+ const blob = new Blob([label], { type: "audio/wav" });
+ blobLabels.set(blob, label);
+ return blob;
+};
+
+const request = (markdown: string, messageId = "message-1") => ({
+ markdown,
+ messageId,
+ projectId: "project-1",
+});
+
+describe("useSpeechPlayback", () => {
+ const revokeObjectURL = vi.fn();
+
+ beforeEach(() => {
+ FakeAudio.instances = [];
+ speechApiMock.synthesizeSpeech.mockReset();
+ revokeObjectURL.mockReset();
+ vi.stubGlobal("Audio", FakeAudio);
+ vi.stubGlobal("URL", {
+ createObjectURL: vi.fn((blob: Blob) => `blob:${blobLabels.get(blob) ?? "unknown"}`),
+ revokeObjectURL,
+ });
+ });
+
+ afterEach(() => {
+ cleanup();
+ vi.unstubAllGlobals();
+ });
+
+ it("starts the first sentence alone, then overlaps a bounded lookahead with playback", async () => {
+ const chunks = [deferred(), deferred(), deferred(), deferred()];
+ speechApiMock.synthesizeSpeech.mockImplementation((text: string) => {
+ const index = ["first", "second", "third", "fourth"].indexOf(text);
+ return chunks[index].promise;
+ });
+ const { result } = renderHook(() => useSpeechPlayback());
+
+ act(() => {
+ void result.current.play(request("first | second | third | fourth"));
+ });
+
+ expect(speechApiMock.synthesizeSpeech).toHaveBeenCalledTimes(1);
+ expect(speechApiMock.synthesizeSpeech).toHaveBeenNthCalledWith(
+ 1,
+ "first",
+ "project-1",
+ undefined,
+ expect.any(AbortSignal),
+ );
+
+ await act(async () => {
+ chunks[0].resolve(speechBlob("first"));
+ await chunks[0].promise;
+ });
+
+ expect(FakeAudio.instances.map((audio) => audio.src)).toEqual(["blob:first"]);
+ expect(FakeAudio.instances[0].onPlay).toHaveBeenCalledTimes(1);
+ expect(speechApiMock.synthesizeSpeech.mock.calls.map(([text]) => text)).toEqual([
+ "first",
+ "second",
+ "third",
+ ]);
+ });
+
+ it("stores out-of-order synthesis results but plays every chunk in strict order", async () => {
+ const chunks = [deferred(), deferred(), deferred(), deferred()];
+ speechApiMock.synthesizeSpeech.mockImplementation((text: string) => (
+ chunks[["first", "second", "third", "fourth"].indexOf(text)].promise
+ ));
+ const { result } = renderHook(() => useSpeechPlayback());
+ let playback!: Promise;
+
+ act(() => {
+ playback = result.current.play(request("first | second | third | fourth"));
+ });
+ await act(async () => {
+ chunks[0].resolve(speechBlob("first"));
+ await chunks[0].promise;
+ });
+ await act(async () => {
+ chunks[2].resolve(speechBlob("third"));
+ await chunks[2].promise;
+ FakeAudio.instances[0].end();
+ await Promise.resolve();
+ });
+ expect(FakeAudio.instances.map((audio) => audio.src)).toEqual(["blob:first"]);
+
+ await act(async () => {
+ chunks[1].resolve(speechBlob("second"));
+ await chunks[1].promise;
+ });
+ expect(FakeAudio.instances.map((audio) => audio.src)).toEqual(["blob:first", "blob:second"]);
+ expect(speechApiMock.synthesizeSpeech.mock.calls.map(([text]) => text)).toEqual([
+ "first",
+ "second",
+ "third",
+ "fourth",
+ ]);
+
+ await act(async () => {
+ chunks[3].resolve(speechBlob("fourth"));
+ await chunks[3].promise;
+ FakeAudio.instances[1].end();
+ await Promise.resolve();
+ });
+ expect(FakeAudio.instances.map((audio) => audio.src)).toEqual([
+ "blob:first",
+ "blob:second",
+ "blob:third",
+ ]);
+
+ await act(async () => {
+ FakeAudio.instances[2].end();
+ await Promise.resolve();
+ });
+ expect(FakeAudio.instances.map((audio) => audio.src)).toEqual([
+ "blob:first",
+ "blob:second",
+ "blob:third",
+ "blob:fourth",
+ ]);
+ await act(async () => {
+ FakeAudio.instances[3].end();
+ await playback;
+ });
+
+ expect(result.current.activeMessageId).toBeNull();
+ expect(result.current.error).toBeNull();
+ expect(revokeObjectURL.mock.calls.map(([url]) => url)).toEqual([
+ "blob:first",
+ "blob:second",
+ "blob:third",
+ "blob:fourth",
+ ]);
+ });
+
+ it("aborts synthesis and releases the current audio when stopped or unmounted", async () => {
+ const first = deferred();
+ const later = [deferred(), deferred()];
+ const unmountSynthesis = deferred();
+ speechApiMock.synthesizeSpeech
+ .mockReturnValueOnce(first.promise)
+ .mockReturnValueOnce(later[0].promise)
+ .mockReturnValueOnce(later[1].promise)
+ .mockReturnValueOnce(unmountSynthesis.promise);
+ const { result, unmount } = renderHook(() => useSpeechPlayback());
+ let playback!: Promise;
+
+ act(() => {
+ playback = result.current.play(request("first | second | third"));
+ });
+ await act(async () => {
+ first.resolve(speechBlob("first"));
+ await first.promise;
+ });
+ const signals = speechApiMock.synthesizeSpeech.mock.calls.map((call) => call[3] as AbortSignal);
+
+ act(() => result.current.stop());
+ await playback;
+
+ expect(signals).toHaveLength(3);
+ expect(signals.every((signal) => signal.aborted)).toBe(true);
+ expect(FakeAudio.instances[0].onPause).toHaveBeenCalledTimes(1);
+ expect(FakeAudio.instances[0].onended).toBeNull();
+ expect(FakeAudio.instances[0].onerror).toBeNull();
+ expect(revokeObjectURL).toHaveBeenCalledWith("blob:first");
+ expect(result.current.activeMessageId).toBeNull();
+
+ act(() => {
+ void result.current.play(request("unmount", "message-2"));
+ });
+ const unmountSignal = speechApiMock.synthesizeSpeech.mock.calls.at(-1)?.[3] as AbortSignal;
+ unmount();
+ expect(unmountSignal.aborted).toBe(true);
+ });
+
+ it("suppresses late synthesis from a replaced run", async () => {
+ const oldSynthesis = deferred();
+ const replacementSynthesis = deferred();
+ speechApiMock.synthesizeSpeech
+ .mockReturnValueOnce(oldSynthesis.promise)
+ .mockReturnValueOnce(replacementSynthesis.promise);
+ const { result } = renderHook(() => useSpeechPlayback());
+
+ act(() => {
+ void result.current.play(request("old", "old-message"));
+ void result.current.play(request("replacement", "new-message"));
+ });
+ const oldSignal = speechApiMock.synthesizeSpeech.mock.calls[0][3] as AbortSignal;
+ expect(oldSignal.aborted).toBe(true);
+
+ await act(async () => {
+ replacementSynthesis.resolve(speechBlob("replacement"));
+ await replacementSynthesis.promise;
+ oldSynthesis.resolve(speechBlob("old"));
+ await oldSynthesis.promise;
+ });
+
+ expect(FakeAudio.instances.map((audio) => audio.src)).toEqual(["blob:replacement"]);
+ expect(result.current.activeMessageId).toBe("new-message");
+ });
+
+ it("surfaces synthesis and browser playback errors and stops the run", async () => {
+ const synthesisFailure = deferred();
+ speechApiMock.synthesizeSpeech.mockReturnValueOnce(synthesisFailure.promise);
+ const { result } = renderHook(() => useSpeechPlayback());
+
+ act(() => {
+ void result.current.play(request("synthesis failure"));
+ });
+ await act(async () => {
+ synthesisFailure.reject(new Error("Configured voice is unavailable."));
+ await synthesisFailure.promise.catch(() => undefined);
+ });
+ await waitFor(() => expect(result.current.error).toBe("Configured voice is unavailable."));
+ expect(result.current.activeMessageId).toBeNull();
+
+ const playable = deferred();
+ speechApiMock.synthesizeSpeech.mockReturnValueOnce(playable.promise);
+ act(() => {
+ void result.current.play(request("playback failure", "message-2"));
+ });
+ await act(async () => {
+ playable.resolve(speechBlob("playback failure"));
+ await playable.promise;
+ });
+ act(() => FakeAudio.instances[0].fail());
+
+ await waitFor(() => expect(result.current.error).toBe("The browser could not play the generated audio."));
+ expect(result.current.activeMessageId).toBeNull();
+ expect(FakeAudio.instances[0].onended).toBeNull();
+ expect(FakeAudio.instances[0].onerror).toBeNull();
+ });
+});
diff --git a/dashboard/src/v2/hooks/use-speech-playback.ts b/dashboard/src/v2/hooks/use-speech-playback.ts
index 0f33a1d5ff..530edbb925 100644
--- a/dashboard/src/v2/hooks/use-speech-playback.ts
+++ b/dashboard/src/v2/hooks/use-speech-playback.ts
@@ -2,6 +2,82 @@ import { useCallback, useEffect, useRef, useState } from "preact/hooks";
import { synthesizeSpeech } from "../lib/speech-api.js";
import { speechTextFromMarkdown, splitSpeechPlaybackText } from "../lib/speech-playback.js";
+const SPEECH_PREFETCH_AHEAD = 2;
+
+interface SpeechPlaybackRun {
+ abortController: AbortController;
+ activeAudio: HTMLAudioElement | null;
+ activeUrl: string | null;
+ cancelled: boolean;
+ settleAudio: (() => void) | null;
+}
+
+interface SynthesisOutcome {
+ blob?: Blob;
+ error?: unknown;
+}
+
+interface AudioOutcome {
+ completed: boolean;
+ error?: string;
+}
+
+const readPlaybackError = (error: unknown, fallback: string): string => (
+ error instanceof Error && error.message ? error.message : fallback
+);
+
+const cancelPlaybackRun = (run: SpeechPlaybackRun): void => {
+ if (run.cancelled) return;
+ run.cancelled = true;
+ run.abortController.abort();
+ run.activeAudio?.pause();
+ run.settleAudio?.();
+ run.settleAudio = null;
+ run.activeAudio = null;
+ if (run.activeUrl) {
+ URL.revokeObjectURL(run.activeUrl);
+ run.activeUrl = null;
+ }
+};
+
+const playAudioBlob = (run: SpeechPlaybackRun, blob: Blob): Promise => (
+ new Promise((resolve) => {
+ const url = URL.createObjectURL(blob);
+ run.activeUrl = url;
+ const audio = new Audio(url);
+ run.activeAudio = audio;
+ let settled = false;
+
+ const finish = (completed: boolean, error?: string): void => {
+ if (settled) return;
+ settled = true;
+ audio.onended = null;
+ audio.onerror = null;
+ URL.revokeObjectURL(url);
+ if (run.activeUrl === url) run.activeUrl = null;
+ if (run.activeAudio === audio) run.activeAudio = null;
+ if (run.settleAudio === cancelAudio) run.settleAudio = null;
+ resolve({ completed, error });
+ };
+ const cancelAudio = (): void => finish(false);
+
+ run.settleAudio = cancelAudio;
+ audio.onended = () => finish(true);
+ audio.onerror = () => finish(false, "The browser could not play the generated audio.");
+ try {
+ void audio.play().catch((error: unknown) => finish(
+ false,
+ readPlaybackError(error, "The browser blocked audio playback. Use the voice button and try again."),
+ ));
+ } catch (error) {
+ finish(
+ false,
+ readPlaybackError(error, "The browser blocked audio playback. Use the voice button and try again."),
+ );
+ }
+ })
+);
+
export interface SpeechPlaybackRequest {
markdown: string;
messageId: string;
@@ -20,21 +96,12 @@ export interface SpeechPlaybackController {
export const useSpeechPlayback = (): SpeechPlaybackController => {
const [activeMessageId, setActiveMessageId] = useState(null);
const [error, setError] = useState(null);
- const audioRef = useRef(null);
- const audioUrlRef = useRef(null);
- const settleAudioRef = useRef<(() => void) | null>(null);
- const requestSequenceRef = useRef(0);
+ const currentRunRef = useRef(null);
const stop = useCallback((): void => {
- requestSequenceRef.current += 1;
- audioRef.current?.pause();
- settleAudioRef.current?.();
- settleAudioRef.current = null;
- audioRef.current = null;
- if (audioUrlRef.current) {
- URL.revokeObjectURL(audioUrlRef.current);
- audioUrlRef.current = null;
- }
+ const currentRun = currentRunRef.current;
+ currentRunRef.current = null;
+ if (currentRun) cancelPlaybackRun(currentRun);
setActiveMessageId(null);
setError(null);
}, []);
@@ -45,58 +112,84 @@ export const useSpeechPlayback = (): SpeechPlaybackController => {
if (chunks.length === 0) return;
stop();
- const requestSequence = requestSequenceRef.current;
+ const run: SpeechPlaybackRun = {
+ abortController: new AbortController(),
+ activeAudio: null,
+ activeUrl: null,
+ cancelled: false,
+ settleAudio: null,
+ };
+ currentRunRef.current = run;
setActiveMessageId(messageId);
+ const isCurrentRun = (): boolean => currentRunRef.current === run && !run.cancelled;
+ const failRun = (playbackError: unknown, fallback: string): void => {
+ if (!isCurrentRun()) return;
+ currentRunRef.current = null;
+ cancelPlaybackRun(run);
+ setError(readPlaybackError(playbackError, fallback));
+ setActiveMessageId(null);
+ };
+ const synthesisByIndex = new Map>();
+ const startSynthesis = (index: number): void => {
+ if (!isCurrentRun() || synthesisByIndex.has(index) || index >= chunks.length) return;
+ const synthesis = synthesizeSpeech(
+ chunks[index],
+ projectId,
+ undefined,
+ run.abortController.signal,
+ ).then(
+ (blob): SynthesisOutcome => ({ blob }),
+ (synthesisError): SynthesisOutcome => {
+ failRun(synthesisError, "Speech synthesis failed.");
+ return { error: synthesisError };
+ },
+ );
+ synthesisByIndex.set(index, synthesis);
+ };
+ const prefetchAfter = (index: number): void => {
+ const lastPrefetchIndex = Math.min(chunks.length - 1, index + SPEECH_PREFETCH_AHEAD);
+ for (let prefetchIndex = index + 1; prefetchIndex <= lastPrefetchIndex; prefetchIndex += 1) {
+ startSynthesis(prefetchIndex);
+ }
+ };
+
+ // Only the first sentence is requested before playback. Later chunks begin
+ // synthesizing once the current audio has started, bounded by the lookahead.
+ startSynthesis(0);
+
try {
- for (const chunk of chunks) {
- const audioBlob = await synthesizeSpeech(chunk, projectId);
- if (requestSequence !== requestSequenceRef.current) return;
-
- const outcome = await new Promise<{ completed: boolean; error?: string }>((resolve) => {
- const url = URL.createObjectURL(audioBlob);
- audioUrlRef.current = url;
- const audio = new Audio(url);
- audioRef.current = audio;
- let settled = false;
- const finish = (playedToEnd: boolean, playbackError?: string): void => {
- if (settled) return;
- settled = true;
- if (audioUrlRef.current === url) {
- URL.revokeObjectURL(url);
- audioUrlRef.current = null;
- }
- if (audioRef.current === audio) audioRef.current = null;
- settleAudioRef.current = null;
- resolve({ completed: playedToEnd, error: playbackError });
- };
- settleAudioRef.current = () => finish(false);
- audio.onended = () => finish(true);
- audio.onerror = () => finish(false, "The browser could not play the generated audio.");
- void audio.play().catch((playbackError: unknown) => finish(
- false,
- playbackError instanceof Error && playbackError.message
- ? playbackError.message
- : "The browser blocked audio playback. Use the voice button and try again.",
- ));
- });
- if (!outcome.completed || requestSequence !== requestSequenceRef.current) {
- if (outcome.error && requestSequence === requestSequenceRef.current) setError(outcome.error);
+ for (let index = 0; index < chunks.length; index += 1) {
+ const synthesis = synthesisByIndex.get(index);
+ if (!synthesis) return;
+ const synthesisOutcome = await synthesis;
+ synthesisByIndex.delete(index);
+ if (!isCurrentRun() || !synthesisOutcome.blob) return;
+
+ const audioOutcomePromise = playAudioBlob(run, synthesisOutcome.blob);
+ prefetchAfter(index);
+ const audioOutcome = await audioOutcomePromise;
+ if (!isCurrentRun()) return;
+ if (!audioOutcome.completed) {
+ failRun(audioOutcome.error, "The browser could not play the generated audio.");
return;
}
}
} catch (playbackError) {
- if (requestSequence === requestSequenceRef.current) {
- setError(playbackError instanceof Error && playbackError.message
- ? playbackError.message
- : "Speech synthesis failed.");
- }
+ failRun(playbackError, "Speech playback failed.");
} finally {
- if (requestSequence === requestSequenceRef.current) setActiveMessageId(null);
+ if (currentRunRef.current === run) {
+ currentRunRef.current = null;
+ setActiveMessageId(null);
+ }
}
}, [stop]);
- useEffect(() => stop, [stop]);
+ useEffect(() => () => {
+ const currentRun = currentRunRef.current;
+ currentRunRef.current = null;
+ if (currentRun) cancelPlaybackRun(currentRun);
+ }, []);
return { activeMessageId, error, play, stop };
};
diff --git a/dashboard/src/v2/lib/__tests__/speech-api.test.ts b/dashboard/src/v2/lib/__tests__/speech-api.test.ts
index 5fa1e4d678..611018af09 100644
--- a/dashboard/src/v2/lib/__tests__/speech-api.test.ts
+++ b/dashboard/src/v2/lib/__tests__/speech-api.test.ts
@@ -1,5 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import { transcribeSpeechAudio } from "../speech-api.js";
+import { synthesizeSpeech, transcribeSpeechAudio } from "../speech-api.js";
const createJsonResponse = (body: unknown, init: ResponseInit = {}): Response => (
new Response(JSON.stringify(body), {
@@ -151,4 +151,20 @@ describe("speech-api", () => {
},
});
});
+
+ it("passes cancellation through to speech synthesis without changing its request body", async () => {
+ const audio = new Blob(["audio"], { type: "audio/wav" });
+ vi.mocked(fetch).mockResolvedValueOnce(new Response(audio, { status: 200 }));
+ const signal = new AbortController().signal;
+
+ const result = await synthesizeSpeech("Read this", "project-1", "voice-1", signal);
+
+ expect(result).toEqual(audio);
+ expect(fetch).toHaveBeenCalledWith("/api/speech/synthesis", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ text: "Read this", projectId: "project-1", voice: "voice-1" }),
+ signal,
+ });
+ });
});
diff --git a/dashboard/src/v2/lib/speech-api.ts b/dashboard/src/v2/lib/speech-api.ts
index d8188ed3f8..d46124b166 100644
--- a/dashboard/src/v2/lib/speech-api.ts
+++ b/dashboard/src/v2/lib/speech-api.ts
@@ -168,11 +168,17 @@ export async function deleteSpeechModel(modelId: string): Promise {
if (!response.ok) throw new Error("Speech model could not be deleted.");
}
-export async function synthesizeSpeech(text: string, projectId?: string | null, voice?: string | null): Promise {
+export async function synthesizeSpeech(
+ text: string,
+ projectId?: string | null,
+ voice?: string | null,
+ signal?: AbortSignal,
+): Promise {
const response = await fetch("/api/speech/synthesis", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ text, projectId, voice }),
+ signal,
});
if (!response.ok) {
const body = await response.json().catch(() => ({})) as { error?: { message?: string } | string };
diff --git a/docs-web/architecture/speech-output.md b/docs-web/architecture/speech-output.md
index 3ea2dfa9e1..ab5b52cde4 100644
--- a/docs-web/architecture/speech-output.md
+++ b/docs-web/architecture/speech-output.md
@@ -33,7 +33,9 @@ Local model and voice settings resolve as a compatible pair across system, proje
When TTS is active, the volume icon in the avatar nameplate control dock starts enabled. New project-manager replies are synthesized and played once; refreshing, opening existing history, or changing threads does not replay them. Before synthesis, Code UX silently removes dashboard-only rich-widget fences and fenced code while preserving the surrounding visible prose, so widget payloads and artificial omission notices are never spoken. Click the icon to mute or unmute. The adjacent microphone dictates into the 3D Chat draft. Both controls remain outside the composer. Muting stops current playback and is remembered per project in that browser without disabling the saved TTS runtime for other clients. Synthesis or browser playback failures appear as an accessible inline voice error instead of being silently ignored.
-Assistant prose messages include a small accessible replay control in 3D Chat, Threads, and invocation transcripts. Replay is explicit in Threads and Invocations: transcript loading and live updates never start speech. Long replies play as deterministic sequential requests: the first complete sentence is synthesized independently for a quick start, later sentences are grouped when they fit, and oversized or unpunctuated passages use bounded word-aware splits. Every request stays within the 8,000-character synthesis limit without reordering or omitting spoken content, and starting another clip stops the previous clip on that surface.
+Assistant prose messages include a small accessible replay control in 3D Chat, Threads, and invocation transcripts. Replay is explicit in Threads and Invocations: transcript loading and live updates never start speech. For long replies, the first complete sentence is synthesized immediately and begins playing as soon as it is ready. While it plays, Code UX prefetches at most two later chunks, retains results by index, and plays only the next contiguous chunk. Later sentences are grouped when they fit, and oversized or unpunctuated passages use bounded word-aware splits. Every request stays within the 8,000-character synthesis limit without reordering or omitting spoken content.
+
+Stopping, muting, starting another replay, changing thread or Chat mode, or leaving the surface aborts pending synthesis and releases active audio resources. Late results cannot restart a cancelled run. A synthesis or browser playback failure stops the ordered run and appears in the surface's accessible voice or transcript status without hiding the written reply.
## Local files and endpoints
diff --git a/docs-web/content/docs/architecture-speech-output.mdx b/docs-web/content/docs/architecture-speech-output.mdx
index acd3653e0f..ecaefccf87 100644
--- a/docs-web/content/docs/architecture-speech-output.mdx
+++ b/docs-web/content/docs/architecture-speech-output.mdx
@@ -33,7 +33,9 @@ Local model and voice settings resolve as a compatible pair across system, proje
When TTS is active, the volume icon in the avatar nameplate control dock starts enabled. New project-manager replies are synthesized and played once; refreshing, opening existing history, or changing threads does not replay them. Before synthesis, Code UX silently removes dashboard-only rich-widget fences and fenced code while preserving the surrounding visible prose, so widget payloads and artificial omission notices are never spoken. Click the icon to mute or unmute. The adjacent microphone dictates into the 3D Chat draft. Both controls remain outside the composer. Muting stops current playback and is remembered per project in that browser without disabling the saved TTS runtime for other clients. Synthesis or browser playback failures appear as an accessible inline voice error instead of being silently ignored.
-Assistant prose messages include a small accessible replay control in 3D Chat, Threads, and invocation transcripts. Replay is explicit in Threads and Invocations: transcript loading and live updates never start speech. Long replies play as deterministic sequential requests: the first complete sentence is synthesized independently for a quick start, later sentences are grouped when they fit, and oversized or unpunctuated passages use bounded word-aware splits. Every request stays within the 8,000-character synthesis limit without reordering or omitting spoken content, and starting another clip stops the previous clip on that surface.
+Assistant prose messages include a small accessible replay control in 3D Chat, Threads, and invocation transcripts. Replay is explicit in Threads and Invocations: transcript loading and live updates never start speech. For long replies, the first complete sentence is synthesized immediately and begins playing as soon as it is ready. While it plays, Code UX prefetches at most two later chunks, retains results by index, and plays only the next contiguous chunk. Later sentences are grouped when they fit, and oversized or unpunctuated passages use bounded word-aware splits. Every request stays within the 8,000-character synthesis limit without reordering or omitting spoken content.
+
+Stopping, muting, starting another replay, changing thread or Chat mode, or leaving the surface aborts pending synthesis and releases active audio resources. Late results cannot restart a cancelled run. A synthesis or browser playback failure stops the ordered run and appears in the surface's accessible voice or transcript status without hiding the written reply.
## Local files and endpoints
diff --git a/docs-web/content/docs/user-dashboard-chat.mdx b/docs-web/content/docs/user-dashboard-chat.mdx
index 90c16e8003..a884146b0b 100644
--- a/docs-web/content/docs/user-dashboard-chat.mdx
+++ b/docs-web/content/docs/user-dashboard-chat.mdx
@@ -23,7 +23,9 @@ The thought area turns known runtime fields into compact cues for container star
During the selected Project Manager's provider-working phase, the avatar rotates through five work tools every seven seconds: Power screwdriver (`screwdriver`), Jackhammer (`jackhammer`), Open-end wrench (`wrench`), Claw hammer (`hammer`), and Welding torch (`torch`). Container startup keeps the thinking state without a tool. Background work never equips a tool. Maintainers can pin a valid catalog identifier with `/chat?stageTool=` for design review; missing or unsupported values leave normal runtime selection in place. Reduced-motion and WebGL fallback surfaces replace tool animation with the visible tool label and an accessible avatar description.
-When a text-to-speech model or API is active under **Settings -> AI Models**, 3D Chat reads new Project Manager replies aloud. A compact control dock beneath the avatar identity holds the microphone and agent mute/unmute buttons, outside the composer. Voice defaults on, shows synthesis activity, and can be muted immediately. The preference is remembered per project in the current browser; opening an existing thread does not replay its latest historical message. If synthesis or browser playback fails, 3D Chat shows the voice error beside the controls so a silent failure can be corrected.
+When a text-to-speech model or API is active under **Settings -> AI Models**, 3D Chat reads new Project Manager replies aloud. A compact control dock beneath the avatar identity holds the microphone and agent mute/unmute buttons, outside the composer. Voice defaults on, shows synthesis activity, and can be muted immediately. The preference is remembered per project in the current browser; opening an existing thread does not replay its latest historical message. Long replies start with the first ready sentence while a bounded two-chunk lookahead is synthesized, then continue through every chunk in transcript order. Muting, changing threads or Chat mode, leaving the page, or starting another replay cancels pending speech and releases the active audio. If synthesis or browser playback fails, playback stops and the accessible voice or transcript status reports the error without hiding the written reply.
+
+Assistant replies in 3D Chat, Threads, and invocation transcripts include an accessible replay button. Threads and Invocations never autoplay loaded or newly refreshed transcript history; speech there begins only when you request replay.
## No-project assistant
diff --git a/docs-web/user/dashboard/chat.md b/docs-web/user/dashboard/chat.md
index 90c16e8003..a884146b0b 100644
--- a/docs-web/user/dashboard/chat.md
+++ b/docs-web/user/dashboard/chat.md
@@ -23,7 +23,9 @@ The thought area turns known runtime fields into compact cues for container star
During the selected Project Manager's provider-working phase, the avatar rotates through five work tools every seven seconds: Power screwdriver (`screwdriver`), Jackhammer (`jackhammer`), Open-end wrench (`wrench`), Claw hammer (`hammer`), and Welding torch (`torch`). Container startup keeps the thinking state without a tool. Background work never equips a tool. Maintainers can pin a valid catalog identifier with `/chat?stageTool=` for design review; missing or unsupported values leave normal runtime selection in place. Reduced-motion and WebGL fallback surfaces replace tool animation with the visible tool label and an accessible avatar description.
-When a text-to-speech model or API is active under **Settings -> AI Models**, 3D Chat reads new Project Manager replies aloud. A compact control dock beneath the avatar identity holds the microphone and agent mute/unmute buttons, outside the composer. Voice defaults on, shows synthesis activity, and can be muted immediately. The preference is remembered per project in the current browser; opening an existing thread does not replay its latest historical message. If synthesis or browser playback fails, 3D Chat shows the voice error beside the controls so a silent failure can be corrected.
+When a text-to-speech model or API is active under **Settings -> AI Models**, 3D Chat reads new Project Manager replies aloud. A compact control dock beneath the avatar identity holds the microphone and agent mute/unmute buttons, outside the composer. Voice defaults on, shows synthesis activity, and can be muted immediately. The preference is remembered per project in the current browser; opening an existing thread does not replay its latest historical message. Long replies start with the first ready sentence while a bounded two-chunk lookahead is synthesized, then continue through every chunk in transcript order. Muting, changing threads or Chat mode, leaving the page, or starting another replay cancels pending speech and releases the active audio. If synthesis or browser playback fails, playback stops and the accessible voice or transcript status reports the error without hiding the written reply.
+
+Assistant replies in 3D Chat, Threads, and invocation transcripts include an accessible replay button. Threads and Invocations never autoplay loaded or newly refreshed transcript history; speech there begins only when you request replay.
## No-project assistant
diff --git a/docs/architecture/speech-output.md b/docs/architecture/speech-output.md
index 59ea6c0638..f1126a6286 100644
--- a/docs/architecture/speech-output.md
+++ b/docs/architecture/speech-output.md
@@ -37,7 +37,9 @@ Provider selection is explicit: local mode never sends text to an external provi
3D Chat establishes the loaded thread as a silent history baseline, then watches for a newly appended project-manager message. When voice is enabled, it removes Markdown-only decoration, dashboard-only `codeux:*` rich-widget fences, and ordinary fenced code before requesting audio for the active project scope. Those non-spoken blocks are removed silently: neither their payloads nor an artificial "output omitted" notice reaches the speech provider. Human-facing prose around the blocks remains in reading order. Each new reply plays once; refreshing, opening 3D Chat, or changing threads never speaks loaded history.
-Assistant prose messages expose a small accessible replay control in 3D Chat, Threads, and invocation transcripts. Replay is always explicit outside 3D Chat, so thread and invocation loads or live updates never start speech. Long replies are normalized and chunked deterministically: the first complete sentence is synthesized independently for low startup latency, later sentences are grouped when they fit, and oversized or unpunctuated passages fall back to bounded word-aware splits. Every request stays within the backend's 8,000-character request bound without reordering or omitting spoken content. Starting another clip stops the previous clip on that transcript surface.
+Assistant prose messages expose a small accessible replay control in 3D Chat, Threads, and invocation transcripts. Replay is always explicit outside 3D Chat, so thread and invocation loads or live updates never start speech. Long replies are normalized and chunked deterministically: the first complete sentence is synthesized immediately and starts playing as soon as it is ready. While that audio plays, the browser prefetches at most two later chunks, stores them by index, and only plays the next contiguous result. Later sentences are grouped when they fit, and oversized or unpunctuated passages fall back to bounded word-aware splits. Every request stays within the backend's 8,000-character request bound without reordering or omitting spoken content.
+
+Each playback run owns one abort controller, active audio element, and object URL. Stopping, muting, replaying another message, changing thread or Chat mode, and unmounting abort pending synthesis and release browser audio resources. A stale or failed synthesis result cannot restart playback. Synthesis and browser playback failures stop the whole ordered run; 3D Chat reports them beside its voice controls, while Threads and Invocations report replay failures in their existing accessible status surfaces without hiding the transcript.
The avatar nameplate includes a compact microphone button and volume icon, outside the composer. Dictation uses the same caret-aware insertion behavior as Threads mode. Voice defaults on when saved TTS settings are active. Muting stops playback immediately and stores a per-project browser preference; it does not disable the saved TTS model for other clients. If no TTS model/API is active, the volume icon is disabled and its accessible help points the operator to Settings -> AI Models.