From 18da34bc5590b839f56ee1da1cab771ef402a230 Mon Sep 17 00:00:00 2001 From: 04mg Date: Sun, 19 Jul 2026 20:30:07 -0300 Subject: [PATCH 01/36] feat: add voice mode with speech-to-text input for terminals --- .../settings/components/SettingsDialog.tsx | 49 +++++- .../features/shared/components/AppLayout.tsx | 4 +- .../status-bar/components/StatusBar.tsx | 76 ++++++++- .../voice-mode/components/VoiceBubble.tsx | 64 +++++++ .../voice-mode/hooks/useSpeechRecognition.ts | 158 ++++++++++++++++++ src/frontend/src/features/voice-mode/types.ts | 1 + src/frontend/src/index.css | 85 ++++++++++ 7 files changed, 432 insertions(+), 5 deletions(-) create mode 100644 src/frontend/src/features/voice-mode/components/VoiceBubble.tsx create mode 100644 src/frontend/src/features/voice-mode/hooks/useSpeechRecognition.ts create mode 100644 src/frontend/src/features/voice-mode/types.ts diff --git a/src/frontend/src/features/settings/components/SettingsDialog.tsx b/src/frontend/src/features/settings/components/SettingsDialog.tsx index c3fd0b0..4c3d361 100644 --- a/src/frontend/src/features/settings/components/SettingsDialog.tsx +++ b/src/frontend/src/features/settings/components/SettingsDialog.tsx @@ -2,7 +2,7 @@ import { useState, useEffect, useCallback, useRef, type ElementType } from 'reac import { Dialog, DialogContent, DialogTitle, DialogClose } from '@/components/dialog' import { Slider } from '@/components/slider' -import { Palette, Bot, Terminal, Check, Moon, Sun, Monitor, ChartSpline, ArrowLeft, LogIn, ExternalLink, Loader2, Folder, Settings as SettingsIcon, X, Bell } from 'lucide-react' +import { Palette, Bot, Terminal, Check, Moon, Sun, Monitor, ChartSpline, ArrowLeft, LogIn, ExternalLink, Loader2, Folder, Settings as SettingsIcon, X, Bell, Mic } from 'lucide-react' import { Antigravity, OpenCode, Ollama, Claude, Codex, GithubCopilot, OpenRouter } from '@lobehub/icons' import { agentTypes, getAgentCmdOverrides, setAgentCmdOverride } from '@/features/agents/services/agentTypes' import { setAllTerminalFontSizes, setAllTerminalThemes } from '@/features/terminal/services/terminalRegistry' @@ -15,7 +15,7 @@ interface SettingsDialogProps { initialSection?: string } -type Section = 'appearance' | 'agents' | 'terminal' | 'workspaces' | 'limits' | 'notifications' +type Section = 'appearance' | 'agents' | 'terminal' | 'workspaces' | 'limits' | 'notifications' | 'voice' export function SettingsDialog({ open, onOpenChange, initialSection }: SettingsDialogProps) { const [activeSection, setActiveSection] = useState
('appearance') @@ -81,6 +81,7 @@ export function SettingsDialog({ open, onOpenChange, initialSection }: SettingsD const [pushBusy, setPushBusy] = useState(false) const [pushError, setPushError] = useState('') const [soundEnabled, setSoundEnabled] = useState(true) + const [voiceLanguage, setVoiceLanguage] = useState('') const isSecureContext = typeof window !== 'undefined' && (window.isSecureContext || window.location.hostname === 'localhost') const loadQuotaSettings = useCallback(async () => { @@ -187,6 +188,7 @@ export function SettingsDialog({ open, onOpenChange, initialSection }: SettingsD // Load push notification state setSoundEnabled(localStorage.getItem('caw:soundEnabled') !== '0') + setVoiceLanguage(localStorage.getItem('caw:voiceLanguage') || '') if (pushSupported && 'Notification' in window) { setPushPermission(Notification.permission) } else if (!pushSupported && !pushIOSPWA) { @@ -390,6 +392,7 @@ export function SettingsDialog({ open, onOpenChange, initialSection }: SettingsD { id: 'notifications', label: 'Notifications', icon: Bell }, { id: 'workspaces', label: 'Workspaces', icon: Folder }, { id: 'terminal', label: 'Terminal', icon: Terminal }, + { id: 'voice', label: 'Voice', icon: Mic }, { id: 'agents', label: 'Agents', icon: Bot }, { id: 'limits', label: 'Limits', icon: ChartSpline }, ] @@ -755,6 +758,48 @@ export function SettingsDialog({ open, onOpenChange, initialSection }: SettingsD )} + {activeSection === 'voice' && ( +
+
+

Voice

+

Configure voice input language for speech recognition.

+
+ +
+
+ + +

Language used for speech recognition. "System Default" uses your browser's default language.

+
+
+
+ )} + {activeSection === 'workspaces' && (
diff --git a/src/frontend/src/features/shared/components/AppLayout.tsx b/src/frontend/src/features/shared/components/AppLayout.tsx index f43ebc1..5eaa2a2 100644 --- a/src/frontend/src/features/shared/components/AppLayout.tsx +++ b/src/frontend/src/features/shared/components/AppLayout.tsx @@ -27,7 +27,7 @@ import { import { type Workspace, type TabGroupsNode } from '@/features/workspaces/types' import { TabGroupTree } from '@/features/workspaces/components/TabGroupTree' import { ensureTabGroups, findGroupById, collectGroups, collectTabIds, moveTabToGroup, removeTabFromTree, splitGroup, getTopRightGroupId, findGroupWithTab } from '@/features/workspaces/utils/tabGroups' -import { destroyTerminal, releaseTerminal, setOnTerminalExit } from '@/features/terminal/services/terminalRegistry' +import { destroyTerminal, releaseTerminal, setOnTerminalExit, sendTerminalInput } from '@/features/terminal/services/terminalRegistry' import { useHotkeys } from '@/hooks/useHotkeys' import { Folder, Menu, Plus, SquareTerminal, GitBranch, FileCode, Terminal, Settings, PanelRight, X } from 'lucide-react' import { Button } from '@/components/button' @@ -1672,6 +1672,7 @@ export function AppLayout() { setSettingsOpen(true) }} hideControlCenter + onSendText={(text) => { if (activePaneId) sendTerminalInput(activePaneId, text) }} />
) : ( @@ -1877,6 +1878,7 @@ export function AppLayout() { setSettingsOpen(true) }} controlCenterButtonRef={controlCenterBtnRef} + onSendText={(text) => { if (activePaneId) sendTerminalInput(activePaneId, text) }} /> )} diff --git a/src/frontend/src/features/status-bar/components/StatusBar.tsx b/src/frontend/src/features/status-bar/components/StatusBar.tsx index 32b418d..bc07e9e 100644 --- a/src/frontend/src/features/status-bar/components/StatusBar.tsx +++ b/src/frontend/src/features/status-bar/components/StatusBar.tsx @@ -5,10 +5,13 @@ import { DropdownMenuContent, DropdownMenuSeparator, } from '@/components/dropdown-menu' -import { RefreshCw, Key, Check, Loader2, ChevronUp, Workflow, Folder, SquareKanban, Settings } from 'lucide-react' +import { RefreshCw, Key, Check, Loader2, ChevronUp, Workflow, Folder, SquareKanban, Settings, Mic } from 'lucide-react' import { Antigravity, OpenCode, Ollama, Claude, Codex, GithubCopilot, OpenRouter } from '@lobehub/icons' import { cn } from '@/features/shared/utils/utils' import { Tooltip, TooltipTrigger, TooltipContent } from '@/components/tooltip' +import { useSpeechRecognition, isVoiceSupported } from '@/features/voice-mode/hooks/useSpeechRecognition' +import { VoiceBubble } from '@/features/voice-mode/components/VoiceBubble' +import type { VoicePhase } from '@/features/voice-mode/types' interface Quota { used: number @@ -88,14 +91,17 @@ interface StatusBarProps { onOpenSettings: (section?: string) => void hideControlCenter?: boolean controlCenterButtonRef?: React.Ref + onSendText?: (text: string) => void } -export function StatusBar({ workspaceName, worktreeBranch, agentBoardOpen, onToggleAgentBoard, onOpenSettings, hideControlCenter, controlCenterButtonRef }: StatusBarProps) { +export function StatusBar({ workspaceName, worktreeBranch, agentBoardOpen, onToggleAgentBoard, onOpenSettings, hideControlCenter, controlCenterButtonRef, onSendText }: StatusBarProps) { const [quotas, setQuotas] = useState(null) const [settings, setSettings] = useState>>({}) const [isLoading, setIsLoading] = useState(false) const [selectedView, setSelectedView] = useState('') const [isMobile, setIsMobile] = useState(() => typeof window !== 'undefined' && window.innerWidth < 768) + const [voicePhase, setVoicePhase] = useState('idle') + const speech = useSpeechRecognition() useEffect(() => { const onResize = () => setIsMobile(window.innerWidth < 768) @@ -166,6 +172,33 @@ export function StatusBar({ workspaceName, worktreeBranch, agentBoardOpen, onTog localStorage.setItem('caw:quota:selected_view', view) } + const handleToggleVoice = () => { + if (!isVoiceSupported()) { + speech.reset() + return + } + if (voicePhase === 'idle') { + speech.reset() + speech.start() + setVoicePhase('listening') + } else if (voicePhase === 'listening') { + speech.stop() + setVoicePhase('review') + } + } + + const handleSendVoice = () => { + const text = speech.transcript.trim() + if (text) onSendText?.(text) + speech.reset() + setVoicePhase('idle') + } + + const handleDiscardVoice = () => { + speech.reset() + setVoicePhase('idle') + } + const hasClaude = settings.claude?.installed !== 'false' && !(quotas && quotas.claude?.error) const hasCodex = settings.codex?.installed !== 'false' && !(quotas && quotas.codex?.error) const hasCopilot = !!settings.copilot?.token @@ -365,6 +398,44 @@ export function StatusBar({ workspaceName, worktreeBranch, agentBoardOpen, onTog )}
+
+ {voicePhase !== 'idle' && ( + + )} + {!isMobile && ( + <> + + + + + + Voice Mode + + + + + )} +
) } diff --git a/src/frontend/src/features/voice-mode/components/VoiceBubble.tsx b/src/frontend/src/features/voice-mode/components/VoiceBubble.tsx new file mode 100644 index 0000000..5956eb5 --- /dev/null +++ b/src/frontend/src/features/voice-mode/components/VoiceBubble.tsx @@ -0,0 +1,64 @@ +import { createPortal } from 'react-dom' +import { Check, X } from 'lucide-react' +import { cn } from '@/features/shared/utils/utils' + +interface VoiceBubbleProps { + transcript: string + error: string | null + isListening: boolean + onSend: () => void + onDiscard: () => void +} + +export function VoiceBubble({ transcript, error, isListening, onSend, onDiscard }: VoiceBubbleProps) { + const hasContent = transcript.trim().length > 0 + + return createPortal( +
+
+
+ {error ? ( + {error} + ) : transcript ? ( + {transcript} + ) : isListening ? ( + Listening... + ) : null} + + {isListening && ( +
+ )} +
+ + {!isListening && ( +
+ + +
+ )} +
+
, + document.body + ) +} diff --git a/src/frontend/src/features/voice-mode/hooks/useSpeechRecognition.ts b/src/frontend/src/features/voice-mode/hooks/useSpeechRecognition.ts new file mode 100644 index 0000000..b91aeb6 --- /dev/null +++ b/src/frontend/src/features/voice-mode/hooks/useSpeechRecognition.ts @@ -0,0 +1,158 @@ +import { useState, useRef, useCallback, useEffect } from 'react' + +interface SpeechRecognitionResult { + isFinal: boolean + [key: number]: { transcript: string } + length: number +} + +interface SpeechRecognitionResultList { + length: number + item(index: number): SpeechRecognitionResult + [key: number]: SpeechRecognitionResult +} + +interface SpeechRecognitionResultEvent extends Event { + results: SpeechRecognitionResultList + resultIndex: number +} + +interface SpeechRecognitionErrorEvent extends Event { + error: string + message: string +} + +interface SpeechRecognitionInstance extends EventTarget { + continuous: boolean + interimResults: boolean + lang: string + onresult: ((event: SpeechRecognitionResultEvent) => void) | null + onerror: ((event: SpeechRecognitionErrorEvent) => void) | null + onend: (() => void) | null + start(): void + stop(): void + abort(): void +} + +interface SpeechRecognitionConstructor { + new(): SpeechRecognitionInstance +} + +declare global { + interface Window { + SpeechRecognition?: SpeechRecognitionConstructor + webkitSpeechRecognition?: SpeechRecognitionConstructor + } +} + +const SpeechRecognitionAPI = + typeof window !== 'undefined' + ? window.SpeechRecognition || window.webkitSpeechRecognition + : null + +export function isVoiceSupported(): boolean { + return !!SpeechRecognitionAPI +} + +export function useSpeechRecognition() { + const [isListening, setIsListening] = useState(false) + const [transcript, setTranscript] = useState('') + const [error, setError] = useState(null) + const recognitionRef = useRef(null) + const shouldRestartRef = useRef(false) + + const reset = useCallback(() => { + setTranscript('') + setError(null) + }, []) + + const stop = useCallback(() => { + shouldRestartRef.current = false + if (recognitionRef.current) { + try { + recognitionRef.current.stop() + } catch { + // already stopped + } + } + setIsListening(false) + }, []) + + const start = useCallback(() => { + if (!SpeechRecognitionAPI) { + setError('Voice input is not supported in this browser') + return + } + + const lang = localStorage.getItem('caw:voiceLanguage') || '' + const recognition = new SpeechRecognitionAPI() + recognition.continuous = true + recognition.interimResults = true + if (lang) recognition.lang = lang + + recognition.onresult = (event: SpeechRecognitionResultEvent) => { + let finalText = '' + let interimText = '' + for (let i = event.resultIndex; i < event.results.length; i++) { + const result = event.results[i] + if (result.isFinal) { + finalText += result[0].transcript + } else { + interimText += result[0].transcript + } + } + setTranscript((prev) => { + const base = prev.replace(/\u200B$/, '') + if (finalText) return base + finalText + return base + interimText + }) + } + + recognition.onerror = (event: SpeechRecognitionErrorEvent) => { + if (event.error === 'no-speech' || event.error === 'aborted') return + setError(event.error === 'not-allowed' + ? 'Microphone permission denied' + : `Speech error: ${event.error}`) + setIsListening(false) + shouldRestartRef.current = false + } + + recognition.onend = () => { + if (shouldRestartRef.current) { + try { + recognition.start() + } catch { + setIsListening(false) + shouldRestartRef.current = false + } + return + } + setIsListening(false) + } + + try { + recognition.start() + recognitionRef.current = recognition + shouldRestartRef.current = true + setIsListening(true) + setError(null) + } catch { + setError('Failed to start voice recognition') + } + }, []) + + useEffect(() => { + return () => { + shouldRestartRef.current = false + if (recognitionRef.current) { + try { + recognitionRef.current.stop() + } catch { + // ignore + } + } + } + }, []) + + return { isListening, transcript, error, start, stop, reset, setTranscript } +} diff --git a/src/frontend/src/features/voice-mode/types.ts b/src/frontend/src/features/voice-mode/types.ts new file mode 100644 index 0000000..ada6372 --- /dev/null +++ b/src/frontend/src/features/voice-mode/types.ts @@ -0,0 +1 @@ +export type VoicePhase = 'idle' | 'listening' | 'review' diff --git a/src/frontend/src/index.css b/src/frontend/src/index.css index ba81660..5ec09fe 100644 --- a/src/frontend/src/index.css +++ b/src/frontend/src/index.css @@ -270,6 +270,91 @@ html, body, #root { 100% { background-position: 100% 50%; } } +/* ========================================================================== + Voice Mode + ========================================================================== */ + +.lava-lamp-mic { + stroke: url(#lava-gradient) !important; + filter: drop-shadow(0 0 3px rgba(168, 85, 247, 0.5)); +} + +.voice-bubble-container { + position: fixed; + bottom: 44px; + left: 50%; + transform: translateX(-50%); + z-index: 50; + pointer-events: auto; +} + +.voice-bubble { + background: hsl(var(--popover)); + border: 1px solid hsl(var(--border)); + border-radius: 16px; + box-shadow: 0 8px 30px rgba(0, 0, 0, 0.25), 0 2px 8px rgba(0, 0, 0, 0.15); + padding: 10px 14px; + min-width: 180px; + max-width: 420px; + animation: voice-bubble-in 0.2s ease-out; +} + +@keyframes voice-bubble-in { + 0% { + opacity: 0; + transform: translateY(6px) scale(0.96); + } + 100% { + opacity: 1; + transform: translateY(0) scale(1); + } +} + +.voice-bubble-content { + display: flex; + align-items: center; + gap: 8px; +} + +.voice-bubble-pulse { + width: 6px; + height: 6px; + border-radius: 50%; + background: #a855f7; + animation: voice-pulse 1.2s ease-in-out infinite; + flex-shrink: 0; +} + +@keyframes voice-pulse { + 0%, 100% { opacity: 1; transform: scale(1); } + 50% { opacity: 0.4; transform: scale(0.7); } +} + +.voice-bubble-actions { + display: flex; + align-items: center; + gap: 6px; + margin-left: 8px; + flex-shrink: 0; +} + +.voice-bubble-btn { + width: 30px; + height: 30px; + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + background: hsl(var(--secondary)); + border: 1px solid hsl(var(--border)); + cursor: pointer; + transition: all 0.15s ease; +} + +.voice-bubble-btn:hover { + transform: scale(1.08); +} + /* ========================================================================== Monaco Editor overrides to match Shadcn UI context menus & floats ========================================================================== */ From 4c053ba6248612057830152885bb6ba3c52011ae Mon Sep 17 00:00:00 2001 From: 04mg Date: Sun, 19 Jul 2026 20:42:14 -0300 Subject: [PATCH 02/36] fix: hide voice mode button when browser unsupported, show support status in settings --- .../features/settings/components/SettingsDialog.tsx | 13 +++++++++++++ .../features/status-bar/components/StatusBar.tsx | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/frontend/src/features/settings/components/SettingsDialog.tsx b/src/frontend/src/features/settings/components/SettingsDialog.tsx index 4c3d361..6261209 100644 --- a/src/frontend/src/features/settings/components/SettingsDialog.tsx +++ b/src/frontend/src/features/settings/components/SettingsDialog.tsx @@ -6,6 +6,7 @@ import { Palette, Bot, Terminal, Check, Moon, Sun, Monitor, ChartSpline, ArrowLe import { Antigravity, OpenCode, Ollama, Claude, Codex, GithubCopilot, OpenRouter } from '@lobehub/icons' import { agentTypes, getAgentCmdOverrides, setAgentCmdOverride } from '@/features/agents/services/agentTypes' import { setAllTerminalFontSizes, setAllTerminalThemes } from '@/features/terminal/services/terminalRegistry' +import { isVoiceSupported } from '@/features/voice-mode/hooks/useSpeechRecognition' import { SettingsItem } from './SettingsItem' @@ -765,6 +766,18 @@ export function SettingsDialog({ open, onOpenChange, initialSection }: SettingsD

Configure voice input language for speech recognition.

+ {isVoiceSupported() ? ( +
+
+ Voice Mode is supported in this browser +
+ ) : ( +
+
+ Voice Mode is not supported in this browser. Use Chrome, Edge, or Safari for speech recognition. +
+ )} +
diff --git a/src/frontend/src/features/status-bar/components/StatusBar.tsx b/src/frontend/src/features/status-bar/components/StatusBar.tsx index bc07e9e..5f73a51 100644 --- a/src/frontend/src/features/status-bar/components/StatusBar.tsx +++ b/src/frontend/src/features/status-bar/components/StatusBar.tsx @@ -408,7 +408,7 @@ export function StatusBar({ workspaceName, worktreeBranch, agentBoardOpen, onTog onDiscard={handleDiscardVoice} /> )} - {!isMobile && ( + {!isMobile && isVoiceSupported() && ( <> From 445bc4e7099a8982cc18b2344a7b40fa9dd8ef4a Mon Sep 17 00:00:00 2001 From: 04mg Date: Sun, 19 Jul 2026 20:47:46 -0300 Subject: [PATCH 03/36] fix: show transcript only after first word, move send/discard buttons outside bubble --- .../voice-mode/components/VoiceBubble.tsx | 60 ++++++++++--------- src/frontend/src/index.css | 7 ++- 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/src/frontend/src/features/voice-mode/components/VoiceBubble.tsx b/src/frontend/src/features/voice-mode/components/VoiceBubble.tsx index 5956eb5..2a72ac8 100644 --- a/src/frontend/src/features/voice-mode/components/VoiceBubble.tsx +++ b/src/frontend/src/features/voice-mode/components/VoiceBubble.tsx @@ -12,6 +12,8 @@ interface VoiceBubbleProps { export function VoiceBubble({ transcript, error, isListening, onSend, onDiscard }: VoiceBubbleProps) { const hasContent = transcript.trim().length > 0 + const words = transcript.trim().split(/\s+/).filter(Boolean) + const showTranscript = words.length > 0 return createPortal(
@@ -19,7 +21,7 @@ export function VoiceBubble({ transcript, error, isListening, onSend, onDiscard
{error ? ( {error} - ) : transcript ? ( + ) : showTranscript ? ( {transcript} ) : isListening ? ( Listening... @@ -29,35 +31,35 @@ export function VoiceBubble({ transcript, error, isListening, onSend, onDiscard
)}
- - {!isListening && ( -
- - -
- )}
+ + {!isListening && ( +
+ + +
+ )}
, document.body ) diff --git a/src/frontend/src/index.css b/src/frontend/src/index.css index 5ec09fe..ca3957f 100644 --- a/src/frontend/src/index.css +++ b/src/frontend/src/index.css @@ -286,6 +286,9 @@ html, body, #root { transform: translateX(-50%); z-index: 50; pointer-events: auto; + display: flex; + align-items: center; + gap: 8px; } .voice-bubble { @@ -332,9 +335,9 @@ html, body, #root { .voice-bubble-actions { display: flex; + flex-direction: column; align-items: center; - gap: 6px; - margin-left: 8px; + gap: 4px; flex-shrink: 0; } From d6ded29189dfd4e62f4d384965720596f1b07bde Mon Sep 17 00:00:00 2001 From: 04mg Date: Sun, 19 Jul 2026 20:53:17 -0300 Subject: [PATCH 04/36] fix: deduplicate transcript text, remove Listening placeholder, match button colors to bubble --- .../voice-mode/components/VoiceBubble.tsx | 15 +++------------ .../voice-mode/hooks/useSpeechRecognition.ts | 19 ++++++++++--------- src/frontend/src/index.css | 4 +++- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/src/frontend/src/features/voice-mode/components/VoiceBubble.tsx b/src/frontend/src/features/voice-mode/components/VoiceBubble.tsx index 2a72ac8..b8bb8fd 100644 --- a/src/frontend/src/features/voice-mode/components/VoiceBubble.tsx +++ b/src/frontend/src/features/voice-mode/components/VoiceBubble.tsx @@ -12,8 +12,6 @@ interface VoiceBubbleProps { export function VoiceBubble({ transcript, error, isListening, onSend, onDiscard }: VoiceBubbleProps) { const hasContent = transcript.trim().length > 0 - const words = transcript.trim().split(/\s+/).filter(Boolean) - const showTranscript = words.length > 0 return createPortal(
@@ -21,10 +19,8 @@ export function VoiceBubble({ transcript, error, isListening, onSend, onDiscard
{error ? ( {error} - ) : showTranscript ? ( + ) : hasContent ? ( {transcript} - ) : isListening ? ( - Listening... ) : null} {isListening && ( @@ -37,10 +33,7 @@ export function VoiceBubble({ transcript, error, isListening, onSend, onDiscard
- {voicePhase !== 'idle' && ( + {voice.phase !== 'idle' && ( @@ -417,14 +407,14 @@ export function StatusBar({ workspaceName, worktreeBranch, agentBoardOpen, onTog data-testid="status-bar-voice-mode" className={cn( "shrink-0 transition-colors cursor-pointer", - voicePhase === 'listening' + voice.phase === 'listening' ? "text-primary" : "text-muted-foreground hover:text-foreground" )} > diff --git a/src/frontend/src/features/terminal/components/MobileControlBar.tsx b/src/frontend/src/features/terminal/components/MobileControlBar.tsx index b70f083..d04ed4e 100644 --- a/src/frontend/src/features/terminal/components/MobileControlBar.tsx +++ b/src/frontend/src/features/terminal/components/MobileControlBar.tsx @@ -9,7 +9,9 @@ import { resetStickyModifiers, sendTerminalInput } from '@/features/terminal/services/terminalRegistry' -import { ChevronUp, ChevronDown, ChevronLeft, ChevronRight, CornerDownLeft } from 'lucide-react' +import { ChevronUp, ChevronDown, ChevronLeft, ChevronRight, CornerDownLeft, Mic } from 'lucide-react' +import { useVoiceMode, isVoiceSupported } from '@/features/voice-mode/hooks/useVoiceMode' +import { cn } from '@/features/shared/utils/utils' interface MobileControlBarProps { terminalId: string @@ -17,6 +19,7 @@ interface MobileControlBarProps { export function MobileControlBar({ terminalId }: MobileControlBarProps) { const [sticky, setSticky] = useState({ ctrl: false, alt: false, shift: false }) + const voice = useVoiceMode() useEffect(() => { return subscribeStickyModifiers(() => { @@ -28,15 +31,38 @@ export function MobileControlBar({ terminalId }: MobileControlBarProps) { sendTerminalInput(terminalId, sequence) } + const handleToggleVoice = () => { + if (!isVoiceSupported()) return + if (voice.phase === 'idle') { + voice.start() + } else if (voice.phase === 'listening') { + voice.stop() + } + } + return (
e.preventDefault()}> {/* Sticky Modifiers with toggled color states */}
+ {isVoiceSupported() && ( + + )} - -
- )} + {!isListening && ( +
+ + +
+ )} +
, document.body ) diff --git a/src/frontend/src/index.css b/src/frontend/src/index.css index 4fce75f..f2ff6eb 100644 --- a/src/frontend/src/index.css +++ b/src/frontend/src/index.css @@ -286,9 +286,6 @@ html, body, #root { transform: translateX(-50%); z-index: 50; pointer-events: auto; - display: flex; - align-items: center; - gap: 8px; } @media (max-width: 767px) { @@ -305,16 +302,18 @@ html, body, #root { background: hsl(var(--popover)); border: 1px solid hsl(var(--border)); border-radius: 16px; - box-shadow: 0 8px 30px rgba(0, 0, 0, 0.25), 0 2px 8px rgba(0, 0, 0, 0.15); + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.12); padding: 10px 14px; min-width: 180px; max-width: 420px; animation: voice-bubble-in 0.2s ease-out; + display: flex; + flex-direction: column; + gap: 8px; } @media (max-width: 767px) { .voice-bubble { - flex: 1; min-width: 0; max-width: none; } @@ -351,12 +350,13 @@ html, body, #root { 50% { opacity: 0.4; transform: scale(0.7); } } -.voice-bubble-actions { +.voice-bubble-footer { display: flex; - flex-direction: row; align-items: center; + justify-content: flex-end; gap: 6px; - flex-shrink: 0; + padding-top: 6px; + border-top: 1px solid hsl(var(--border)); } .voice-bubble-btn { From 62c69e24bef1b312a8102cd967b2f0e07acf3e78 Mon Sep 17 00:00:00 2001 From: 04mg Date: Sun, 19 Jul 2026 21:29:58 -0300 Subject: [PATCH 12/36] fix: position bubble above mic button on desktop, overlay footer buttons on bottom edge --- .../features/status-bar/components/StatusBar.tsx | 12 +++++++++++- .../features/voice-mode/components/VoiceBubble.tsx | 14 ++++++++++++-- src/frontend/src/index.css | 7 ++++--- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/frontend/src/features/status-bar/components/StatusBar.tsx b/src/frontend/src/features/status-bar/components/StatusBar.tsx index 7847f5a..ec5f45f 100644 --- a/src/frontend/src/features/status-bar/components/StatusBar.tsx +++ b/src/frontend/src/features/status-bar/components/StatusBar.tsx @@ -1,4 +1,4 @@ -import { useState, useEffect, useCallback } from 'react' +import { useState, useEffect, useCallback, useRef } from 'react' import { DropdownMenu, DropdownMenuTrigger, @@ -100,6 +100,14 @@ export function StatusBar({ workspaceName, worktreeBranch, agentBoardOpen, onTog const [selectedView, setSelectedView] = useState('') const [isMobile, setIsMobile] = useState(() => typeof window !== 'undefined' && window.innerWidth < 768) const voice = useVoiceMode() + const micBtnRef = useRef(null) + const [micRect, setMicRect] = useState(null) + + useEffect(() => { + if (voice.phase !== 'idle' && micBtnRef.current) { + setMicRect(micBtnRef.current.getBoundingClientRect()) + } + }, [voice.phase]) useEffect(() => { const onResize = () => setIsMobile(window.innerWidth < 768) @@ -396,6 +404,7 @@ export function StatusBar({ workspaceName, worktreeBranch, agentBoardOpen, onTog isListening={voice.phase === 'listening'} onSend={handleSendVoice} onDiscard={handleDiscardVoice} + targetRect={micRect} /> )} {!isMobile && isVoiceSupported() && ( @@ -403,6 +412,7 @@ export function StatusBar({ workspaceName, worktreeBranch, agentBoardOpen, onTog - -
- )}
+ + {!isListening && ( +
+ + +
+ )}
, document.body ) -} +} \ No newline at end of file diff --git a/src/frontend/src/index.css b/src/frontend/src/index.css index 0aeac2e..f30de8b 100644 --- a/src/frontend/src/index.css +++ b/src/frontend/src/index.css @@ -286,6 +286,9 @@ html, body, #root { transform: translateX(-50%); z-index: 50; pointer-events: auto; + display: flex; + flex-direction: column; + align-items: center; } @media (max-width: 767px) { @@ -294,6 +297,7 @@ html, body, #root { left: 12px; right: 12px; transform: none; + align-items: stretch; } } @@ -306,9 +310,6 @@ html, body, #root { min-width: 180px; max-width: 420px; animation: voice-bubble-in 0.2s ease-out; - display: flex; - flex-direction: column; - gap: 8px; } @media (max-width: 767px) { @@ -350,14 +351,12 @@ html, body, #root { } .voice-bubble-footer { + position: absolute; + bottom: -15px; + right: 0; display: flex; align-items: center; - justify-content: flex-end; gap: 6px; - margin: 0 -14px -10px; - padding: 6px 14px 8px; - border-radius: 0 0 16px 16px; - background: hsl(var(--popover)); } .voice-bubble-btn { From 60933ece2f0902ed6ad4f6d0716ff1e69040a3ea Mon Sep 17 00:00:00 2001 From: 04mg Date: Sun, 19 Jul 2026 21:48:06 -0300 Subject: [PATCH 14/36] feat: desktop auto-sends voice input, mobile shows review bubble with max-height scroll fade --- .../status-bar/components/StatusBar.tsx | 20 ++--- .../voice-mode/components/VoiceBubble.tsx | 78 +++++++++---------- .../features/voice-mode/hooks/useVoiceMode.ts | 11 ++- src/frontend/src/index.css | 54 ++++++------- 4 files changed, 75 insertions(+), 88 deletions(-) diff --git a/src/frontend/src/features/status-bar/components/StatusBar.tsx b/src/frontend/src/features/status-bar/components/StatusBar.tsx index ec5f45f..eb839d6 100644 --- a/src/frontend/src/features/status-bar/components/StatusBar.tsx +++ b/src/frontend/src/features/status-bar/components/StatusBar.tsx @@ -1,4 +1,4 @@ -import { useState, useEffect, useCallback, useRef } from 'react' +import { useState, useEffect, useCallback } from 'react' import { DropdownMenu, DropdownMenuTrigger, @@ -100,14 +100,6 @@ export function StatusBar({ workspaceName, worktreeBranch, agentBoardOpen, onTog const [selectedView, setSelectedView] = useState('') const [isMobile, setIsMobile] = useState(() => typeof window !== 'undefined' && window.innerWidth < 768) const voice = useVoiceMode() - const micBtnRef = useRef(null) - const [micRect, setMicRect] = useState(null) - - useEffect(() => { - if (voice.phase !== 'idle' && micBtnRef.current) { - setMicRect(micBtnRef.current.getBoundingClientRect()) - } - }, [voice.phase]) useEffect(() => { const onResize = () => setIsMobile(window.innerWidth < 768) @@ -183,7 +175,11 @@ export function StatusBar({ workspaceName, worktreeBranch, agentBoardOpen, onTog if (voice.phase === 'idle') { voice.start() } else if (voice.phase === 'listening') { - voice.stop() + if (isMobile) { + voice.stop() + } else { + voice.stop({ send: (text) => onSendText?.(text) }) + } } } @@ -397,14 +393,13 @@ export function StatusBar({ workspaceName, worktreeBranch, agentBoardOpen, onTog
- {voice.phase !== 'idle' && ( + {isMobile && voice.phase !== 'idle' && ( )} {!isMobile && isVoiceSupported() && ( @@ -412,7 +407,6 @@ export function StatusBar({ workspaceName, worktreeBranch, agentBoardOpen, onTog - +
- )} + + {!isListening && ( +
+ + +
+ )} +
, document.body ) diff --git a/src/frontend/src/features/voice-mode/hooks/useVoiceMode.ts b/src/frontend/src/features/voice-mode/hooks/useVoiceMode.ts index 0a5bd4a..c00d912 100644 --- a/src/frontend/src/features/voice-mode/hooks/useVoiceMode.ts +++ b/src/frontend/src/features/voice-mode/hooks/useVoiceMode.ts @@ -130,14 +130,21 @@ export function useVoiceMode() { } }, []) - const stop = useCallback(() => { + const stop = useCallback((autoSend?: { send: (text: string) => void }) => { if (speechRef.current) { try { speechRef.current.stop() } catch {} speechRef.current = null } - if (!globalState.transcript.trim()) { + const text = globalState.transcript.trim() + if (!text) { + setState({ phase: 'idle', transcript: '', error: null }) + return + } + if (autoSend) { + autoSend.send(text) + finalizedRef.current = '' setState({ phase: 'idle', transcript: '', error: null }) } else { setState({ phase: 'review' }) diff --git a/src/frontend/src/index.css b/src/frontend/src/index.css index f30de8b..a12f9b0 100644 --- a/src/frontend/src/index.css +++ b/src/frontend/src/index.css @@ -281,42 +281,23 @@ html, body, #root { .voice-bubble-container { position: fixed; - bottom: 44px; - left: 50%; - transform: translateX(-50%); + bottom: 100px; + left: 12px; + right: 12px; z-index: 50; pointer-events: auto; - display: flex; - flex-direction: column; - align-items: center; -} - -@media (max-width: 767px) { - .voice-bubble-container { - bottom: 100px; - left: 12px; - right: 12px; - transform: none; - align-items: stretch; - } } .voice-bubble { background: hsl(var(--popover)); - border: 1px solid hsl(var(--border)); - border-radius: 16px; + border: 1px solid hsl(var(--border) / 0.5); + border-radius: 10px; box-shadow: 0 4px 12px rgba(0, 0, 0, 0.12); padding: 10px 14px; - min-width: 180px; - max-width: 420px; animation: voice-bubble-in 0.2s ease-out; -} - -@media (max-width: 767px) { - .voice-bubble { - min-width: 0; - max-width: none; - } + display: flex; + flex-direction: column; + gap: 8px; } @keyframes voice-bubble-in { @@ -330,6 +311,17 @@ html, body, #root { } } +.voice-bubble-scroll { + max-height: 140px; + overflow-y: auto; + scrollbar-width: none; + position: relative; + -webkit-mask-image: linear-gradient(to bottom, transparent 0, #000 14px, #000 calc(100% - 14px), transparent 100%); + mask-image: linear-gradient(to bottom, transparent 0, #000 14px, #000 calc(100% - 14px), transparent 100%); +} + +.voice-bubble-scroll::-webkit-scrollbar { display: none; } + .voice-bubble-content { display: flex; align-items: center; @@ -351,12 +343,12 @@ html, body, #root { } .voice-bubble-footer { - position: absolute; - bottom: -15px; - right: 0; display: flex; align-items: center; + justify-content: flex-end; gap: 6px; + padding-top: 6px; + border-top: 1px solid hsl(var(--border) / 0.4); } .voice-bubble-btn { @@ -367,7 +359,7 @@ html, body, #root { align-items: center; justify-content: center; background: hsl(var(--popover)); - border: 1px solid hsl(var(--border)); + border: 1px solid hsl(var(--border) / 0.5); color: hsl(var(--muted-foreground)); cursor: pointer; transition: all 0.15s ease; From 8f4bbbef9cb07a97457250aeea7df5f760ac1a61 Mon Sep 17 00:00:00 2001 From: 04mg Date: Sun, 19 Jul 2026 21:52:55 -0300 Subject: [PATCH 15/36] fix: only show scroll fade when content overflows --- README.md | 1 - .../features/voice-mode/components/VoiceBubble.tsx | 12 +++++++++++- src/frontend/src/index.css | 7 +++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0569c5d..7698dae 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,6 @@

Latest release GitHub stars - Total downloads License

diff --git a/src/frontend/src/features/voice-mode/components/VoiceBubble.tsx b/src/frontend/src/features/voice-mode/components/VoiceBubble.tsx index 9d5b39e..92a294c 100644 --- a/src/frontend/src/features/voice-mode/components/VoiceBubble.tsx +++ b/src/frontend/src/features/voice-mode/components/VoiceBubble.tsx @@ -1,3 +1,4 @@ +import { useRef, useState, useEffect } from 'react' import { createPortal } from 'react-dom' import { Check, X } from 'lucide-react' import { cn } from '@/features/shared/utils/utils' @@ -12,13 +13,22 @@ interface VoiceBubbleProps { export function VoiceBubble({ transcript, error, isListening, onSend, onDiscard }: VoiceBubbleProps) { const hasContent = transcript.trim().length > 0 + const scrollRef = useRef(null) + const [isOverflowing, setIsOverflowing] = useState(false) + + useEffect(() => { + const el = scrollRef.current + if (el) { + setIsOverflowing(el.scrollHeight > el.clientHeight) + } + }, [transcript]) if (!hasContent && !error) return null return createPortal(
-
+
{error ? ( {error} diff --git a/src/frontend/src/index.css b/src/frontend/src/index.css index a12f9b0..85b7146 100644 --- a/src/frontend/src/index.css +++ b/src/frontend/src/index.css @@ -316,12 +316,15 @@ html, body, #root { overflow-y: auto; scrollbar-width: none; position: relative; - -webkit-mask-image: linear-gradient(to bottom, transparent 0, #000 14px, #000 calc(100% - 14px), transparent 100%); - mask-image: linear-gradient(to bottom, transparent 0, #000 14px, #000 calc(100% - 14px), transparent 100%); } .voice-bubble-scroll::-webkit-scrollbar { display: none; } +.voice-bubble-scroll-fade { + -webkit-mask-image: linear-gradient(to bottom, transparent 0, #000 14px, #000 calc(100% - 14px), transparent 100%); + mask-image: linear-gradient(to bottom, transparent 0, #000 14px, #000 calc(100% - 14px), transparent 100%); +} + .voice-bubble-content { display: flex; align-items: center; From 8622561aace8cfe125a666c9d7ab4ee4f3a18502 Mon Sep 17 00:00:00 2001 From: 04mg Date: Sun, 19 Jul 2026 22:09:31 -0300 Subject: [PATCH 16/36] feat: add local Kroko ASR voice mode with model download from settings --- .../settings/components/SettingsDialog.tsx | 288 +++++++++++++++--- .../voice-mode/hooks/useKrokoVoiceMode.ts | 200 ++++++++++++ .../features/voice-mode/hooks/useVoiceMode.ts | 59 +++- .../features/voice-mode/services/krokoAsr.ts | 256 ++++++++++++++++ 4 files changed, 751 insertions(+), 52 deletions(-) create mode 100644 src/frontend/src/features/voice-mode/hooks/useKrokoVoiceMode.ts create mode 100644 src/frontend/src/features/voice-mode/services/krokoAsr.ts diff --git a/src/frontend/src/features/settings/components/SettingsDialog.tsx b/src/frontend/src/features/settings/components/SettingsDialog.tsx index a39eed9..819bb4e 100644 --- a/src/frontend/src/features/settings/components/SettingsDialog.tsx +++ b/src/frontend/src/features/settings/components/SettingsDialog.tsx @@ -2,11 +2,25 @@ import { useState, useEffect, useCallback, useRef, type ElementType } from 'reac import { Dialog, DialogContent, DialogTitle, DialogClose } from '@/components/dialog' import { Slider } from '@/components/slider' -import { Palette, Bot, Terminal, Check, Moon, Sun, Monitor, ChartSpline, ArrowLeft, LogIn, ExternalLink, Loader2, Folder, Settings as SettingsIcon, X, Bell, Mic } from 'lucide-react' +import { Palette, Bot, Terminal, Check, Moon, Sun, Monitor, ChartSpline, ArrowLeft, LogIn, ExternalLink, Loader2, Folder, Settings as SettingsIcon, X, Bell, Mic, Download, HardDrive, Globe, Trash2 } from 'lucide-react' import { Antigravity, OpenCode, Ollama, Claude, Codex, GithubCopilot, OpenRouter } from '@lobehub/icons' import { agentTypes, getAgentCmdOverrides, setAgentCmdOverride } from '@/features/agents/services/agentTypes' import { setAllTerminalFontSizes, setAllTerminalThemes } from '@/features/terminal/services/terminalRegistry' import { isVoiceSupported } from '@/features/voice-mode/hooks/useVoiceMode' +import { + getVoiceMode, + setVoiceMode, + fetchLanguages, + fetchModels, + downloadModel, + isModelCached, + deleteModel, + isKrokoSupported, + getKrokoLanguage, + setKrokoLanguage, + type KrokoModel, + type KrokoLanguage, +} from '@/features/voice-mode/services/krokoAsr' import { SettingsItem } from './SettingsItem' @@ -83,6 +97,14 @@ export function SettingsDialog({ open, onOpenChange, initialSection }: SettingsD const [pushError, setPushError] = useState('') const [soundEnabled, setSoundEnabled] = useState(true) const [voiceLanguage, setVoiceLanguage] = useState('') + const [voiceMode, setVoiceModeState] = useState<'browser' | 'local'>('browser') + const [krokoLanguages, setKrokoLanguages] = useState([]) + const [krokoModels, setKrokoModels] = useState([]) + const [krokoLanguage, setKrokoLanguageState] = useState('') + const [krokoModelCache, setKrokoModelCache] = useState>({}) + const [krokoDownloading, setKrokoDownloading] = useState(null) + const [krokoDownloadStatus, setKrokoDownloadStatus] = useState('') + const [krokoLoading, setKrokoLoading] = useState(true) const isSecureContext = typeof window !== 'undefined' && (window.isSecureContext || window.location.hostname === 'localhost') const loadQuotaSettings = useCallback(async () => { @@ -190,6 +212,36 @@ export function SettingsDialog({ open, onOpenChange, initialSection }: SettingsD // Load push notification state setSoundEnabled(localStorage.getItem('caw:soundEnabled') !== '0') setVoiceLanguage(localStorage.getItem('caw:voiceLanguage') || '') + setVoiceModeState(getVoiceMode()) + setKrokoLanguageState(getKrokoLanguage()) + + // Load Kroko models and languages + if (isKrokoSupported()) { + setKrokoLoading(true) + Promise.all([fetchLanguages(), fetchModels()]) + .then(([langs, models]) => { + setKrokoLanguages(langs) + setKrokoModels(models) + const lang = getKrokoLanguage() + if (lang && langs.some((l) => l.iso === lang)) { + setKrokoLanguageState(lang) + } else if (langs.length > 0) { + setKrokoLanguageState(langs[0].iso) + setKrokoLanguage(langs[0].iso) + } + // Check cache status for all models + Promise.all(models.map((m) => isModelCached(m.url).then((c) => [m.url, c] as const))) + .then((results) => { + const cacheMap: Record = {} + for (const [url, cached] of results) cacheMap[url] = cached + setKrokoModelCache(cacheMap) + }) + .catch(() => {}) + }) + .catch(() => {}) + .finally(() => setKrokoLoading(false)) + } + if (pushSupported && 'Notification' in window) { setPushPermission(Notification.permission) } else if (!pushSupported && !pushIOSPWA) { @@ -763,53 +815,205 @@ export function SettingsDialog({ open, onOpenChange, initialSection }: SettingsD

Voice

-

Configure voice input language for speech recognition.

+

Configure voice input engine and language for speech recognition.

- {isVoiceSupported() ? ( -
-
- Voice Mode is supported in this browser -
- ) : ( -
-
- Voice Mode is not supported in this browser. Use Chrome, Edge, or Safari for speech recognition. -
- )} - -
-
- - -

Language used for speech recognition. "System Default" uses your browser's default language.

+ + Browser + +
+

+ {voiceMode === 'browser' + ? 'Uses the browser\'s built-in speech recognition. Requires Chrome, Edge, or Safari.' + : 'Runs speech recognition locally in your browser. Models are downloaded and cached on your device.'} +

+ + {voiceMode === 'browser' && ( + <> + {isVoiceSupported() ? ( +
+
+ Voice Mode is supported in this browser +
+ ) : ( +
+
+ Voice Mode is not supported in this browser. Use Chrome, Edge, or Safari for speech recognition. +
+ )} + +
+ + +

Language used for speech recognition. "System Default" uses your browser's default language.

+
+ + )} + + {voiceMode === 'local' && ( + <> + {!isKrokoSupported() ? ( +
+
+ Local voice mode is not supported in this browser. Cache API is required. +
+ ) : krokoLoading ? ( +
+ + Loading available models... +
+ ) : ( + <> +
+ + +

Language for the local speech recognition model.

+
+ +
+ +
+ {krokoModels + .filter((m) => m.language_iso === krokoLanguage) + .map((model) => { + const sizeMB = Math.round(model.file_size / 1000 / 1000) + const isDownloaded = krokoModelCache[model.url] || false + const isDownloading = krokoDownloading === model.url + return ( +
+
+ {model.name} + {sizeMB}MB +
+
+ {isDownloaded ? ( + <> +
+
+ Downloaded +
+ + + ) : ( + + )} +
+
+ ) + })} + {krokoModels.filter((m) => m.language_iso === krokoLanguage).length === 0 && ( +

No models available for this language.

+ )} +
+
+ + )} + + )}
)} diff --git a/src/frontend/src/features/voice-mode/hooks/useKrokoVoiceMode.ts b/src/frontend/src/features/voice-mode/hooks/useKrokoVoiceMode.ts new file mode 100644 index 0000000..0757982 --- /dev/null +++ b/src/frontend/src/features/voice-mode/hooks/useKrokoVoiceMode.ts @@ -0,0 +1,200 @@ +import { useState, useEffect, useCallback, useRef } from 'react' +import { + createKrokoRecognizer, + downsampleBuffer, + loadKrokoSdk, +} from '../services/krokoAsr' + +type VoicePhase = 'idle' | 'listening' | 'review' + +interface KrokoVoiceState { + phase: VoicePhase + transcript: string + error: string | null +} + +const EXPECTED_SAMPLE_RATE = 16000 + +let globalState: KrokoVoiceState = { phase: 'idle', transcript: '', error: null } +let listeners: Array<() => void> = [] + +function notify() { + for (const fn of listeners) fn() +} + +function setState(partial: Partial) { + globalState = { ...globalState, ...partial } + notify() +} + +export function getKrokoVoicePhase(): VoicePhase { + return globalState.phase +} + +let recognizerCache: any = null + +export async function ensureRecognizer(): Promise { + if (recognizerCache) return recognizerCache + await loadKrokoSdk() + recognizerCache = await createKrokoRecognizer() + return recognizerCache +} + +export function useKrokoVoiceMode() { + const [, forceRender] = useState(0) + const audioCtxRef = useRef(null) + const mediaStreamRef = useRef(null) + const processorRef = useRef(null) + const recognizerRef = useRef(null) + const streamRef = useRef(null) + const finalizedRef = useRef('') + const activeRef = useRef(false) + + useEffect(() => { + const listener = () => forceRender((n) => n + 1) + listeners.push(listener) + return () => { + listeners = listeners.filter((l) => l !== listener) + } + }, []) + + const cleanup = useCallback(() => { + activeRef.current = false + if (processorRef.current) { + try { + processorRef.current.disconnect() + } catch {} + processorRef.current = null + } + if (mediaStreamRef.current) { + mediaStreamRef.current.getTracks().forEach((t) => t.stop()) + mediaStreamRef.current = null + } + if (audioCtxRef.current) { + try { + audioCtxRef.current.close() + } catch {} + audioCtxRef.current = null + } + streamRef.current = null + }, []) + + const start = useCallback(async () => { + try { + setState({ phase: 'listening', transcript: '', error: null }) + finalizedRef.current = '' + + const recognizer = await ensureRecognizer() + recognizerRef.current = recognizer + + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + sampleRate: EXPECTED_SAMPLE_RATE, + } as any, + }) + mediaStreamRef.current = stream + + const audioCtx = new AudioContext({ sampleRate: EXPECTED_SAMPLE_RATE }) + audioCtxRef.current = audioCtx + + const source = audioCtx.createMediaStreamSource(stream) + const bufferSize = 4096 + const processor = audioCtx.createScriptProcessor(bufferSize, 1, 1) + processorRef.current = processor + + let recognizerStream: any = null + streamRef.current = null + + processor.onaudioprocess = async (e: AudioProcessingEvent) => { + if (!activeRef.current) return + + let samples: Float32Array = new Float32Array(e.inputBuffer.getChannelData(0)) + const recordRate = audioCtx.sampleRate + if (recordRate !== EXPECTED_SAMPLE_RATE) { + samples = downsampleBuffer(samples, recordRate, EXPECTED_SAMPLE_RATE) + } + + try { + if (!recognizerStream) { + recognizerStream = await recognizer.createStream() + streamRef.current = recognizerStream + } + + await recognizerStream.acceptWaveform(EXPECTED_SAMPLE_RATE, samples) + + while (await recognizer.isReady(recognizerStream)) { + await recognizer.decode(recognizerStream) + } + + const isEndpoint = await recognizer.isEndpoint(recognizerStream) + const result = await recognizer.getResult(recognizerStream) + const text = result.text || '' + + if (text.length > 0 && finalizedRef.current + text !== globalState.transcript) { + setState({ transcript: finalizedRef.current + text }) + } + + if (isEndpoint) { + if (text.length > 0) { + finalizedRef.current += text + } + await recognizer.reset(recognizerStream) + recognizerStream = null + streamRef.current = null + } + } catch { + // recognizer errors during processing — ignore, keep listening + } + } + + source.connect(processor) + processor.connect(audioCtx.destination) + activeRef.current = true + } catch (err: any) { + cleanup() + setState({ + error: + err.name === 'NotAllowedError' + ? 'Microphone permission denied' + : err.message || 'Failed to start local voice recognition', + phase: 'idle', + }) + } + }, [cleanup]) + + const stop = useCallback( + (autoSend?: { send: (text: string) => void }) => { + cleanup() + const text = globalState.transcript.trim() + if (!text) { + setState({ phase: 'idle', transcript: '', error: null }) + return + } + if (autoSend) { + autoSend.send(text) + finalizedRef.current = '' + setState({ phase: 'idle', transcript: '', error: null }) + } else { + setState({ phase: 'review' }) + } + }, + [cleanup], + ) + + const reset = useCallback(() => { + cleanup() + finalizedRef.current = '' + setState({ phase: 'idle', transcript: '', error: null }) + }, [cleanup]) + + return { + phase: globalState.phase, + transcript: globalState.transcript, + error: globalState.error, + start, + stop, + reset, + } +} diff --git a/src/frontend/src/features/voice-mode/hooks/useVoiceMode.ts b/src/frontend/src/features/voice-mode/hooks/useVoiceMode.ts index c00d912..692ae18 100644 --- a/src/frontend/src/features/voice-mode/hooks/useVoiceMode.ts +++ b/src/frontend/src/features/voice-mode/hooks/useVoiceMode.ts @@ -1,4 +1,6 @@ import { useState, useEffect, useCallback, useRef } from 'react' +import { useKrokoVoiceMode } from './useKrokoVoiceMode' +import { getVoiceMode } from '../services/krokoAsr' type VoicePhase = 'idle' | 'listening' | 'review' @@ -53,22 +55,14 @@ const SpeechRecognitionAPI = : null export function isVoiceSupported(): boolean { + if (getVoiceMode() === 'local') return true return !!SpeechRecognitionAPI } -export function useVoiceMode() { - const [, forceRender] = useState(0) +function BrowserVoiceMode() { const speechRef = useRef(null) const finalizedRef = useRef('') - useEffect(() => { - const listener = () => forceRender((n) => n + 1) - listeners.push(listener) - return () => { - listeners = listeners.filter((l) => l !== listener) - } - }, []) - const start = useCallback(async () => { if (!SpeechRecognitionAPI) { setState({ error: 'Voice input is not supported in this browser' }) @@ -162,6 +156,51 @@ export function useVoiceMode() { setState({ phase: 'idle', transcript: '', error: null }) }, []) + return { start, stop, reset } +} + +export function useVoiceMode() { + const [, forceRender] = useState(0) + const kroko = useKrokoVoiceMode() + const browser = useRef(BrowserVoiceMode()).current + + useEffect(() => { + const listener = () => forceRender((n) => n + 1) + listeners.push(listener) + return () => { + listeners = listeners.filter((l) => l !== listener) + } + }, []) + + const mode = getVoiceMode() + + const start = useCallback(async () => { + if (mode === 'local') { + await kroko.start() + } else { + await browser.start() + } + }, [mode, kroko, browser]) + + const stop = useCallback( + (autoSend?: { send: (text: string) => void }) => { + if (mode === 'local') { + kroko.stop(autoSend) + } else { + browser.stop(autoSend) + } + }, + [mode, kroko, browser], + ) + + const reset = useCallback(() => { + if (mode === 'local') { + kroko.reset() + } else { + browser.reset() + } + }, [mode, kroko, browser]) + return { phase: globalState.phase, transcript: globalState.transcript, diff --git a/src/frontend/src/features/voice-mode/services/krokoAsr.ts b/src/frontend/src/features/voice-mode/services/krokoAsr.ts new file mode 100644 index 0000000..f9ac7fb --- /dev/null +++ b/src/frontend/src/features/voice-mode/services/krokoAsr.ts @@ -0,0 +1,256 @@ +const KROKO_SDK_URL = + 'https://huggingface.co/spaces/Banafo/Kroko-Streaming-ASR-Wasm/resolve/main/kroko-sdk.js' +const KROKO_API_BASE = 'https://license.kroko.ai/api/public/v1' +const CACHE_NAME = 'kroko-sdk' + +let sdkLoaded = false +let sdkLoadPromise: Promise | null = null + +export interface KrokoModel { + model_id: string + name: string + language_iso: string + file_size: number + type: string + url: string + streaming: boolean +} + +export interface KrokoLanguage { + iso: string + name: string +} + +let cachedLanguages: KrokoLanguage[] | null = null +let cachedModels: KrokoModel[] | null = null + +export function loadKrokoSdk(): Promise { + if (sdkLoaded) return Promise.resolve() + if (sdkLoadPromise) return sdkLoadPromise + + sdkLoadPromise = new Promise((resolve, reject) => { + if (typeof document === 'undefined') { + reject(new Error('Kroko SDK requires a browser environment')) + return + } + const existing = document.querySelector(`script[src="${KROKO_SDK_URL}"]`) + if (existing) { + sdkLoaded = true + resolve() + return + } + const script = document.createElement('script') + script.src = KROKO_SDK_URL + script.onload = () => { + sdkLoaded = true + resolve() + } + script.onerror = () => { + sdkLoadPromise = null + reject(new Error('Failed to load Kroko SDK')) + } + document.head.appendChild(script) + }) + return sdkLoadPromise +} + +export async function fetchLanguages(): Promise { + if (cachedLanguages) return cachedLanguages + const res = await fetch(`${KROKO_API_BASE}/languages`) + if (!res.ok) throw new Error('Failed to fetch languages') + const data: KrokoLanguage[] = await res.json() + cachedLanguages = data.sort((a, b) => a.name.localeCompare(b.name)) + return cachedLanguages +} + +export async function fetchModels(): Promise { + if (cachedModels) return cachedModels + const res = await fetch(`${KROKO_API_BASE}/models`) + if (!res.ok) throw new Error('Failed to fetch models') + const data: KrokoModel[] = await res.json() + cachedModels = data + .filter((m) => m.streaming && m.type === 'free') + .sort((a, b) => a.name.localeCompare(b.name)) + return cachedModels +} + +async function fetchWithCache(url: string): Promise { + const cache = await caches.open(CACHE_NAME) + const cached = await cache.match(url) + if (cached) return cached + const response = await fetch(url) + if (!response.ok) throw new Error(`Failed to fetch: ${url}`) + await cache.put(url, response.clone()) + return response +} + +async function createCacheEntry(name: string, contents: Uint8Array): Promise { + const dummyUrl = `${KROKO_SDK_URL}/${name}` + const cache = await caches.open(CACHE_NAME) + const existing = await cache.match(dummyUrl) + if (!existing) { + const response = new Response(new Uint8Array(contents), { status: 200, statusText: 'OK' }) + await cache.put(dummyUrl, response.clone()) + } + return dummyUrl +} + +interface ModelData { + header: Record + blob: Uint8Array + encoder: Uint8Array + decoder: Uint8Array + joiner: Uint8Array + tokens: Uint8Array +} + +async function unpackModel(url: string): Promise<[string, string, string, string]> { + const res = await fetchWithCache(url) + const arrayBuf = await res.arrayBuffer() + const data = new Uint8Array(arrayBuf) + + if (data.byteLength < 4) throw new Error('Invalid model file') + + const view = new DataView(data.buffer) + const headerLen = view.getUint32(0, true) + if (data.byteLength < 4 + headerLen) throw new Error('Invalid model header') + + const headerBytes = data.slice(4, 4 + headerLen) + const header = JSON.parse(new TextDecoder().decode(headerBytes)) + const blob = data.slice(4 + headerLen) + + const modelData: ModelData = { + header, + blob, + encoder: new Uint8Array(0), + decoder: new Uint8Array(0), + joiner: new Uint8Array(0), + tokens: new Uint8Array(0), + } + + if (blob.byteLength < 4) throw new Error('Invalid model payload') + + let offset = 0 + const readBlock = (): Uint8Array => { + if (offset + 4 > blob.length) throw new Error('Invalid block header') + const len = new DataView(blob.buffer, blob.byteOffset).getUint32(offset, true) + offset += 4 + if (offset + len > blob.length) throw new Error('Block size mismatch') + const buf = blob.slice(offset, offset + len) + offset += len + return buf + } + + modelData.encoder = readBlock() + modelData.decoder = readBlock() + modelData.joiner = readBlock() + modelData.tokens = readBlock() + + return Promise.all([ + createCacheEntry('encoder', modelData.encoder), + createCacheEntry('decoder', modelData.decoder), + createCacheEntry('joiner', modelData.joiner), + createCacheEntry('tokens', modelData.tokens), + ]) +} + +export async function downloadModel( + url: string, + onProgress?: (status: string) => void, +): Promise { + onProgress?.('Downloading model...') + const [encoder, decoder, joiner, tokens] = await unpackModel(url) + void encoder + void decoder + void joiner + void tokens + onProgress?.('Model ready') +} + +export async function isModelCached(url: string): Promise { + try { + const cache = await caches.open(CACHE_NAME) + const res = await cache.match(url) + return !!res + } catch { + return false + } +} + +export async function deleteModel(): Promise { + await caches.delete(CACHE_NAME) +} + +export async function createKrokoRecognizer(): Promise { + await loadKrokoSdk() + + const KrokoWorker = (window as any).KrokoWorker + if (!KrokoWorker) throw new Error('KrokoWorker not available') + + const worker = new KrokoWorker() + const encoderUrl = `${KROKO_SDK_URL}/encoder` + const decoderUrl = `${KROKO_SDK_URL}/decoder` + const joinerUrl = `${KROKO_SDK_URL}/joiner` + const tokensUrl = `${KROKO_SDK_URL}/tokens` + + const recognizer = await worker.createOnlineRecognizer({ + modelConfig: { + transducer: { + encoder: encoderUrl, + decoder: decoderUrl, + joiner: joinerUrl, + }, + tokens: tokensUrl, + }, + }) + + return recognizer +} + +export function getVoiceMode(): 'browser' | 'local' { + if (typeof localStorage === 'undefined') return 'browser' + return (localStorage.getItem('caw:voiceMode') as 'browser' | 'local') || 'browser' +} + +export function setVoiceMode(mode: 'browser' | 'local'): void { + localStorage.setItem('caw:voiceMode', mode) +} + +export function getKrokoLanguage(): string { + if (typeof localStorage === 'undefined') return '' + return localStorage.getItem('caw:krokoLanguage') || '' +} + +export function setKrokoLanguage(iso: string): void { + localStorage.setItem('caw:krokoLanguage', iso) +} + +export function isKrokoSupported(): boolean { + return typeof window !== 'undefined' && typeof caches !== 'undefined' +} + +export function downsampleBuffer( + buffer: Float32Array, + recordSampleRate: number, + exportSampleRate: number, +): Float32Array { + if (exportSampleRate === recordSampleRate) return buffer + const sampleRateRatio = recordSampleRate / exportSampleRate + const newLength = Math.round(buffer.length / sampleRateRatio) + const result = new Float32Array(newLength) + let offsetResult = 0 + let offsetBuffer = 0 + while (offsetResult < result.length) { + const nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio) + let accum = 0 + let count = 0 + for (let i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) { + accum += buffer[i] + count++ + } + result[offsetResult] = accum / count + offsetResult++ + offsetBuffer = nextOffsetBuffer + } + return result +} From 14cb9c88ccf5d78fef7d22dd7ded98bc345640eb Mon Sep 17 00:00:00 2001 From: 04mg Date: Sun, 19 Jul 2026 22:43:29 -0300 Subject: [PATCH 17/36] feat: add shadcn Select component for voice settings --- src/frontend/package-lock.json | 289 ++++++++++++++++++ src/frontend/package.json | 1 + src/frontend/src/components/select.tsx | 82 +++++ .../settings/components/SettingsDialog.tsx | 67 ++-- .../voice-mode/hooks/useKrokoVoiceMode.ts | 11 + .../features/voice-mode/hooks/useVoiceMode.ts | 62 ++-- .../features/voice-mode/services/krokoAsr.ts | 40 +-- 7 files changed, 467 insertions(+), 85 deletions(-) create mode 100644 src/frontend/src/components/select.tsx diff --git a/src/frontend/package-lock.json b/src/frontend/package-lock.json index 0d7e90a..b308424 100644 --- a/src/frontend/package-lock.json +++ b/src/frontend/package-lock.json @@ -13,6 +13,7 @@ "@radix-ui/react-dialog": "^1.1.18", "@radix-ui/react-dropdown-menu": "^2.1.19", "@radix-ui/react-scroll-area": "^1.2.13", + "@radix-ui/react-select": "^2.3.4", "@radix-ui/react-slider": "^1.4.2", "@radix-ui/react-tooltip": "^1.2.11", "@tailwindcss/vite": "^4.3.2", @@ -2001,6 +2002,294 @@ } } }, + "node_modules/@radix-ui/react-select": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-select/-/react-select-2.3.4.tgz", + "integrity": "sha512-E2JxqAvaTUEhWtBptWo02g8FnLYPymv9ahEvW/cZQPPV4ySeyo0M8n3sXccsLUAIfMbexnfXt91qF7UjTbTMMg==", + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.2", + "@radix-ui/primitive": "1.1.6", + "@radix-ui/react-collection": "1.1.12", + "@radix-ui/react-compose-refs": "1.1.3", + "@radix-ui/react-context": "1.2.0", + "@radix-ui/react-direction": "1.1.2", + "@radix-ui/react-dismissable-layer": "1.1.16", + "@radix-ui/react-focus-guards": "1.1.4", + "@radix-ui/react-focus-scope": "1.1.13", + "@radix-ui/react-id": "1.1.2", + "@radix-ui/react-popper": "1.3.4", + "@radix-ui/react-portal": "1.1.14", + "@radix-ui/react-presence": "1.1.8", + "@radix-ui/react-primitive": "2.1.7", + "@radix-ui/react-slot": "1.3.0", + "@radix-ui/react-use-callback-ref": "1.1.2", + "@radix-ui/react-use-controllable-state": "1.2.4", + "@radix-ui/react-use-layout-effect": "1.1.2", + "@radix-ui/react-use-previous": "1.1.2", + "@radix-ui/react-visually-hidden": "1.2.8", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select/node_modules/@radix-ui/primitive": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.6.tgz", + "integrity": "sha512-w9hl+724uYEgCGR3bhuRepjBtrNB/6gkhCnAf58Ke+SLbHPPQqVZZB59z60roB+5H+nh3nWTcdJhQdFMEydWmw==", + "license": "MIT" + }, + "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-arrow": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.12.tgz", + "integrity": "sha512-ltXCE0glRomMZ9+u10d9o1Go+edqa1aLxufH59JRNNM3Yz1uvaeNWSaS1HeVh1X64agtdBG5JA1W1I6ySqWiwA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.7" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-collection": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collection/-/react-collection-1.1.12.tgz", + "integrity": "sha512-nb67INpE0IahJKN7EYPp9m9YGwYeKlnzxT3MwXVkgCskaSJia97kG4T0ywpjNUSSnoJk/uvk12V8vbrEHEj+/Q==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.3", + "@radix-ui/react-context": "1.2.0", + "@radix-ui/react-primitive": "2.1.7", + "@radix-ui/react-slot": "1.3.0" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-context": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.2.0.tgz", + "integrity": "sha512-fOE+JtN9rygNZkCnHRBEP0TAvLldlhyOxMsbwFvTP4nAs+nBmfnna+o/Zski2wkmY1YMrFC0aSzsHoLY47iLrg==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-dismissable-layer": { + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.16.tgz", + "integrity": "sha512-t45h68IjFx0ccBnPJqk0X6ecv69LkCFWd6DNCFQX56mUnVEXZbNOLCH/u9fHlAjFZ1RrFdl8/m4zev7B7NyhXQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.6", + "@radix-ui/react-compose-refs": "1.1.3", + "@radix-ui/react-primitive": "2.1.7", + "@radix-ui/react-use-callback-ref": "1.1.2", + "@radix-ui/react-use-effect-event": "0.0.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-focus-scope": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.13.tgz", + "integrity": "sha512-dE04aPEuP9rvKKT0d0KjSOtTEYNg6bmCYFsoSJpfC+y91Hic28ZfDCGgv6aJ+2Kw/LBXYipMZpyqVj/OD3Z8Gg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.3", + "@radix-ui/react-primitive": "2.1.7", + "@radix-ui/react-use-callback-ref": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-popper": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.3.4.tgz", + "integrity": "sha512-PXnCa3XgTQk0FegMctxgqJXtFLZe4IFJdbUkB7jKSCKEpb6utEO4S9Vog/pkyCfEPdzM331gvE4xpztmBAfMng==", + "license": "MIT", + "dependencies": { + "@floating-ui/react-dom": "^2.0.0", + "@radix-ui/react-arrow": "1.1.12", + "@radix-ui/react-compose-refs": "1.1.3", + "@radix-ui/react-context": "1.2.0", + "@radix-ui/react-primitive": "2.1.7", + "@radix-ui/react-use-callback-ref": "1.1.2", + "@radix-ui/react-use-layout-effect": "1.1.2", + "@radix-ui/react-use-rect": "1.1.2", + "@radix-ui/react-use-size": "1.1.2", + "@radix-ui/rect": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-portal": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.14.tgz", + "integrity": "sha512-REwjAGPMa3J9oyDE4cuWkZbwnCbbyky66NurquQklXMSDn67cl6oGFx2gO7KZhPtFNbNw9xTWNrti3VIhgluYw==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.7", + "@radix-ui/react-use-layout-effect": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-presence": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.8.tgz", + "integrity": "sha512-0hhyrQdXMaATgq4ammLG9+iPqsXxzZkgTSIxdrJHdfLnXO4Uo5L7BoO3/Xf0AEaettadGZWGGJMw6ujzQvIpGA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-use-controllable-state": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.2.4.tgz", + "integrity": "sha512-cx2DixxmSfjCcEoRvDvy1NLd6SWK94XFcEEOZUcharUlXbmahFQGKCfwdKZL2ub34iIwOPOEFVF80xb+yfLYiA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.6", + "@radix-ui/react-use-effect-event": "0.0.3", + "@radix-ui/react-use-layout-effect": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-visually-hidden": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.2.8.tgz", + "integrity": "sha512-FjsQEpkNBJJYiPSat6jh2LGKLPX2jAoDVS3AZSBNX3cOUoEGhw/f+z2FCY8Cf1NkoYIbytJ1f4mlWPQpR+MjVg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.7" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, "node_modules/@radix-ui/react-slider": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/@radix-ui/react-slider/-/react-slider-1.4.2.tgz", diff --git a/src/frontend/package.json b/src/frontend/package.json index 2932fe0..1b102f8 100644 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -15,6 +15,7 @@ "@radix-ui/react-dialog": "^1.1.18", "@radix-ui/react-dropdown-menu": "^2.1.19", "@radix-ui/react-scroll-area": "^1.2.13", + "@radix-ui/react-select": "^2.3.4", "@radix-ui/react-slider": "^1.4.2", "@radix-ui/react-tooltip": "^1.2.11", "@tailwindcss/vite": "^4.3.2", diff --git a/src/frontend/src/components/select.tsx b/src/frontend/src/components/select.tsx new file mode 100644 index 0000000..77d0158 --- /dev/null +++ b/src/frontend/src/components/select.tsx @@ -0,0 +1,82 @@ +import * as React from 'react' +import * as SelectPrimitive from '@radix-ui/react-select' +import { Check, ChevronDown } from 'lucide-react' +import { cn } from '@/features/shared/utils/utils' + +const Select = SelectPrimitive.Root +const SelectGroup = SelectPrimitive.Group +const SelectValue = SelectPrimitive.Value + +const SelectTrigger = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, children, ...props }, ref) => ( + span]:line-clamp-1 cursor-pointer', + className + )} + {...props} + > + {children} + + + + +)) +SelectTrigger.displayName = SelectPrimitive.Trigger.displayName + +const SelectContent = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, children, position = 'popper', ...props }, ref) => ( + + + + {children} + + + +)) +SelectContent.displayName = SelectPrimitive.Content.displayName + +const SelectItem = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, children, ...props }, ref) => ( + + + + + + + {children} + +)) +SelectItem.displayName = SelectPrimitive.Item.displayName + +export { Select, SelectGroup, SelectValue, SelectTrigger, SelectContent, SelectItem } diff --git a/src/frontend/src/features/settings/components/SettingsDialog.tsx b/src/frontend/src/features/settings/components/SettingsDialog.tsx index 819bb4e..8e9b361 100644 --- a/src/frontend/src/features/settings/components/SettingsDialog.tsx +++ b/src/frontend/src/features/settings/components/SettingsDialog.tsx @@ -1,6 +1,7 @@ import { useState, useEffect, useCallback, useRef, type ElementType } from 'react' import { Dialog, DialogContent, DialogTitle, DialogClose } from '@/components/dialog' import { Slider } from '@/components/slider' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/select' import { Palette, Bot, Terminal, Check, Moon, Sun, Monitor, ChartSpline, ArrowLeft, LogIn, ExternalLink, Loader2, Folder, Settings as SettingsIcon, X, Bell, Mic, Download, HardDrive, Globe, Trash2 } from 'lucide-react' import { Antigravity, OpenCode, Ollama, Claude, Codex, GithubCopilot, OpenRouter } from '@lobehub/icons' @@ -877,32 +878,36 @@ export function SettingsDialog({ open, onOpenChange, initialSection }: SettingsD
- + + + + + System Default + English (US) + English (UK) + Spanish + French + German + Japanese + Korean + Chinese (Simplified) + Portuguese (Brazil) + Italian + Dutch + Russian + +

Language used for speech recognition. "System Default" uses your browser's default language.

@@ -924,18 +929,22 @@ export function SettingsDialog({ open, onOpenChange, initialSection }: SettingsD <>
- + + + + + {krokoLanguages.map((lang) => ( + {lang.name} + ))} + +

Language for the local speech recognition model.

diff --git a/src/frontend/src/features/voice-mode/hooks/useKrokoVoiceMode.ts b/src/frontend/src/features/voice-mode/hooks/useKrokoVoiceMode.ts index 0757982..fab6236 100644 --- a/src/frontend/src/features/voice-mode/hooks/useKrokoVoiceMode.ts +++ b/src/frontend/src/features/voice-mode/hooks/useKrokoVoiceMode.ts @@ -31,6 +31,17 @@ export function getKrokoVoicePhase(): VoicePhase { return globalState.phase } +export function getKrokoVoiceState(): KrokoVoiceState { + return globalState +} + +export function subscribeKrokoVoice(cb: () => void): () => void { + listeners.push(cb) + return () => { + listeners = listeners.filter((l) => l !== cb) + } +} + let recognizerCache: any = null export async function ensureRecognizer(): Promise { diff --git a/src/frontend/src/features/voice-mode/hooks/useVoiceMode.ts b/src/frontend/src/features/voice-mode/hooks/useVoiceMode.ts index 692ae18..9b1c8d3 100644 --- a/src/frontend/src/features/voice-mode/hooks/useVoiceMode.ts +++ b/src/frontend/src/features/voice-mode/hooks/useVoiceMode.ts @@ -1,5 +1,5 @@ import { useState, useEffect, useCallback, useRef } from 'react' -import { useKrokoVoiceMode } from './useKrokoVoiceMode' +import { useKrokoVoiceMode, subscribeKrokoVoice, getKrokoVoiceState } from './useKrokoVoiceMode' import { getVoiceMode } from '../services/krokoAsr' type VoicePhase = 'idle' | 'listening' | 'review' @@ -33,20 +33,22 @@ declare global { } } -let globalState: VoiceModeState = { phase: 'idle', transcript: '', error: null } -let listeners: Array<() => void> = [] +let browserGlobalState: VoiceModeState = { phase: 'idle', transcript: '', error: null } +let browserListeners: Array<() => void> = [] -function notify() { - for (const fn of listeners) fn() +function browserNotify() { + for (const fn of browserListeners) fn() } -function setState(partial: Partial) { - globalState = { ...globalState, ...partial } - notify() +function browserSetState(partial: Partial) { + browserGlobalState = { ...browserGlobalState, ...partial } + browserNotify() } export function getVoicePhase(): VoicePhase { - return globalState.phase + const mode = getVoiceMode() + if (mode === 'local') return getKrokoVoiceState().phase + return browserGlobalState.phase } const SpeechRecognitionAPI = @@ -65,7 +67,7 @@ function BrowserVoiceMode() { const start = useCallback(async () => { if (!SpeechRecognitionAPI) { - setState({ error: 'Voice input is not supported in this browser' }) + browserSetState({ error: 'Voice input is not supported in this browser' }) return } @@ -91,12 +93,12 @@ function BrowserVoiceMode() { interimText += result[0].transcript } } - setState({ transcript: finalizedRef.current + interimText }) + browserSetState({ transcript: finalizedRef.current + interimText }) } recognition.onerror = (event: any) => { if (event.error === 'no-speech' || event.error === 'aborted') return - setState({ + browserSetState({ error: event.error === 'not-allowed' ? 'Microphone permission denied' : `Speech error: ${event.error}`, @@ -105,11 +107,11 @@ function BrowserVoiceMode() { } recognition.onend = () => { - if (globalState.phase === 'listening') { + if (browserGlobalState.phase === 'listening') { try { recognition.start() } catch { - setState({ phase: 'idle' }) + browserSetState({ phase: 'idle' }) } return } @@ -118,9 +120,9 @@ function BrowserVoiceMode() { try { recognition.start() speechRef.current = recognition - setState({ phase: 'listening', transcript: '', error: null }) + browserSetState({ phase: 'listening', transcript: '', error: null }) } catch { - setState({ error: 'Failed to start voice recognition' }) + browserSetState({ error: 'Failed to start voice recognition' }) } }, []) @@ -131,17 +133,17 @@ function BrowserVoiceMode() { } catch {} speechRef.current = null } - const text = globalState.transcript.trim() + const text = browserGlobalState.transcript.trim() if (!text) { - setState({ phase: 'idle', transcript: '', error: null }) + browserSetState({ phase: 'idle', transcript: '', error: null }) return } if (autoSend) { autoSend.send(text) finalizedRef.current = '' - setState({ phase: 'idle', transcript: '', error: null }) + browserSetState({ phase: 'idle', transcript: '', error: null }) } else { - setState({ phase: 'review' }) + browserSetState({ phase: 'review' }) } }, []) @@ -153,7 +155,7 @@ function BrowserVoiceMode() { speechRef.current = null } finalizedRef.current = '' - setState({ phase: 'idle', transcript: '', error: null }) + browserSetState({ phase: 'idle', transcript: '', error: null }) }, []) return { start, stop, reset } @@ -165,10 +167,14 @@ export function useVoiceMode() { const browser = useRef(BrowserVoiceMode()).current useEffect(() => { - const listener = () => forceRender((n) => n + 1) - listeners.push(listener) + const browserListener = () => forceRender((n) => n + 1) + browserListeners.push(browserListener) + + const unsubscribeKroko = subscribeKrokoVoice(() => forceRender((n) => n + 1)) + return () => { - listeners = listeners.filter((l) => l !== listener) + browserListeners = browserListeners.filter((l) => l !== browserListener) + unsubscribeKroko() } }, []) @@ -201,10 +207,12 @@ export function useVoiceMode() { } }, [mode, kroko, browser]) + const activeState = mode === 'local' ? getKrokoVoiceState() : browserGlobalState + return { - phase: globalState.phase, - transcript: globalState.transcript, - error: globalState.error, + phase: activeState.phase, + transcript: activeState.transcript, + error: activeState.error, start, stop, reset, diff --git a/src/frontend/src/features/voice-mode/services/krokoAsr.ts b/src/frontend/src/features/voice-mode/services/krokoAsr.ts index f9ac7fb..95e142d 100644 --- a/src/frontend/src/features/voice-mode/services/krokoAsr.ts +++ b/src/frontend/src/features/voice-mode/services/krokoAsr.ts @@ -3,8 +3,8 @@ const KROKO_SDK_URL = const KROKO_API_BASE = 'https://license.kroko.ai/api/public/v1' const CACHE_NAME = 'kroko-sdk' -let sdkLoaded = false -let sdkLoadPromise: Promise | null = null +let sdkModule: any = null +let sdkLoadPromise: Promise | null = null export interface KrokoModel { model_id: string @@ -24,33 +24,15 @@ export interface KrokoLanguage { let cachedLanguages: KrokoLanguage[] | null = null let cachedModels: KrokoModel[] | null = null -export function loadKrokoSdk(): Promise { - if (sdkLoaded) return Promise.resolve() +export async function loadKrokoSdk(): Promise { + if (sdkModule) return sdkModule if (sdkLoadPromise) return sdkLoadPromise - sdkLoadPromise = new Promise((resolve, reject) => { - if (typeof document === 'undefined') { - reject(new Error('Kroko SDK requires a browser environment')) - return - } - const existing = document.querySelector(`script[src="${KROKO_SDK_URL}"]`) - if (existing) { - sdkLoaded = true - resolve() - return - } - const script = document.createElement('script') - script.src = KROKO_SDK_URL - script.onload = () => { - sdkLoaded = true - resolve() - } - script.onerror = () => { - sdkLoadPromise = null - reject(new Error('Failed to load Kroko SDK')) - } - document.head.appendChild(script) - }) + sdkLoadPromise = (async () => { + const mod = await import(KROKO_SDK_URL) + sdkModule = mod + return mod + })() return sdkLoadPromise } @@ -182,9 +164,9 @@ export async function deleteModel(): Promise { } export async function createKrokoRecognizer(): Promise { - await loadKrokoSdk() + const mod = await loadKrokoSdk() - const KrokoWorker = (window as any).KrokoWorker + const KrokoWorker = mod.KrokoWorker if (!KrokoWorker) throw new Error('KrokoWorker not available') const worker = new KrokoWorker() From c628b2b41efe083306f2207e6818ff15584a561e Mon Sep 17 00:00:00 2001 From: 04mg Date: Sun, 19 Jul 2026 23:08:54 -0300 Subject: [PATCH 18/36] fix: prevent stale transcript race condition in Kroko async processor handler --- .../voice-mode/hooks/useKrokoVoiceMode.ts | 52 +++++++++++-------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/src/frontend/src/features/voice-mode/hooks/useKrokoVoiceMode.ts b/src/frontend/src/features/voice-mode/hooks/useKrokoVoiceMode.ts index fab6236..e82c73a 100644 --- a/src/frontend/src/features/voice-mode/hooks/useKrokoVoiceMode.ts +++ b/src/frontend/src/features/voice-mode/hooks/useKrokoVoiceMode.ts @@ -88,6 +88,7 @@ export function useKrokoVoiceMode() { audioCtxRef.current = null } streamRef.current = null + finalizedRef.current = '' }, []) const start = useCallback(async () => { @@ -127,35 +128,40 @@ export function useKrokoVoiceMode() { samples = downsampleBuffer(samples, recordRate, EXPECTED_SAMPLE_RATE) } - try { - if (!recognizerStream) { - recognizerStream = await recognizer.createStream() - streamRef.current = recognizerStream - } + try { + if (!recognizerStream) { + recognizerStream = await recognizer.createStream() + if (!activeRef.current) return + streamRef.current = recognizerStream + } - await recognizerStream.acceptWaveform(EXPECTED_SAMPLE_RATE, samples) + await recognizerStream.acceptWaveform(EXPECTED_SAMPLE_RATE, samples) + if (!activeRef.current) return - while (await recognizer.isReady(recognizerStream)) { - await recognizer.decode(recognizerStream) - } + while (await recognizer.isReady(recognizerStream)) { + if (!activeRef.current) return + await recognizer.decode(recognizerStream) + } - const isEndpoint = await recognizer.isEndpoint(recognizerStream) - const result = await recognizer.getResult(recognizerStream) - const text = result.text || '' + const isEndpoint = await recognizer.isEndpoint(recognizerStream) + if (!activeRef.current) return + const result = await recognizer.getResult(recognizerStream) + const text = result.text || '' - if (text.length > 0 && finalizedRef.current + text !== globalState.transcript) { - setState({ transcript: finalizedRef.current + text }) - } + if (text.length > 0 && finalizedRef.current + text !== globalState.transcript) { + setState({ transcript: finalizedRef.current + text }) + } - if (isEndpoint) { - if (text.length > 0) { - finalizedRef.current += text - } - await recognizer.reset(recognizerStream) - recognizerStream = null - streamRef.current = null + if (isEndpoint) { + if (text.length > 0) { + finalizedRef.current += text } - } catch { + await recognizer.reset(recognizerStream) + if (!activeRef.current) return + recognizerStream = null + streamRef.current = null + } + } catch { // recognizer errors during processing — ignore, keep listening } } From 6a235cf2d7a5b40b89e1ef6511cca4adb18fac14 Mon Sep 17 00:00:00 2001 From: 04mg Date: Sun, 19 Jul 2026 23:13:27 -0300 Subject: [PATCH 19/36] fix: use session generation token to invalidate stale Kroko async handlers --- .../voice-mode/hooks/useKrokoVoiceMode.ts | 78 ++++++++++++------- 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/src/frontend/src/features/voice-mode/hooks/useKrokoVoiceMode.ts b/src/frontend/src/features/voice-mode/hooks/useKrokoVoiceMode.ts index e82c73a..db42a54 100644 --- a/src/frontend/src/features/voice-mode/hooks/useKrokoVoiceMode.ts +++ b/src/frontend/src/features/voice-mode/hooks/useKrokoVoiceMode.ts @@ -60,6 +60,7 @@ export function useKrokoVoiceMode() { const streamRef = useRef(null) const finalizedRef = useRef('') const activeRef = useRef(false) + const sessionGenRef = useRef(0) useEffect(() => { const listener = () => forceRender((n) => n + 1) @@ -71,7 +72,11 @@ export function useKrokoVoiceMode() { const cleanup = useCallback(() => { activeRef.current = false + sessionGenRef.current++ // invalidate any in-flight handler from this session if (processorRef.current) { + try { + processorRef.current.onaudioprocess = null + } catch {} try { processorRef.current.disconnect() } catch {} @@ -92,11 +97,14 @@ export function useKrokoVoiceMode() { }, []) const start = useCallback(async () => { + const gen = ++sessionGenRef.current try { setState({ phase: 'listening', transcript: '', error: null }) finalizedRef.current = '' + activeRef.current = false const recognizer = await ensureRecognizer() + if (gen !== sessionGenRef.current) return recognizerRef.current = recognizer const stream = await navigator.mediaDevices.getUserMedia({ @@ -106,9 +114,18 @@ export function useKrokoVoiceMode() { sampleRate: EXPECTED_SAMPLE_RATE, } as any, }) + if (gen !== sessionGenRef.current) { + stream.getTracks().forEach((t) => t.stop()) + return + } mediaStreamRef.current = stream const audioCtx = new AudioContext({ sampleRate: EXPECTED_SAMPLE_RATE }) + if (gen !== sessionGenRef.current) { + stream.getTracks().forEach((t) => t.stop()) + audioCtx.close() + return + } audioCtxRef.current = audioCtx const source = audioCtx.createMediaStreamSource(stream) @@ -120,7 +137,7 @@ export function useKrokoVoiceMode() { streamRef.current = null processor.onaudioprocess = async (e: AudioProcessingEvent) => { - if (!activeRef.current) return + if (gen !== sessionGenRef.current) return let samples: Float32Array = new Float32Array(e.inputBuffer.getChannelData(0)) const recordRate = audioCtx.sampleRate @@ -128,40 +145,42 @@ export function useKrokoVoiceMode() { samples = downsampleBuffer(samples, recordRate, EXPECTED_SAMPLE_RATE) } - try { - if (!recognizerStream) { - recognizerStream = await recognizer.createStream() - if (!activeRef.current) return - streamRef.current = recognizerStream - } + try { + if (!recognizerStream) { + recognizerStream = await recognizer.createStream() + if (gen !== sessionGenRef.current) return + streamRef.current = recognizerStream + } - await recognizerStream.acceptWaveform(EXPECTED_SAMPLE_RATE, samples) - if (!activeRef.current) return + await recognizerStream.acceptWaveform(EXPECTED_SAMPLE_RATE, samples) + if (gen !== sessionGenRef.current) return - while (await recognizer.isReady(recognizerStream)) { - if (!activeRef.current) return - await recognizer.decode(recognizerStream) - } + while (await recognizer.isReady(recognizerStream)) { + if (gen !== sessionGenRef.current) return + await recognizer.decode(recognizerStream) + } + if (gen !== sessionGenRef.current) return - const isEndpoint = await recognizer.isEndpoint(recognizerStream) - if (!activeRef.current) return - const result = await recognizer.getResult(recognizerStream) - const text = result.text || '' + const isEndpoint = await recognizer.isEndpoint(recognizerStream) + if (gen !== sessionGenRef.current) return + const result = await recognizer.getResult(recognizerStream) + if (gen !== sessionGenRef.current) return + const text = result.text || '' - if (text.length > 0 && finalizedRef.current + text !== globalState.transcript) { - setState({ transcript: finalizedRef.current + text }) - } + if (text.length > 0 && finalizedRef.current + text !== globalState.transcript) { + setState({ transcript: finalizedRef.current + text }) + } - if (isEndpoint) { - if (text.length > 0) { - finalizedRef.current += text + if (isEndpoint) { + if (text.length > 0) { + finalizedRef.current += text + } + await recognizer.reset(recognizerStream) + if (gen !== sessionGenRef.current) return + recognizerStream = null + streamRef.current = null } - await recognizer.reset(recognizerStream) - if (!activeRef.current) return - recognizerStream = null - streamRef.current = null - } - } catch { + } catch { // recognizer errors during processing — ignore, keep listening } } @@ -170,6 +189,7 @@ export function useKrokoVoiceMode() { processor.connect(audioCtx.destination) activeRef.current = true } catch (err: any) { + if (gen !== sessionGenRef.current) return cleanup() setState({ error: From df40a3fe03adcefe9a20c64f61668a2a3cb25475 Mon Sep 17 00:00:00 2001 From: 04mg Date: Sun, 19 Jul 2026 23:18:02 -0300 Subject: [PATCH 20/36] feat: show last spoken word next to mic and display model download progress in MB --- .../settings/components/SettingsDialog.tsx | 18 ++++--- .../status-bar/components/StatusBar.tsx | 8 +++ .../features/voice-mode/services/krokoAsr.ts | 53 ++++++++++++++----- 3 files changed, 60 insertions(+), 19 deletions(-) diff --git a/src/frontend/src/features/settings/components/SettingsDialog.tsx b/src/frontend/src/features/settings/components/SettingsDialog.tsx index 8e9b361..7b5e2e7 100644 --- a/src/frontend/src/features/settings/components/SettingsDialog.tsx +++ b/src/frontend/src/features/settings/components/SettingsDialog.tsx @@ -104,7 +104,7 @@ export function SettingsDialog({ open, onOpenChange, initialSection }: SettingsD const [krokoLanguage, setKrokoLanguageState] = useState('') const [krokoModelCache, setKrokoModelCache] = useState>({}) const [krokoDownloading, setKrokoDownloading] = useState(null) - const [krokoDownloadStatus, setKrokoDownloadStatus] = useState('') + const [krokoDownloadProgress, setKrokoDownloadProgress] = useState<{ downloaded: number, total: number } | null>(null) const [krokoLoading, setKrokoLoading] = useState(true) const isSecureContext = typeof window !== 'undefined' && (window.isSecureContext || window.location.hostname === 'localhost') @@ -988,15 +988,17 @@ export function SettingsDialog({ open, onOpenChange, initialSection }: SettingsD )}
diff --git a/src/frontend/src/features/status-bar/components/StatusBar.tsx b/src/frontend/src/features/status-bar/components/StatusBar.tsx index eb839d6..041a790 100644 --- a/src/frontend/src/features/status-bar/components/StatusBar.tsx +++ b/src/frontend/src/features/status-bar/components/StatusBar.tsx @@ -404,6 +404,14 @@ export function StatusBar({ workspaceName, worktreeBranch, agentBoardOpen, onTog )} {!isMobile && isVoiceSupported() && ( <> + {voice.phase === 'listening' && ( + + {voice.transcript.trim().split(/\s+/).filter(Boolean).slice(-1).join(' ') || ''} + + )} - Voice Mode + {voice.phase === 'loading' ? 'Loading voice engine…' : 'Voice Mode'} diff --git a/src/frontend/src/features/terminal/components/MobileControlBar.tsx b/src/frontend/src/features/terminal/components/MobileControlBar.tsx index d04ed4e..780dc1e 100644 --- a/src/frontend/src/features/terminal/components/MobileControlBar.tsx +++ b/src/frontend/src/features/terminal/components/MobileControlBar.tsx @@ -9,7 +9,7 @@ import { resetStickyModifiers, sendTerminalInput } from '@/features/terminal/services/terminalRegistry' -import { ChevronUp, ChevronDown, ChevronLeft, ChevronRight, CornerDownLeft, Mic } from 'lucide-react' +import { ChevronUp, ChevronDown, ChevronLeft, ChevronRight, CornerDownLeft, Mic, Loader2 } from 'lucide-react' import { useVoiceMode, isVoiceSupported } from '@/features/voice-mode/hooks/useVoiceMode' import { cn } from '@/features/shared/utils/utils' @@ -54,8 +54,13 @@ export function MobileControlBar({ terminalId }: MobileControlBarProps) { : 'border-border/30 text-muted-foreground' )} onClick={handleToggleVoice} + disabled={voice.phase === 'loading'} > - + {voice.phase === 'loading' ? ( + + ) : ( + + )} )}