feat(stage-tamagotchi,stage-web): sync transcription implementation, improve config drawer

Author: nekomeowww

apps/stage-tamagotchi/src/renderer/components/HearingPermissionStatus.vue

@@ -0,0 +1,62 @@
+<script setup lang="ts">
+import { HearingConfigDialog } from '@proj-airi/stage-ui/components'
+import { useAudioAnalyzer, useAudioContextFromStream } from '@proj-airi/stage-ui/composables'
+import { useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/settings'
+import { useAsyncState } from '@vueuse/core'
+import { storeToRefs } from 'pinia'
+import { onMounted, onUnmounted, watch } from 'vue'
+
+import { electron } from '../../../../shared/electron'
+import { useElectronEventaInvoke } from '../../../composables/electron-vueuse/use-electron-eventa-context'
+
+const show = defineModel('show', { type: Boolean, default: false })
+
+const settingsAudioDeviceStore = useSettingsAudioDevice()
+const { enabled, selectedAudioInput, stream, audioInputs } = storeToRefs(settingsAudioDeviceStore)
+const { startStream, stopStream } = settingsAudioDeviceStore
+
+const getMediaAccessStatus = useElectronEventaInvoke(electron.systemPreferences.getMediaAccessStatus)
+const { state: mediaAccessStatus, execute: refreshMediaAccessStatus } = useAsyncState(() => getMediaAccessStatus(['microphone']), 'not-determined')
+
+const { audioContext, initialize, dispose, pause } = useAudioContextFromStream(stream)
+const { volumeLevel, startAnalyzer, stopAnalyzer } = useAudioAnalyzer()
+
+watch(enabled, (val) => {
+  if (val) {
+    startStream()
+    initialize().then(() => startAnalyzer(audioContext.value!))
+  }
+  else {
+    stopStream()
+    pause()
+  }
+}, { immediate: true })
+
+onMounted(async () => {
+  await refreshMediaAccessStatus()
+  if (audioContext.value) {
+    await startAnalyzer(audioContext.value)
+  }
+})
+
+onUnmounted(async () => {
+  await stopAnalyzer()
+  await dispose()
+})
+</script>
+
+<template>
+  <HearingConfigDialog
+    v-model:show="show"
+    v-model:enabled="enabled"
+    v-model:selected-audio-input="selectedAudioInput"
+    :granted="mediaAccessStatus !== 'denied' && mediaAccessStatus !== 'restricted'"
+    :audio-input-options="audioInputs"
+    :volume-level="volumeLevel"
+  >
+    <slot />
+    <template #extra>
+      <slot name="extra" />
+    </template>
+  </HearingConfigDialog>
+</template>

apps/stage-tamagotchi/src/renderer/components/Widgets/ControlsIsland/ControlsIslandHearingConfig.vue

@@ -1,14 +1,13 @@
 <script setup lang="ts">
-import { HearingConfigDialog } from '@proj-airi/stage-ui/components'
 import { useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/settings'
 import { defineInvoke } from '@unbird/eventa'
 import { useDark, useToggle } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
 import { ref } from 'vue'
 
-import HearingPermissionStatus from '../../../components/HearingPermissionStatus.vue'
 import ControlButton from './ControlButton.vue'
 import ControlButtonTooltip from './ControlButtonTooltip.vue'
+import ControlsIslandHearingConfig from './ControlsIslandHearingConfig.vue'
 import IndicatorMicVolume from './IndicatorMicVolume.vue'
 
 import { electronOpenChat, electronOpenSettings, electronStartDraggingWindow } from '../../../../shared/eventa'
@@ -61,7 +60,7 @@ defineExpose({ hearingDialogOpen })
       </ControlButtonTooltip>
 
       <ControlButtonTooltip>
-        <HearingConfigDialog v-model:show="hearingDialogOpen">
+        <ControlsIslandHearingConfig v-model:show="hearingDialogOpen">
           <div class="relative">
             <ControlButton>
               <Transition name="fade" mode="out-in">
@@ -70,10 +69,7 @@ defineExpose({ hearingDialogOpen })
               </Transition>
             </ControlButton>
           </div>
-          <template #extra>
-            <HearingPermissionStatus />
-          </template>
-        </HearingConfigDialog>
+        </ControlsIslandHearingConfig>
 
         <template #tooltip>
           Open hearing controls

apps/stage-tamagotchi/src/renderer/components/Widgets/ControlsIsland/index.vue

@@ -1,11 +1,8 @@
 <script setup lang="ts">
 import type { ChatProvider } from '@xsai-ext/shared-providers'
 
-import WhisperWorker from '@proj-airi/stage-ui/libs/workers/worker?worker&url'
-
-import { toWAVBase64 } from '@proj-airi/audio'
 import { HearingConfigDialog } from '@proj-airi/stage-ui/components'
-import { useMicVAD, useWhisper } from '@proj-airi/stage-ui/composables'
+import { useAudioAnalyzer } from '@proj-airi/stage-ui/composables'
 import { useAudioContext } from '@proj-airi/stage-ui/stores/audio'
 import { useChatStore } from '@proj-airi/stage-ui/stores/chat'
 import { useConsciousnessStore } from '@proj-airi/stage-ui/stores/modules/consciousness'
@@ -21,7 +18,6 @@ import ChatHistory from '../Widgets/ChatHistory.vue'
 import IndicatorMicVolume from '../Widgets/IndicatorMicVolume.vue'
 
 const messageInput = ref('')
-const listening = ref(false)
 const hearingDialogOpen = ref(false)
 const isComposing = ref(false)
 
@@ -30,29 +26,15 @@ const { activeProvider, activeModel } = storeToRefs(useConsciousnessStore())
 const { themeColorsHueDynamic } = storeToRefs(useSettings())
 
 const { askPermission } = useSettingsAudioDevice()
-const { enabled, selectedAudioInput } = storeToRefs(useSettingsAudioDevice())
+const { enabled, selectedAudioInput, stream, audioInputs } = storeToRefs(useSettingsAudioDevice())
 const { send, onAfterMessageComposed, discoverToolsCompatibility, cleanupMessages } = useChatStore()
 const { messages } = storeToRefs(useChatStore())
 const { audioContext } = useAudioContext()
 const { t } = useI18n()
 
 const isDark = useDark({ disableTransition: false })
 
-const { transcribe: generate, terminate } = useWhisper(WhisperWorker, {
-  onComplete: async (res) => {
-    if (!res || !res.trim()) {
-      return
-    }
-
-    const providerConfig = providersStore.getProviderConfig(activeProvider.value)
-
-    await send(res, {
-      chatProvider: await providersStore.getProviderInstance(activeProvider.value) as ChatProvider,
-      model: activeModel.value,
-      providerConfig,
-    })
-  },
-})
+// Legacy whisper pipeline removed; audio pipeline handled at page level
 
 async function handleSend() {
   if (!messageInput.value.trim() || isComposing.value) {
@@ -77,46 +59,7 @@ async function handleSend() {
   }
 }
 
-const { destroy, start } = useMicVAD(selectedAudioInput, {
-  onSpeechStart: () => {
-    // TODO: interrupt the playback
-    // TODO: interrupt any of the ongoing TTS
-    // TODO: interrupt any of the ongoing LLM requests
-    // TODO: interrupt any of the ongoing animation of Live2D or VRM
-    // TODO: once interrupted, we should somehow switch to listen or thinking
-    //       emotion / expression?
-    listening.value = true
-  },
-  // VAD misfire means while speech end is detected but
-  // the frames of the segment of the audio buffer
-  // is not enough to be considered as a speech segment
-  // which controlled by the `minSpeechFrames` parameter
-  onVADMisfire: () => {
-    // TODO: do audio buffer send to whisper
-    listening.value = false
-  },
-  onSpeechEnd: (buffer) => {
-    // TODO: do audio buffer send to whisper
-    listening.value = false
-    handleTranscription(buffer.buffer)
-  },
-  auto: false,
-})
-
-async function handleTranscription(buffer: ArrayBufferLike) {
-  await audioContext.resume()
-
-  // Convert Float32Array to WAV format
-  const audioBase64 = await toWAVBase64(buffer, audioContext.sampleRate)
-  generate({ type: 'generate', data: { audio: audioBase64, language: 'en' } })
-}
-
-watch(enabled, async (value) => {
-  if (value === false) {
-    destroy()
-    terminate()
-  }
-})
+// No inline VAD/whisper here; see pages/index.vue pipeline
 
 watch(hearingDialogOpen, async (value) => {
   if (value) {
@@ -130,14 +73,38 @@ watch([activeProvider, activeModel], async () => {
   }
 })
 
-onMounted(() => {
-  // loadWhisper()
-  start()
-})
+onMounted(() => {})
 
 onAfterMessageComposed(async () => {
   messageInput.value = ''
 })
+
+const { startAnalyzer, stopAnalyzer, volumeLevel } = useAudioAnalyzer()
+let analyzerSource: MediaStreamAudioSourceNode | undefined
+
+function teardownAnalyzer() {
+  try { analyzerSource?.disconnect() }
+  catch {}
+  analyzerSource = undefined
+  stopAnalyzer()
+}
+
+async function setupAnalyzer() {
+  teardownAnalyzer()
+  if (!hearingDialogOpen.value || !enabled.value || !stream.value)
+    return
+  if (audioContext.state === 'suspended')
+    await audioContext.resume()
+  const analyser = startAnalyzer(audioContext)
+  if (!analyser)
+    return
+  analyzerSource = audioContext.createMediaStreamSource(stream.value)
+  analyzerSource.connect(analyser)
+}
+
+watch([hearingDialogOpen, enabled, stream], () => {
+  setupAnalyzer()
+}, { immediate: true })
 </script>
 
 <template>
@@ -167,12 +134,23 @@ onAfterMessageComposed(async () => {
             @compositionend="isComposing = false"
           />
 
-          <HearingConfigDialog v-model:show="hearingDialogOpen" :overlay-dim="true" :overlay-blur="true">
+          <HearingConfigDialog
+            v-model:show="hearingDialogOpen"
+            :overlay-dim="true"
+            :overlay-blur="true"
+            :enabled="enabled"
+            :audio-input-options="audioInputs"
+            :selected-audio-input="selectedAudioInput"
+            :has-devices="(audioInputs || []).length > 0"
+            :volume-level="volumeLevel"
+            @toggle-enabled="enabled = !enabled"
+            @update:selected-audio-input="val => selectedAudioInput = val as string"
+          >
             <button
               class="max-h-[10lh] min-h-[1lh]"
               bg="neutral-100 dark:neutral-800"
               text="lg neutral-500 dark:neutral-400"
-              :class="{ 'ring-2 ring-primary-400/60 ring-offset-2 dark:ring-offset-neutral-900': listening }"
+              :class="{ 'ring-2 ring-primary-400/60 ring-offset-2 dark:ring-offset-neutral-900': enabled }"
               flex items-center justify-center rounded-md p-2 outline-none
               transition="colors duration-200, transform duration-100" active:scale-95
               :title="t('settings.hearing.title')"

apps/stage-web/src/components/Layouts/InteractiveArea.vue

@@ -9,8 +9,14 @@ import HearingConfig from './hearing-config.vue'
 const props = defineProps<{
   overlayDim?: boolean
   overlayBlur?: boolean
+  granted?: boolean
+  audioInputOptions?: MediaDeviceInfo[]
+  volumeLevel?: number
 }>()
+
 const showDialog = defineModel('show', { type: Boolean, default: false, required: false })
+const selectedAudioInput = defineModel<string>('selectedAudioInput')
+const enabled = defineModel<boolean>('enabled', { default: false })
 
 const isDesktop = useMediaQuery('(min-width: 768px)')
 const screenSafeArea = useScreenSafeArea()
@@ -36,7 +42,13 @@ onMounted(() => screenSafeArea.update())
         <VisuallyHidden>
           <DialogTitle>Hearing Input</DialogTitle>
         </VisuallyHidden>
-        <HearingConfig @close="showDialog = false" />
+        <HearingConfig
+          v-model:enabled="enabled"
+          v-model:selected-audio-input="selectedAudioInput"
+          :audio-input-options="props.audioInputOptions"
+          :granted="props.granted"
+          :volume-level="props.volumeLevel"
+        />
         <slot name="extra" />
       </DialogContent>
     </DialogPortal>
@@ -47,9 +59,15 @@ onMounted(() => screenSafeArea.update())
     </DrawerTrigger>
     <DrawerPortal>
       <DrawerOverlay class="fixed inset-0" />
-      <DrawerContent class="fixed bottom-0 left-0 right-0 z-1000 mt-20 h-full max-h-[50%] flex flex-col rounded-t-2xl bg-neutral-50 px-4 pt-4 outline-none backdrop-blur-md dark:bg-neutral-900/95" :style="{ paddingBottom: `${Math.max(Number.parseFloat(screenSafeArea.bottom.value.replace('px', '')), 24)}px` }">
-        <DrawerHandle />
-        <HearingConfig @close="showDialog = false" />
+      <DrawerContent class="fixed bottom-0 left-0 right-0 z-1000 mt-20 h-full max-h-[45%] flex flex-col rounded-t-2xl bg-neutral-50 px-4 pt-4 outline-none backdrop-blur-md dark:bg-neutral-900/95" :style="{ paddingBottom: `${Math.max(Number.parseFloat(screenSafeArea.bottom.value.replace('px', '')), 24)}px` }">
+        <DrawerHandle my-2 />
+        <HearingConfig
+          v-model:enabled="enabled"
+          v-model:selected-audio-input="selectedAudioInput"
+          :audio-input-options="props.audioInputOptions"
+          :granted="props.granted"
+          :volume-level="props.volumeLevel"
+        />
         <slot name="extra" />
       </DrawerContent>
     </DrawerPortal>