feat(stage-web): voice & microphone for mobile web

Author: nekomeowww

apps/stage-web/src/components/Layouts/InteractiveArea.vue

@@ -4,6 +4,7 @@ import type { ChatProvider } from '@xsai-ext/shared-providers'
 import WhisperWorker from '@proj-airi/stage-ui/libs/workers/worker?worker&url'
 
 import { toWAVBase64 } from '@proj-airi/audio'
+import { HearingConfigDialog } from '@proj-airi/stage-ui/components'
 import { useMicVAD, useWhisper } from '@proj-airi/stage-ui/composables'
 import { useAudioContext } from '@proj-airi/stage-ui/stores/audio'
 import { useChatStore } from '@proj-airi/stage-ui/stores/chat'
@@ -17,10 +18,11 @@ import { onMounted, ref, watch } from 'vue'
 import { useI18n } from 'vue-i18n'
 
 import ChatHistory from '../Widgets/ChatHistory.vue'
+import IndicatorMicVolume from '../Widgets/IndicatorMicVolume.vue'
 
 const messageInput = ref('')
 const listening = ref(false)
-const showMicrophoneSelect = ref(false)
+const hearingDialogOpen = ref(false)
 const isComposing = ref(false)
 
 const providersStore = useProvidersStore()
@@ -116,7 +118,7 @@ watch(enabled, async (value) => {
   }
 })
 
-watch(showMicrophoneSelect, async (value) => {
+watch(hearingDialogOpen, async (value) => {
   if (value) {
     await askPermission()
   }
@@ -164,6 +166,23 @@ onAfterMessageComposed(async () => {
             @compositionstart="isComposing = true"
             @compositionend="isComposing = false"
           />
+
+          <HearingConfigDialog v-model:show="hearingDialogOpen" :overlay-dim="true" :overlay-blur="true">
+            <button
+              class="max-h-[10lh] min-h-[1lh]"
+              bg="neutral-100 dark:neutral-800"
+              text="lg neutral-500 dark:neutral-400"
+              :class="{ 'ring-2 ring-primary-400/60 ring-offset-2 dark:ring-offset-neutral-900': listening }"
+              flex items-center justify-center rounded-md p-2 outline-none
+              transition="colors duration-200, transform duration-100" active:scale-95
+              :title="t('settings.hearing.title')"
+            >
+              <Transition name="fade" mode="out-in">
+                <IndicatorMicVolume v-if="enabled" />
+                <div v-else class="i-ph:microphone-slash" />
+              </Transition>
+            </button>
+          </HearingConfigDialog>
         </div>
       </div>
     </div>

apps/stage-web/src/components/Layouts/MobileInteractiveArea.vue

@@ -1,6 +1,7 @@
 <script setup lang="ts">
 import type { ChatProvider } from '@xsai-ext/shared-providers'
 
+import { HearingConfigDialog } from '@proj-airi/stage-ui/components'
 import { useMicVAD } from '@proj-airi/stage-ui/composables'
 import { useChatStore } from '@proj-airi/stage-ui/stores/chat'
 import { useConsciousnessStore } from '@proj-airi/stage-ui/stores/modules/consciousness'
@@ -19,6 +20,7 @@ import ActionViewControls from './InteractiveArea/Actions/ViewControls.vue'
 import ViewControlInputs from './ViewControls/Inputs.vue'
 
 const isDark = useDark({ disableTransition: false })
+const hearingDialogOpen = ref(false)
 
 const viewControlsActiveMode = ref<'x' | 'y' | 'z' | 'scale'>('scale')
 const viewControlsInputsRef = useTemplateRef<InstanceType<typeof ViewControlInputs>>('viewControlsInputs')
@@ -140,6 +142,19 @@ onMounted(() => {
       <div translate-y="[-100%]" absolute right-0 w-full px-3 pb-3 font-sans>
         <div flex="~ col" w-full gap-1>
           <ActionAbout />
+          <HearingConfigDialog v-model:show="hearingDialogOpen">
+            <button
+              border="2 solid neutral-100/60 dark:neutral-800/30"
+              bg="neutral-50/70 dark:neutral-800/70"
+              w-fit flex items-center self-end justify-center rounded-xl p-2 backdrop-blur-md
+              title="Hearing"
+            >
+              <Transition name="fade" mode="out-in">
+                <div v-if="enabled" i-solar:microphone-3-bold-duotone size-5 text="neutral-500 dark:neutral-400" />
+                <div v-else i-solar:microphone-3-outline size-5 text="neutral-500 dark:neutral-400" />
+              </Transition>
+            </button>
+          </HearingConfigDialog>
           <button border="2 solid neutral-100/60 dark:neutral-800/30" bg="neutral-50/70 dark:neutral-800/70" w-fit flex items-center self-end justify-center rounded-xl p-2 backdrop-blur-md title="Theme" @click="isDark = !isDark">
             <Transition name="fade" mode="out-in">
               <div v-if="isDark" i-solar:moon-outline size-5 text="neutral-500 dark:neutral-400" />

apps/stage-web/src/components/Widgets/IndicatorMicVolume.vue

@@ -0,0 +1,78 @@
+<script setup lang="ts">
+import { useAudioAnalyzer } from '@proj-airi/stage-ui/composables'
+import { useAudioContext } from '@proj-airi/stage-ui/stores/audio'
+import { useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/settings'
+import { storeToRefs } from 'pinia'
+import { computed, onMounted, onUnmounted, watch } from 'vue'
+
+const props = withDefaults(defineProps<{ colorClass?: string }>(), { colorClass: 'text-primary-500 dark:text-primary-200' })
+const settingsAudio = useSettingsAudioDevice()
+const { stream, enabled } = storeToRefs(settingsAudio)
+
+const { audioContext } = storeToRefs(useAudioContext())
+const { startAnalyzer, stopAnalyzer, volumeLevel } = useAudioAnalyzer()
+
+let source: MediaStreamAudioSourceNode | undefined
+
+const normalized = computed(() => Math.min(1, (volumeLevel.value ?? 0) / 100))
+
+function teardown() {
+  try {
+    source?.disconnect()
+  }
+  catch {
+    // ignore
+  }
+
+  source = undefined
+  stopAnalyzer()
+}
+
+async function setup() {
+  teardown()
+  if (!enabled.value || !stream.value)
+    return
+
+  const ctx = audioContext.value
+  if (ctx.state === 'suspended')
+    await ctx.resume()
+  const analyser = startAnalyzer(ctx)
+  if (!analyser)
+    return
+  source = ctx.createMediaStreamSource(stream.value)
+  source.connect(analyser)
+}
+
+onMounted(() => {
+  watch([enabled, stream], () => setup(), { immediate: true })
+})
+
+onUnmounted(() => teardown())
+</script>
+
+<template>
+  <div :class="['flex items-center justify-center', props.colorClass]">
+    <svg width="24" height="24" viewBox="0 0 256 256" aria-hidden="true">
+      <defs>
+        <linearGradient id="micLevel" x1="0%" y1="0%" x2="0%" y2="100%">
+          <stop offset="0%" stop-color="currentColor" stop-opacity="0" />
+          <stop :offset="`${100 - Math.round(normalized * 100)}%`" stop-color="currentColor" stop-opacity="0" />
+          <stop :offset="`${100 - Math.round(normalized * 100)}%`" stop-color="currentColor" stop-opacity="0.95" />
+          <stop offset="100%" stop-color="currentColor" stop-opacity="0.95" />
+        </linearGradient>
+      </defs>
+      <path
+        fill="url(#micLevel)"
+        d="M128 176a48.05 48.05 0 0 0 48-48V64a48 48 0 0 0-96 0v64a48.05 48.05 0 0 0 48 48M96 64a32 32 0 0 1 64 0v64a32 32 0 0 1-64 0Zm40 143.6V240a8 8 0 0 1-16 0v-32.4A80.11 80.11 0 0 1 48 128a8 8 0 0 1 16 0a64 64 0 0 0 128 0a8 8 0 0 1 16 0a80.11 80.11 0 0 1-72 79.6"
+      />
+      <path
+        fill="none"
+        stroke="currentColor"
+        stroke-opacity="1"
+        stroke-width="2"
+        d="M128 176a48.05 48.05 0 0 0 48-48V64a48 48 0 0 0-96 0v64a48.05 48.05 0 0 0 48 48M96 64a32 32 0 0 1 64 0v64a32 32 0 0 1-64 0Zm40 143.6V240a8 8 0 0 1-16 0v-32.4A80.11 80.11 0 0 1 48 128a8 8 0 0 1 16 0a64 64 0 0 0 128 0a8 8 0 0 1 16 0a80.11 80.11 0 0 1-72 79.6"
+      />
+    </svg>
+  </div>
+</template>
+

apps/stage-web/src/pages/index.vue

@@ -1,9 +1,20 @@
 <script setup lang="ts">
+import type { ChatProvider } from '@xsai-ext/shared-providers'
+
+import workletUrl from '@proj-airi/stage-ui/workers/vad/process.worklet?worker&url'
+
 import { WidgetStage } from '@proj-airi/stage-ui/components/scenes'
+import { useAudioRecorder } from '@proj-airi/stage-ui/composables/audio/audio-recorder'
+import { useVAD } from '@proj-airi/stage-ui/stores/ai/models/vad'
+import { useChatStore } from '@proj-airi/stage-ui/stores/chat'
 import { useLive2d } from '@proj-airi/stage-ui/stores/live2d'
+import { useConsciousnessStore } from '@proj-airi/stage-ui/stores/modules/consciousness'
+import { useHearingSpeechInputPipeline } from '@proj-airi/stage-ui/stores/modules/hearing'
+import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
+import { useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/settings'
 import { breakpointsTailwind, useBreakpoints, useDark, useMouse } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
-import { onMounted, ref, watch } from 'vue'
+import { onMounted, onUnmounted, ref, watch } from 'vue'
 
 import Cross from '../components/Backgrounds/Cross.vue'
 import Header from '../components/Layouts/Header.vue'
@@ -29,6 +40,91 @@ const isMobile = breakpoints.smaller('md')
 const { updateThemeColor } = useThemeColor(themeColorFromPropertyOf('.widgets.top-widgets .colored-area', 'background-color'))
 watch(dark, () => updateThemeColor(), { immediate: true })
 onMounted(() => updateThemeColor())
+
+// Audio + transcription pipeline (mirrors stage-tamagotchi)
+const settingsAudioDeviceStore = useSettingsAudioDevice()
+const { stream, enabled } = storeToRefs(settingsAudioDeviceStore)
+const { startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream)
+const { transcribeForRecording } = useHearingSpeechInputPipeline()
+const providersStore = useProvidersStore()
+const consciousnessStore = useConsciousnessStore()
+const { activeProvider: activeChatProvider, activeModel: activeChatModel } = storeToRefs(consciousnessStore)
+const chatStore = useChatStore()
+
+const {
+  init: initVAD,
+  dispose: disposeVAD,
+  start: startVAD,
+  loaded: vadLoaded,
+} = useVAD(workletUrl, {
+  threshold: ref(0.6),
+  onSpeechStart: () => startRecord(),
+  onSpeechEnd: () => stopRecord(),
+})
+
+let stopOnStopRecord: (() => void) | undefined
+
+async function startAudioInteraction() {
+  try {
+    await initVAD()
+    if (stream.value)
+      await startVAD(stream.value)
+
+    // Hook once
+    stopOnStopRecord = onStopRecord(async (recording) => {
+      const text = await transcribeForRecording(recording)
+      if (!text || !text.trim())
+        return
+
+      try {
+        const provider = await providersStore.getProviderInstance(activeChatProvider.value)
+        if (!provider || !activeChatModel.value)
+          return
+
+        await chatStore.send(text, { model: activeChatModel.value, chatProvider: provider as ChatProvider })
+      }
+      catch (err) {
+        console.error('Failed to send chat from voice:', err)
+      }
+    })
+  }
+  catch (e) {
+    console.error('Audio interaction init failed:', e)
+  }
+}
+
+function stopAudioInteraction() {
+  try {
+    stopOnStopRecord?.()
+    stopOnStopRecord = undefined
+    disposeVAD()
+  }
+  catch {}
+}
+
+watch(enabled, async (val) => {
+  if (val) {
+    await startAudioInteraction()
+  }
+  else {
+    stopAudioInteraction()
+  }
+}, { immediate: true })
+
+onUnmounted(() => {
+  stopAudioInteraction()
+})
+
+watch([stream, () => vadLoaded.value], async ([s, loaded]) => {
+  if (enabled.value && loaded && s) {
+    try {
+      await startVAD(s)
+    }
+    catch (e) {
+      console.error('Failed to start VAD with stream:', e)
+    }
+  }
+})
 </script>
 
 <template>

cspell.config.yaml

@@ -89,11 +89,13 @@ words:
   - DuckDBWASMQuery
   - eastasia
   - elevenlabs
+  - elfutils
   - Equirectangular
   - esaxx
   - eventa
   - Factorio
   - feaxios
+  - Flathub
   - flexsearch
   - formkit
   - frontmatter