feat(stage-ui): Synchronised Emotion Expression and Delay Animation to the TTS AIRI Speaking (#741)

* feat(stage-ui): Synchronised emotion expression and delay animation with the tts speaking

* [autofix.ci] apply automated fixes

* Update packages/stage-ui/src/utils/tts.ts

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* feat(stage-ui): Synchronised emotion expression and delay animation with the tts speaking - code review

* feat(stage-ui): Synchronised emotion expression and delay animation with the tts speaking - code review

* feat(stage-ui): Synchronised emotion expression and delay animation with the tts speaking - code review

* feat(stage-ui): Synchronised emotion expression and delay animation with the tts speaking - code review

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

Author: 3 people

packages/stage-ui/src/components/scenarios/providers/SpeechStreamingPlayground.vue

@@ -67,11 +67,11 @@ async function handleSpeechGeneration(ctx: { data: string }) {
 const ttsQueue = createQueue<string>({ handlers: [handleSpeechGeneration] })
 
 onTextSegmented((chunk) => {
-  ttsQueue.enqueue(chunk)
+  ttsQueue.enqueue(chunk.chunk)
 })
 
 async function testStreaming() {
-  textSegmentationQueue.value.enqueue(props.text)
+  textSegmentationQueue.value.enqueue({ type: 'literal', value: props.text })
 }
 
 async function testChunking() {

packages/stage-ui/src/components/scenes/Stage.vue

@@ -3,7 +3,9 @@ import type { DuckDBWasmDrizzleDatabase } from '@proj-airi/drizzle-duckdb-wasm'
 import type { SpeechProviderWithExtraOptions } from '@xsai-ext/shared-providers'
 import type { UnElevenLabsOptions } from 'unspeech'
 
+import type { TextSegmentationItem } from '../../composables/queues'
 import type { Emotion } from '../../constants/emotions'
+import type { TTSChunkItem } from '../../utils/tts'
 
 import { drizzle } from '@proj-airi/drizzle-duckdb-wasm'
 import { getImportUrlBundles } from '@proj-airi/drizzle-duckdb-wasm/bundles/import-url-browser'
@@ -65,7 +67,7 @@ const { textSegmentationQueue } = storeToRefs(textSegmentationStore)
 clearTextSegmentationHooks()
 
 const characterSpeechPlaybackQueue = usePipelineCharacterSpeechPlaybackQueueStore()
-const { connectAudioContext, connectAudioAnalyser, clearAll, onPlaybackStarted } = characterSpeechPlaybackQueue
+const { connectAudioContext, connectAudioAnalyser, clearAll, onPlaybackStarted, onPlaybackFinished } = characterSpeechPlaybackQueue
 const { currentAudioSource, playbackQueue } = storeToRefs(characterSpeechPlaybackQueue)
 
 const settingsStore = useSettings()
@@ -123,7 +125,48 @@ const lipSyncStarted = ref(false)
 const speechStore = useSpeechStore()
 const { ssmlEnabled, activeSpeechProvider, activeSpeechModel, activeSpeechVoice, pitch } = storeToRefs(speechStore)
 
-async function handleSpeechGeneration(ctx: { data: string }) {
+const { currentMotion } = storeToRefs(useLive2d())
+
+const emotionsQueue = createQueue<Emotion>({
+  handlers: [
+    async (ctx) => {
+      if (stageModelRenderer.value === 'vrm') {
+        // console.debug("VRM emotion anime: ", ctx.data)
+        const value = EMOTION_VRMExpressionName_value[ctx.data]
+        if (!value)
+          return
+
+        await vrmViewerRef.value!.setExpression(value)
+      }
+      else if (stageModelRenderer.value === 'live2d') {
+        currentMotion.value = { group: EMOTION_EmotionMotionName_value[ctx.data] }
+      }
+    },
+  ],
+})
+
+const emotionMessageContentQueue = useEmotionsMessageQueue(emotionsQueue)
+emotionMessageContentQueue.onHandlerEvent('emotion', (emotion) => {
+  // eslint-disable-next-line no-console
+  console.debug('emotion detected', emotion)
+})
+
+const delaysQueue = useDelayMessageQueue()
+delaysQueue.onHandlerEvent('delay', (delay) => {
+  // eslint-disable-next-line no-console
+  console.debug('delay detected', delay)
+})
+
+// Play special token: delay or emotion
+function playSpecialToken(special: string) {
+  delaysQueue.enqueue(special)
+  emotionMessageContentQueue.enqueue(special)
+}
+onPlaybackFinished(({ special }) => {
+  playSpecialToken(special)
+})
+
+async function handleSpeechGeneration(ctx: { data: TTSChunkItem }) {
   try {
     if (!activeSpeechProvider.value) {
       console.warn('No active speech provider configured')
@@ -141,11 +184,21 @@ async function handleSpeechGeneration(ctx: { data: string }) {
       return
     }
 
+    // console.debug("ctx.data.chunk is empty? ", ctx.data.chunk === "")
+    // console.debug("ctx.data.special: ", ctx.data.special)
+    if (ctx.data.chunk === '' && !ctx.data.special)
+      return
+    // If special token only and chunk = ""
+    if (ctx.data.chunk === '' && ctx.data.special) {
+      playSpecialToken(ctx.data.special)
+      return
+    }
+
     const providerConfig = providersStore.getProviderConfig(activeSpeechProvider.value)
 
     const input = ssmlEnabled.value
-      ? speechStore.generateSSML(ctx.data, activeSpeechVoice.value, { ...providerConfig, pitch: pitch.value })
-      : ctx.data
+      ? speechStore.generateSSML(ctx.data.chunk, activeSpeechVoice.value, { ...providerConfig, pitch: pitch.value })
+      : ctx.data.chunk
 
     const res = await generateSpeech({
       ...provider.speech(activeSpeechModel.value, providerConfig),
@@ -154,52 +207,21 @@ async function handleSpeechGeneration(ctx: { data: string }) {
     })
 
     const audioBuffer = await audioContext.decodeAudioData(res)
-    playbackQueue.value.enqueue({ audioBuffer, text: ctx.data })
+    playbackQueue.value.enqueue({ audioBuffer, text: ctx.data.chunk, special: ctx.data.special })
   }
   catch (error) {
     console.error('Speech generation failed:', error)
   }
 }
 
-const ttsQueue = createQueue<string>({
+const ttsQueue = createQueue<TTSChunkItem>({
   handlers: [
     handleSpeechGeneration,
   ],
 })
 
-onTextSegmented((chunk) => {
-  ttsQueue.enqueue(chunk)
-})
-
-const { currentMotion } = storeToRefs(useLive2d())
-
-const emotionsQueue = createQueue<Emotion>({
-  handlers: [
-    async (ctx) => {
-      if (stageModelRenderer.value === 'vrm') {
-        const value = EMOTION_VRMExpressionName_value[ctx.data]
-        if (!value)
-          return
-
-        await vrmViewerRef.value!.setExpression(value)
-      }
-      else if (stageModelRenderer.value === 'live2d') {
-        currentMotion.value = { group: EMOTION_EmotionMotionName_value[ctx.data] }
-      }
-    },
-  ],
-})
-
-const emotionMessageContentQueue = useEmotionsMessageQueue(emotionsQueue)
-emotionMessageContentQueue.onHandlerEvent('emotion', (emotion) => {
-  // eslint-disable-next-line no-console
-  console.debug('emotion detected', emotion)
-})
-
-const delaysQueue = useDelayMessageQueue()
-delaysQueue.onHandlerEvent('delay', (delay) => {
-  // eslint-disable-next-line no-console
-  console.debug('delay detected', delay)
+onTextSegmented((chunkItem) => {
+  ttsQueue.enqueue(chunkItem)
 })
 
 function getVolumeWithMinMaxNormalizeWithFrameUpdates() {
@@ -241,12 +263,14 @@ onBeforeSend(async () => {
 
 onTokenLiteral(async (literal) => {
   // Only push to segmentation; visual presentation happens on playback start
-  textSegmentationQueue.value.enqueue(literal)
+  textSegmentationQueue.value.enqueue({ type: 'literal', value: literal } as TextSegmentationItem)
 })
 
 onTokenSpecial(async (special) => {
-  delaysQueue.enqueue(special)
-  emotionMessageContentQueue.enqueue(special)
+  // delaysQueue.enqueue(special)
+  // emotionMessageContentQueue.enqueue(special)
+  // Also push special token to the queue for emotion animation/delay and TTS playback synchronisation
+  textSegmentationQueue.value.enqueue({ type: 'special', value: special } as TextSegmentationItem)
 })
 
 onStreamEnd(async () => {

packages/stage-ui/src/composables/queues.ts

@@ -1,5 +1,6 @@
 import type { Emotion } from '../constants/emotions'
 import type { UseQueueReturn } from '../utils/queue'
+import type { TTSChunkItem } from '../utils/tts'
 
 import { sleep } from '@moeru/std'
 import { invoke } from '@vueuse/core'
@@ -9,7 +10,12 @@ import { ref, shallowRef } from 'vue'
 import { EMOTION_VALUES } from '../constants/emotions'
 import { createQueue } from '../utils/queue'
 import { createControllableStream } from '../utils/stream'
-import { chunkEmitter } from '../utils/tts'
+import { chunkEmitter, TTS_SPECIAL_TOKEN } from '../utils/tts'
+
+export interface TextSegmentationItem {
+  type: 'literal' | 'special'
+  value: string
+}
 
 export function useEmotionsMessageQueue(emotionsQueue: UseQueueReturn<Emotion>) {
   function splitEmotion(content: string) {
@@ -106,13 +112,13 @@ export function useDelayMessageQueue() {
 export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelines:character:speech', () => {
   // Hooks
   const onPlaybackStartedHooks = ref<Array<(payload: { text: string }) => Promise<void> | void>>([])
-  const onPlaybackFinishedHooks = ref<Array<(payload: { text: string }) => Promise<void> | void>>([])
+  const onPlaybackFinishedHooks = ref<Array<(payload: { special: string }) => Promise<void> | void>>([])
 
   // Hooks registers
   function onPlaybackStarted(hook: (payload: { text: string }) => Promise<void> | void) {
     onPlaybackStartedHooks.value.push(hook)
   }
-  function onPlaybackFinished(hook: (payload: { text: string }) => Promise<void> | void) {
+  function onPlaybackFinished(hook: (payload: { special: string }) => Promise<void> | void) {
     onPlaybackFinishedHooks.value.push(hook)
   }
 
@@ -141,7 +147,7 @@ export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelin
   }
 
   const playbackQueue = ref(invoke(() => {
-    return createQueue<{ audioBuffer: AudioBuffer, text: string }>({
+    return createQueue<{ audioBuffer: AudioBuffer, text: string, special: string | null }>({
       handlers: [
         (ctx) => {
           return new Promise((resolve) => {
@@ -167,11 +173,15 @@ export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelin
             currentAudioSource.value = source
             source.start(0)
             source.onended = () => {
-              for (const hook of onPlaybackFinishedHooks.value) hook({ text: ctx.data.text })
+              // Play special token: delay or emotion
+              if (ctx.data.special) {
+                for (const hook of onPlaybackFinishedHooks.value)
+                  hook({ special: ctx.data.special })
+              }
+
               if (currentAudioSource.value === source) {
                 currentAudioSource.value = undefined
               }
-
               resolve()
             }
           })
@@ -206,10 +216,10 @@ export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelin
 
 export const usePipelineWorkflowTextSegmentationStore = defineStore('pipelines:workflows:text-segmentation', () => {
   // Hooks
-  const onTextSegmentedHooks = ref<Array<(segment: string) => Promise<void> | void>>([])
+  const onTextSegmentedHooks = ref<Array<(segment: TTSChunkItem) => Promise<void> | void>>([])
 
   // Hooks registers
-  function onTextSegmented(hook: (segment: string) => Promise<void> | void) {
+  function onTextSegmented(hook: (segment: TTSChunkItem) => Promise<void> | void) {
     onTextSegmentedHooks.value.push(hook)
   }
 
@@ -226,17 +236,27 @@ export const usePipelineWorkflowTextSegmentationStore = defineStore('pipelines:w
     const { stream, controller } = createControllableStream<Uint8Array>()
     textSegmentationStream.value = stream
     textSegmentationStreamController.value = controller
+    // This is the queue for pending special tokens
+    const pendingSpecials: string[] = []
 
-    chunkEmitter(stream.getReader(), async (chunk) => {
+    chunkEmitter(stream.getReader(), pendingSpecials, async (chunk) => {
       for (const hook of onTextSegmentedHooks.value) {
         await hook(chunk)
       }
     })
 
-    return createQueue<string>({
+    return createQueue<TextSegmentationItem>({
       handlers: [
         async (ctx) => {
-          controller.enqueue(encoder.encode(ctx.data))
+          if (ctx.data.type === 'literal') {
+            controller.enqueue(encoder.encode(ctx.data.value))
+          }
+          else {
+            // Special literal, need to be flushed in tts rechunking
+            // console.debug("TextSegmentationQueue: Special enqueue", encoder.encode(TTS_SPECIAL_TOKEN))
+            pendingSpecials.push(ctx.data.value)
+            controller.enqueue(encoder.encode(TTS_SPECIAL_TOKEN))
+          }
         },
       ],
     })

packages/stage-ui/src/constants/emotions.ts

@@ -1,51 +1,56 @@
-export const EMOTION_HAPPY = '<|EMOTE_HAPPY|>'
-export const EMOTION_SAD = '<|EMOTE_SAD|>'
-export const EMOTION_ANGRY = '<|EMOTE_ANGRY|>'
-export const EMOTION_THINK = '<|EMOTE_THINK|>'
-export const EMOTION_SURPRISE = '<|EMOTE_SURPRISE|>'
-export const EMOTION_AWKWARD = '<|EMOTE_AWKWARD|>'
-export const EMOTION_QUESTION = '<|EMOTE_QUESTION|>'
-
-export enum Emotion {
-  Idle = '<|EMOTE_NEUTRAL|>',
-  Happy = '<|EMOTE_HAPPY|>',
-  Sad = '<|EMOTE_SAD|>',
-  Angry = '<|EMOTE_ANGRY|>',
-  Think = '<|EMOTE_THINK|>',
-  Surprise = '<|EMOTE_SURPRISE|>',
-  Awkward = '<|EMOTE_AWKWARD|>',
-  Question = '<|EMOTE_QUESTION|>',
-}
-
-export const EMOTION_VALUES = Object.values(Emotion)
-
-export const EmotionHappyMotionName = 'Happy'
-export const EmotionSadMotionName = 'Sad'
-export const EmotionAngryMotionName = 'Angry'
-export const EmotionAwkwardMotionName = 'Awkward'
-export const EmotionThinkMotionName = 'Think'
-export const EmotionSurpriseMotionName = 'Surprise'
-export const EmotionQuestionMotionName = 'Question'
-export const EmotionNeutralMotionName = 'Idle'
-
-export const EMOTION_EmotionMotionName_value = {
-  [Emotion.Happy]: EmotionHappyMotionName,
-  [Emotion.Sad]: EmotionSadMotionName,
-  [Emotion.Angry]: EmotionAngryMotionName,
-  [Emotion.Think]: EmotionThinkMotionName,
-  [Emotion.Surprise]: EmotionSurpriseMotionName,
-  [Emotion.Awkward]: EmotionAwkwardMotionName,
-  [Emotion.Question]: EmotionQuestionMotionName,
-  [Emotion.Idle]: EmotionNeutralMotionName,
-}
-
-export const EMOTION_VRMExpressionName_value = {
-  [Emotion.Happy]: 'happy',
-  [Emotion.Sad]: 'sad',
-  [Emotion.Angry]: 'angry',
-  [Emotion.Think]: undefined,
-  [Emotion.Surprise]: 'surprised',
-  [Emotion.Awkward]: undefined,
-  [Emotion.Question]: undefined,
-  [Emotion.Idle]: undefined,
-} satisfies Record<Emotion, string | undefined>
+export const EMOTION_HAPPY = '<|EMOTE_HAPPY|>'
+export const EMOTION_SAD = '<|EMOTE_SAD|>'
+export const EMOTION_ANGRY = '<|EMOTE_ANGRY|>'
+export const EMOTION_THINK = '<|EMOTE_THINK|>'
+export const EMOTION_SURPRISE = '<|EMOTE_SURPRISED|>'
+export const EMOTION_AWKWARD = '<|EMOTE_AWKWARD|>'
+export const EMOTION_QUESTION = '<|EMOTE_QUESTION|>'
+export const EMOTION_CURIOUS = '<|EMOTE_CURIOUS|>'
+
+export enum Emotion {
+  Idle = '<|EMOTE_NEUTRAL|>',
+  Happy = '<|EMOTE_HAPPY|>',
+  Sad = '<|EMOTE_SAD|>',
+  Angry = '<|EMOTE_ANGRY|>',
+  Think = '<|EMOTE_THINK|>',
+  Surprise = '<|EMOTE_SURPRISED|>',
+  Awkward = '<|EMOTE_AWKWARD|>',
+  Question = '<|EMOTE_QUESTION|>',
+  Curious = '<|EMOTE_CURIOUS|>',
+}
+
+export const EMOTION_VALUES = Object.values(Emotion)
+
+export const EmotionHappyMotionName = 'Happy'
+export const EmotionSadMotionName = 'Sad'
+export const EmotionAngryMotionName = 'Angry'
+export const EmotionAwkwardMotionName = 'Awkward'
+export const EmotionThinkMotionName = 'Think'
+export const EmotionSurpriseMotionName = 'Surprise'
+export const EmotionQuestionMotionName = 'Question'
+export const EmotionNeutralMotionName = 'Idle'
+export const EmotionCuriousMotionName = 'Curious'
+
+export const EMOTION_EmotionMotionName_value = {
+  [Emotion.Happy]: EmotionHappyMotionName,
+  [Emotion.Sad]: EmotionSadMotionName,
+  [Emotion.Angry]: EmotionAngryMotionName,
+  [Emotion.Think]: EmotionThinkMotionName,
+  [Emotion.Surprise]: EmotionSurpriseMotionName,
+  [Emotion.Awkward]: EmotionAwkwardMotionName,
+  [Emotion.Question]: EmotionQuestionMotionName,
+  [Emotion.Idle]: EmotionNeutralMotionName,
+  [Emotion.Curious]: EmotionCuriousMotionName,
+}
+
+export const EMOTION_VRMExpressionName_value = {
+  [Emotion.Happy]: 'happy',
+  [Emotion.Sad]: 'sad',
+  [Emotion.Angry]: 'angry',
+  [Emotion.Think]: undefined,
+  [Emotion.Surprise]: 'surprised',
+  [Emotion.Awkward]: undefined,
+  [Emotion.Question]: undefined,
+  [Emotion.Idle]: undefined,
+  [Emotion.Curious]: 'surprised',
+} satisfies Record<Emotion, string | undefined>