Skip to content

Commit 09d084b

Browse files
sumimakitosumimakito
authored
feat(stage-web,stage-ui): beat sync (#621)
1 parent 13f3225 commit 09d084b

File tree

9 files changed

+558
-334
lines changed

9 files changed

+558
-334
lines changed

‎apps/stage-web/package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
"@huggingface/transformers": "^3.7.3",
2323
"@llama-flow/core": "^0.4.4",
2424
"@moeru/std": "catalog:",
25+
"@nekopaw/tempora": "0.3.1-alpha.1",
2526
"@proj-airi/audio": "workspace:^",
2627
"@proj-airi/ccc": "workspace:^",
2728
"@proj-airi/drizzle-duckdb-wasm": "catalog:",
@@ -53,6 +54,7 @@
5354
"@xsai/shared-chat": "catalog:",
5455
"@xsai/stream-text": "catalog:",
5556
"@xsai/utils-chat": "catalog:",
57+
"animejs": "^4.1.3",
5658
"colorjs.io": "^0.5.2",
5759
"culori": "^4.0.2",
5860
"date-fns": "^4.1.0",

‎apps/stage-web/src/pages/settings/modules/beat-sync.vue‎

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
<script setup lang="ts">
2+
import type { AnalyserBeatEvent, AnalyserWorkletParameters } from '@nekopaw/tempora'
3+
4+
import { DEFAULT_ANALYSER_WORKLET_PARAMS } from '@nekopaw/tempora'
5+
import { Button } from '@proj-airi/stage-ui/components'
6+
import { useBeatSyncStore } from '@proj-airi/stage-ui/stores/beat-sync'
7+
import { FieldCheckbox, FieldRange } from '@proj-airi/ui'
8+
import { createTimeline } from 'animejs'
9+
import { nanoid } from 'nanoid'
10+
import { onMounted, onUnmounted, ref, watchEffect } from 'vue'
11+
import { useI18n } from 'vue-i18n'
12+
13+
const beatSyncStore = useBeatSyncStore()
14+
const { t } = useI18n()
15+
16+
const beatsHistory = ref<Array<{
17+
id: string
18+
energy: number
19+
normalizedEnergy: number
20+
}>>([])
21+
22+
const parameters = ref<AnalyserWorkletParameters>({ ...DEFAULT_ANALYSER_WORKLET_PARAMS })
23+
24+
watchEffect(() => {
25+
beatSyncStore.updateParameters(parameters.value)
26+
})
27+
28+
function normalizeEnergy(energy: number) {
29+
const base = 2
30+
const a = 0.5
31+
return ((base ** energy - 1) / (base - 1)) ** a
32+
}
33+
34+
onMounted(() => {
35+
const onBeat = ({ energy }: AnalyserBeatEvent) => {
36+
beatsHistory.value.unshift({
37+
id: nanoid(),
38+
energy,
39+
normalizedEnergy: normalizeEnergy(energy),
40+
})
41+
}
42+
43+
beatSyncStore.on('beat', onBeat)
44+
45+
onUnmounted(() => {
46+
beatSyncStore.off('beat', onBeat)
47+
})
48+
})
49+
50+
function onRippleEnter(el: Element, done: () => void) {
51+
const beatId = (el as HTMLElement).dataset.beatId
52+
createTimeline()
53+
.set(el, {
54+
opacity: 1,
55+
scale: 0,
56+
})
57+
.add(el, {
58+
opacity: 0,
59+
scale: 1,
60+
duration: 2000,
61+
delay: 0,
62+
ease: 'out(5)',
63+
onComplete: () => {
64+
if (!beatId)
65+
return
66+
67+
const idx = beatsHistory.value.findIndex(b => b.id === beatId)
68+
if (idx >= 0)
69+
beatsHistory.value.splice(idx, 1)
70+
71+
done()
72+
},
73+
})
74+
}
75+
76+
function resetDefaultParameters() {
77+
parameters.value = { ...DEFAULT_ANALYSER_WORKLET_PARAMS }
78+
}
79+
</script>
80+
81+
<template>
82+
<div flex="~ col md:row gap-6">
83+
<div bg="neutral-100 dark:[rgba(0,0,0,0.3)]" rounded-xl p-4 flex="~ col gap-4" class="h-fit w-full md:w-[40%]">
84+
<div flex="~ col gap-6">
85+
<div flex="~ col gap-4">
86+
<div>
87+
<h2 class="text-lg text-neutral-500 md:text-2xl dark:text-neutral-500">
88+
{{ t('settings.pages.modules.beat_sync.sections.audio_source.title') }}
89+
</h2>
90+
<div text="neutral-400 dark:neutral-400">
91+
<span>{{ t('settings.pages.modules.beat_sync.sections.audio_source.description') }}</span>
92+
</div>
93+
</div>
94+
95+
<div max-w-full flex="~ row gap-4 wrap">
96+
<template v-if="beatSyncStore.isActive">
97+
<Button @click="beatSyncStore.stop">
98+
{{ t('settings.pages.modules.beat_sync.sections.audio_source.actions.stop') }}
99+
</Button>
100+
</template>
101+
102+
<template v-else>
103+
<Button @click="beatSyncStore.startFromScreenCapture">
104+
{{ t('settings.pages.modules.beat_sync.sections.audio_source.actions.start_screen_capture') }}
105+
</Button>
106+
</template>
107+
</div>
108+
</div>
109+
110+
<div flex="~ col gap-4">
111+
<div flex="~ row" items-center justify-between>
112+
<div>
113+
<h2 class="text-lg text-neutral-500 md:text-2xl dark:text-neutral-500">
114+
{{ t('settings.pages.modules.beat_sync.sections.parameters.title') }}
115+
</h2>
116+
<div text="neutral-400 dark:neutral-400">
117+
<span>{{ t('settings.pages.modules.beat_sync.sections.parameters.description') }}</span>
118+
</div>
119+
</div>
120+
121+
<button
122+
title="Reset settings"
123+
flex items-center justify-center rounded-full p-2
124+
transition="all duration-250 ease-in-out"
125+
text="neutral-500 dark:neutral-400"
126+
bg="transparent dark:transparent hover:neutral-200 dark:hover:neutral-800 active:neutral-300 dark:active:neutral-700"
127+
@click="resetDefaultParameters"
128+
>
129+
<div i-solar:refresh-bold-duotone text-xl />
130+
</button>
131+
</div>
132+
133+
<div max-w-full flex="~ col gap-4">
134+
<FieldRange
135+
v-model="parameters.sensitivity"
136+
:label="t('settings.pages.modules.beat_sync.sections.parameters.parameters.sensitivity.label')"
137+
:min="0"
138+
:max="1"
139+
:step="0.01"
140+
:format-value="value => value.toFixed(1)"
141+
/>
142+
143+
<FieldRange
144+
v-model="parameters.minBeatInterval"
145+
:label="t('settings.pages.modules.beat_sync.sections.parameters.parameters.min_beat_interval.label')"
146+
:description="t('settings.pages.modules.beat_sync.sections.parameters.parameters.min_beat_interval.description')"
147+
:min="0.05"
148+
:max="1"
149+
:step="0.01"
150+
:format-value="value => `${(60 / value).toFixed(1)} BPM / ${value.toFixed(2)} s`"
151+
/>
152+
153+
<div>
154+
<h3 class="text text-neutral-500 md:text-xl dark:text-neutral-500">
155+
{{ t('settings.pages.modules.beat_sync.sections.parameters.advanced_parameters') }}
156+
</h3>
157+
</div>
158+
159+
<FieldRange
160+
v-model="parameters.lowpassFilterFrequency"
161+
:label="t('settings.pages.modules.beat_sync.sections.parameters.parameters.lowpass_filter_frequency.label')"
162+
:description="t('settings.pages.modules.beat_sync.sections.parameters.parameters.lowpass_filter_frequency.description')"
163+
:min="20"
164+
:max="600"
165+
:step="10"
166+
:format-value="value => `${value.toFixed(0)} Hz`"
167+
/>
168+
169+
<FieldRange
170+
v-model="parameters.highpassFilterFrequency"
171+
:label="t('settings.pages.modules.beat_sync.sections.parameters.parameters.highpass_filter_frequency.label')"
172+
:description="t('settings.pages.modules.beat_sync.sections.parameters.parameters.highpass_filter_frequency.description')"
173+
:min="150"
174+
:max="2000"
175+
:step="10"
176+
:format-value="value => `${value.toFixed(0)} Hz`"
177+
/>
178+
179+
<FieldRange
180+
v-model="parameters.envelopeFilterFrequency"
181+
:label="t('settings.pages.modules.beat_sync.sections.parameters.parameters.envelope_filter_frequency.label')"
182+
:description="t('settings.pages.modules.beat_sync.sections.parameters.parameters.envelope_filter_frequency.description')"
183+
:min="20"
184+
:max="200"
185+
:step="10"
186+
:format-value="value => `${value.toFixed(0)} Hz`"
187+
/>
188+
189+
<FieldCheckbox
190+
v-model="parameters.warmup"
191+
:label="t('settings.pages.modules.beat_sync.sections.parameters.parameters.warmup.label')"
192+
:description="t('settings.pages.modules.beat_sync.sections.parameters.parameters.warmup.description')"
193+
/>
194+
195+
<FieldCheckbox
196+
v-model="parameters.adaptiveThreshold"
197+
:label="t('settings.pages.modules.beat_sync.sections.parameters.parameters.adaptive_threshold.label')"
198+
:description="t('settings.pages.modules.beat_sync.sections.parameters.parameters.adaptive_threshold.description')"
199+
/>
200+
201+
<FieldCheckbox
202+
v-model="parameters.spectralFlux"
203+
:label="t('settings.pages.modules.beat_sync.sections.parameters.parameters.spectral_flux.label')"
204+
:description="t('settings.pages.modules.beat_sync.sections.parameters.parameters.spectral_flux.description')"
205+
/>
206+
207+
<FieldRange
208+
v-model="parameters.bufferDuration"
209+
:label="t('settings.pages.modules.beat_sync.sections.parameters.parameters.buffer_duration.label')"
210+
:description="t('settings.pages.modules.beat_sync.sections.parameters.parameters.buffer_duration.description')"
211+
:min="2"
212+
:max="10"
213+
:step="0.5"
214+
:format-value="value => `${value.toFixed(1)} s`"
215+
/>
216+
</div>
217+
</div>
218+
</div>
219+
</div>
220+
221+
<div flex="~ col gap-6" class="w-full md:w-[60%]">
222+
<div w-full rounded-xl flex="~ col gap-4">
223+
<h2 class="mb-4 text-lg text-neutral-500 md:text-2xl dark:text-neutral-400" w-full>
224+
<div class="inline-flex items-center gap-4">
225+
{{ t('settings.pages.modules.beat_sync.sections.beat_visualizer.title') }}
226+
</div>
227+
</h2>
228+
229+
<div flex="~ col gap-4 items-center">
230+
<TransitionGroup
231+
tag="div"
232+
bg="neutral/10"
233+
relative box-border aspect-square h-full max-h-400px max-w-400px w-full rounded-2xl
234+
flex="~ row gap-2 wrap items-center"
235+
:css="false"
236+
@enter="onRippleEnter"
237+
>
238+
<div
239+
v-for="beat in beatsHistory"
240+
:key="beat.id"
241+
:data-beat-id="beat.id"
242+
absolute h-full w-full
243+
rounded-full bg="primary/50"
244+
/>
245+
</TransitionGroup>
246+
</div>
247+
</div>
248+
</div>
249+
</div>
250+
</template>
251+
252+
<route lang="yaml">
253+
meta:
254+
layout: settings
255+
stageTransition:
256+
name: slide
257+
</route>

‎apps/stage-web/src/pages/settings/modules/index.vue‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
<script setup lang="ts">
22
import { IconStatusItem } from '@proj-airi/stage-ui/components'
3+
import { useBeatSyncStore } from '@proj-airi/stage-ui/stores/beat-sync'
34
import { useConsciousnessStore } from '@proj-airi/stage-ui/stores/modules/consciousness'
45
import { useSpeechStore } from '@proj-airi/stage-ui/stores/modules/speech'
56
import { computed } from 'vue'
@@ -22,6 +23,8 @@ interface Module {
2223
configured: boolean
2324
}
2425
26+
const beatSyncStore = useBeatSyncStore()
27+
2528
// TODO: categorize modules, such as essential, messaging, gaming, etc.
2629
const modulesList = computed<Module[]>(() => [
2730
{
@@ -104,6 +107,14 @@ const modulesList = computed<Module[]>(() => [
104107
to: '',
105108
configured: false,
106109
},
110+
{
111+
id: 'beat-sync',
112+
name: t('settings.pages.modules.beat_sync.title'),
113+
description: t('settings.pages.modules.beat_sync.description'),
114+
icon: 'i-solar:music-notes-bold-duotone',
115+
to: '/settings/modules/beat-sync',
116+
configured: beatSyncStore.isActive,
117+
},
107118
])
108119
109120
const {

‎cspell.config.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ words:
105105
- hfspaceup
106106
- hfsup
107107
- hfup
108+
- highpass
108109
- higress
109110
- histoire
110111
- hiyori
@@ -136,6 +137,7 @@ words:
136137
- lobehub
137138
- logg
138139
- Lorebook
140+
- lowpass
139141
- lucide
140142
- luoling
141143
- Maekawa
@@ -243,6 +245,7 @@ words:
243245
- tamagotchi
244246
- tauri
245247
- taze
248+
- tempora
246249
- togetherapi
247250
- tolist
248251
- tresjs

‎packages/i18n/src/locales/en/settings.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,49 @@ pages:
165165
description: Configure 3D VRM models and settings
166166
scene: Scene
167167
modules:
168+
beat_sync:
169+
description: Vibe with beats from the audio source
170+
title: Beat Sync
171+
sections:
172+
audio_source:
173+
actions:
174+
start_screen_capture: Start screen capture
175+
stop: Stop
176+
description: Select an audio source to detect beats from.
177+
title: Audio source
178+
beat_visualizer:
179+
title: Beat visualizer
180+
parameters:
181+
advanced_parameters: Advanced Parameters
182+
description: Tweak the beat detection parameters.
183+
title: Parameters
184+
parameters:
185+
adaptive_threshold:
186+
description: Whether to apply adaptive thresholds based on signal variance over time.
187+
label: Adaptive threshold
188+
buffer_duration:
189+
description: Duration of the internal analysis buffer.
190+
label: Buffer duration
191+
envelope_filter_frequency:
192+
description: Frequency for the envelope filter applied to smooth energy changes.
193+
label: Envelope filter frequency
194+
highpass_filter_frequency:
195+
description: Frequency for the highpass filter applied to reduce low frequencies like sub-bass noises.
196+
label: Highpass filter frequency
197+
lowpass_filter_frequency:
198+
description: Frequency for the lowpass filter applied to reduce high frequencies like vocals.
199+
label: Lowpass filter frequency
200+
min_beat_interval:
201+
description: Maximum BPM or minimum interval between detected beats.
202+
label: Max BPM / Min beat interval
203+
sensitivity:
204+
label: Sensitivity
205+
spectral_flux:
206+
description: Whether to enable spectral flux-based onset detection.
207+
label: Spectral flux
208+
warmup:
209+
description: Whether to warm up before detecting beats for better accuracy.
210+
label: Warmup
168211
consciousness:
169212
description: Personality, desired model, etc.
170213
sections:

‎packages/stage-ui/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
"@formkit/auto-animate": "^0.9.0",
4545
"@huggingface/transformers": "^3.7.3",
4646
"@lemonneko/crop-empty-pixels": "0.1.1",
47+
"@nekopaw/tempora": "0.3.1-alpha.1",
4748
"@pixi/app": "^6.5.10",
4849
"@pixi/constants": "^6.5.10",
4950
"@pixi/core": "^6.5.10",

0 commit comments

Comments
 (0)