From 93c6e7dddb0847cd76c77ae746952ee3f4a50d02 Mon Sep 17 00:00:00 2001 From: sliterok <12751644+sliterok@users.noreply.github.com> Date: Fri, 13 Jun 2025 23:55:07 +0200 Subject: [PATCH 1/4] fix pulse bpm sync --- src/backend/pattern/extra.ts | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/backend/pattern/extra.ts b/src/backend/pattern/extra.ts index 3603b41..c7dcb93 100644 --- a/src/backend/pattern/extra.ts +++ b/src/backend/pattern/extra.ts @@ -46,13 +46,25 @@ export const getStrobeColor: IColorGetter = (_, time) => { } export const getPulseColor: IColorGetter = (_, time) => { - const cycle = settings.syncToMusic && audioState.bpm ? 60000 / audioState.bpm : 1000 - const t = (time * settings.effectSpeed) % cycle - if (t < lastPulse) { + const target = settings.syncToMusic && audioState.bpm ? 60000 / audioState.bpm : 1000 + + if (!lastPulseTime) { + lastPulseTime = time + pulseCycle = target + } + + const dt = (time - lastPulseTime) * settings.effectSpeed + lastPulseTime = time + pulsePhase += dt / pulseCycle + + if (pulsePhase >= 1) { + pulsePhase %= 1 pulseColor = hslToRgb(Math.random() * 360, 1, 0.5) } - lastPulse = t - const intensity = Math.sin((t / cycle) * Math.PI) + + pulseCycle = target + + const intensity = Math.sin(pulsePhase * Math.PI) return pulseColor.map(c => Math.round(c * intensity)) as IArrColor } @@ -82,7 +94,9 @@ export const getMultiPulseColor: IColorGetter = (index, time) => { let strobeColor: IArrColor = [255, 255, 255] let lastStrobe = 0 let pulseColor: IArrColor = [255, 0, 0] -let lastPulse = 0 +let pulseCycle = 1000 +let pulsePhase = 0 +let lastPulseTime = 0 let multiPulseColors: IArrColor[] = Array(4) .fill(null) .map(() => hslToRgb(Math.random() * 360, 1, 0.5)) @@ -95,7 +109,9 @@ export function resetExtraPatterns() { strobeColor = [255, 255, 255] lastStrobe = 0 pulseColor = [255, 0, 0] - lastPulse = 0 + pulseCycle = 1000 + pulsePhase = 0 + lastPulseTime = 0 multiPulseColors = Array(4) .fill(null) .map(() => hslToRgb(Math.random() * 360, 1, 0.5)) From 7e2a92d12e06437d469f9cbc6c986fc18a011838 Mon Sep 17 00:00:00 2001 From: sliterok <12751644+sliterok@users.noreply.github.com> Date: Sat, 14 Jun 2025 11:18:18 +0200 Subject: [PATCH 2/4] feat: mic bpm --- mic-sender.js | 49 ++++++++++++++++++++++++++++++------ package.json | 3 ++- pnpm-lock.yaml | 36 +++++++++++++++++--------- src/backend/pattern/extra.ts | 10 +++++--- src/backend/wsAudio.ts | 11 +++++++- 5 files changed, 84 insertions(+), 25 deletions(-) diff --git a/mic-sender.js b/mic-sender.js index 6829ec2..34f5e7a 100644 --- a/mic-sender.js +++ b/mic-sender.js @@ -1,10 +1,45 @@ -import record from 'node-record-lpcm16' +import { Essentia, arrayToVector } from 'essentia.js' +import mic from 'mic' import WebSocket from 'ws' -const url = process.argv[2] || 'ws://localhost:8081' -const ws = new WebSocket(url) +async function main() { + const url = process.argv[2] || 'ws://localhost:8081' + const ws = new WebSocket(url) + await new Promise(resolve => ws.once('open', resolve)) + + const essentia = await Essentia() + const sampleRate = 44100 + const bufferSizeSec = 5 + const bufferSize = sampleRate * bufferSizeSec + let buffer = new Float32Array(0) + + const rhythmAlgo = (signal) => + essentia.RhythmExtractor2013({ signal, sampleRate }) + + const micInst = mic({ rate: sampleRate.toString(), channels: '1' }) + const micStream = micInst.getAudioStream() + + micInst.start() + + micStream.on('data', (chunk) => { + ws.send(chunk) + const pcm = new Float32Array(chunk.buffer, chunk.byteOffset, chunk.length / 4) + const tmp = new Float32Array(buffer.length + pcm.length) + tmp.set(buffer) + tmp.set(pcm, buffer.length) + buffer = tmp + if (buffer.length >= bufferSize) { + const sigVec = arrayToVector(buffer) + const res = rhythmAlgo(sigVec) + const bpm = res.bpm + ws.send(JSON.stringify({ bpm })) + console.log(`BPM: ${bpm.toFixed(2)}, ticks: ${res.ticks.length}`) + buffer = buffer.subarray(bufferSize) + } + }) + + micStream.on('error', console.error) +} + +main().catch(console.error) -ws.on('open', () => { - const mic = record.start({ sampleRate: 44100, channels: 1 }) - mic.on('data', chunk => ws.send(chunk)) -}) diff --git a/package.json b/package.json index 74ca811..d1d915a 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,8 @@ "express": "^4.18.2", "fft-js": "^0.0.12", "music-tempo": "^1.0.3", - "node-record-lpcm16": "^1.0.1", + "mic": "^2.1.1", + "essentia.js": "^0.1.3", "react": "^19.1.0", "react-colorful": "^5.6.1", "react-dom": "^19.1.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7fef3ba..854ebcb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32,18 +32,21 @@ importers: dotenv: specifier: ^16.4.5 version: 16.5.0 + essentia.js: + specifier: ^0.1.3 + version: 0.1.3 express: specifier: ^4.18.2 version: 4.21.2 fft-js: specifier: ^0.0.12 version: 0.0.12 + mic: + specifier: ^2.1.1 + version: 2.1.2 music-tempo: specifier: ^1.0.3 version: 1.0.3 - node-record-lpcm16: - specifier: ^1.0.1 - version: 1.0.1 react: specifier: ^19.1.0 version: 19.1.0 @@ -1623,6 +1626,9 @@ packages: resolution: {integrity: sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==} engines: {node: '>=4.0'} + essentia.js@0.1.3: + resolution: {integrity: sha512-vVEPgeVMEBLRXbM5o5H5Rgu53EPHu25vyFKYg+flWLzI/nEoegJQez9FKRv8GR/KxIBwm+fXDEFL+MkQeoHaLw==} + estraverse@5.3.0: resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} engines: {node: '>=4.0'} @@ -2247,6 +2253,9 @@ packages: resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} engines: {node: '>= 0.6'} + mic@2.1.2: + resolution: {integrity: sha512-rpl4tgdXX24sAzYwjRc5OZfGNAuhUIIjdd0cw8+Ubq7rp3iGhi40AdqcwurDWhEZADk60tPOxb3E2MpoeLeyxw==} + micromatch@4.0.8: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} @@ -2337,12 +2346,13 @@ packages: node-int64@0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} - node-record-lpcm16@1.0.1: - resolution: {integrity: sha512-H75GMOP8ErnF67m21+qSgj4USnzv5RLfm7OkEItdIi+soNKoJZpMQPX6umM8Cn9nVPSgd/dBUtc1msst5MmABA==} - node-releases@2.0.19: resolution: {integrity: sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==} + node-wav@0.0.2: + resolution: {integrity: sha512-M6Rm/bbG6De/gKGxOpeOobx/dnGuP0dz40adqx38boqHhlWssBJZgLCPBNtb9NkrmnKYiV04xELq+R6PFOnoLA==} + engines: {node: '>=4.4.0'} + normalize-path@3.0.0: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} engines: {node: '>=0.10.0'} @@ -4619,6 +4629,10 @@ snapshots: dependencies: estraverse: 5.3.0 + essentia.js@0.1.3: + dependencies: + node-wav: 0.0.2 + estraverse@5.3.0: {} estree-walker@2.0.2: {} @@ -5458,6 +5472,8 @@ snapshots: methods@1.1.2: {} + mic@2.1.2: {} + micromatch@4.0.8: dependencies: braces: 3.0.3 @@ -5523,14 +5539,10 @@ snapshots: node-int64@0.4.0: {} - node-record-lpcm16@1.0.1: - dependencies: - debug: 2.6.9 - transitivePeerDependencies: - - supports-color - node-releases@2.0.19: {} + node-wav@0.0.2: {} + normalize-path@3.0.0: {} npm-run-path@4.0.1: diff --git a/src/backend/pattern/extra.ts b/src/backend/pattern/extra.ts index c7dcb93..875c249 100644 --- a/src/backend/pattern/extra.ts +++ b/src/backend/pattern/extra.ts @@ -55,15 +55,17 @@ export const getPulseColor: IColorGetter = (_, time) => { const dt = (time - lastPulseTime) * settings.effectSpeed lastPulseTime = time - pulsePhase += dt / pulseCycle + + const elapsed = pulsePhase * pulseCycle + dt + pulseCycle = target + pulsePhase = elapsed / pulseCycle if (pulsePhase >= 1) { - pulsePhase %= 1 + const cycles = Math.floor(pulsePhase) + pulsePhase -= cycles pulseColor = hslToRgb(Math.random() * 360, 1, 0.5) } - pulseCycle = target - const intensity = Math.sin(pulsePhase * Math.PI) return pulseColor.map(c => Math.round(c * intensity)) as IArrColor } diff --git a/src/backend/wsAudio.ts b/src/backend/wsAudio.ts index 0690789..40f6245 100644 --- a/src/backend/wsAudio.ts +++ b/src/backend/wsAudio.ts @@ -20,7 +20,16 @@ export function startAudioServer(port = 8081) { const wss = new WebSocketServer({ port }) wss.on('connection', ws => { ws.on('message', data => { - if (Buffer.isBuffer(data)) processAudio(data) + if (Buffer.isBuffer(data)) { + processAudio(data) + } else { + try { + const msg = JSON.parse(data.toString()) + if (typeof msg.bpm === 'number') audioState.bpm = msg.bpm + } catch { + // ignore non-json messages + } + } }) }) } From ce500a927e6aa7c5897612b7c89e5679919fdc3a Mon Sep 17 00:00:00 2001 From: sliterok <12751644+sliterok@users.noreply.github.com> Date: Sat, 14 Jun 2025 11:18:24 +0200 Subject: [PATCH 3/4] Server bpm compute --- mic-sender.js | 43 ++++++------------------------------------ package.json | 4 ++-- src/backend/wsAudio.ts | 11 +---------- 3 files changed, 9 insertions(+), 49 deletions(-) diff --git a/mic-sender.js b/mic-sender.js index 34f5e7a..679c132 100644 --- a/mic-sender.js +++ b/mic-sender.js @@ -1,45 +1,14 @@ -import { Essentia, arrayToVector } from 'essentia.js' import mic from 'mic' import WebSocket from 'ws' -async function main() { - const url = process.argv[2] || 'ws://localhost:8081' - const ws = new WebSocket(url) - await new Promise(resolve => ws.once('open', resolve)) +const url = process.argv[2] || 'ws://localhost:8081' +const ws = new WebSocket(url) - const essentia = await Essentia() - const sampleRate = 44100 - const bufferSizeSec = 5 - const bufferSize = sampleRate * bufferSizeSec - let buffer = new Float32Array(0) - - const rhythmAlgo = (signal) => - essentia.RhythmExtractor2013({ signal, sampleRate }) - - const micInst = mic({ rate: sampleRate.toString(), channels: '1' }) +ws.on('open', () => { + const micInst = mic({ rate: '44100', channels: '1' }) const micStream = micInst.getAudioStream() - micInst.start() - - micStream.on('data', (chunk) => { - ws.send(chunk) - const pcm = new Float32Array(chunk.buffer, chunk.byteOffset, chunk.length / 4) - const tmp = new Float32Array(buffer.length + pcm.length) - tmp.set(buffer) - tmp.set(pcm, buffer.length) - buffer = tmp - if (buffer.length >= bufferSize) { - const sigVec = arrayToVector(buffer) - const res = rhythmAlgo(sigVec) - const bpm = res.bpm - ws.send(JSON.stringify({ bpm })) - console.log(`BPM: ${bpm.toFixed(2)}, ticks: ${res.ticks.length}`) - buffer = buffer.subarray(bufferSize) - } - }) - + micStream.on('data', chunk => ws.send(chunk)) micStream.on('error', console.error) -} - -main().catch(console.error) +}) diff --git a/package.json b/package.json index d1d915a..04c661d 100644 --- a/package.json +++ b/package.json @@ -52,8 +52,8 @@ "d3-interpolate": "^3.0.1", "dotenv": "^16.4.5", "express": "^4.18.2", - "fft-js": "^0.0.12", - "music-tempo": "^1.0.3", + "fft-js": "^0.0.12", + "music-tempo": "^1.0.3", "mic": "^2.1.1", "essentia.js": "^0.1.3", "react": "^19.1.0", diff --git a/src/backend/wsAudio.ts b/src/backend/wsAudio.ts index 40f6245..0690789 100644 --- a/src/backend/wsAudio.ts +++ b/src/backend/wsAudio.ts @@ -20,16 +20,7 @@ export function startAudioServer(port = 8081) { const wss = new WebSocketServer({ port }) wss.on('connection', ws => { ws.on('message', data => { - if (Buffer.isBuffer(data)) { - processAudio(data) - } else { - try { - const msg = JSON.parse(data.toString()) - if (typeof msg.bpm === 'number') audioState.bpm = msg.bpm - } catch { - // ignore non-json messages - } - } + if (Buffer.isBuffer(data)) processAudio(data) }) }) } From 2dd7acb3c2998c9b82a458a1f1acf622fc809f2b Mon Sep 17 00:00:00 2001 From: sliterok <12751644+sliterok@users.noreply.github.com> Date: Sat, 14 Jun 2025 15:55:11 +0200 Subject: [PATCH 4/4] Use essentia bpm --- package.json | 1 - pnpm-lock.yaml | 7 ------- src/backend/wsAudio.ts | 11 +++++++---- src/types/essentia.js.d.ts | 1 + tests/wsAudio.test.ts | 8 ++++---- 5 files changed, 12 insertions(+), 16 deletions(-) create mode 100644 src/types/essentia.js.d.ts diff --git a/package.json b/package.json index 04c661d..223f713 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,6 @@ "dotenv": "^16.4.5", "express": "^4.18.2", "fft-js": "^0.0.12", - "music-tempo": "^1.0.3", "mic": "^2.1.1", "essentia.js": "^0.1.3", "react": "^19.1.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 854ebcb..8f30897 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44,9 +44,6 @@ importers: mic: specifier: ^2.1.1 version: 2.1.2 - music-tempo: - specifier: ^1.0.3 - version: 1.0.3 react: specifier: ^19.1.0 version: 19.1.0 @@ -2309,9 +2306,6 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - music-tempo@1.0.3: - resolution: {integrity: sha512-qAocTKLp3jaSeJLeGs98mkkpLYDFM1VCevA1OPFJvLPkHlzwTLUxChXMgnxZRHKSJQ13DuaT+Fr51BEUb2D4pQ==} - mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} @@ -5515,7 +5509,6 @@ snapshots: ms@2.1.3: {} - music-tempo@1.0.3: {} mz@2.7.0: dependencies: diff --git a/src/backend/wsAudio.ts b/src/backend/wsAudio.ts index 0690789..c258c6a 100644 --- a/src/backend/wsAudio.ts +++ b/src/backend/wsAudio.ts @@ -1,6 +1,6 @@ import { WebSocketServer } from 'ws' import fftjs from 'fft-js' -import MusicTempo from 'music-tempo' +import { Essentia, EssentiaWASM } from 'essentia.js' // eslint-disable-next-line import/default import RingBufferTs from 'ring-buffer-ts' const RingBuffer = RingBufferTs.RingBuffer @@ -29,8 +29,9 @@ const sampleRateDefault = 44100 const bpmWindow = sampleRateDefault * 8 const sampleBuffer = new RingBuffer(bpmWindow) let lastBpmUpdate = 0 +const essentiaPromise = Promise.resolve(new Essentia(EssentiaWASM)) -export function processAudio(buffer: Buffer, sampleRate = sampleRateDefault) { +export async function processAudio(buffer: Buffer, sampleRate = sampleRateDefault) { const samples = new Int16Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 2) const input = Array.from(samples, s => s / 32768) for (const s of input) sampleBuffer.add(s) @@ -56,8 +57,10 @@ export function processAudio(buffer: Buffer, sampleRate = sampleRateDefault) { if (now - lastBpmUpdate > 2000 && sampleBuffer.getBufferLength() >= sampleRate * 4) { lastBpmUpdate = now try { - const mt = new MusicTempo(Float32Array.from(sampleBuffer.toArray())) - const tempo = parseFloat(String(mt.tempo)) + const essentia = await essentiaPromise + const signal = essentia.arrayToVector(Float32Array.from(sampleBuffer.toArray())) + const res = essentia.RhythmExtractor2013(signal, 208, 'multifeature', 40) + const tempo = res.bpm if (!Number.isNaN(tempo)) audioState.bpm = tempo } catch { // ignore errors diff --git a/src/types/essentia.js.d.ts b/src/types/essentia.js.d.ts new file mode 100644 index 0000000..d48130b --- /dev/null +++ b/src/types/essentia.js.d.ts @@ -0,0 +1 @@ +declare module 'essentia.js' diff --git a/tests/wsAudio.test.ts b/tests/wsAudio.test.ts index b52fefa..d01f72e 100644 --- a/tests/wsAudio.test.ts +++ b/tests/wsAudio.test.ts @@ -24,16 +24,16 @@ describe('processAudio', () => { beforeEach(() => { resetAudioState() }) - test('detects frequency', () => { + test('detects frequency', async () => { const buf = genSine(440, 1234, 44100) - processAudio(buf) + await processAudio(buf) expect(audioState.freq).toBeGreaterThan(430) expect(audioState.freq).toBeLessThan(450) }) - test('detects bpm', () => { + test('detects bpm', async () => { const buf = genBeat(120, 4, 44100) - processAudio(buf) + await processAudio(buf) expect(audioState.bpm).toBeGreaterThan(110) expect(audioState.bpm).toBeLessThan(130) })