Skip to content

Commit

Permalink
bncm: 统一了音频可视化数据源
Browse files Browse the repository at this point in the history
  • Loading branch information
@Steve-xmh
Steve-xmh committed Dec 18, 2023
1 parent 5c47f87 commit dfe5278
Show file tree
Hide file tree
Showing 3 changed files with 129 additions and 107 deletions.
4 changes: 4 additions & 0 deletions packages/bncm/src/injector/index.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { AMLLConfig } from "../components/config";
import { AMLLGuide } from "../player/common/guide";
import { FC, PropsWithChildren, Suspense, useEffect } from "react";
import { warn } from "../utils/logger";
import { AudioFFTContext } from "../player/common/fft-context";

export const mainViewElement: HTMLDivElement = document.createElement("div");
mainViewElement.id = "amll-view";
Expand Down Expand Up @@ -58,6 +59,9 @@ export function initLyricPage() {
<SuspenseLogger text="WebSocketWrapper">
<WebSocketWrapper />
</SuspenseLogger>
<SuspenseLogger text="AudioFFTContext">
<AudioFFTContext />
</SuspenseLogger>
<SuspenseLogger text="LyricProvider">
<LyricProvider />
</SuspenseLogger>
Expand Down
116 changes: 116 additions & 0 deletions packages/bncm/src/player/common/fft-context.tsx
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
import { atom, useAtomValue, useSetAtom } from "jotai";
import { FC, useLayoutEffect, useRef } from "react";
import { musicContextAtom, playStatusAtom } from "../../music-context/wrapper";
import { processBarFFTAtom } from "../../components/config/atoms";
import { SoundProcessor } from "../../utils/fft";
import { MusicStatusGetterEvents, PlayState } from "../../music-context";
import PCMPlayer from "../../utils/pcm-player";

export const fftDataAtom = atom([] as number[]);

export const AudioFFTContext: FC = () => {
const musicCtx = useAtomValue(musicContextAtom);
const processBarFFT = useAtomValue(processBarFFTAtom);
const playstate = useAtomValue(playStatusAtom);
const setFFTData = useSetAtom(fftDataAtom);
const soundProcessor = useRef<SoundProcessor>();
const amllFFT = useRef<PCMPlayer>();

useLayoutEffect(() => {
amllFFT.current = new PCMPlayer({
inputCodec: "Int16",
channels: 2,
sampleRate: 48000,
flushTime: 50,
});
soundProcessor.current = new SoundProcessor({
filterParams: {
sigma: 1,
radius: 2,
},
sampleRate: 48000,
fftSize: 1024,
outBandsQty: 512,
startFrequency: 150,
endFrequency: 4500,
aWeight: true,
});
return () => {
amllFFT.current?.destroy();
amllFFT.current = undefined;
soundProcessor.current = undefined;
};
}, []);

useLayoutEffect(() => {
if (playstate === PlayState.Playing) {
amllFFT.current?.continue();
}
}, [playstate]);

useLayoutEffect(() => {
const fft = amllFFT.current
if (!fft) return;
let isFFTMode = false;
let ctxFFTData: number[] = [];

const onAudioData = (evt: MusicStatusGetterEvents["audio-data"]) => {
isFFTMode = false;
fft.feed(evt.detail.data);
fft.continue();
};

const onFFTData = (evt: MusicStatusGetterEvents["fft-data"]) => {
isFFTMode = true;
ctxFFTData = evt.detail.data;
};

musicCtx?.addEventListener("audio-data", onAudioData);
musicCtx?.addEventListener("fft-data", onFFTData);

let stopped = false;

const fftData = fft.createFrequencyData();
soundProcessor.current = new SoundProcessor({
filterParams: {
sigma: 1,
radius: 1,
},
sampleRate: 48000,
fftSize: fftData.length,
outBandsQty: 61,
startFrequency: 100,
endFrequency: 14000,
aWeight: true,
});

function onFrame() {
if (stopped) return;
let processed: number[];

if (isFFTMode) {
processed = ctxFFTData;
} else {
amllFFT.current?.getByteFrequencyData(fftData);
if (processBarFFT)
processed = soundProcessor.current?.process(fftData) ?? [...fftData];
else {
processed = soundProcessor.current?.divide(fftData) ?? [...fftData];
}
}
setFFTData(processed);

requestAnimationFrame(onFrame);
}

onFrame();

return () => {
musicCtx?.removeEventListener("audio-data", onAudioData);
musicCtx?.removeEventListener("fft-data", onFFTData);
stopped = true;
};
}, [musicCtx, amllFFT.current, processBarFFT]);

return null;
};
116 changes: 9 additions & 107 deletions packages/bncm/src/player/horizonal/audio-fft-control.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,73 +1,18 @@
import { useRef, FC, useLayoutEffect, MutableRefObject } from "react";
import { SoundProcessor } from "../../utils/fft";
import { useRef, FC, useLayoutEffect } from "react";
import { useAtomValue } from "jotai";
import { musicContextAtom, playStatusAtom } from "../../music-context/wrapper";
import { MusicStatusGetterEvents, PlayState } from "../../music-context";
import { musicContextAtom } from "../../music-context/wrapper";

import PCMPlayer from "../../utils/pcm-player";
import { processBarFFTAtom } from "../../components/config/atoms";
import { globalStore } from "../../injector";
import { fftDataAtom } from "../common/fft-context";

export const AudioFFTControl: FC = () => {
const canvasRef = useRef<HTMLCanvasElement>(null);
const musicCtx = useAtomValue(musicContextAtom);
const processBarFFT = useAtomValue(processBarFFTAtom);
const playstate = useAtomValue(playStatusAtom);

// const fftWeightingMethod = useConfigValue("fftWeightingMethod", "");
const soundProcessor = useRef() as MutableRefObject<SoundProcessor>;
if (soundProcessor.current === undefined) {
soundProcessor.current = new SoundProcessor({
filterParams: {
sigma: 1,
radius: 2,
},
sampleRate: 48000,
fftSize: 1024,
outBandsQty: 512,
startFrequency: 150,
endFrequency: 4500,
aWeight: true,
});
}
const amllFFT = useRef<PCMPlayer>();

useLayoutEffect(() => {
amllFFT.current = new PCMPlayer({
inputCodec: "Int16",
channels: 2,
sampleRate: 48000,
flushTime: 50,
});
return () => {
amllFFT.current?.destroy();
(amllFFT.current as unknown) = undefined;
};
}, []);

useLayoutEffect(() => {
const canvas = canvasRef.current;

if (canvas) {
const ctx = canvas.getContext("2d");
if (ctx) {
musicCtx?.acquireAudioData();
return () => {
musicCtx?.releaseAudioData();
};
}
}
}, [canvasRef.current]);

useLayoutEffect(() => {
if (playstate === PlayState.Playing) {
amllFFT.current?.continue();
}
}, [playstate]);

useLayoutEffect(() => {
const canvas = canvasRef.current;

if (canvas && amllFFT.current) {
const ctx = canvas.getContext("2d");
if (ctx) {
const obs = new ResizeObserver((sizes) => {
Expand All @@ -78,62 +23,19 @@ export const AudioFFTControl: FC = () => {
}
});

// const audioStream: number[] = [];
let isFFTMode = false;
let ctxFFTData: number[] = [];

const onAudioData = (evt: MusicStatusGetterEvents["audio-data"]) => {
isFFTMode = false;
amllFFT.current?.feed(evt.detail.data);
amllFFT.current?.continue();
};

const onFFTData = (evt: MusicStatusGetterEvents["fft-data"]) => {
isFFTMode = true;
ctxFFTData = evt.detail.data.map((v) => v * 50);
};

musicCtx?.addEventListener("audio-data", onAudioData);
musicCtx?.addEventListener("fft-data", onFFTData);

obs.observe(canvas);

let maxValue = 100;
let stopped = false;

const fftData = amllFFT.current.createFrequencyData();
soundProcessor.current = new SoundProcessor({
filterParams: {
sigma: 1,
radius: 1,
},
sampleRate: 48000,
fftSize: fftData.length,
outBandsQty: 61,
startFrequency: 100,
endFrequency: 14000,
aWeight: true,
});
let maxValue = 1;

console.log("onFrame");

function onFrame() {
if (!(canvas && ctx) || stopped) return;
const width = canvas.width;
const height = canvas.height;

let processed: number[];

if (isFFTMode) {
processed = ctxFFTData;
} else {
amllFFT.current?.getByteFrequencyData(fftData);
if (processBarFFT)
processed = soundProcessor.current.process(fftData);
else {
processed = soundProcessor.current.divide(fftData);
}
}
let processed = globalStore.get(fftDataAtom);

ctx.clearRect(0, 0, width, height);
{
Expand Down Expand Up @@ -170,15 +72,15 @@ export const AudioFFTControl: FC = () => {

onFrame();

musicCtx?.acquireAudioData();
return () => {
obs.disconnect();
musicCtx?.removeEventListener("audio-data", onAudioData);
musicCtx?.removeEventListener("fft-data", onFFTData);
musicCtx?.releaseAudioData();
stopped = true;
};
}
}
}, [canvasRef.current, musicCtx, amllFFT.current, processBarFFT]);
}, [canvasRef.current, musicCtx]);

return (
<canvas
Expand Down

0 comments on commit dfe5278

Please sign in to comment.