Skip to content

Commit

Permalink
two channel experimentation
Browse files Browse the repository at this point in the history
  • Loading branch information
JonnyBurger committed May 27, 2022
1 parent 479673b commit 75ddfce
Show file tree
Hide file tree
Showing 9 changed files with 181 additions and 38 deletions.
1 change: 1 addition & 0 deletions packages/cli/src/editor/components/AudioWaveform.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ export const AudioWaveform: React.FC<{
startTimeInSeconds: startFrom / fps,
durationInSeconds: durationInFrames / fps,
numberOfSamples,
channel: 0,
outputRange: 'zero-to-one',
});
}, [durationInFrames, fps, metadata, startFrom, visualizationWidth]);
Expand Down
1 change: 1 addition & 0 deletions packages/example/src/AudioVisualization/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ const AudioVisualization: React.FC = () => {
audioData: audioDataVoice,
numberOfSamples: WAVEFORM_SAMPLES,
windowInSeconds: 1 / fps,
channel: 0,
});
console.log({waveform});

Expand Down
30 changes: 21 additions & 9 deletions packages/example/src/Video.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import {VideoOnCanvas} from './VideoOnCanvas';
import {Greenscreen} from './VideoOnCanvas/greenscreen';
import {VideoSpeed} from './VideoSpeed';
import {VideoTesting} from './VideoTesting';
import {TwoChannelVoiceVisualization} from './voice-visualization/two-channel';

// Use it to test that UI does not regress on weird CSS
// import './weird-css.css';
Expand Down Expand Up @@ -351,15 +352,26 @@ export const Index: React.FC = () => {
fps={30}
durationInFrames={180 * 30}
/>
<Composition
id="voice-visualization"
lazyComponent={() => import('./voice-visualization')}
width={1080}
height={1080}
fps={30}
durationInFrames={180 * 30}
defaultProps={{}}
/>
<Folder name="VoiceVisualization">
<Composition
id="rolling-waveform"
lazyComponent={() => import('./voice-visualization')}
width={1080}
height={1080}
fps={30}
durationInFrames={30 * 30}
defaultProps={{}}
/>
<Composition
id="twochannel"
component={TwoChannelVoiceVisualization}
width={1080}
height={1080}
fps={30}
durationInFrames={30 * 30}
defaultProps={{}}
/>
</Folder>
</Folder>
<Folder name="three">
<Still id="Orb" component={OrbScene} width={2000} height={2000} />
Expand Down
36 changes: 14 additions & 22 deletions packages/example/src/voice-visualization/index.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import {
createSmoothSvgPath,
smoothenSvgPath,
useAudioData,
visualizeAudioWaveform,
} from '@remotion/media-utils';
Expand All @@ -14,13 +14,6 @@ const FullSize = styled(AbsoluteFill)`
align-items: center;
`;

const Orb = styled.div`
display: flex;
justify-content: center;
align-items: center;
flex: 1;
`;

const VoiceVisualization: React.FC = () => {
const frame = useCurrentFrame();
const {fps} = useVideoConfig();
Expand All @@ -37,15 +30,16 @@ const VoiceVisualization: React.FC = () => {
fps,
frame: posterized,
audioData: audioDataVoice,
numberOfSamples: 64,
windowInSeconds: 0.5,
numberOfSamples: 16,
windowInSeconds: 1 / fps,
channel: 0,
});

const p = createSmoothSvgPath(
const p = smoothenSvgPath(
waveform.map((y, i) => {
return [
(i / (waveform.length - 1)) * width,
height / 2 + ((y * height) / 2) * 5,
height / 2 + (y * height) / 2,
];
})
);
Expand All @@ -54,16 +48,14 @@ const VoiceVisualization: React.FC = () => {
<div style={{flex: 1}}>
<Audio src={voice} />
<FullSize>
<Orb>
<svg
style={{backgroundColor: 'white'}}
viewBox={`0 0 ${width} ${height}`}
width={width}
height={height}
>
<path fill="none" stroke="#0b84f3" strokeWidth={10} d={p} />
</svg>
</Orb>
<svg
style={{backgroundColor: 'white'}}
viewBox={`0 0 ${width} ${height}`}
width={width}
height={height}
>
<path fill="none" stroke="#0b84f3" strokeWidth={10} d={p} />
</svg>
</FullSize>
</div>
);
Expand Down
105 changes: 105 additions & 0 deletions packages/example/src/voice-visualization/two-channel.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import {
smoothenSvgPath,
useAudioData,
visualizeAudioWaveform,
} from '@remotion/media-utils';
import React from 'react';
import {AbsoluteFill, Audio, useCurrentFrame, useVideoConfig} from 'remotion';
import styled from 'styled-components';
import voice from '../resources/voice-short.mp3';

const FullSize = styled(AbsoluteFill)`
display: flex;
justify-content: center;
align-items: center;
`;

const makeDuoform = (form: number[], width: number, height: number) => {
const p2Path = form.map((y, i) => {
return [
(i / (form.length - 1)) * width,
height / 2 + (Math.max(0.015, Math.abs(y)) * height) / 2,
] as [number, number];
});
const p2PathReverse = form
.map((y, i) => {
return [
(i / (form.length - 1)) * width,
height / 2 - (Math.max(0.015, Math.abs(y)) * height) / 2,
] as [number, number];
})
.reverse();

const p2 = smoothenSvgPath(p2Path);
const p2Reverse = smoothenSvgPath(p2PathReverse);

const joined = [
p2,
`L ${p2PathReverse[0][0]} ${p2PathReverse[0][1]}`,
p2Reverse,
`L ${p2Path[0][0]} ${p2Path[0][1]}`,
].join(' ');

return joined;
};

export const TwoChannelVoiceVisualization: React.FC = () => {
const frame = useCurrentFrame();
const {fps} = useVideoConfig();
const audioDataVoice = useAudioData(voice);
const {width, height} = useVideoConfig();

if (!audioDataVoice) {
return null;
}

const posterized = Math.round(frame / 3) * 3;

const waveform = visualizeAudioWaveform({
fps,
frame: posterized + 0.5 * fps,
audioData: audioDataVoice,
numberOfSamples: 16,
windowInSeconds: 1,
channel: 0,
});
const waveform2 = visualizeAudioWaveform({
fps,
frame: posterized,
audioData: audioDataVoice,
numberOfSamples: 16,
windowInSeconds: 1,
channel: 1,
});

const joined = makeDuoform(waveform, width, height);
const joined2 = makeDuoform(waveform2, width, height);

return (
<div style={{flex: 1, backgroundColor: 'white'}}>
<Audio src={voice} />
<FullSize style={{opacity: 1}}>
<svg viewBox={`0 0 ${width} ${height}`} width={width} height={height}>
<defs>
<linearGradient id="myGradient2" gradientTransform="rotate(90)">
<stop offset="0%" stopColor="#adffff" />
<stop offset="100%" stopColor="rgb(173, 255, 255)" />
</linearGradient>
</defs>
<path fill="url(#myGradient2)" fillRule="nonzero" d={joined} />
</svg>
</FullSize>
<FullSize style={{opacity: 0.5, mixBlendMode: 'exclusion'}}>
<svg viewBox={`0 0 ${width} ${height}`} width={width} height={height}>
<defs>
<linearGradient id="myGradient" gradientTransform="rotate(90)">
<stop offset="5%" stopColor="#ff0000" />
<stop offset="95%" stopColor="#ff7d19" />
</linearGradient>
</defs>
<path fill="url(#myGradient)" fillRule="nonzero" d={joined2} />
</svg>
</FullSize>{' '}
</div>
);
};
10 changes: 5 additions & 5 deletions packages/media-utils/src/get-wave-form-samples.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,13 @@ export const getWaveformSamples = ({
const blockStart = blockSize * i; // the location of the first sample in the block
let sum = 0;
for (let j = 0; j < blockSize; j++) {
sum +=
outputRange === 'minus-one-to-one'
? audioBuffer[blockStart + j]
: Math.abs(audioBuffer[blockStart + j]); // find the sum of all the samples in the block
sum += Math.abs(audioBuffer[blockStart + j]); // find the sum of all the samples in the block
}

filteredData.push(sum / blockSize); // divide the sum by the block size to get the average
filteredData.push(
(sum / blockSize) *
(i % 2 === 0 && outputRange === 'minus-one-to-one' ? -1 : 1)
); // divide the sum by the block size to get the average
}

return filteredData;
Expand Down
8 changes: 6 additions & 2 deletions packages/media-utils/src/get-waveform-portion.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import {getWaveformSamples, SampleOutputRange} from './get-wave-form-samples';
import {AudioData} from './types';
import {validateChannel} from './validate-channel';

type Bar = {
index: number;
Expand Down Expand Up @@ -34,15 +35,19 @@ const getWaveformPortion = ({
startTimeInSeconds,
durationInSeconds,
numberOfSamples,
channel,
outputRange = 'zero-to-one',
}: {
audioData: AudioData;
startTimeInSeconds: number;
durationInSeconds: number;
numberOfSamples: number;
channel: number;
outputRange?: SampleOutputRange;
}): Bar[] => {
const waveform = audioData.channelWaveforms[0];
validateChannel(channel, audioData.numberOfChannels);

const waveform = audioData.channelWaveforms[channel];

const startSample = Math.floor(
(startTimeInSeconds / audioData.durationInSeconds) * waveform.length
Expand All @@ -64,7 +69,6 @@ const getWaveformPortion = ({
: null;
const padEnd =
samplesAfterEnd > 0 ? new Float32Array(samplesAfterEnd).fill(0) : null;
console.log({samplesBeforeStart, samplesAfterEnd});
const arrs = [
padStart,
waveform.slice(clampedStart, clampedEnd),
Expand Down
25 changes: 25 additions & 0 deletions packages/media-utils/src/validate-channel.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
export const validateChannel = (channel: unknown, numberOfChannels: number) => {
if (typeof channel !== 'number') {
throw new TypeError(`"channel" must be a number`);
}

if (channel % 1 !== 0) {
throw new TypeError(`"channel" must an integer, got ${channel}`);
}

if (Number.isNaN(channel)) {
throw new TypeError(`The channel parameter is NaN.`);
}

if (channel < 0) {
throw new TypeError('"channel" cannot be negative');
}

if (channel > numberOfChannels - 1) {
throw new TypeError(
`"channel" must be ${
numberOfChannels - 1
} or lower. The audio has ${numberOfChannels} channels`
);
}
};
3 changes: 3 additions & 0 deletions packages/media-utils/src/visualize-audio-waveform.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ type FnParameters = {
fps: number;
windowInSeconds: number;
numberOfSamples: number;
channel: number;
};

const visualizeAudioWaveformFrame = ({
Expand All @@ -17,6 +18,7 @@ const visualizeAudioWaveformFrame = ({
fps,
numberOfSamples,
windowInSeconds,
channel,
}: FnParameters) => {
if (windowInSeconds * audioData.sampleRate < numberOfSamples) {
throw new TypeError(
Expand All @@ -43,6 +45,7 @@ const visualizeAudioWaveformFrame = ({
durationInSeconds: windowInSeconds,
numberOfSamples,
outputRange: 'minus-one-to-one',
channel,
});
};

Expand Down

0 comments on commit 75ddfce

Please sign in to comment.