From 93fc9df1c0a3ce1d19ceaffbabe3f6a58c308ad6 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Tue, 11 Sep 2018 15:55:40 +0530 Subject: [PATCH 1/7] Add AnalyserNode --- audio/src/analyser_node.rs | 37 +++++++++++++++++++++++++++++++++++++ audio/src/lib.rs | 1 + audio/src/node.rs | 2 +- audio/src/render_thread.rs | 2 ++ 4 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 audio/src/analyser_node.rs diff --git a/audio/src/analyser_node.rs b/audio/src/analyser_node.rs new file mode 100644 index 00000000..2df9cf0c --- /dev/null +++ b/audio/src/analyser_node.rs @@ -0,0 +1,37 @@ +use block::{Block, Chunk}; +use node::AudioNodeEngine; +use node::BlockInfo; +use node::{AudioNodeType, ChannelInfo, ChannelInterpretation}; +use std::sync::mpsc::Sender; + + +#[derive(AudioNodeCommon)] +pub(crate) struct AnalyserNode { + channel_info: ChannelInfo, + sender: Sender +} + +impl AnalyserNode { + pub fn new(sender: Sender, channel_info: ChannelInfo) -> Self { + Self { sender, channel_info } + } + +} + +impl AudioNodeEngine for AnalyserNode { + fn node_type(&self) -> AudioNodeType { + AudioNodeType::AnalyserNode + } + + fn process(&mut self, inputs: Chunk, _: &BlockInfo) -> Chunk { + debug_assert!(inputs.len() == 1); + + let mut push = inputs.blocks[0].clone(); + push.mix(1, ChannelInterpretation::Speakers); + + let _ = self.sender.send(push); + + // analyser node doesn't modify the inputs + inputs + } +} diff --git a/audio/src/lib.rs b/audio/src/lib.rs index c441d33c..5e321c6c 100644 --- a/audio/src/lib.rs +++ b/audio/src/lib.rs @@ -10,6 +10,7 @@ extern crate smallvec; #[macro_use] pub mod macros; +pub mod analyser_node; pub mod block; pub mod buffer_source_node; pub mod channel_node; diff --git a/audio/src/node.rs b/audio/src/node.rs index 1128da66..d848fc78 100644 --- a/audio/src/node.rs +++ b/audio/src/node.rs @@ -11,7 +11,7 @@ use std::sync::mpsc::Sender; /// Information required to construct an audio node #[derive(Debug, Clone)] pub enum AudioNodeInit { - AnalyserNode, + AnalyserNode(Sender), BiquadFilterNode, AudioBuffer, AudioBufferSourceNode(AudioBufferSourceNodeOptions), diff --git a/audio/src/render_thread.rs b/audio/src/render_thread.rs index 573908d8..357de814 100644 --- a/audio/src/render_thread.rs +++ b/audio/src/render_thread.rs @@ -1,3 +1,4 @@ +use analyser_node::AnalyserNode; use block::{Chunk, Tick, FRAMES_PER_BLOCK}; use buffer_source_node::AudioBufferSourceNode; use channel_node::{ChannelMergerNode, ChannelSplitterNode}; @@ -129,6 +130,7 @@ impl AudioRenderThread { fn create_node(&mut self, node_type: AudioNodeInit, ch: ChannelInfo) -> NodeId { let mut needs_listener = false; let node: Box = match node_type { + AudioNodeInit::AnalyserNode(sender) => Box::new(AnalyserNode::new(sender, ch)), AudioNodeInit::AudioBufferSourceNode(options) => { Box::new(AudioBufferSourceNode::new(options, ch)) } From 55706b4c07caadb680b388994a4439ca23d73c38 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Tue, 11 Sep 2018 20:52:38 +0530 Subject: [PATCH 2/7] Basic AnalysisEngine --- audio/src/analyser_node.rs | 72 +++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/audio/src/analyser_node.rs b/audio/src/analyser_node.rs index 2df9cf0c..50fb83a8 100644 --- a/audio/src/analyser_node.rs +++ b/audio/src/analyser_node.rs @@ -1,4 +1,4 @@ -use block::{Block, Chunk}; +use block::{Block, Chunk, FRAMES_PER_BLOCK_USIZE}; use node::AudioNodeEngine; use node::BlockInfo; use node::{AudioNodeType, ChannelInfo, ChannelInterpretation}; @@ -35,3 +35,73 @@ impl AudioNodeEngine for AnalyserNode { inputs } } + +/// From https://webaudio.github.io/web-audio-api/#dom-analysernode-fftsize +pub const MAX_FFT_SIZE: usize = 32768; +pub const MAX_BLOCK_COUNT: usize = MAX_FFT_SIZE / FRAMES_PER_BLOCK_USIZE; + +/// The actual analysis is done on the DOM side. We provide +/// the actual base functionality in this struct, so the DOM +/// just has to do basic shimming +pub struct AnalysisEngine { + /// The number of past sample-frames to consider in the FFT + fft_size: usize, + /// This is a ring buffer containing the last MAX_FFT_SIZE + /// sample-frames + data: Box<[f32; MAX_FFT_SIZE]>, + /// The index of the current block + current_block: usize, + /// Have we computed the FFT already? + fft_computed: bool, +} + +impl AnalysisEngine { + pub fn new(fft_size: usize) -> Self { + Self { + fft_size, + data: Box::new([0.; MAX_FFT_SIZE]), + current_block: MAX_BLOCK_COUNT - 1, + fft_computed: false, + } + } + + fn advance(&mut self) { + self.current_block += 1; + if self.current_block >= MAX_BLOCK_COUNT { + self.current_block = 0; + } + } + + /// Wrap around the index of a block `offset` elements in the past + fn block_index(&self, offset: usize) -> usize { + debug_assert!(offset < MAX_BLOCK_COUNT); + if offset > self.current_block { + MAX_BLOCK_COUNT - offset + self.current_block + } else { + self.current_block - offset + } + } + + /// Get the data of a block. `offset` tells us how far back to go + fn block_mut(&mut self, offset: usize) -> &mut [f32] { + let index = FRAMES_PER_BLOCK_USIZE * self.block_index(offset); + &mut self.data[index..(index + FRAMES_PER_BLOCK_USIZE)] + } + + pub fn push(&mut self, mut block: Block) { + debug_assert!(block.chan_count() == 1); + self.advance(); + if !block.is_silence() { + self.block_mut(0).copy_from_slice(block.data_mut()); + } + self.fft_computed = false; + } + + fn compute_fft(&mut self) { + if self.fft_computed { + return; + } + self.fft_computed = true; + // ... + } +} From 18d7b52bbac47bddd77e87b083bf05b937142a1a Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Tue, 11 Sep 2018 22:07:37 +0530 Subject: [PATCH 3/7] Add blackman windowing --- audio/src/analyser_node.rs | 54 +++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/audio/src/analyser_node.rs b/audio/src/analyser_node.rs index 50fb83a8..01ab2004 100644 --- a/audio/src/analyser_node.rs +++ b/audio/src/analyser_node.rs @@ -2,8 +2,9 @@ use block::{Block, Chunk, FRAMES_PER_BLOCK_USIZE}; use node::AudioNodeEngine; use node::BlockInfo; use node::{AudioNodeType, ChannelInfo, ChannelInterpretation}; +use std::f32::consts::PI; use std::sync::mpsc::Sender; - +use std::mem; #[derive(AudioNodeCommon)] pub(crate) struct AnalyserNode { @@ -53,6 +54,19 @@ pub struct AnalysisEngine { current_block: usize, /// Have we computed the FFT already? fft_computed: bool, + /// Cached blackman window data + blackman_windows: Vec, + /// The computed FFT data (in frequency domain) + computed_fft_data: Vec, + + // these two vectors are for temporary buffers + // that we keep around for efficiency + + /// The windowed time domain data + /// Used during FFT computation + windowed: Vec, + /// Scratch space used during the actual FFT computation + tmp_transformed: Vec } impl AnalysisEngine { @@ -62,6 +76,10 @@ impl AnalysisEngine { data: Box::new([0.; MAX_FFT_SIZE]), current_block: MAX_BLOCK_COUNT - 1, fft_computed: false, + blackman_windows: Vec::with_capacity(fft_size), + computed_fft_data: Vec::with_capacity(fft_size / 2), + windowed: Vec::with_capacity(fft_size), + tmp_transformed: Vec::with_capacity(fft_size / 2), } } @@ -97,11 +115,45 @@ impl AnalysisEngine { self.fft_computed = false; } + /// https://webaudio.github.io/web-audio-api/#blackman-window + fn compute_blackman_windows(&mut self) { + if self.blackman_windows.len() == self.fft_size { + return; + } + const ALPHA: f32 = 0.16; + const ALPHA_0: f32 = (1. - ALPHA) / 2.; + const ALPHA_1: f32 = 1. / 2.; + const ALPHA_2: f32 = ALPHA / 2.; + self.blackman_windows.resize(self.fft_size, 0.); + let coeff = PI * 2. / self.fft_size as f32; + for n in 0..self.fft_size { + self.blackman_windows[n] = ALPHA_0 - ALPHA_1 * (coeff * n as f32).cos() + + ALPHA_2 * (2. * coeff * n as f32).cos(); + } + } + + fn apply_blackman_window(&mut self) { + self.compute_blackman_windows(); + // avoids conflicting borrows with data_mut + let mut windowed = mem::replace(&mut self.windowed, Vec::new()); + windowed.resize(self.fft_size, 0.); + let mut n = 0; + for offset in (0..self.fft_size).rev() { + let data = self.block_mut(offset); + for frame in 0..FRAMES_PER_BLOCK_USIZE { + windowed[n] = data[frame]; + n += 1; + } + } + self.windowed = windowed; + } + fn compute_fft(&mut self) { if self.fft_computed { return; } self.fft_computed = true; + self.apply_blackman_window(); // ... } } From bfd69be46425cd1904441d432d5122d6e31a8a75 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Fri, 14 Sep 2018 11:50:53 +0530 Subject: [PATCH 4/7] Finish implementing fourier transform --- audio/src/analyser_node.rs | 160 +++++++++++++++++++++++++++++++++---- 1 file changed, 144 insertions(+), 16 deletions(-) diff --git a/audio/src/analyser_node.rs b/audio/src/analyser_node.rs index 01ab2004..7cce5b78 100644 --- a/audio/src/analyser_node.rs +++ b/audio/src/analyser_node.rs @@ -4,7 +4,7 @@ use node::BlockInfo; use node::{AudioNodeType, ChannelInfo, ChannelInterpretation}; use std::f32::consts::PI; use std::sync::mpsc::Sender; -use std::mem; +use std::{cmp, mem}; #[derive(AudioNodeCommon)] pub(crate) struct AnalyserNode { @@ -47,6 +47,9 @@ pub const MAX_BLOCK_COUNT: usize = MAX_FFT_SIZE / FRAMES_PER_BLOCK_USIZE; pub struct AnalysisEngine { /// The number of past sample-frames to consider in the FFT fft_size: usize, + smoothing_constant: f64, + min_decibels: f64, + max_decibels: f64, /// This is a ring buffer containing the last MAX_FFT_SIZE /// sample-frames data: Box<[f32; MAX_FFT_SIZE]>, @@ -56,33 +59,78 @@ pub struct AnalysisEngine { fft_computed: bool, /// Cached blackman window data blackman_windows: Vec, - /// The computed FFT data (in frequency domain) + /// The smoothed FFT data (in frequency domain) + smoothed_fft_data: Vec, + /// The computed FFT data, in decibels computed_fft_data: Vec, - - // these two vectors are for temporary buffers - // that we keep around for efficiency - /// The windowed time domain data /// Used during FFT computation windowed: Vec, - /// Scratch space used during the actual FFT computation - tmp_transformed: Vec } impl AnalysisEngine { - pub fn new(fft_size: usize) -> Self { + pub fn new(fft_size: usize, smoothing_constant: f64, + min_decibels: f64, max_decibels: f64) -> Self { + debug_assert!(fft_size >= 32 && fft_size <= 32768); + // must be a power of two + debug_assert!(fft_size & fft_size - 1 == 0); + debug_assert!(smoothing_constant <= 1. && smoothing_constant >= 0.); + debug_assert!(max_decibels > min_decibels); Self { fft_size, + smoothing_constant, + min_decibels, + max_decibels, data: Box::new([0.; MAX_FFT_SIZE]), current_block: MAX_BLOCK_COUNT - 1, fft_computed: false, blackman_windows: Vec::with_capacity(fft_size), computed_fft_data: Vec::with_capacity(fft_size / 2), + smoothed_fft_data: Vec::with_capacity(fft_size / 2), windowed: Vec::with_capacity(fft_size), - tmp_transformed: Vec::with_capacity(fft_size / 2), } } + pub fn set_fft_size(&mut self, fft_size: usize) { + debug_assert!(fft_size >= 32 && fft_size <= 32768); + // must be a power of two + debug_assert!(fft_size & fft_size - 1 == 0); + self.fft_size = fft_size; + self.fft_computed = false; + } + + pub fn get_fft_size(&self) -> usize { + self.fft_size + } + + pub fn set_smoothing_constant(&mut self, smoothing_constant: f64) { + debug_assert!(smoothing_constant <= 1. && smoothing_constant >= 0.); + self.smoothing_constant = smoothing_constant; + self.fft_computed = false; + } + + pub fn get_smoothing_constant(&self) -> f64 { + self.smoothing_constant + } + + pub fn set_min_decibels(&mut self, min_decibels: f64) { + debug_assert!(min_decibels < self.max_decibels); + self.min_decibels = min_decibels; + } + + pub fn get_min_decibels(&self) -> f64 { + self.min_decibels + } + + pub fn set_max_decibels(&mut self, max_decibels: f64) { + debug_assert!(self.min_decibels < max_decibels); + self.max_decibels = max_decibels; + } + + pub fn get_max_decibels(&self) -> f64 { + self.max_decibels + } + fn advance(&mut self) { self.current_block += 1; if self.current_block >= MAX_BLOCK_COUNT { @@ -106,6 +154,12 @@ impl AnalysisEngine { &mut self.data[index..(index + FRAMES_PER_BLOCK_USIZE)] } + /// Get the data of a block. `offset` tells us how far back to go + fn block(&self, offset: usize) -> &[f32] { + let index = FRAMES_PER_BLOCK_USIZE * self.block_index(offset); + &self.data[index..(index + FRAMES_PER_BLOCK_USIZE)] + } + pub fn push(&mut self, mut block: Block) { debug_assert!(block.chan_count() == 1); self.advance(); @@ -139,11 +193,9 @@ impl AnalysisEngine { windowed.resize(self.fft_size, 0.); let mut n = 0; for offset in (0..self.fft_size).rev() { - let data = self.block_mut(offset); - for frame in 0..FRAMES_PER_BLOCK_USIZE { - windowed[n] = data[frame]; - n += 1; - } + let data = self.block(offset); + windowed[n..n+FRAMES_PER_BLOCK_USIZE].copy_from_slice(&data); + n += FRAMES_PER_BLOCK_USIZE; } self.windowed = windowed; } @@ -154,6 +206,82 @@ impl AnalysisEngine { } self.fft_computed = true; self.apply_blackman_window(); - // ... + self.computed_fft_data.resize(self.fft_size / 2, 0.); + self.smoothed_fft_data.resize(self.fft_size / 2, 0.); + + for k in 0..(self.fft_size / 2) { + let mut sum_real = 0.; + let mut sum_imaginary = 0.; + let factor = - 2. * PI * k as f32 / self.fft_size as f32; + for n in 0..(self.fft_size) { + sum_real += self.windowed[n] * (factor * n as f32).cos(); + sum_imaginary += self.windowed[n] * (factor * n as f32).sin(); + } + let sum_real = sum_real / self.fft_size as f32; + let sum_imaginary = sum_imaginary / self.fft_size as f32; + let magnitude = (sum_real * sum_real + sum_imaginary * sum_imaginary).sqrt(); + self.smoothed_fft_data[k] = (self.smoothing_constant * self.smoothed_fft_data[k] as f64 + + (1. - self.smoothing_constant) * magnitude as f64) as f32; + self.computed_fft_data[k] = 20. * self.smoothed_fft_data[k].log(10.); + } + } + + pub fn fill_time_domain_data(&self, dest: &mut [f32]) { + let mut n = 0; + for offset in (0..self.fft_size).rev() { + let data = self.block(offset); + let mut end = n + FRAMES_PER_BLOCK_USIZE; + if n >= dest.len() { + break; + } else if end > dest.len() { + end = dest.len(); + }; + let offset = end - n; + dest[n..end].copy_from_slice(&data[0..offset]); + + n += FRAMES_PER_BLOCK_USIZE; + } + } + + pub fn fill_byte_time_domain_data(&self, dest: &mut [u8]) { + let mut n = 0; + for offset in (0..self.fft_size).rev() { + let data = self.block(offset); + if n >= dest.len() { + break; + } + let end = cmp::min(FRAMES_PER_BLOCK_USIZE, dest.len() - n - FRAMES_PER_BLOCK_USIZE); + for frame in 0..end { + let result = 128. * (1. + data[frame]); + dest[n] = clamp_255(result); + n += 1; + } + } + } + + pub fn fill_frequency_data(&mut self, dest: &mut [f32]) { + self.compute_fft(); + let len = cmp::min(dest.len(), self.computed_fft_data.len()); + dest[0..len].copy_from_slice(&mut self.computed_fft_data[0..len]); + } + + pub fn fill_byte_frequency_data(&mut self, dest: &mut [u8]) { + self.compute_fft(); + let len = cmp::min(dest.len(), self.computed_fft_data.len()); + let ratio = 255. / (self.max_decibels - self.min_decibels); + for freq in 0..len { + let result = ratio * (self.computed_fft_data[freq] as f64 - self.min_decibels); + dest[freq] = clamp_255(result as f32); + } } } + +fn clamp_255(val: f32) -> u8 { + if val > 255. { + 255 + } else if val < 0. { + 0 + } else { + val as u8 + } +} \ No newline at end of file From fe882349dd84b705e4729a312891707be0cdae4b Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Fri, 14 Sep 2018 12:22:52 +0530 Subject: [PATCH 5/7] Use a callback function --- audio/src/analyser_node.rs | 9 ++++----- audio/src/node.rs | 3 +-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/audio/src/analyser_node.rs b/audio/src/analyser_node.rs index 7cce5b78..ee9c9fe0 100644 --- a/audio/src/analyser_node.rs +++ b/audio/src/analyser_node.rs @@ -3,18 +3,17 @@ use node::AudioNodeEngine; use node::BlockInfo; use node::{AudioNodeType, ChannelInfo, ChannelInterpretation}; use std::f32::consts::PI; -use std::sync::mpsc::Sender; use std::{cmp, mem}; #[derive(AudioNodeCommon)] pub(crate) struct AnalyserNode { channel_info: ChannelInfo, - sender: Sender + callback: Box, } impl AnalyserNode { - pub fn new(sender: Sender, channel_info: ChannelInfo) -> Self { - Self { sender, channel_info } + pub fn new(callback: Box, channel_info: ChannelInfo) -> Self { + Self { callback, channel_info } } } @@ -30,7 +29,7 @@ impl AudioNodeEngine for AnalyserNode { let mut push = inputs.blocks[0].clone(); push.mix(1, ChannelInterpretation::Speakers); - let _ = self.sender.send(push); + (self.callback)(push); // analyser node doesn't modify the inputs inputs diff --git a/audio/src/node.rs b/audio/src/node.rs index d848fc78..1b9b3908 100644 --- a/audio/src/node.rs +++ b/audio/src/node.rs @@ -9,9 +9,8 @@ use param::{Param, ParamRate, ParamType, UserAutomationEvent}; use std::sync::mpsc::Sender; /// Information required to construct an audio node -#[derive(Debug, Clone)] pub enum AudioNodeInit { - AnalyserNode(Sender), + AnalyserNode(Box), BiquadFilterNode, AudioBuffer, AudioBufferSourceNode(AudioBufferSourceNodeOptions), From 7f4761df3cbf54a34fa7bc991f5c886ca096b11c Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Fri, 14 Sep 2018 14:14:49 +0530 Subject: [PATCH 6/7] Add serde impls for Block --- Cargo.lock | 2 + audio/Cargo.toml | 2 + audio/src/analyser_node.rs | 77 ++++++++++++++++++-------------------- audio/src/block.rs | 2 +- audio/src/lib.rs | 3 ++ 5 files changed, 45 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0f9ab2cb..5d712bc9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1214,6 +1214,8 @@ dependencies = [ "euclid 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", "servo_media_derive 0.1.0", "smallvec 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/audio/Cargo.toml b/audio/Cargo.toml index e9826b48..1740bb6e 100644 --- a/audio/Cargo.toml +++ b/audio/Cargo.toml @@ -10,6 +10,8 @@ name = "servo_media_audio" [dependencies] boxfnonce = "0.1" euclid = "0.19.0" +serde_derive = "1.0.66" +serde = "1.0.66" servo_media_derive = { path = "../servo-media-derive" } smallvec = "0.6.1" diff --git a/audio/src/analyser_node.rs b/audio/src/analyser_node.rs index ee9c9fe0..e9969891 100644 --- a/audio/src/analyser_node.rs +++ b/audio/src/analyser_node.rs @@ -3,7 +3,7 @@ use node::AudioNodeEngine; use node::BlockInfo; use node::{AudioNodeType, ChannelInfo, ChannelInterpretation}; use std::f32::consts::PI; -use std::{cmp, mem}; +use std::cmp; #[derive(AudioNodeCommon)] pub(crate) struct AnalyserNode { @@ -153,10 +153,24 @@ impl AnalysisEngine { &mut self.data[index..(index + FRAMES_PER_BLOCK_USIZE)] } - /// Get the data of a block. `offset` tells us how far back to go - fn block(&self, offset: usize) -> &[f32] { - let index = FRAMES_PER_BLOCK_USIZE * self.block_index(offset); - &self.data[index..(index + FRAMES_PER_BLOCK_USIZE)] + /// Given an index from 0 to fft_size, convert it into an index into + /// the backing array + fn convert_index(&self, index: usize) -> usize { + let offset = self.fft_size - index; + let current_block_idx = self.current_block * FRAMES_PER_BLOCK_USIZE; + if offset > current_block_idx { + MAX_FFT_SIZE - offset + current_block_idx + } else { + current_block_idx - offset + } + } + + /// Given an index into the backing array, increment it + fn advance_index(&self, index: &mut usize) { + *index += 1; + if *index >= MAX_FFT_SIZE { + *index = 0; + } } pub fn push(&mut self, mut block: Block) { @@ -187,16 +201,13 @@ impl AnalysisEngine { fn apply_blackman_window(&mut self) { self.compute_blackman_windows(); - // avoids conflicting borrows with data_mut - let mut windowed = mem::replace(&mut self.windowed, Vec::new()); - windowed.resize(self.fft_size, 0.); - let mut n = 0; - for offset in (0..self.fft_size).rev() { - let data = self.block(offset); - windowed[n..n+FRAMES_PER_BLOCK_USIZE].copy_from_slice(&data); - n += FRAMES_PER_BLOCK_USIZE; + self.windowed.resize(self.fft_size, 0.); + + let mut data_idx = self.convert_index(0); + for n in 0..self.fft_size { + self.windowed[n] = self.blackman_windows[n] * self.data[data_idx]; + self.advance_index(&mut data_idx); } - self.windowed = windowed; } fn compute_fft(&mut self) { @@ -226,35 +237,21 @@ impl AnalysisEngine { } pub fn fill_time_domain_data(&self, dest: &mut [f32]) { - let mut n = 0; - for offset in (0..self.fft_size).rev() { - let data = self.block(offset); - let mut end = n + FRAMES_PER_BLOCK_USIZE; - if n >= dest.len() { - break; - } else if end > dest.len() { - end = dest.len(); - }; - let offset = end - n; - dest[n..end].copy_from_slice(&data[0..offset]); - - n += FRAMES_PER_BLOCK_USIZE; + let mut data_idx = self.convert_index(0); + let end = cmp::min(self.fft_size, dest.len()); + for n in 0..end { + dest[n] = self.data[data_idx]; + self.advance_index(&mut data_idx); } } pub fn fill_byte_time_domain_data(&self, dest: &mut [u8]) { - let mut n = 0; - for offset in (0..self.fft_size).rev() { - let data = self.block(offset); - if n >= dest.len() { - break; - } - let end = cmp::min(FRAMES_PER_BLOCK_USIZE, dest.len() - n - FRAMES_PER_BLOCK_USIZE); - for frame in 0..end { - let result = 128. * (1. + data[frame]); - dest[n] = clamp_255(result); - n += 1; - } + let mut data_idx = self.convert_index(0); + let end = cmp::min(self.fft_size, dest.len()); + for n in 0..end { + let result = 128. * (1. + self.data[data_idx]); + dest[n] = clamp_255(result); + self.advance_index(&mut data_idx) } } @@ -283,4 +280,4 @@ fn clamp_255(val: f32) -> u8 { } else { val as u8 } -} \ No newline at end of file +} diff --git a/audio/src/block.rs b/audio/src/block.rs index fddeb908..6f41fd56 100644 --- a/audio/src/block.rs +++ b/audio/src/block.rs @@ -45,7 +45,7 @@ impl Chunk { /// We render audio in blocks of size FRAMES_PER_BLOCK /// /// A single block may contain multiple channels -#[derive(Clone)] +#[derive(Clone, Serialize, Deserialize)] pub struct Block { /// The number of channels in this block channels: u8, diff --git a/audio/src/lib.rs b/audio/src/lib.rs index 5e321c6c..20d2d2c0 100644 --- a/audio/src/lib.rs +++ b/audio/src/lib.rs @@ -1,3 +1,6 @@ +#[macro_use] +extern crate serde_derive; + #[macro_use] extern crate servo_media_derive; From 92e285e5557ae8bce40ceb6a74af5518ba1935af Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Fri, 14 Sep 2018 18:11:42 +0530 Subject: [PATCH 7/7] Fix ring buffer calculations --- audio/src/analyser_node.rs | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/audio/src/analyser_node.rs b/audio/src/analyser_node.rs index e9969891..45322ecc 100644 --- a/audio/src/analyser_node.rs +++ b/audio/src/analyser_node.rs @@ -137,19 +137,9 @@ impl AnalysisEngine { } } - /// Wrap around the index of a block `offset` elements in the past - fn block_index(&self, offset: usize) -> usize { - debug_assert!(offset < MAX_BLOCK_COUNT); - if offset > self.current_block { - MAX_BLOCK_COUNT - offset + self.current_block - } else { - self.current_block - offset - } - } - - /// Get the data of a block. `offset` tells us how far back to go - fn block_mut(&mut self, offset: usize) -> &mut [f32] { - let index = FRAMES_PER_BLOCK_USIZE * self.block_index(offset); + /// Get the data of the current block + fn curent_block_mut(&mut self) -> &mut [f32] { + let index = FRAMES_PER_BLOCK_USIZE * self.current_block; &mut self.data[index..(index + FRAMES_PER_BLOCK_USIZE)] } @@ -157,11 +147,11 @@ impl AnalysisEngine { /// the backing array fn convert_index(&self, index: usize) -> usize { let offset = self.fft_size - index; - let current_block_idx = self.current_block * FRAMES_PER_BLOCK_USIZE; - if offset > current_block_idx { - MAX_FFT_SIZE - offset + current_block_idx + let last_element = (1 + self.current_block) * FRAMES_PER_BLOCK_USIZE - 1; + if offset > last_element { + MAX_FFT_SIZE - offset + last_element } else { - current_block_idx - offset + last_element - offset } } @@ -177,7 +167,7 @@ impl AnalysisEngine { debug_assert!(block.chan_count() == 1); self.advance(); if !block.is_silence() { - self.block_mut(0).copy_from_slice(block.data_mut()); + self.curent_block_mut().copy_from_slice(block.data_mut()); } self.fft_computed = false; }