Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add sinc_resampler #329

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions livekit/src/audio/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
pub mod ring_buffer;
pub mod sinc_resampler;
124 changes: 124 additions & 0 deletions livekit/src/audio/ring_buffer.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
pub struct Fixed<T: Default + Copy> {
data: Box<[T]>,
write_pos: usize,
read_pos: usize,
same_wrap: bool,
}

impl<T: Default + Copy> Fixed<T> {
pub fn new(len: usize) -> Self {
Self {
data: vec![T::default(); len].into_boxed_slice(),
write_pos: 0,
read_pos: 0,
same_wrap: true,
}
}

pub fn write(&mut self, data: &[T]) -> usize {
let free = self.available_write();
let write = std::cmp::min(free, data.len());
let margin = self.data.len() - self.write_pos;

let mut n = write;
if write > margin {
self.data[0..margin].copy_from_slice(&data[0..margin]);
self.write_pos = 0;
self.same_wrap = false;
n -= margin;
}

self.write_pos += n;
write
}

pub fn read(&mut self, len: usize, dst: &mut [T]) -> usize {
let mut index1 = 0;
let mut len1 = 0;
let mut index2 = 0;
let mut len2 = 0;
let read = self.read_regions(len, &mut index1, &mut len1, &mut index2, &mut len2);
self.move_read_ptr(read);

if len2 > 0 {
// borrow from dst
dst[0..len1].clone_from_slice(&self.data[index1..index1 + len1]);
dst[len1..len].clone_from_slice(&self.data[index2..index2 + len2]);
} else {
// borrow from self.data
dst[0..len].clone_from_slice(&self.data[index1..index1 + len1]);
}

read
}

fn move_read_ptr(&mut self, mut len: usize) -> usize {
let free = self.available_write();
let read = self.available_read();

if len > read {
len = read;
}

if len > free {
len = free;
}

let mut read_pos = self.read_pos as isize;
let data_len = self.data.len() as isize;
read_pos += len as isize;
if read_pos >= data_len {
read_pos -= data_len;
self.same_wrap = true;
}

if read_pos < 0 {
read_pos += data_len;
self.same_wrap = false;
}

self.read_pos = read_pos as usize;
len
}

fn read_regions(
&self,
len: usize,
index1: &mut usize,
len1: &mut usize,
index2: &mut usize,
len2: &mut usize,
) -> usize {
let readable = self.available_read();
let read = std::cmp::min(readable, len);
let margin = self.data.len() - self.read_pos;

if read > margin {
// Data is not contiguous
*index1 = self.read_pos;
*len1 = margin;
*index2 = 0;
*len2 = read - margin;
} else {
// Data is contiguous
*index1 = self.read_pos;
*len1 = read;
*index2 = 0;
*len2 = 0;
}

read
}

fn available_read(&self) -> usize {
if self.same_wrap {
self.write_pos - self.read_pos
} else {
self.data.len() - self.read_pos + self.write_pos
}
}

fn available_write(&self) -> usize {
self.data.len() - self.write_pos
}
}
217 changes: 217 additions & 0 deletions livekit/src/audio/sinc_resampler.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
use std::{f32, f64};

// Implementation based on https://chromium.googlesource.com/chromium/src.git/+/refs/heads/main/media/base/sinc_resampler.cc

#[derive(Debug)]
pub struct SincResampler {
kernel_storage: Box<[f32]>,
kernel_pre_sinc_storage: Box<[f32]>,
kernel_window_storage: Box<[f32]>,
input_buffer: Box<[f32]>,
kernel_size: usize,
io_sample_rate_ratio: f64,
request_frames: usize,
}

impl SincResampler {
const MAX_KERNEL_SIZE: usize = 64;
const MIN_KERNEL_SIZE: usize = 32;
const DEFAULT_REQUEST_SIZE: usize = 512;
const SMALL_REQUEST_SIZE: usize = Self::MAX_KERNEL_SIZE * 2;
const KERNEL_OFFSET_COUNT: usize = 32;

// Blackman window parameters
// See https://en.wikipedia.org/wiki/Window_function
const A0: f64 = 0.42;
const A1: f64 = 0.5;
const A2: f64 = 0.08;

pub fn new(io_sample_rate_ratio: f64, request_frames: usize) -> Self {
let kernel_size = kernel_size_from_request_frames(request_frames);
let kernel_storage_size = kernel_size * (Self::KERNEL_OFFSET_COUNT + 1);
let input_buffer_size = request_frames + kernel_size;

// Initialize kernel
let mut kernel_storage = vec![0.0; kernel_storage_size].into_boxed_slice();
let mut kernel_pre_sinc_storage = vec![0.0; kernel_storage_size].into_boxed_slice();
let mut kernel_window_storage = vec![0.0; kernel_storage_size].into_boxed_slice();
let input_buffer = vec![0.0; input_buffer_size].into_boxed_slice();

let sinc_scale_factor = sinc_scale_factor(io_sample_rate_ratio, kernel_size);
for offset_idx in 0..Self::KERNEL_OFFSET_COUNT + 1 {
let subsample_offset = offset_idx as f32 / Self::KERNEL_OFFSET_COUNT as f32;
for i in 0..kernel_size {
let idx = i + offset_idx * kernel_size;
let pre_sinc =
f32::consts::PI * (i as f32 - kernel_size as f32 / 2.0 - subsample_offset);

kernel_pre_sinc_storage[idx] = pre_sinc;

// Blackman window
let x = (i as f64 - subsample_offset as f64) / kernel_size as f64;
let window = (Self::A0 - Self::A1 * (2.0 * f64::consts::PI * x).cos()
+ Self::A2 * (4.0 * f64::consts::PI * x).cos())
as f32;
kernel_window_storage[idx] = window;

// Compute the sinc with offset and the window
let a = if pre_sinc != 0.0 {
(sinc_scale_factor as f32 * pre_sinc).sin() / pre_sinc
} else {
sinc_scale_factor as f32
};
kernel_storage[idx] = a;
}
}

Self {
kernel_storage,
kernel_pre_sinc_storage,
kernel_window_storage,
input_buffer,
kernel_size,
io_sample_rate_ratio,
request_frames,
}
}

pub fn update_regions(&mut self, second_load: bool) {
self.r0 = if second_load { self.kernel_size } else { self.kernel_size / 2 };
self.r3 = self.r0 + self.request_frames - self.kernel_size;
self.r4 = self.r0 + self.request_frames - self.kernel_size / 2;

self.block_size = self.r4 - self.r2;
self.chunk_size = calculate_chunk_size(self.block_size, self.io_sample_rate_ratio);
}

pub fn resample(&mut self, dst: &mut [f32]) {
let dst_len = (self.request_frames as f64 / self.io_sample_rate_ratio) as usize;

for x in 0..dst_len {
let virtual_index = x as f64 * self.io_sample_rate_ratio;
let virtual_offset =
(virtual_index - virtual_index.floor()) * Self::KERNEL_OFFSET_COUNT as f64;

let offset = virtual_offset as usize; // subsample kernel index

let k1 = offset * self.kernel_size;
let k2 = k1 + self.kernel_size; // End of the subsample kernel

let input_index = virtual_index as usize;
let kernel_interpolation_factor = virtual_offset - offset as f64;

dst[x] = convolve(
self.kernel_size,
&self.input_buffer[input_index..],
&self.kernel_storage[k1..k2],
&self.kernel_storage[k2..k2 + self.kernel_size],
kernel_interpolation_factor,
);
}
}

pub fn set_ratio(&mut self, io_sample_rate_ratio: f64) {
if (self.io_sample_rate_ratio - io_sample_rate_ratio).abs() < f64::EPSILON {
return;
}

self.io_sample_rate_ratio = io_sample_rate_ratio;
let sinc_scale_factor = sinc_scale_factor(io_sample_rate_ratio, self.kernel_size);
}

/*pub fn prime_with_silence(&mut self) {
self.update_regions(true);
}

pub fn flush(&mut self) {
self.buffer_primed = false;
self.virtual_source_idx = 0.0;
self.input_buffer.fill(0.0);
self.update_regions(false);
}

pub fn max_input_frames_requested(&self, output_frames_requested: usize) -> usize {
let num_chunks: usize =
(output_frames_requested as f32 / self.chunk_size as f32).ceil() as usize;
num_chunks * self.request_frames
}*/

pub fn kernel_size(&self) -> usize {
self.kernel_size
}
}

// TODO(theomonnom): SIMD implementation
fn convolve(
kernel_size: usize,
input: &[f32],
k1: &[f32],
k2: &[f32],
kernel_interpolation_factor: f64,
) -> f32 {
let mut sum1 = 0.0_f32;
let mut sum2 = 0.0_f32;

for i in 0..kernel_size {
sum1 += input[i] * k1[i];
sum2 += input[i] * k2[i];
}

((1.0 - kernel_interpolation_factor) * sum1 as f64 + kernel_interpolation_factor * sum2 as f64)
as f32
}

fn sinc_scale_factor(io_ratio: f64, kernel_size: usize) -> f64 {
let mut sinc_scale_factor = if io_ratio > 1.0 { 1.0 / io_ratio } else { 1.0 };

if kernel_size == SincResampler::MAX_KERNEL_SIZE {
sinc_scale_factor *= 0.92;
} else if kernel_size == SincResampler::MIN_KERNEL_SIZE {
sinc_scale_factor *= 0.90;
}

sinc_scale_factor
}

fn kernel_size_from_request_frames(request_frames: usize) -> usize {
const SMALL_KERNEL_LIMIT: usize = SincResampler::MAX_KERNEL_SIZE * 3 / 2;
if request_frames <= SMALL_KERNEL_LIMIT {
SincResampler::MIN_KERNEL_SIZE
} else {
SincResampler::MAX_KERNEL_SIZE
}
}

#[cfg(test)]
mod tests {
use super::*;
use std::fs::File;
use std::io::Write;

// Write the kernel buffers into a CSV file for plotting.
#[test]
pub fn plot_kernels() {
let mut file = File::create("plot_kernels.csv").unwrap();
let resampler = SincResampler::new(1.0, SincResampler::DEFAULT_REQUEST_SIZE);

writeln!(file, "x, kernel_pre_sinc_storage, kernel_window_storage, kernel_storage")
.unwrap();

for x in 0..resampler.kernel_storage.len() {
// Convert each float to a string with commas instead of dots
let kernel_pre_sinc_storage = resampler.kernel_pre_sinc_storage[x];
let kernel_window_storage = resampler.kernel_window_storage[x];
let kernel_storage = resampler.kernel_storage[x];

writeln!(
file,
"\"{}\",\"{}\",\"{}\",\"{}\"",
x,
kernel_pre_sinc_storage.to_string().replace(".", ","),
kernel_window_storage.to_string().replace(".", ","),
kernel_storage.to_string().replace(".", ","),
)
.unwrap();
}
}
}
1 change: 1 addition & 0 deletions livekit/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

pub mod audio;
pub mod proto;
mod room;
mod rtc_engine;
Expand Down
Loading