Do low pass filtering by sample to smooth out pitch wobbling

Source/Core/AudioCommon/Mixer.cpp

@@ -63,22 +63,18 @@ void Mixer::DoState(PointerWrap& p)
 unsigned int Mixer::MixerFifo::Mix(s16* samples, unsigned int num_samples,
                                    bool consider_frame_limit, float audio_speed)
 {
-  // Cache access in non-volatile variable
-  // This is the only function changing the read value,
-  // so it's safe to cache and write to it locally.
-  // The writing pointer will only increase, so we can
-  // safely ignore new written data while interpolating.
-  // Without this cache, the compiler wouldn't be allowed
-  // to optimize the interpolation loop.
+  // Cache access in non-volatile variable. This is the only function changing the read value, so
+  // it's safe to cache and write to it locally. The writing pointer will only increase, so we can
+  // safely ignore new written data while interpolating. Without this cache, the compiler wouldn't
+  // be allowed to optimize the interpolation loop.
   const u32 indexW = m_indexW.load();  // Write Index in Circular Buffer
   u32 indexR = m_indexR.load();        // Read Index in Circular Buffer
 
   // Volumes of left and right audio channels
   const float lvolume = float(m_LVolume.load()) / float(1 << 8);
   const float rvolume = float(m_RVolume.load()) / float(1 << 8);
 
-  // Read from buffer with proper endianess while
-  // masking the index to the buffer size
+  // Read from buffer with proper endianess while masking the index to the buffer size
   const auto read_buffer = [this](auto index) -> s16 {
     return m_little_endian ? m_buffer[index & INDEX_MASK] :
                              Common::swap16(m_buffer[index & INDEX_MASK]);
@@ -97,6 +93,11 @@ unsigned int Mixer::MixerFifo::Mix(s16* samples, unsigned int num_samples,
   // The CPU Thread will attempt to keep timings within this variance
   const u32 timing_variance = m_mixer->m_config_timing_variance;
 
+  // Amount of time that has passed since the last frame. This is also used to calculate the low
+  // pass filter gain for smoothing out playback speed.
+  const float time_delta = num_samples / float(m_mixer->m_sampleRate);
+  const float lpf_gain = 1.0f - std::exp(-time_delta / (num_samples * SAMPLE_RATE_LPF));
+
   // This is the sample rate of the device this is resampling from
   float aid_sample_rate = FIXED_SAMPLE_RATE_DIVIDEND / float(m_input_sample_rate_divisor);
 
@@ -109,35 +110,26 @@ unsigned int Mixer::MixerFifo::Mix(s16* samples, unsigned int num_samples,
                                       (u64(m_input_sample_rate_divisor) * 1000)));
 
     // Calculate the number of samples remaining after this frame
-    const float time_delta = num_samples / float(m_mixer->m_sampleRate);
     const float requested_samples = time_delta * aid_sample_rate;
     const float numLeft = getRemainingSamples();
 
-    // Ideally, numLeft - low_watermark should be the number of samples requested
-    // which would make the multiplier here 1.0. However if the speed is different,
-    // we need to adjust the multiplier to match the requested samples.
+    // Ideally, numLeft - low_watermark should be the number of samples requested which would make
+    // the multiplier here 1.0. However if the speed is different, we need to adjust the multiplier
+    // to match the requested samples.
     float multiplier = (numLeft - low_watermark) / requested_samples;
-    multiplier = std::clamp(multiplier, MIN_PITCH_SHIFT, MAX_PITCH_SHIFT);
-    aid_sample_rate *= multiplier * audio_speed;
-
-    // Sudden changes in sample rate can cause pitch wobbling,
-    // so we apply a low-pass filter to smooth out the changes.
-    const float lpf_gain = 1.0f - std::exp(-time_delta / SAMPLE_RATE_LPF);
-    m_aidSampleRate += (aid_sample_rate - m_aidSampleRate) * lpf_gain;
-  }
-  else
-  {
-    m_aidSampleRate = aid_sample_rate;
+    if (0.0f < audio_speed)
+      multiplier =
+          std::clamp(multiplier, audio_speed * MIN_PITCH_SHIFT, audio_speed * MAX_PITCH_SHIFT);
+    aid_sample_rate *= multiplier;
   }
 
-  // This ratio is how many emulator samples are needed to produce one host sample
-  // In the majority of cases, this will be < 1.0, meaning we will need to upsample
-  // the audio to match the host sample rate. This is done by interpolating the audio
-  // using a sinc filter. However in the case where this is > 1.0, i.e. the emulators
-  // is running faster than native speed, we will need to adjust the sinc window
-  // to be wider to avoid aliasing. We also need to define pi because windows
+  // This ratio is how many emulator samples are needed to produce one host sample. In the majority
+  // of cases, this will be < 1.0, meaning we will need to upsample the audio to match the host
+  // sample rate. This is done by interpolating the audio using a sinc filter. However in the case
+  // where this is > 1.0, i.e. the emulators is running faster than native speed, we will need to
+  // adjust the sinc window to be wider to avoid aliasing. We also need to define pi because windows
   // doesn't define it in the global namespace >:(
-  const float ratio = float(m_aidSampleRate) / float(m_mixer->m_sampleRate);
+  const float ratio = float(aid_sample_rate) / float(m_mixer->m_sampleRate);
   const float sinc_ratio = 1.0f / std::max(1.0f, ratio);
   const float pi = 3.14159265358979323846f;
 
@@ -156,8 +148,8 @@ unsigned int Mixer::MixerFifo::Mix(s16* samples, unsigned int num_samples,
       // Distance from the target sample
       float x = i - m_frac;
 
-      // Initialize weight with value of Hanning Window
-      // When tested against Kaiser window, this was found to do better.
+      // Initialize weight with value of Hanning Window. When tested against Kaiser window, this was
+      // found to do better.
       float w = 0.5f + 0.5f * std::cos(pi * x / float(SINC_WINDOW_WIDTH));
 
       // Apply sinc function (avoid division by zero)
@@ -177,16 +169,20 @@ unsigned int Mixer::MixerFifo::Mix(s16* samples, unsigned int num_samples,
     samples[2 * sample + 0] = clamp(samples[2 * sample + 0] + s32(r_sample));
     samples[2 * sample + 1] = clamp(samples[2 * sample + 1] + s32(l_sample));
 
-    m_frac += ratio;
+    // We need to smooth out the ratio in order to mask sudden pitch changes
+    m_ratio += (ratio - m_ratio) * lpf_gain;
+
+    // Increment the fractional part of the index, and if it exceeds 1.0, increment the index by 2
+    // and subtract 1.0 from the fractional part.
+    m_frac += m_ratio;
     s32 inc = s32(m_frac);
     indexR += 2 * inc;
     m_frac -= inc;
   }
 
-  // Actual number of samples read from the buffer. In almost all cases,
-  // this will be the same number of samples requested, but in the case
-  // where we run out of samples in the buffer, we will return the actual
-  // number of samples read.
+  // Actual number of samples read from the buffer. In almost all cases, this will be the same
+  // number of samples requested, but in the case where we run out of samples in the buffer, we will
+  // return the actual number of samples read.
   unsigned int actual_sample_count = sample;
 
   // Padding

Source/Core/AudioCommon/Mixer.h

@@ -93,9 +93,7 @@ class Mixer final
     std::atomic<s32> m_LVolume{256};
     std::atomic<s32> m_RVolume{256};
 
-    // This gets updated so this just
-    // needs to be a ball park figure
-    float m_aidSampleRate = 32000.0f;
+    float m_ratio = 0.0f;
     float m_frac = 0.0f;
   };
 

Source/Core/VideoCommon/PerformanceTracker.cpp

@@ -16,7 +16,7 @@
 #include "Core/Core.h"
 #include "VideoCommon/VideoConfig.h"
 
-static constexpr double SAMPLE_RC_RATIO = 0.33;
+static constexpr double SAMPLE_RC_RATIO = 0.25;
 
 PerformanceTracker::PerformanceTracker(const std::optional<std::string> log_name,
                                        const std::optional<s64> sample_window_us)