VPIO requires the same sample rate for microphone and speaker formats

https://bugs.webkit.org/show_bug.cgi?id=239956

Reviewed by Eric Carlson.

VPIO requires the speaker sample rate and microphone sample rate to be the same.
This is not guaranteed to be true : getUserMedia/applyConstraints may set the microphone sample rate to different values.
If audio rendering already started at capture start time, we use the audio rendering sample rate.
This is not strictly compliant, given required constraints but they are not widely used.
Applications that want to make sure to use a specific sample rate would need to pause audio rendering when starting to capture.
We reset exposed settings when starting to capture to expose the actual sample rate to JavaScript through MediaStreamTrack.getSettings().

Manually tested.

* platform/mediastream/mac/BaseAudioSharedUnit.h:
* platform/mediastream/mac/CoreAudioCaptureSource.cpp:

Canonical link: https://commits.webkit.org/250168@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@293664 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/WebCore/ChangeLog

@@ -1,3 +1,22 @@
+2022-05-02  Youenn Fablet  <youenn@apple.com>
+
+        VPIO requires the same sample rate for microphone and speaker formats
+        https://bugs.webkit.org/show_bug.cgi?id=239956
+
+        Reviewed by Eric Carlson.
+
+        VPIO requires the speaker sample rate and microphone sample rate to be the same.
+        This is not guaranteed to be true : getUserMedia/applyConstraints may set the microphone sample rate to different values.
+        If audio rendering already started at capture start time, we use the audio rendering sample rate.
+        This is not strictly compliant, given required constraints but they are not widely used.
+        Applications that want to make sure to use a specific sample rate would need to pause audio rendering when starting to capture.
+        We reset exposed settings when starting to capture to expose the actual sample rate to JavaScript through MediaStreamTrack.getSettings().
+
+        Manually tested.
+
+        * platform/mediastream/mac/BaseAudioSharedUnit.h:
+        * platform/mediastream/mac/CoreAudioCaptureSource.cpp:
+
 2022-05-01  Philippe Normand  <philn@igalia.com>
 
         [GStreamer][Debug] media-recorder tests hitting an ASSERT

Source/WebCore/platform/mediastream/mac/BaseAudioSharedUnit.h

@@ -77,6 +77,7 @@ class BaseAudioSharedUnit : public CanMakeWeakPtr<BaseAudioSharedUnit> {
     void setCaptureDevice(String&&, uint32_t);
 
     virtual CapabilityValueOrRange sampleRateCapacities() const = 0;
+    virtual int actualSampleRate() const { return sampleRate(); }
 
     void devicesChanged(const Vector<CaptureDevice>&);
     void whenAudioCaptureUnitIsNotRunning(Function<void()>&&);

Source/WebCore/platform/mediastream/mac/CoreAudioCaptureSource.cpp

@@ -101,9 +101,10 @@ class CoreAudioSharedUnit final : public BaseAudioSharedUnit {
     bool isProducingData() const final { return m_ioUnitStarted; }
     void isProducingMicrophoneSamplesChanged() final;
     void validateOutputDevice(uint32_t deviceID) final;
+    int actualSampleRate() const final;
 
-    OSStatus configureSpeakerProc();
-    OSStatus configureMicrophoneProc();
+    OSStatus configureSpeakerProc(int sampleRate);
+    OSStatus configureMicrophoneProc(int sampleRate);
     OSStatus defaultOutputDevice(uint32_t*);
     OSStatus defaultInputDevice(uint32_t*);
 
@@ -149,7 +150,7 @@ class CoreAudioSharedUnit final : public BaseAudioSharedUnit {
     Timer m_verifyCapturingTimer;
 
     bool m_isReconfiguring { false };
-    Lock m_speakerSamplesProducerLock;
+    mutable Lock m_speakerSamplesProducerLock;
     CoreAudioSpeakerSamplesProducer* m_speakerSamplesProducer WTF_GUARDED_BY_LOCK(m_speakerSamplesProducerLock) { nullptr };
 };
 
@@ -258,11 +259,13 @@ OSStatus CoreAudioSharedUnit::setupAudioUnit()
     setOutputDeviceID(!err ? defaultOutputDeviceID : 0);
 #endif
 
-    err = configureMicrophoneProc();
+    // FIXME: Add support for different speaker/microphone sample rates.
+    int actualSampleRate = this->actualSampleRate();
+    err = configureMicrophoneProc(actualSampleRate);
     if (err)
         return err;
 
-    err = configureSpeakerProc();
+    err = configureSpeakerProc(actualSampleRate);
     if (err)
         return err;
 
@@ -285,8 +288,16 @@ void CoreAudioSharedUnit::unduck()
         AudioDeviceDuck(outputDevice, 1.0, nullptr, 0);
 }
 
-OSStatus CoreAudioSharedUnit::configureMicrophoneProc()
+int CoreAudioSharedUnit::actualSampleRate() const
 {
+    Locker locker { m_speakerSamplesProducerLock };
+    return m_speakerSamplesProducer ? m_speakerSamplesProducer->format().streamDescription().mSampleRate : sampleRate();
+}
+
+OSStatus CoreAudioSharedUnit::configureMicrophoneProc(int sampleRate)
+{
+    ASSERT(isMainThread());
+
     if (!isProducingMicrophoneSamples())
         return noErr;
 
@@ -306,7 +317,7 @@ OSStatus CoreAudioSharedUnit::configureMicrophoneProc()
         return err;
     }
 
-    microphoneProcFormat.mSampleRate = sampleRate();
+    microphoneProcFormat.mSampleRate = sampleRate;
     err = PAL::AudioUnitSetProperty(m_ioUnit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, inputBus, &microphoneProcFormat, size);
     if (err) {
         RELEASE_LOG_ERROR(WebRTC, "CoreAudioSharedUnit::configureMicrophoneProc(%p) unable to set output stream format, error %d (%.4s)", this, (int)err, (char*)&err);
@@ -319,8 +330,10 @@ OSStatus CoreAudioSharedUnit::configureMicrophoneProc()
     return err;
 }
 
-OSStatus CoreAudioSharedUnit::configureSpeakerProc()
+OSStatus CoreAudioSharedUnit::configureSpeakerProc(int sampleRate)
 {
+    ASSERT(isMainThread());
+
     AURenderCallbackStruct callback = { speakerCallback, this };
     auto err = PAL::AudioUnitSetProperty(m_ioUnit, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Input, outputBus, &callback, sizeof(callback));
     if (err) {
@@ -332,17 +345,18 @@ OSStatus CoreAudioSharedUnit::configureSpeakerProc()
     UInt32 size = sizeof(speakerProcFormat);
     {
         Locker locker { m_speakerSamplesProducerLock };
-        if (m_speakerSamplesProducer)
+        if (m_speakerSamplesProducer) {
+            ASSERT(speakerProcFormat.mSampleRate == sampleRate);
             speakerProcFormat = m_speakerSamplesProducer->format().streamDescription();
-        else {
+        } else {
             err = PAL::AudioUnitGetProperty(m_ioUnit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, outputBus, &speakerProcFormat, &size);
             if (err) {
                 RELEASE_LOG_ERROR(WebRTC, "CoreAudioSharedUnit::configureSpeakerProc(%p) unable to get input stream format, error %d (%.4s)", this, (int)err, (char*)&err);
                 return err;
             }
-            speakerProcFormat.mSampleRate = sampleRate();
         }
     }
+    speakerProcFormat.mSampleRate = sampleRate;
 
     err = PAL::AudioUnitSetProperty(m_ioUnit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, outputBus, &speakerProcFormat, size);
     if (err) {
@@ -493,6 +507,8 @@ OSStatus CoreAudioSharedUnit::reconfigureAudioUnit()
 
 OSStatus CoreAudioSharedUnit::startInternal()
 {
+    ASSERT(isMainThread());
+
     setIsProducingMicrophoneSamples(true);
 
     OSStatus err;
@@ -573,6 +589,8 @@ void CoreAudioSharedUnit::verifyIsCapturing()
 
 void CoreAudioSharedUnit::stopInternal()
 {
+    ASSERT(isMainThread());
+
     m_verifyCapturingTimer.stop();
 
     if (!m_ioUnit || !m_ioUnitStarted)
@@ -632,6 +650,8 @@ OSStatus CoreAudioSharedUnit::defaultOutputDevice(uint32_t* deviceID)
 
 void CoreAudioSharedUnit::registerSpeakerSamplesProducer(CoreAudioSpeakerSamplesProducer& producer)
 {
+    ASSERT(isMainThread());
+
     setIsRenderingAudio(true);
 
     CoreAudioSpeakerSamplesProducer* oldProducer;
@@ -649,6 +669,8 @@ void CoreAudioSharedUnit::registerSpeakerSamplesProducer(CoreAudioSpeakerSamples
 
 void CoreAudioSharedUnit::unregisterSpeakerSamplesProducer(CoreAudioSpeakerSamplesProducer& producer)
 {
+    ASSERT(isMainThread());
+
     {
         Locker locker { m_speakerSamplesProducerLock };
         if (m_speakerSamplesProducer != &producer)
@@ -819,6 +841,7 @@ void CoreAudioCaptureSource::startProducingData()
 {
     initializeToStartProducingData();
     unit().startProducingData();
+    m_currentSettings = { };
 }
 
 void CoreAudioCaptureSource::stopProducingData()
@@ -845,7 +868,7 @@ const RealtimeMediaSourceSettings& CoreAudioCaptureSource::settings()
     if (!m_currentSettings) {
         RealtimeMediaSourceSettings settings;
         settings.setVolume(volume());
-        settings.setSampleRate(sampleRate());
+        settings.setSampleRate(unit().actualSampleRate());
         settings.setDeviceId(hashedId());
         settings.setLabel(name());
         settings.setEchoCancellation(echoCancellation());