Skip to content
Permalink
Browse files
VPIO requires the same sample rate for microphone and speaker formats
https://bugs.webkit.org/show_bug.cgi?id=239956

Reviewed by Eric Carlson.

VPIO requires the speaker sample rate and microphone sample rate to be the same.
This is not guaranteed to be true : getUserMedia/applyConstraints may set the microphone sample rate to different values.
If audio rendering already started at capture start time, we use the audio rendering sample rate.
This is not strictly compliant, given required constraints but they are not widely used.
Applications that want to make sure to use a specific sample rate would need to pause audio rendering when starting to capture.
We reset exposed settings when starting to capture to expose the actual sample rate to JavaScript through MediaStreamTrack.getSettings().

Manually tested.

* platform/mediastream/mac/BaseAudioSharedUnit.h:
* platform/mediastream/mac/CoreAudioCaptureSource.cpp:

Canonical link: https://commits.webkit.org/250168@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@293664 268f45cc-cd09-0410-ab3c-d52691b4dbfc
  • Loading branch information
youennf committed May 2, 2022
1 parent d2a33ae commit ccee887db38323b9e0cd8fb5d3b96def66103c6d
Showing 3 changed files with 55 additions and 12 deletions.
@@ -1,3 +1,22 @@
2022-05-02 Youenn Fablet <youenn@apple.com>

VPIO requires the same sample rate for microphone and speaker formats
https://bugs.webkit.org/show_bug.cgi?id=239956

Reviewed by Eric Carlson.

VPIO requires the speaker sample rate and microphone sample rate to be the same.
This is not guaranteed to be true : getUserMedia/applyConstraints may set the microphone sample rate to different values.
If audio rendering already started at capture start time, we use the audio rendering sample rate.
This is not strictly compliant, given required constraints but they are not widely used.
Applications that want to make sure to use a specific sample rate would need to pause audio rendering when starting to capture.
We reset exposed settings when starting to capture to expose the actual sample rate to JavaScript through MediaStreamTrack.getSettings().

Manually tested.

* platform/mediastream/mac/BaseAudioSharedUnit.h:
* platform/mediastream/mac/CoreAudioCaptureSource.cpp:

2022-05-01 Philippe Normand <philn@igalia.com>

[GStreamer][Debug] media-recorder tests hitting an ASSERT
@@ -77,6 +77,7 @@ class BaseAudioSharedUnit : public CanMakeWeakPtr<BaseAudioSharedUnit> {
void setCaptureDevice(String&&, uint32_t);

virtual CapabilityValueOrRange sampleRateCapacities() const = 0;
virtual int actualSampleRate() const { return sampleRate(); }

void devicesChanged(const Vector<CaptureDevice>&);
void whenAudioCaptureUnitIsNotRunning(Function<void()>&&);
@@ -101,9 +101,10 @@ class CoreAudioSharedUnit final : public BaseAudioSharedUnit {
bool isProducingData() const final { return m_ioUnitStarted; }
void isProducingMicrophoneSamplesChanged() final;
void validateOutputDevice(uint32_t deviceID) final;
int actualSampleRate() const final;

OSStatus configureSpeakerProc();
OSStatus configureMicrophoneProc();
OSStatus configureSpeakerProc(int sampleRate);
OSStatus configureMicrophoneProc(int sampleRate);
OSStatus defaultOutputDevice(uint32_t*);
OSStatus defaultInputDevice(uint32_t*);

@@ -149,7 +150,7 @@ class CoreAudioSharedUnit final : public BaseAudioSharedUnit {
Timer m_verifyCapturingTimer;

bool m_isReconfiguring { false };
Lock m_speakerSamplesProducerLock;
mutable Lock m_speakerSamplesProducerLock;
CoreAudioSpeakerSamplesProducer* m_speakerSamplesProducer WTF_GUARDED_BY_LOCK(m_speakerSamplesProducerLock) { nullptr };
};

@@ -258,11 +259,13 @@ OSStatus CoreAudioSharedUnit::setupAudioUnit()
setOutputDeviceID(!err ? defaultOutputDeviceID : 0);
#endif

err = configureMicrophoneProc();
// FIXME: Add support for different speaker/microphone sample rates.
int actualSampleRate = this->actualSampleRate();
err = configureMicrophoneProc(actualSampleRate);
if (err)
return err;

err = configureSpeakerProc();
err = configureSpeakerProc(actualSampleRate);
if (err)
return err;

@@ -285,8 +288,16 @@ void CoreAudioSharedUnit::unduck()
AudioDeviceDuck(outputDevice, 1.0, nullptr, 0);
}

OSStatus CoreAudioSharedUnit::configureMicrophoneProc()
int CoreAudioSharedUnit::actualSampleRate() const
{
Locker locker { m_speakerSamplesProducerLock };
return m_speakerSamplesProducer ? m_speakerSamplesProducer->format().streamDescription().mSampleRate : sampleRate();
}

OSStatus CoreAudioSharedUnit::configureMicrophoneProc(int sampleRate)
{
ASSERT(isMainThread());

if (!isProducingMicrophoneSamples())
return noErr;

@@ -306,7 +317,7 @@ OSStatus CoreAudioSharedUnit::configureMicrophoneProc()
return err;
}

microphoneProcFormat.mSampleRate = sampleRate();
microphoneProcFormat.mSampleRate = sampleRate;
err = PAL::AudioUnitSetProperty(m_ioUnit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, inputBus, &microphoneProcFormat, size);
if (err) {
RELEASE_LOG_ERROR(WebRTC, "CoreAudioSharedUnit::configureMicrophoneProc(%p) unable to set output stream format, error %d (%.4s)", this, (int)err, (char*)&err);
@@ -319,8 +330,10 @@ OSStatus CoreAudioSharedUnit::configureMicrophoneProc()
return err;
}

OSStatus CoreAudioSharedUnit::configureSpeakerProc()
OSStatus CoreAudioSharedUnit::configureSpeakerProc(int sampleRate)
{
ASSERT(isMainThread());

AURenderCallbackStruct callback = { speakerCallback, this };
auto err = PAL::AudioUnitSetProperty(m_ioUnit, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Input, outputBus, &callback, sizeof(callback));
if (err) {
@@ -332,17 +345,18 @@ OSStatus CoreAudioSharedUnit::configureSpeakerProc()
UInt32 size = sizeof(speakerProcFormat);
{
Locker locker { m_speakerSamplesProducerLock };
if (m_speakerSamplesProducer)
if (m_speakerSamplesProducer) {
ASSERT(speakerProcFormat.mSampleRate == sampleRate);
speakerProcFormat = m_speakerSamplesProducer->format().streamDescription();
else {
} else {
err = PAL::AudioUnitGetProperty(m_ioUnit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, outputBus, &speakerProcFormat, &size);
if (err) {
RELEASE_LOG_ERROR(WebRTC, "CoreAudioSharedUnit::configureSpeakerProc(%p) unable to get input stream format, error %d (%.4s)", this, (int)err, (char*)&err);
return err;
}
speakerProcFormat.mSampleRate = sampleRate();
}
}
speakerProcFormat.mSampleRate = sampleRate;

err = PAL::AudioUnitSetProperty(m_ioUnit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, outputBus, &speakerProcFormat, size);
if (err) {
@@ -493,6 +507,8 @@ OSStatus CoreAudioSharedUnit::reconfigureAudioUnit()

OSStatus CoreAudioSharedUnit::startInternal()
{
ASSERT(isMainThread());

setIsProducingMicrophoneSamples(true);

OSStatus err;
@@ -573,6 +589,8 @@ void CoreAudioSharedUnit::verifyIsCapturing()

void CoreAudioSharedUnit::stopInternal()
{
ASSERT(isMainThread());

m_verifyCapturingTimer.stop();

if (!m_ioUnit || !m_ioUnitStarted)
@@ -632,6 +650,8 @@ OSStatus CoreAudioSharedUnit::defaultOutputDevice(uint32_t* deviceID)

void CoreAudioSharedUnit::registerSpeakerSamplesProducer(CoreAudioSpeakerSamplesProducer& producer)
{
ASSERT(isMainThread());

setIsRenderingAudio(true);

CoreAudioSpeakerSamplesProducer* oldProducer;
@@ -649,6 +669,8 @@ void CoreAudioSharedUnit::registerSpeakerSamplesProducer(CoreAudioSpeakerSamples

void CoreAudioSharedUnit::unregisterSpeakerSamplesProducer(CoreAudioSpeakerSamplesProducer& producer)
{
ASSERT(isMainThread());

{
Locker locker { m_speakerSamplesProducerLock };
if (m_speakerSamplesProducer != &producer)
@@ -819,6 +841,7 @@ void CoreAudioCaptureSource::startProducingData()
{
initializeToStartProducingData();
unit().startProducingData();
m_currentSettings = { };
}

void CoreAudioCaptureSource::stopProducingData()
@@ -845,7 +868,7 @@ const RealtimeMediaSourceSettings& CoreAudioCaptureSource::settings()
if (!m_currentSettings) {
RealtimeMediaSourceSettings settings;
settings.setVolume(volume());
settings.setSampleRate(sampleRate());
settings.setSampleRate(unit().actualSampleRate());
settings.setDeviceId(hashedId());
settings.setLabel(name());
settings.setEchoCancellation(echoCancellation());

0 comments on commit ccee887

Please sign in to comment.