From 46c451d2df00395930f2b6f08846f2f541812110 Mon Sep 17 00:00:00 2001 From: CloudWebRTC Date: Sat, 11 Sep 2021 19:27:32 +0800 Subject: [PATCH] Merge pull request #2 from webrtc-sdk/listen-only-audio-session allow listen-only mode in AudioUnit, adjust when category changes --- .../audio/RTCAudioSession+Configuration.mm | 6 +- .../audio/RTCAudioSession+Private.h | 3 + sdk/objc/components/audio/RTCAudioSession.h | 3 + sdk/objc/components/audio/RTCAudioSession.mm | 20 ++++ .../RTCNativeAudioSessionDelegateAdapter.mm | 5 + sdk/objc/native/src/audio/audio_device_ios.h | 2 + sdk/objc/native/src/audio/audio_device_ios.mm | 100 +++++++++++++++++- .../native/src/audio/audio_session_observer.h | 2 + .../src/audio/voice_processing_audio_unit.mm | 34 +++--- 9 files changed, 159 insertions(+), 16 deletions(-) diff --git a/sdk/objc/components/audio/RTCAudioSession+Configuration.mm b/sdk/objc/components/audio/RTCAudioSession+Configuration.mm index 449f31e9dd..b123e2002e 100644 --- a/sdk/objc/components/audio/RTCAudioSession+Configuration.mm +++ b/sdk/objc/components/audio/RTCAudioSession+Configuration.mm @@ -55,7 +55,8 @@ - (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configur if (![self setCategory:configuration.category withOptions:configuration.categoryOptions error:&categoryError]) { - RTCLogError(@"Failed to set category: %@", + RTCLogError(@"Failed to set category to %@: %@", + self.category, categoryError.localizedDescription); error = categoryError; } else { @@ -66,7 +67,8 @@ - (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configur if (self.mode != configuration.mode) { NSError *modeError = nil; if (![self setMode:configuration.mode error:&modeError]) { - RTCLogError(@"Failed to set mode: %@", + RTCLogError(@"Failed to set mode to %@: %@", + self.mode, modeError.localizedDescription); error = modeError; } else { diff --git a/sdk/objc/components/audio/RTCAudioSession+Private.h b/sdk/objc/components/audio/RTCAudioSession+Private.h index 2be1b9fb3d..c38f7715d8 100644 --- a/sdk/objc/components/audio/RTCAudioSession+Private.h +++ b/sdk/objc/components/audio/RTCAudioSession+Private.h @@ -35,6 +35,9 @@ NS_ASSUME_NONNULL_BEGIN */ @property(nonatomic, assign) BOOL isInterrupted; +/** if the current category could allow recording */ +@property(nonatomic, assign) BOOL isRecordingEnabled; + /** Adds the delegate to the list of delegates, and places it at the front of * the list. This delegate will be notified before other delegates of * audio events. diff --git a/sdk/objc/components/audio/RTCAudioSession.h b/sdk/objc/components/audio/RTCAudioSession.h index 3b83b27ba5..43d1431c13 100644 --- a/sdk/objc/components/audio/RTCAudioSession.h +++ b/sdk/objc/components/audio/RTCAudioSession.h @@ -102,6 +102,9 @@ RTC_OBJC_EXPORT - (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession audioUnitStartFailedWithError:(NSError *)error; +/** Called when audio session changed from output-only to input & output */ +- (void)audioSessionWillRecord:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession; + @end /** This is a protocol used to inform RTCAudioSession when the audio session diff --git a/sdk/objc/components/audio/RTCAudioSession.mm b/sdk/objc/components/audio/RTCAudioSession.mm index 550a426d36..cda9671dcb 100644 --- a/sdk/objc/components/audio/RTCAudioSession.mm +++ b/sdk/objc/components/audio/RTCAudioSession.mm @@ -114,6 +114,8 @@ - (instancetype)initWithAudioSession:(id)audioSession { options:NSKeyValueObservingOptionNew | NSKeyValueObservingOptionOld context:(__bridge void *)RTC_OBJC_TYPE(RTCAudioSession).class]; + self.isRecordingEnabled = [_session.category isEqualToString:AVAudioSessionCategoryPlayAndRecord]; + RTCLog(@"RTC_OBJC_TYPE(RTCAudioSession) (%p): init.", self); } return self; @@ -542,6 +544,13 @@ - (void)handleRouteChangeNotification:(NSNotification *)notification { case AVAudioSessionRouteChangeReasonCategoryChange: RTCLog(@"Audio route changed: CategoryChange to :%@", self.session.category); + if (!self.isRecordingEnabled && [self.session.category isEqualToString:AVAudioSessionCategoryPlayAndRecord]) { + self.isRecordingEnabled = true; + [self notifyWillRecord]; + } + if (self.isRecordingEnabled && [self.session.category isEqualToString:AVAudioSessionCategoryPlayback]) { + self.isRecordingEnabled = false; + } break; case AVAudioSessionRouteChangeReasonOverride: RTCLog(@"Audio route changed: Override"); @@ -773,6 +782,7 @@ - (BOOL)unconfigureWebRTCSession:(NSError **)outError { } RTCLog(@"Unconfiguring audio session for WebRTC."); [self setActive:NO error:outError]; + self.isRecordingEnabled = NO; return YES; } @@ -997,4 +1007,14 @@ - (void)notifyFailedToSetActive:(BOOL)active error:(NSError *)error { } } +- (void)notifyWillRecord { + for (auto delegate : self.delegates) { + SEL sel = @selector(audioSessionWillRecord:); + if ([delegate respondsToSelector:sel]) { + [delegate audioSessionWillRecord:self]; + } + } +} + + @end diff --git a/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm b/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm index daddf314a4..2f0f094262 100644 --- a/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm +++ b/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm @@ -86,4 +86,9 @@ - (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession _observer->OnChangedOutputVolume(); } +- (void)audioSessionWillRecord:(RTC_OBJC_TYPE(RTCAudioSession) *)session { + // re-trigger audio unit init, by using interrupt ended callback + _observer->OnAudioWillRecord(); +} + @end diff --git a/sdk/objc/native/src/audio/audio_device_ios.h b/sdk/objc/native/src/audio/audio_device_ios.h index a86acb56fe..99482fddfc 100644 --- a/sdk/objc/native/src/audio/audio_device_ios.h +++ b/sdk/objc/native/src/audio/audio_device_ios.h @@ -147,6 +147,7 @@ class AudioDeviceIOS : public AudioDeviceGeneric, void OnValidRouteChange() override; void OnCanPlayOrRecordChange(bool can_play_or_record) override; void OnChangedOutputVolume() override; + void OnAudioWillRecord() override; // VoiceProcessingAudioUnitObserver methods. OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, @@ -171,6 +172,7 @@ class AudioDeviceIOS : public AudioDeviceGeneric, void HandleSampleRateChange(); void HandlePlayoutGlitchDetected(); void HandleOutputVolumeChange(); + void HandleAudioWillRecord(); // Uses current `playout_parameters_` and `record_parameters_` to inform the // audio device buffer (ADB) about our internal audio parameters. diff --git a/sdk/objc/native/src/audio/audio_device_ios.mm b/sdk/objc/native/src/audio/audio_device_ios.mm index dd2c11bdd2..4a717422e5 100644 --- a/sdk/objc/native/src/audio/audio_device_ios.mm +++ b/sdk/objc/native/src/audio/audio_device_ios.mm @@ -61,6 +61,16 @@ const UInt16 kFixedPlayoutDelayEstimate = 30; const UInt16 kFixedRecordDelayEstimate = 30; +enum AudioDeviceMessageType : uint32_t { + kMessageTypeInterruptionBegin, + kMessageTypeInterruptionEnd, + kMessageTypeValidRouteChange, + kMessageTypeCanPlayOrRecordChange, + kMessageTypePlayoutGlitchDetected, + kMessageOutputVolumeChange, + kMessageTypeAudioWillRecord, +}; + using ios::CheckAndLogError; #if !defined(NDEBUG) @@ -360,6 +370,11 @@ static void LogDeviceInfo() { thread_->PostTask(SafeTask(safety_, [this] { HandleOutputVolumeChange(); })); } +void AudioDeviceIOS::OnAudioWillRecord() { + RTC_DCHECK(thread_); + thread_->Post(RTC_FROM_HERE, this, kMessageTypeAudioWillRecord); +} + OSStatus AudioDeviceIOS::OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, const AudioTimeStamp* time_stamp, UInt32 bus_number, @@ -445,7 +460,7 @@ static void LogDeviceInfo() { // Exclude extreme delta values since they do most likely not correspond // to a real glitch. Instead, the most probable cause is that a headset // has been plugged in or out. There are more direct ways to detect - // audio device changes (see HandleValidRouteChange()) but experiments + // audio device changes (see ValidRouteChange()) but experiments // show that using it leads to more complex implementations. // TODO(henrika): more tests might be needed to come up with an even // better upper limit. @@ -467,6 +482,34 @@ static void LogDeviceInfo() { return noErr; } +void AudioDeviceIOS::OnMessage(rtc::Message* msg) { + switch (msg->message_id) { + case kMessageTypeInterruptionBegin: + HandleInterruptionBegin(); + break; + case kMessageTypeInterruptionEnd: + HandleInterruptionEnd(); + break; + case kMessageTypeValidRouteChange: + HandleValidRouteChange(); + break; + case kMessageTypeCanPlayOrRecordChange: { + rtc::TypedMessageData* data = static_cast*>(msg->pdata); + HandleCanPlayOrRecordChange(data->data()); + delete data; + break; + } + case kMessageTypePlayoutGlitchDetected: + HandlePlayoutGlitchDetected(); + break; + case kMessageOutputVolumeChange: + HandleOutputVolumeChange(); + break; + case kMessageTypeAudioWillRecord: + HandleAudioWillRecord(); + } +} + void AudioDeviceIOS::HandleInterruptionBegin() { RTC_DCHECK_RUN_ON(thread_); RTCLog(@"Interruption begin. IsInterrupted changed from %d to 1.", is_interrupted_); @@ -633,6 +676,61 @@ static void LogDeviceInfo() { last_output_volume_change_time_ = rtc::TimeMillis(); } +void AudioDeviceIOS::HandleAudioWillRecord() { + RTC_DCHECK_RUN_ON(&thread_checker_); + + LOGI() << "HandleAudioWillRecord"; + + // If we don't have an audio unit yet, or the audio unit is uninitialized, + // there is no work to do. + if (!audio_unit_ || audio_unit_->GetState() < VoiceProcessingAudioUnit::kInitialized) { + return; + } + + // The audio unit is already initialized or started. + // Check to see if the sample rate or buffer size has changed. + RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + const double session_sample_rate = session.sampleRate; + + // Extra sanity check to ensure that the new sample rate is valid. + if (session_sample_rate <= 0.0) { + RTCLogError(@"Sample rate is invalid: %f", session_sample_rate); + LOGI() << "Sample rate is invalid " << session_sample_rate; + return; + } + // We need to adjust our format and buffer sizes. + // The stream format is about to be changed and it requires that we first + // stop and uninitialize the audio unit to deallocate its resources. + RTCLog(@"Stopping and uninitializing audio unit to adjust buffers."); + bool restart_audio_unit = false; + if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) { + audio_unit_->Stop(); + restart_audio_unit = true; + PrepareForNewStart(); + } + if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { + audio_unit_->Uninitialize(); + } + + // Allocate new buffers given the new stream format. + SetupAudioBuffersForActiveAudioSession(); + + // Initialize the audio unit again with the new sample rate. + RTC_DCHECK_EQ(playout_parameters_.sample_rate(), session_sample_rate); + if (!audio_unit_->Initialize(session_sample_rate)) { + RTCLogError(@"Failed to initialize the audio unit with sample rate: %f", session_sample_rate); + return; + } + + // Restart the audio unit if it was already running. + if (restart_audio_unit && !audio_unit_->Start()) { + RTCLogError(@"Failed to start audio unit with sample rate: %f", session_sample_rate); + return; + } + + LOGI() << "Successfully enabled audio unit for recording."; +} + void AudioDeviceIOS::UpdateAudioDeviceBuffer() { LOGI() << "UpdateAudioDevicebuffer"; // AttachAudioBuffer() is called at construction by the main class but check diff --git a/sdk/objc/native/src/audio/audio_session_observer.h b/sdk/objc/native/src/audio/audio_session_observer.h index f7c44c8184..c050fb7f5f 100644 --- a/sdk/objc/native/src/audio/audio_session_observer.h +++ b/sdk/objc/native/src/audio/audio_session_observer.h @@ -32,6 +32,8 @@ class AudioSessionObserver { virtual void OnChangedOutputVolume() = 0; + virtual void OnAudioWillRecord() = 0; + protected: virtual ~AudioSessionObserver() {} }; diff --git a/sdk/objc/native/src/audio/voice_processing_audio_unit.mm b/sdk/objc/native/src/audio/voice_processing_audio_unit.mm index 3905b6857a..636db543f3 100644 --- a/sdk/objc/native/src/audio/voice_processing_audio_unit.mm +++ b/sdk/objc/native/src/audio/voice_processing_audio_unit.mm @@ -111,19 +111,6 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { return false; } - // Enable input on the input scope of the input element. - UInt32 enable_input = 1; - result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO, - kAudioUnitScope_Input, kInputBus, &enable_input, - sizeof(enable_input)); - if (result != noErr) { - DisposeAudioUnit(); - RTCLogError(@"Failed to enable input on input scope of input element. " - "Error=%ld.", - (long)result); - return false; - } - // Enable output on the output scope of the output element. UInt32 enable_output = 1; result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO, @@ -204,6 +191,27 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { LogStreamDescription(format); #endif + // Enable input on the input scope of the input element. + // keep it disabled if audio session is configured for playback only + AVAudioSession* session = [AVAudioSession sharedInstance]; + UInt32 enable_input = 0; + if ([session.category isEqualToString: AVAudioSessionCategoryPlayAndRecord] || + [session.category isEqualToString: AVAudioSessionCategoryRecord]) { + enable_input = 1; + } + RTCLog(@"Initializing AudioUnit, category=%@, enable_input=%d", session.category, enable_input); + // LOGI() << "Initialize" << session.category << ", enable_input=" << enable_input; + result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO, + kAudioUnitScope_Input, kInputBus, &enable_input, + sizeof(enable_input)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to enable input on input scope of input element. " + "Error=%ld.", + (long)result); + return false; + } + // Set the format on the output scope of the input element/bus. result = AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat,