From 938a0de60d83a81c0f8bbdcd151a04cda88ccd0c Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sat, 5 Aug 2023 17:39:13 +0900 Subject: [PATCH 01/17] audio renderer protocol --- sdk/BUILD.gn | 3 +++ sdk/objc/base/RTCAudioRenderer.h | 33 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 sdk/objc/base/RTCAudioRenderer.h diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index 5fe0f67b63..7c74df1c87 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -119,6 +119,7 @@ if (is_ios || is_mac) { "objc/base/RTCVideoFrame.mm", "objc/base/RTCVideoFrameBuffer.h", "objc/base/RTCVideoRenderer.h", + "objc/base/RTCAudioRenderer.h", "objc/base/RTCYUVPlanarBuffer.h", ] @@ -1357,6 +1358,7 @@ if (is_ios || is_mac) { "objc/base/RTCVideoFrame.h", "objc/base/RTCVideoFrameBuffer.h", "objc/base/RTCVideoRenderer.h", + "objc/base/RTCAudioRenderer.h", "objc/base/RTCYUVPlanarBuffer.h", "objc/components/audio/RTCAudioDevice.h", "objc/components/audio/RTCAudioSession.h", @@ -1572,6 +1574,7 @@ if (is_ios || is_mac) { "objc/base/RTCVideoFrame.h", "objc/base/RTCVideoFrameBuffer.h", "objc/base/RTCVideoRenderer.h", + "objc/base/RTCAudioRenderer.h", "objc/base/RTCYUVPlanarBuffer.h", "objc/components/capturer/RTCCameraVideoCapturer.h", "objc/components/capturer/RTCFileVideoCapturer.h", diff --git a/sdk/objc/base/RTCAudioRenderer.h b/sdk/objc/base/RTCAudioRenderer.h new file mode 100644 index 0000000000..9cbe558935 --- /dev/null +++ b/sdk/objc/base/RTCAudioRenderer.h @@ -0,0 +1,33 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import +#if TARGET_OS_IPHONE +#import +#endif + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE +(RTCAudioRenderer){} + + - (void)renderSampleBuffer : (CMSampleBufferRef)sampleBuffer; + +@end + +NS_ASSUME_NONNULL_END From 55f98cc1e5c675a2711c052c7a7c5f99788e7ba4 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sat, 5 Aug 2023 18:27:00 +0900 Subject: [PATCH 02/17] basic set up --- .../peerconnection/RTCAudioTrack+Private.h | 9 +-- sdk/objc/api/peerconnection/RTCAudioTrack.h | 7 ++ sdk/objc/api/peerconnection/RTCAudioTrack.mm | 64 ++++++++++++++++++- 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h b/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h index 6495500484..8233f9d3f5 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h +++ b/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h @@ -15,11 +15,12 @@ NS_ASSUME_NONNULL_BEGIN @class RTC_OBJC_TYPE(RTCPeerConnectionFactory); -@interface RTC_OBJC_TYPE (RTCAudioTrack) -() +@interface RTC_OBJC_TYPE (RTCAudioTrack) () - /** AudioTrackInterface created or passed in at construction. */ - @property(nonatomic, readonly) rtc::scoped_refptr nativeAudioTrack; +/** AudioTrackInterface created or passed in at construction. */ +@property(nonatomic, readonly) rtc::scoped_refptr nativeAudioTrack; +/** Accessed on _workerThread */ +@property(nonatomic, readonly) NSMutableArray *renderers; /** Initialize an RTCAudioTrack with an id. */ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.h b/sdk/objc/api/peerconnection/RTCAudioTrack.h index 95eb5d3d48..f1e5f9f30b 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.h +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.h @@ -13,6 +13,7 @@ NS_ASSUME_NONNULL_BEGIN +@protocol RTC_OBJC_TYPE (RTCAudioRenderer); @class RTC_OBJC_TYPE(RTCAudioSource); RTC_OBJC_EXPORT @@ -23,6 +24,12 @@ RTC_OBJC_EXPORT /** The audio source for this audio track. */ @property(nonatomic, readonly) RTC_OBJC_TYPE(RTCAudioSource) * source; +/** ..... */ +- (void)addRenderer:(id)renderer; + +/** ..... */ +- (void)removeRenderer:(id)renderer; + @end NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index 5c1736f436..64177e54b5 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -17,9 +17,40 @@ #include "rtc_base/checks.h" -@implementation RTC_OBJC_TYPE (RTCAudioTrack) +namespace webrtc { +/** + * Captures audio data and converts to CMSampleBuffers + */ +class AudioSinkConverter : public webrtc::AudioTrackSinkInterface { + private: + __weak RTCAudioTrack *audioTrack_; + + public: + AudioSinkConverter(RTCAudioTrack *audioTrack) { + NSLog(@"AudioHook: init Hook in RTCAudioTrack"); + audioTrack_ = audioTrack; + } + + void OnData(const void *audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) override { + // TODO: Convert to CMSampleBuffer... + // audioTrack_.renderers; + } +}; +} // namespace webrtc + +@implementation RTC_OBJC_TYPE (RTCAudioTrack) { + rtc::Thread *_workerThread; + BOOL _IsAudioConverterActive; + std::unique_ptr _audioConverter; +} @synthesize source = _source; +@synthesize renderers = _renderers; - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory source:(RTC_OBJC_TYPE(RTCAudioSource) *)source @@ -43,7 +74,16 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto NSParameterAssert(factory); NSParameterAssert(nativeTrack); NSParameterAssert(type == RTCMediaStreamTrackTypeAudio); - return [super initWithFactory:factory nativeTrack:nativeTrack type:type]; + if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) { + _renderers = [NSMutableArray array]; + _audioConverter.reset(new webrtc::AudioSinkConverter(self)); + } + + return self; +} + +- (void)dealloc { + // TODO: Clean up... } - (RTC_OBJC_TYPE(RTCAudioSource) *)source { @@ -57,6 +97,26 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto return _source; } +- (void)addRenderer:(id)renderer { + [_renderers addObject:renderer]; + + if ([_renderers count] != 0 && !_IsAudioConverterActive) { + self.nativeAudioTrack->AddSink(_audioConverter.get()); + _IsAudioConverterActive + = YES; + } +} + +- (void)removeRenderer:(id)renderer { + [_renderers removeObject:renderer]; + + if ([_renderers count] == 0 && _IsAudioConverterActive) { + self.nativeAudioTrack->RemoveSink(_audioConverter.get()); + _IsAudioConverterActive + = NO; + } +} + #pragma mark - Private - (rtc::scoped_refptr)nativeAudioTrack { From f1e472e4cf6b134e7596b561f47309fe32e75085 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sun, 6 Aug 2023 06:54:34 +0900 Subject: [PATCH 03/17] progress --- .../peerconnection/RTCAudioTrack+Private.h | 3 + sdk/objc/api/peerconnection/RTCAudioTrack.mm | 117 ++++++++++++++++-- sdk/objc/base/RTCAudioRenderer.h | 2 +- 3 files changed, 109 insertions(+), 13 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h b/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h index 8233f9d3f5..41a0f9d536 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h +++ b/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h @@ -8,6 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#import #import "RTCAudioTrack.h" #include "api/media_stream_interface.h" @@ -27,6 +28,8 @@ NS_ASSUME_NONNULL_BEGIN source:(RTC_OBJC_TYPE(RTCAudioSource) *)source trackId:(NSString *)trackId; +- (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer; + @end NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index 64177e54b5..7c0230ddea 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -8,8 +8,11 @@ * be found in the AUTHORS file in the root of the source tree. */ +#import + #import "RTCAudioTrack+Private.h" +#import "RTCAudioRenderer.h" #import "RTCAudioSource+Private.h" #import "RTCMediaStreamTrack+Private.h" #import "RTCPeerConnectionFactory+Private.h" @@ -37,8 +40,78 @@ void OnData(const void *audio_data, size_t number_of_channels, size_t number_of_frames, absl::optional absolute_capture_timestamp_ms) override { - // TODO: Convert to CMSampleBuffer... - // audioTrack_.renderers; + NSLog(@"OnData..."); + // Convert to CMSampleBuffer... + + int64_t elapsed_time_ms = + absolute_capture_timestamp_ms ? absolute_capture_timestamp_ms.value() : rtc::TimeMillis(); + NSLog(@"elapsed_time_ms: %lld", elapsed_time_ms); + + OSStatus status; + + AudioChannelLayout acl; + bzero(&acl, sizeof(acl)); + acl.mChannelLayoutTag = kAudioChannelLayoutTag_Mono; + + AudioStreamBasicDescription audioFormat; + audioFormat.mSampleRate = sample_rate; + audioFormat.mFormatID = kAudioFormatLinearPCM; + audioFormat.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; + audioFormat.mFramesPerPacket = 1; + audioFormat.mChannelsPerFrame = 1; + audioFormat.mBitsPerChannel = 16; + audioFormat.mBytesPerPacket = audioFormat.mFramesPerPacket * audioFormat.mChannelsPerFrame * + audioFormat.mBitsPerChannel / 8; + audioFormat.mBytesPerFrame = audioFormat.mBytesPerPacket / audioFormat.mFramesPerPacket; + + CMSampleTimingInfo timing = { + CMTimeMake(1, sample_rate), CMTimeMake(elapsed_time_ms, 1000), kCMTimeInvalid}; + + CMFormatDescriptionRef format = NULL; + status = CMAudioFormatDescriptionCreate( + kCFAllocatorDefault, &audioFormat, sizeof(acl), &acl, 0, NULL, NULL, &format); + if (status != 0) { + NSLog(@"Failed to create audio format description"); + return; + } + + CMSampleBufferRef buffer; + status = CMSampleBufferCreate(kCFAllocatorDefault, + NULL, + false, + NULL, + NULL, + format, + (CMItemCount)number_of_frames, + 1, + &timing, + 0, + NULL, + &buffer); + if (status != 0) { + NSLog(@"Failed to allocate sample buffer"); + return; + } + + AudioBufferList bufferList; + bufferList.mNumberBuffers = 1; + bufferList.mBuffers[0].mNumberChannels = audioFormat.mChannelsPerFrame; + bufferList.mBuffers[0].mDataByteSize = (UInt32)(number_of_frames * audioFormat.mBytesPerFrame); + bufferList.mBuffers[0].mData = (void *)audio_data; + status = CMSampleBufferSetDataBufferFromAudioBufferList( + buffer, kCFAllocatorDefault, kCFAllocatorDefault, 0, &bufferList); + if (status != 0) { + NSLog(@"Failed to convert audio buffer list into sample buffer"); + return; + } + + // if (!CMSampleBufferIsValid(buffer)) { + NSLog(@"CMSampleBufferIsValid: %d", CMSampleBufferIsValid(buffer)); + + // Report back to RTCAudioTrack + [audioTrack_ didCaptureSampleBuffer:buffer]; + + CFRelease(buffer); } }; } // namespace webrtc @@ -46,7 +119,8 @@ void OnData(const void *audio_data, @implementation RTC_OBJC_TYPE (RTCAudioTrack) { rtc::Thread *_workerThread; BOOL _IsAudioConverterActive; - std::unique_ptr _audioConverter; + // std::unique_ptr _audioConverter; + webrtc::AudioSinkConverter *_audioConverter; } @synthesize source = _source; @@ -76,7 +150,15 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto NSParameterAssert(type == RTCMediaStreamTrackTypeAudio); if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) { _renderers = [NSMutableArray array]; - _audioConverter.reset(new webrtc::AudioSinkConverter(self)); + // Testing + //_workerThread->BlockingCall([self] { + // _audioConverter.reset(new webrtc::AudioSinkConverter(self)); + // self.nativeAudioTrack->AddSink(_audioConverter.get()); + + _audioConverter = new webrtc::AudioSinkConverter(self); + self.nativeAudioTrack->AddSink(_audioConverter); + _IsAudioConverterActive = YES; + //}); } return self; @@ -84,6 +166,7 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto - (void)dealloc { // TODO: Clean up... + NSLog(@"RTCAudioTrack dealloc..."); } - (RTC_OBJC_TYPE(RTCAudioSource) *)source { @@ -98,13 +181,14 @@ - (void)dealloc { } - (void)addRenderer:(id)renderer { - [_renderers addObject:renderer]; + _workerThread->BlockingCall([self, renderer] { + [_renderers addObject:renderer]; - if ([_renderers count] != 0 && !_IsAudioConverterActive) { - self.nativeAudioTrack->AddSink(_audioConverter.get()); - _IsAudioConverterActive - = YES; - } + if ([_renderers count] != 0 && !_IsAudioConverterActive) { + self.nativeAudioTrack->AddSink(_audioConverter.get()); + _IsAudioConverterActive = YES; + } + } } - (void)removeRenderer:(id)renderer { @@ -112,8 +196,7 @@ - (void)removeRenderer:(id)renderer { if ([_renderers count] == 0 && _IsAudioConverterActive) { self.nativeAudioTrack->RemoveSink(_audioConverter.get()); - _IsAudioConverterActive - = NO; + _IsAudioConverterActive = NO; } } @@ -124,4 +207,14 @@ - (void)removeRenderer:(id)renderer { static_cast(self.nativeTrack.get())); } +- (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer { + // ... + _workerThread->BlockingCall([self, sampleBuffer] { + for (id renderer in self.renderers) { + // NSLog(@"%@", renderer); + [renderer renderSampleBuffer:sampleBuffer]; + } + }); +} + @end diff --git a/sdk/objc/base/RTCAudioRenderer.h b/sdk/objc/base/RTCAudioRenderer.h index 9cbe558935..61f1c60dcb 100644 --- a/sdk/objc/base/RTCAudioRenderer.h +++ b/sdk/objc/base/RTCAudioRenderer.h @@ -24,7 +24,7 @@ NS_ASSUME_NONNULL_BEGIN RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE -(RTCAudioRenderer){} +(RTCAudioRenderer) - (void)renderSampleBuffer : (CMSampleBufferRef)sampleBuffer; From c8717a2d0a131119dfcd97584abf0e92f3845e13 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sun, 6 Aug 2023 11:18:32 +0900 Subject: [PATCH 04/17] update header year --- sdk/objc/base/RTCAudioRenderer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/objc/base/RTCAudioRenderer.h b/sdk/objc/base/RTCAudioRenderer.h index 61f1c60dcb..35f84d9603 100644 --- a/sdk/objc/base/RTCAudioRenderer.h +++ b/sdk/objc/base/RTCAudioRenderer.h @@ -1,5 +1,5 @@ /* - * Copyright 2022 LiveKit + * Copyright 2023 LiveKit * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 19c4bda5b2024ef9e0e5a23fbfabf7c72a66cd93 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 8 Aug 2023 23:09:24 +0900 Subject: [PATCH 05/17] impl --- .../peerconnection/RTCAudioTrack+Private.h | 2 +- sdk/objc/api/peerconnection/RTCAudioTrack.mm | 95 ++++++++++--------- 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h b/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h index 41a0f9d536..ea1fa87e50 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h +++ b/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h @@ -20,7 +20,7 @@ NS_ASSUME_NONNULL_BEGIN /** AudioTrackInterface created or passed in at construction. */ @property(nonatomic, readonly) rtc::scoped_refptr nativeAudioTrack; -/** Accessed on _workerThread */ +/** Accessed on _workerThread only */ @property(nonatomic, readonly) NSMutableArray *renderers; /** Initialize an RTCAudioTrack with an id. */ diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index 7c0230ddea..f13754bdef 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -24,13 +24,13 @@ /** * Captures audio data and converts to CMSampleBuffers */ -class AudioSinkConverter : public webrtc::AudioTrackSinkInterface { +class AudioSinkConverter : public rtc::RefCountInterface, public webrtc::AudioTrackSinkInterface { private: __weak RTCAudioTrack *audioTrack_; public: AudioSinkConverter(RTCAudioTrack *audioTrack) { - NSLog(@"AudioHook: init Hook in RTCAudioTrack"); + // Keep weak reference to RTCAudioTrack... audioTrack_ = audioTrack; } @@ -40,12 +40,12 @@ void OnData(const void *audio_data, size_t number_of_channels, size_t number_of_frames, absl::optional absolute_capture_timestamp_ms) override { - NSLog(@"OnData..."); - // Convert to CMSampleBuffer... - + /* + * Convert to CMSampleBuffer + * TODO: Handle case which number_of_channels could be 2 or more. + */ int64_t elapsed_time_ms = absolute_capture_timestamp_ms ? absolute_capture_timestamp_ms.value() : rtc::TimeMillis(); - NSLog(@"elapsed_time_ms: %lld", elapsed_time_ms); OSStatus status; @@ -65,13 +65,17 @@ void OnData(const void *audio_data, audioFormat.mBytesPerFrame = audioFormat.mBytesPerPacket / audioFormat.mFramesPerPacket; CMSampleTimingInfo timing = { - CMTimeMake(1, sample_rate), CMTimeMake(elapsed_time_ms, 1000), kCMTimeInvalid}; + CMTimeMake(1, sample_rate), + CMTimeMake(elapsed_time_ms, 1000), + kCMTimeInvalid, + }; CMFormatDescriptionRef format = NULL; status = CMAudioFormatDescriptionCreate( kCFAllocatorDefault, &audioFormat, sizeof(acl), &acl, 0, NULL, NULL, &format); + if (status != 0) { - NSLog(@"Failed to create audio format description"); + NSLog(@"RTCAudioTrack: Failed to create audio format description"); return; } @@ -89,7 +93,7 @@ void OnData(const void *audio_data, NULL, &buffer); if (status != 0) { - NSLog(@"Failed to allocate sample buffer"); + NSLog(@"RTCAudioTrack: Failed to allocate sample buffer"); return; } @@ -101,13 +105,10 @@ void OnData(const void *audio_data, status = CMSampleBufferSetDataBufferFromAudioBufferList( buffer, kCFAllocatorDefault, kCFAllocatorDefault, 0, &bufferList); if (status != 0) { - NSLog(@"Failed to convert audio buffer list into sample buffer"); + NSLog(@"RTCAudioTrack: Failed to convert audio buffer list into sample buffer"); return; } - // if (!CMSampleBufferIsValid(buffer)) { - NSLog(@"CMSampleBufferIsValid: %d", CMSampleBufferIsValid(buffer)); - // Report back to RTCAudioTrack [audioTrack_ didCaptureSampleBuffer:buffer]; @@ -118,9 +119,8 @@ void OnData(const void *audio_data, @implementation RTC_OBJC_TYPE (RTCAudioTrack) { rtc::Thread *_workerThread; - BOOL _IsAudioConverterActive; - // std::unique_ptr _audioConverter; - webrtc::AudioSinkConverter *_audioConverter; + BOOL _audioSinkAdded; + rtc::scoped_refptr _audioConverter; } @synthesize source = _source; @@ -149,24 +149,27 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto NSParameterAssert(nativeTrack); NSParameterAssert(type == RTCMediaStreamTrackTypeAudio); if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) { + NSLog(@"RTCAudioTrack: init..."); + _workerThread = factory.workerThread; _renderers = [NSMutableArray array]; - // Testing - //_workerThread->BlockingCall([self] { - // _audioConverter.reset(new webrtc::AudioSinkConverter(self)); - // self.nativeAudioTrack->AddSink(_audioConverter.get()); - - _audioConverter = new webrtc::AudioSinkConverter(self); - self.nativeAudioTrack->AddSink(_audioConverter); - _IsAudioConverterActive = YES; - //}); + _audioSinkAdded = NO; + _audioConverter = new rtc::RefCountedObject(self); } return self; } - (void)dealloc { - // TODO: Clean up... - NSLog(@"RTCAudioTrack dealloc..."); + // Clean up... + _workerThread->BlockingCall([self] { + // Remove all renderers... + [_renderers removeAllObjects]; + }); + // Remove sink if added... + if (_audioSinkAdded) { + self.nativeAudioTrack->RemoveSink(_audioConverter.get()); + _audioSinkAdded = NO; + } } - (RTC_OBJC_TYPE(RTCAudioSource) *)source { @@ -181,23 +184,25 @@ - (void)dealloc { } - (void)addRenderer:(id)renderer { - _workerThread->BlockingCall([self, renderer] { - [_renderers addObject:renderer]; - - if ([_renderers count] != 0 && !_IsAudioConverterActive) { - self.nativeAudioTrack->AddSink(_audioConverter.get()); - _IsAudioConverterActive = YES; - } + _workerThread->BlockingCall([self, renderer] { + [_renderers addObject:renderer]; + // Add audio sink if not already added + if ([_renderers count] != 0 && !_audioSinkAdded) { + self.nativeAudioTrack->AddSink(_audioConverter.get()); + _audioSinkAdded = YES; } + }); } - (void)removeRenderer:(id)renderer { - [_renderers removeObject:renderer]; - - if ([_renderers count] == 0 && _IsAudioConverterActive) { - self.nativeAudioTrack->RemoveSink(_audioConverter.get()); - _IsAudioConverterActive = NO; - } + _workerThread->BlockingCall([self, renderer] { + [_renderers removeObject:renderer]; + // Remove audio sink if no more renderers + if ([_renderers count] == 0 && _audioSinkAdded) { + self.nativeAudioTrack->RemoveSink(_audioConverter.get()); + _audioSinkAdded = NO; + } + }); } #pragma mark - Private @@ -208,12 +213,14 @@ - (void)removeRenderer:(id)renderer { } - (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer { - // ... - _workerThread->BlockingCall([self, sampleBuffer] { - for (id renderer in self.renderers) { - // NSLog(@"%@", renderer); + // Retain reference... + CFRetain(sampleBuffer); + _workerThread->PostTask([self, sampleBuffer] { + for (id renderer in _renderers) { [renderer renderSampleBuffer:sampleBuffer]; } + // Release reference... + CFRelease(sampleBuffer); }); } From 0c9367bb5394afb417d5ab0f2402c3f08742e682 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 11 Aug 2023 09:57:57 +0800 Subject: [PATCH 06/17] stereo --- sdk/objc/api/peerconnection/RTCAudioTrack.mm | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index f13754bdef..d8a43b7b80 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -42,23 +42,26 @@ void OnData(const void *audio_data, absl::optional absolute_capture_timestamp_ms) override { /* * Convert to CMSampleBuffer - * TODO: Handle case which number_of_channels could be 2 or more. */ int64_t elapsed_time_ms = absolute_capture_timestamp_ms ? absolute_capture_timestamp_ms.value() : rtc::TimeMillis(); OSStatus status; + // Only mono or stereo is supported currently. + assert(number_of_channels == 1 || number_of_channels == 2); + AudioChannelLayout acl; bzero(&acl, sizeof(acl)); - acl.mChannelLayoutTag = kAudioChannelLayoutTag_Mono; + acl.mChannelLayoutTag = + number_of_channels == 2 ? kAudioChannelLayoutTag_Stereo : kAudioChannelLayoutTag_Mono; AudioStreamBasicDescription audioFormat; audioFormat.mSampleRate = sample_rate; audioFormat.mFormatID = kAudioFormatLinearPCM; audioFormat.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; - audioFormat.mFramesPerPacket = 1; - audioFormat.mChannelsPerFrame = 1; + audioFormat.mFramesPerPacket = number_of_frames; + audioFormat.mChannelsPerFrame = number_of_channels; audioFormat.mBitsPerChannel = 16; audioFormat.mBytesPerPacket = audioFormat.mFramesPerPacket * audioFormat.mChannelsPerFrame * audioFormat.mBitsPerChannel / 8; From 82d88f2f30d0e87c95e10695794cc1e0492fc85c Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 11 Aug 2023 10:02:31 +0800 Subject: [PATCH 07/17] fail gracefully --- sdk/objc/api/peerconnection/RTCAudioTrack.mm | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index d8a43b7b80..8c392a9993 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -43,14 +43,18 @@ void OnData(const void *audio_data, /* * Convert to CMSampleBuffer */ + + if (number_of_channels != 1 && number_of_channels != 2) { + NSLog(@"RTCAudioTrack: Only mono or stereo is supported currently. numberOfChannels: %zu", + number_of_channels); + return; + } + int64_t elapsed_time_ms = absolute_capture_timestamp_ms ? absolute_capture_timestamp_ms.value() : rtc::TimeMillis(); OSStatus status; - // Only mono or stereo is supported currently. - assert(number_of_channels == 1 || number_of_channels == 2); - AudioChannelLayout acl; bzero(&acl, sizeof(acl)); acl.mChannelLayoutTag = From ba3f52d83418eedca2dc3d64ab648189a46fb4de Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 11 Aug 2023 10:04:11 +0800 Subject: [PATCH 08/17] minor fix --- sdk/objc/api/peerconnection/RTCAudioTrack.mm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index 8c392a9993..4c0c264909 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -44,7 +44,7 @@ void OnData(const void *audio_data, * Convert to CMSampleBuffer */ - if (number_of_channels != 1 && number_of_channels != 2) { + if (!(number_of_channels == 1 || number_of_channels == 2)) { NSLog(@"RTCAudioTrack: Only mono or stereo is supported currently. numberOfChannels: %zu", number_of_channels); return; From a4d14d5b9b3128e9b6fc8c8e166c0823cc76d331 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 11 Aug 2023 10:06:53 +0800 Subject: [PATCH 09/17] doc --- sdk/objc/api/peerconnection/RTCAudioTrack.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.h b/sdk/objc/api/peerconnection/RTCAudioTrack.h index f1e5f9f30b..94e2402f3f 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.h +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.h @@ -24,10 +24,10 @@ RTC_OBJC_EXPORT /** The audio source for this audio track. */ @property(nonatomic, readonly) RTC_OBJC_TYPE(RTCAudioSource) * source; -/** ..... */ +/** Register a renderer that will receive all audio CMSampleBuffers on this track */ - (void)addRenderer:(id)renderer; -/** ..... */ +/** Deregister a renderer */ - (void)removeRenderer:(id)renderer; @end From 499f7788bb4e09db920753731d771867de766817 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 23 Aug 2023 18:18:21 +0900 Subject: [PATCH 10/17] optimize AudioStreamBasicDescription --- sdk/objc/api/peerconnection/RTCAudioTrack.mm | 25 ++++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index 4c0c264909..0e165dff36 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -60,16 +60,15 @@ void OnData(const void *audio_data, acl.mChannelLayoutTag = number_of_channels == 2 ? kAudioChannelLayoutTag_Stereo : kAudioChannelLayoutTag_Mono; - AudioStreamBasicDescription audioFormat; - audioFormat.mSampleRate = sample_rate; - audioFormat.mFormatID = kAudioFormatLinearPCM; - audioFormat.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; - audioFormat.mFramesPerPacket = number_of_frames; - audioFormat.mChannelsPerFrame = number_of_channels; - audioFormat.mBitsPerChannel = 16; - audioFormat.mBytesPerPacket = audioFormat.mFramesPerPacket * audioFormat.mChannelsPerFrame * - audioFormat.mBitsPerChannel / 8; - audioFormat.mBytesPerFrame = audioFormat.mBytesPerPacket / audioFormat.mFramesPerPacket; + AudioStreamBasicDescription sd; + sd.mSampleRate = sample_rate; + sd.mFormatID = kAudioFormatLinearPCM; + sd.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; + sd.mFramesPerPacket = number_of_frames; /* 1 */ + sd.mChannelsPerFrame = number_of_channels; + sd.mBitsPerChannel = bits_per_sample; /* 16 */ + sd.mBytesPerFrame = sd.mChannelsPerFrame * (sd.mBitsPerChannel / 8); + sd.mBytesPerPacket = sd.mBytesPerFrame; CMSampleTimingInfo timing = { CMTimeMake(1, sample_rate), @@ -79,7 +78,7 @@ void OnData(const void *audio_data, CMFormatDescriptionRef format = NULL; status = CMAudioFormatDescriptionCreate( - kCFAllocatorDefault, &audioFormat, sizeof(acl), &acl, 0, NULL, NULL, &format); + kCFAllocatorDefault, &sd, sizeof(acl), &acl, 0, NULL, NULL, &format); if (status != 0) { NSLog(@"RTCAudioTrack: Failed to create audio format description"); @@ -106,8 +105,8 @@ void OnData(const void *audio_data, AudioBufferList bufferList; bufferList.mNumberBuffers = 1; - bufferList.mBuffers[0].mNumberChannels = audioFormat.mChannelsPerFrame; - bufferList.mBuffers[0].mDataByteSize = (UInt32)(number_of_frames * audioFormat.mBytesPerFrame); + bufferList.mBuffers[0].mNumberChannels = sd.mChannelsPerFrame; + bufferList.mBuffers[0].mDataByteSize = (UInt32)(number_of_frames * sd.mBytesPerFrame); bufferList.mBuffers[0].mData = (void *)audio_data; status = CMSampleBufferSetDataBufferFromAudioBufferList( buffer, kCFAllocatorDefault, kCFAllocatorDefault, 0, &bufferList); From 8526e1ff907b581d4ee26a11ac6498bbd325f6f1 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 23 Aug 2023 18:45:55 +0900 Subject: [PATCH 11/17] logging --- sdk/objc/api/peerconnection/RTCAudioTrack.mm | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index 0e165dff36..d27fa75172 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -30,10 +30,16 @@ public: AudioSinkConverter(RTCAudioTrack *audioTrack) { + RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter init"; // Keep weak reference to RTCAudioTrack... audioTrack_ = audioTrack; } + ~AudioSinkConverter() { + // + RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter dealloc"; + } + void OnData(const void *audio_data, int bits_per_sample, int sample_rate, @@ -155,7 +161,7 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto NSParameterAssert(nativeTrack); NSParameterAssert(type == RTCMediaStreamTrackTypeAudio); if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) { - NSLog(@"RTCAudioTrack: init..."); + RTC_LOG(LS_INFO) << "RTCAudioTrack init"; _workerThread = factory.workerThread; _renderers = [NSMutableArray array]; _audioSinkAdded = NO; @@ -176,6 +182,7 @@ - (void)dealloc { self.nativeAudioTrack->RemoveSink(_audioConverter.get()); _audioSinkAdded = NO; } + RTC_LOG(LS_INFO) << "RTCAudioTrack dealloc"; } - (RTC_OBJC_TYPE(RTCAudioSource) *)source { From b5dbec751bb0ba52090ba4e4ae841b1af6bba3a8 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 23 Aug 2023 18:50:30 +0900 Subject: [PATCH 12/17] minor refactoring --- sdk/objc/api/peerconnection/RTCAudioTrack.mm | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index d27fa75172..4c0745f1ac 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -131,7 +131,7 @@ void OnData(const void *audio_data, @implementation RTC_OBJC_TYPE (RTCAudioTrack) { rtc::Thread *_workerThread; - BOOL _audioSinkAdded; + BOOL _IsAudioConverterSinkAttached; rtc::scoped_refptr _audioConverter; } @@ -164,7 +164,7 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto RTC_LOG(LS_INFO) << "RTCAudioTrack init"; _workerThread = factory.workerThread; _renderers = [NSMutableArray array]; - _audioSinkAdded = NO; + _IsAudioConverterSinkAttached = NO; _audioConverter = new rtc::RefCountedObject(self); } @@ -178,9 +178,9 @@ - (void)dealloc { [_renderers removeAllObjects]; }); // Remove sink if added... - if (_audioSinkAdded) { + if (_IsAudioConverterSinkAttached) { self.nativeAudioTrack->RemoveSink(_audioConverter.get()); - _audioSinkAdded = NO; + _IsAudioConverterSinkAttached = NO; } RTC_LOG(LS_INFO) << "RTCAudioTrack dealloc"; } @@ -200,9 +200,10 @@ - (void)addRenderer:(id)renderer { _workerThread->BlockingCall([self, renderer] { [_renderers addObject:renderer]; // Add audio sink if not already added - if ([_renderers count] != 0 && !_audioSinkAdded) { + if ([_renderers count] != 0 && !_IsAudioConverterSinkAttached) { + RTC_LOG(LS_INFO) << "RTCAudioTrack attaching sink..."; self.nativeAudioTrack->AddSink(_audioConverter.get()); - _audioSinkAdded = YES; + _IsAudioConverterSinkAttached = YES; } }); } @@ -211,9 +212,10 @@ - (void)removeRenderer:(id)renderer { _workerThread->BlockingCall([self, renderer] { [_renderers removeObject:renderer]; // Remove audio sink if no more renderers - if ([_renderers count] == 0 && _audioSinkAdded) { + if ([_renderers count] == 0 && _IsAudioConverterSinkAttached) { + RTC_LOG(LS_INFO) << "RTCAudioTrack removing sink..."; self.nativeAudioTrack->RemoveSink(_audioConverter.get()); - _audioSinkAdded = NO; + _IsAudioConverterSinkAttached = NO; } }); } From 59060523c230f5787f2d037c5b44fe7162834f62 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 23 Aug 2023 23:32:53 +0900 Subject: [PATCH 13/17] weak reference to delegates --- sdk/objc/api/peerconnection/RTCAudioTrack+Private.h | 2 -- sdk/objc/api/peerconnection/RTCAudioTrack.h | 3 ++- sdk/objc/api/peerconnection/RTCAudioTrack.mm | 7 ++++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h b/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h index ea1fa87e50..38c0bd3b1b 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h +++ b/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h @@ -20,8 +20,6 @@ NS_ASSUME_NONNULL_BEGIN /** AudioTrackInterface created or passed in at construction. */ @property(nonatomic, readonly) rtc::scoped_refptr nativeAudioTrack; -/** Accessed on _workerThread only */ -@property(nonatomic, readonly) NSMutableArray *renderers; /** Initialize an RTCAudioTrack with an id. */ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.h b/sdk/objc/api/peerconnection/RTCAudioTrack.h index 94e2402f3f..c8218ad926 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.h +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.h @@ -24,7 +24,8 @@ RTC_OBJC_EXPORT /** The audio source for this audio track. */ @property(nonatomic, readonly) RTC_OBJC_TYPE(RTCAudioSource) * source; -/** Register a renderer that will receive all audio CMSampleBuffers on this track */ +/** Register a renderer that will receive all audio CMSampleBuffers on this track. + * Does not retain. */ - (void)addRenderer:(id)renderer; /** Deregister a renderer */ diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index 4c0745f1ac..e1f6cf03e0 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -133,10 +133,11 @@ @implementation RTC_OBJC_TYPE (RTCAudioTrack) { rtc::Thread *_workerThread; BOOL _IsAudioConverterSinkAttached; rtc::scoped_refptr _audioConverter; + // Stores weak references to renderers, accessed on _workerThread only. + NSHashTable *_renderers; } @synthesize source = _source; -@synthesize renderers = _renderers; - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory source:(RTC_OBJC_TYPE(RTCAudioSource) *)source @@ -163,7 +164,7 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) { RTC_LOG(LS_INFO) << "RTCAudioTrack init"; _workerThread = factory.workerThread; - _renderers = [NSMutableArray array]; + _renderers = [NSHashTable weakObjectsHashTable]; _IsAudioConverterSinkAttached = NO; _audioConverter = new rtc::RefCountedObject(self); } @@ -231,7 +232,7 @@ - (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer { // Retain reference... CFRetain(sampleBuffer); _workerThread->PostTask([self, sampleBuffer] { - for (id renderer in _renderers) { + for (id renderer in [_renderers allObjects]) { [renderer renderSampleBuffer:sampleBuffer]; } // Release reference... From 1b80bda934fc5709cb0b04954de341fd97315c4c Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 24 Aug 2023 03:22:11 +0900 Subject: [PATCH 14/17] fix timestamp computation --- sdk/objc/api/peerconnection/RTCAudioTrack.mm | 23 +++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index e1f6cf03e0..e944a122cd 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -27,6 +27,7 @@ class AudioSinkConverter : public rtc::RefCountInterface, public webrtc::AudioTrackSinkInterface { private: __weak RTCAudioTrack *audioTrack_; + int64_t total_frames_ = 0; public: AudioSinkConverter(RTCAudioTrack *audioTrack) { @@ -40,12 +41,24 @@ RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter dealloc"; } + void Reset() { + // Reset for creating CMSampleTimingInfo correctly + total_frames_ = 0; + } + void OnData(const void *audio_data, int bits_per_sample, int sample_rate, size_t number_of_channels, size_t number_of_frames, absl::optional absolute_capture_timestamp_ms) override { + RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter OnData bits_per_sample: " + << bits_per_sample << " sample_rate: " << sample_rate + << " number_of_channels: " << number_of_channels + << " number_of_frames: " << number_of_frames + << " absolute_capture_timestamp_ms: " + << (absolute_capture_timestamp_ms ? absolute_capture_timestamp_ms.value() : 0); + /* * Convert to CMSampleBuffer */ @@ -56,9 +69,6 @@ void OnData(const void *audio_data, return; } - int64_t elapsed_time_ms = - absolute_capture_timestamp_ms ? absolute_capture_timestamp_ms.value() : rtc::TimeMillis(); - OSStatus status; AudioChannelLayout acl; @@ -70,7 +80,7 @@ void OnData(const void *audio_data, sd.mSampleRate = sample_rate; sd.mFormatID = kAudioFormatLinearPCM; sd.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; - sd.mFramesPerPacket = number_of_frames; /* 1 */ + sd.mFramesPerPacket = 1; sd.mChannelsPerFrame = number_of_channels; sd.mBitsPerChannel = bits_per_sample; /* 16 */ sd.mBytesPerFrame = sd.mChannelsPerFrame * (sd.mBitsPerChannel / 8); @@ -78,10 +88,12 @@ void OnData(const void *audio_data, CMSampleTimingInfo timing = { CMTimeMake(1, sample_rate), - CMTimeMake(elapsed_time_ms, 1000), + CMTimeMake(total_frames_, sample_rate), kCMTimeInvalid, }; + total_frames_ += number_of_frames; // update the total + CMFormatDescriptionRef format = NULL; status = CMAudioFormatDescriptionCreate( kCFAllocatorDefault, &sd, sizeof(acl), &acl, 0, NULL, NULL, &format); @@ -203,6 +215,7 @@ - (void)addRenderer:(id)renderer { // Add audio sink if not already added if ([_renderers count] != 0 && !_IsAudioConverterSinkAttached) { RTC_LOG(LS_INFO) << "RTCAudioTrack attaching sink..."; + _audioConverter->Reset(); self.nativeAudioTrack->AddSink(_audioConverter.get()); _IsAudioConverterSinkAttached = YES; } From c0d5abf719a286f769a8cfd3b61470dad12e302b Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 24 Aug 2023 03:22:23 +0900 Subject: [PATCH 15/17] change swift delegate signature --- sdk/objc/base/RTCAudioRenderer.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/objc/base/RTCAudioRenderer.h b/sdk/objc/base/RTCAudioRenderer.h index 35f84d9603..def20eac3c 100644 --- a/sdk/objc/base/RTCAudioRenderer.h +++ b/sdk/objc/base/RTCAudioRenderer.h @@ -26,7 +26,8 @@ NS_ASSUME_NONNULL_BEGIN RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE (RTCAudioRenderer) - - (void)renderSampleBuffer : (CMSampleBufferRef)sampleBuffer; + - (void)renderSampleBuffer : (CMSampleBufferRef)sampleBuffer + NS_SWIFT_NAME(render(sampleBuffer:)); @end From d309ecb48bb2432c5ff276b03ee82668fa6bcf92 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 24 Aug 2023 16:54:04 +0900 Subject: [PATCH 16/17] use os_unfair_lock instead --- sdk/objc/api/peerconnection/RTCAudioTrack.mm | 73 ++++++++++---------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index e944a122cd..a4c7aa1936 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -9,6 +9,7 @@ */ #import +#import #import "RTCAudioTrack+Private.h" @@ -142,11 +143,11 @@ void OnData(const void *audio_data, } // namespace webrtc @implementation RTC_OBJC_TYPE (RTCAudioTrack) { - rtc::Thread *_workerThread; BOOL _IsAudioConverterSinkAttached; rtc::scoped_refptr _audioConverter; - // Stores weak references to renderers, accessed on _workerThread only. + // Stores weak references to renderers NSHashTable *_renderers; + os_unfair_lock _lock; } @synthesize source = _source; @@ -175,7 +176,6 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto NSParameterAssert(type == RTCMediaStreamTrackTypeAudio); if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) { RTC_LOG(LS_INFO) << "RTCAudioTrack init"; - _workerThread = factory.workerThread; _renderers = [NSHashTable weakObjectsHashTable]; _IsAudioConverterSinkAttached = NO; _audioConverter = new rtc::RefCountedObject(self); @@ -185,16 +185,13 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto } - (void)dealloc { - // Clean up... - _workerThread->BlockingCall([self] { - // Remove all renderers... - [_renderers removeAllObjects]; - }); // Remove sink if added... + os_unfair_lock_lock(&_lock); if (_IsAudioConverterSinkAttached) { self.nativeAudioTrack->RemoveSink(_audioConverter.get()); - _IsAudioConverterSinkAttached = NO; } + os_unfair_lock_unlock(&_lock); + RTC_LOG(LS_INFO) << "RTCAudioTrack dealloc"; } @@ -210,28 +207,32 @@ - (void)dealloc { } - (void)addRenderer:(id)renderer { - _workerThread->BlockingCall([self, renderer] { - [_renderers addObject:renderer]; - // Add audio sink if not already added - if ([_renderers count] != 0 && !_IsAudioConverterSinkAttached) { - RTC_LOG(LS_INFO) << "RTCAudioTrack attaching sink..."; - _audioConverter->Reset(); - self.nativeAudioTrack->AddSink(_audioConverter.get()); - _IsAudioConverterSinkAttached = YES; - } - }); + os_unfair_lock_lock(&_lock); + [_renderers addObject:renderer]; + NSUInteger renderersCount = _renderers.allObjects.count; + + // Add audio sink if not already added + if (renderersCount != 0 && !_IsAudioConverterSinkAttached) { + RTC_LOG(LS_INFO) << "RTCAudioTrack attaching sink..."; + _audioConverter->Reset(); + self.nativeAudioTrack->AddSink(_audioConverter.get()); + _IsAudioConverterSinkAttached = YES; + } + os_unfair_lock_unlock(&_lock); } - (void)removeRenderer:(id)renderer { - _workerThread->BlockingCall([self, renderer] { - [_renderers removeObject:renderer]; - // Remove audio sink if no more renderers - if ([_renderers count] == 0 && _IsAudioConverterSinkAttached) { - RTC_LOG(LS_INFO) << "RTCAudioTrack removing sink..."; - self.nativeAudioTrack->RemoveSink(_audioConverter.get()); - _IsAudioConverterSinkAttached = NO; - } - }); + os_unfair_lock_lock(&_lock); + [_renderers removeObject:renderer]; + NSUInteger renderersCount = _renderers.allObjects.count; + + // Remove audio sink if no more renderers + if (renderersCount == 0 && _IsAudioConverterSinkAttached) { + RTC_LOG(LS_INFO) << "RTCAudioTrack removing sink..."; + self.nativeAudioTrack->RemoveSink(_audioConverter.get()); + _IsAudioConverterSinkAttached = NO; + } + os_unfair_lock_unlock(&_lock); } #pragma mark - Private @@ -242,15 +243,13 @@ - (void)removeRenderer:(id)renderer { } - (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer { - // Retain reference... - CFRetain(sampleBuffer); - _workerThread->PostTask([self, sampleBuffer] { - for (id renderer in [_renderers allObjects]) { - [renderer renderSampleBuffer:sampleBuffer]; - } - // Release reference... - CFRelease(sampleBuffer); - }); + os_unfair_lock_lock(&_lock); + NSArray *renderers = [_renderers allObjects]; + + for (id renderer in renderers) { + [renderer renderSampleBuffer:sampleBuffer]; + } + os_unfair_lock_unlock(&_lock); } @end From de0ba1f4fc9308b1a7315d101e62e572e43d1d44 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 24 Aug 2023 19:13:49 +0900 Subject: [PATCH 17/17] avoid deadlock --- sdk/objc/api/peerconnection/RTCAudioTrack.mm | 81 +++++++++++++------- 1 file changed, 52 insertions(+), 29 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index a4c7aa1936..065064a1fe 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -27,14 +27,16 @@ */ class AudioSinkConverter : public rtc::RefCountInterface, public webrtc::AudioTrackSinkInterface { private: - __weak RTCAudioTrack *audioTrack_; + os_unfair_lock *lock_; + __weak RTCAudioTrack *audio_track_; int64_t total_frames_ = 0; + bool attached_ = false; public: - AudioSinkConverter(RTCAudioTrack *audioTrack) { + AudioSinkConverter(RTCAudioTrack *audioTrack, os_unfair_lock *lock) { RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter init"; - // Keep weak reference to RTCAudioTrack... - audioTrack_ = audioTrack; + audio_track_ = audioTrack; + lock_ = lock; } ~AudioSinkConverter() { @@ -42,9 +44,28 @@ RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter dealloc"; } - void Reset() { + // Must be called while locked + void TryAttach() { + if (attached_) { + // Already attached + return; + } + RTC_LOG(LS_INFO) << "RTCAudioTrack attaching sink..."; // Reset for creating CMSampleTimingInfo correctly + audio_track_.nativeAudioTrack->AddSink(this); total_frames_ = 0; + attached_ = true; + } + + // Must be called while locked + void TryDetach() { + if (!attached_) { + // Already detached + return; + } + RTC_LOG(LS_INFO) << "RTCAudioTrack detaching sink..."; + audio_track_.nativeAudioTrack->RemoveSink(this); + attached_ = false; } void OnData(const void *audio_data, @@ -60,6 +81,19 @@ void OnData(const void *audio_data, << " absolute_capture_timestamp_ms: " << (absolute_capture_timestamp_ms ? absolute_capture_timestamp_ms.value() : 0); + bool is_locked = os_unfair_lock_trylock(lock_); + if (!is_locked) { + RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter OnData already locked, skipping..."; + return; + } + bool is_attached = attached_; + os_unfair_lock_unlock(lock_); + + if (!is_attached) { + RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter OnData already detached, skipping..."; + return; + } + /* * Convert to CMSampleBuffer */ @@ -135,7 +169,7 @@ void OnData(const void *audio_data, } // Report back to RTCAudioTrack - [audioTrack_ didCaptureSampleBuffer:buffer]; + [audio_track_ didCaptureSampleBuffer:buffer]; CFRelease(buffer); } @@ -143,7 +177,6 @@ void OnData(const void *audio_data, } // namespace webrtc @implementation RTC_OBJC_TYPE (RTCAudioTrack) { - BOOL _IsAudioConverterSinkAttached; rtc::scoped_refptr _audioConverter; // Stores weak references to renderers NSHashTable *_renderers; @@ -177,19 +210,15 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) { RTC_LOG(LS_INFO) << "RTCAudioTrack init"; _renderers = [NSHashTable weakObjectsHashTable]; - _IsAudioConverterSinkAttached = NO; - _audioConverter = new rtc::RefCountedObject(self); + _audioConverter = new rtc::RefCountedObject(self, &_lock); } return self; } - (void)dealloc { - // Remove sink if added... os_unfair_lock_lock(&_lock); - if (_IsAudioConverterSinkAttached) { - self.nativeAudioTrack->RemoveSink(_audioConverter.get()); - } + _audioConverter->TryDetach(); os_unfair_lock_unlock(&_lock); RTC_LOG(LS_INFO) << "RTCAudioTrack dealloc"; @@ -209,15 +238,7 @@ - (void)dealloc { - (void)addRenderer:(id)renderer { os_unfair_lock_lock(&_lock); [_renderers addObject:renderer]; - NSUInteger renderersCount = _renderers.allObjects.count; - - // Add audio sink if not already added - if (renderersCount != 0 && !_IsAudioConverterSinkAttached) { - RTC_LOG(LS_INFO) << "RTCAudioTrack attaching sink..."; - _audioConverter->Reset(); - self.nativeAudioTrack->AddSink(_audioConverter.get()); - _IsAudioConverterSinkAttached = YES; - } + _audioConverter->TryAttach(); os_unfair_lock_unlock(&_lock); } @@ -226,11 +247,9 @@ - (void)removeRenderer:(id)renderer { [_renderers removeObject:renderer]; NSUInteger renderersCount = _renderers.allObjects.count; - // Remove audio sink if no more renderers - if (renderersCount == 0 && _IsAudioConverterSinkAttached) { - RTC_LOG(LS_INFO) << "RTCAudioTrack removing sink..."; - self.nativeAudioTrack->RemoveSink(_audioConverter.get()); - _IsAudioConverterSinkAttached = NO; + if (renderersCount == 0) { + // Detach if no more renderers... + _audioConverter->TryDetach(); } os_unfair_lock_unlock(&_lock); } @@ -243,13 +262,17 @@ - (void)removeRenderer:(id)renderer { } - (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer { - os_unfair_lock_lock(&_lock); + bool is_locked = os_unfair_lock_trylock(&_lock); + if (!is_locked) { + RTC_LOG(LS_INFO) << "RTCAudioTrack didCaptureSampleBuffer already locked, skipping..."; + return; + } NSArray *renderers = [_renderers allObjects]; + os_unfair_lock_unlock(&_lock); for (id renderer in renderers) { [renderer renderSampleBuffer:sampleBuffer]; } - os_unfair_lock_unlock(&_lock); } @end