Expose remote audio sample buffers on RTCAudioTrack

sdk/BUILD.gn

@@ -119,6 +119,7 @@ if (is_ios || is_mac) {
       "objc/base/RTCVideoFrame.mm",
       "objc/base/RTCVideoFrameBuffer.h",
       "objc/base/RTCVideoRenderer.h",
+      "objc/base/RTCAudioRenderer.h",
       "objc/base/RTCYUVPlanarBuffer.h",
     ]
 
@@ -1357,6 +1358,7 @@ if (is_ios || is_mac) {
           "objc/base/RTCVideoFrame.h",
           "objc/base/RTCVideoFrameBuffer.h",
           "objc/base/RTCVideoRenderer.h",
+          "objc/base/RTCAudioRenderer.h",
           "objc/base/RTCYUVPlanarBuffer.h",
           "objc/components/audio/RTCAudioDevice.h",
           "objc/components/audio/RTCAudioSession.h",
@@ -1572,6 +1574,7 @@ if (is_ios || is_mac) {
           "objc/base/RTCVideoFrame.h",
           "objc/base/RTCVideoFrameBuffer.h",
           "objc/base/RTCVideoRenderer.h",
+          "objc/base/RTCAudioRenderer.h",
           "objc/base/RTCYUVPlanarBuffer.h",
           "objc/components/capturer/RTCCameraVideoCapturer.h",
           "objc/components/capturer/RTCFileVideoCapturer.h",

sdk/objc/api/peerconnection/RTCAudioTrack+Private.h

@@ -8,24 +8,28 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
+#import <AVFoundation/AVFoundation.h>
 #import "RTCAudioTrack.h"
 
 #include "api/media_stream_interface.h"
 
 NS_ASSUME_NONNULL_BEGIN
 
 @class RTC_OBJC_TYPE(RTCPeerConnectionFactory);
-@interface RTC_OBJC_TYPE (RTCAudioTrack)
-()
+@interface RTC_OBJC_TYPE (RTCAudioTrack) ()
 
-    /** AudioTrackInterface created or passed in at construction. */
-    @property(nonatomic, readonly) rtc::scoped_refptr<webrtc::AudioTrackInterface> nativeAudioTrack;
+/** AudioTrackInterface created or passed in at construction. */
+@property(nonatomic, readonly) rtc::scoped_refptr<webrtc::AudioTrackInterface> nativeAudioTrack;
+/** Accessed on _workerThread only */
+@property(nonatomic, readonly) NSMutableArray<RTCAudioRenderer> *renderers;
 
 /** Initialize an RTCAudioTrack with an id. */
 - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory
                          source:(RTC_OBJC_TYPE(RTCAudioSource) *)source
                         trackId:(NSString *)trackId;
 
+- (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer;
+
 @end
 
 NS_ASSUME_NONNULL_END

sdk/objc/api/peerconnection/RTCAudioTrack.h

@@ -13,6 +13,7 @@
 
 NS_ASSUME_NONNULL_BEGIN
 
+@protocol RTC_OBJC_TYPE (RTCAudioRenderer);
 @class RTC_OBJC_TYPE(RTCAudioSource);
 
 RTC_OBJC_EXPORT
@@ -23,6 +24,12 @@ RTC_OBJC_EXPORT
 /** The audio source for this audio track. */
 @property(nonatomic, readonly) RTC_OBJC_TYPE(RTCAudioSource) * source;
 
+/** ..... */
+- (void)addRenderer:(id<RTC_OBJC_TYPE(RTCAudioRenderer)>)renderer;
+
+/** ..... */
+- (void)removeRenderer:(id<RTC_OBJC_TYPE(RTCAudioRenderer)>)renderer;
+
 @end
 
 NS_ASSUME_NONNULL_END

sdk/objc/api/peerconnection/RTCAudioTrack.mm

@@ -8,18 +8,123 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
+#import <AVFoundation/AVFoundation.h>
+
 #import "RTCAudioTrack+Private.h"
 
+#import "RTCAudioRenderer.h"
 #import "RTCAudioSource+Private.h"
 #import "RTCMediaStreamTrack+Private.h"
 #import "RTCPeerConnectionFactory+Private.h"
 #import "helpers/NSString+StdString.h"
 
 #include "rtc_base/checks.h"
 
-@implementation RTC_OBJC_TYPE (RTCAudioTrack)
+namespace webrtc {
+/**
+ * Captures audio data and converts to CMSampleBuffers
+ */
+class AudioSinkConverter : public rtc::RefCountInterface, public webrtc::AudioTrackSinkInterface {
+ private:
+  __weak RTCAudioTrack *audioTrack_;
+
+ public:
+  AudioSinkConverter(RTCAudioTrack *audioTrack) {
+    // Keep weak reference to RTCAudioTrack...
+    audioTrack_ = audioTrack;
+  }
+
+  void OnData(const void *audio_data,
+              int bits_per_sample,
+              int sample_rate,
+              size_t number_of_channels,
+              size_t number_of_frames,
+              absl::optional<int64_t> absolute_capture_timestamp_ms) override {
+    /*
+     * Convert to CMSampleBuffer
+     * TODO: Handle case which number_of_channels could be 2 or more.
+     */
+    int64_t elapsed_time_ms =
+        absolute_capture_timestamp_ms ? absolute_capture_timestamp_ms.value() : rtc::TimeMillis();
+
+    OSStatus status;
+
+    AudioChannelLayout acl;
+    bzero(&acl, sizeof(acl));
+    acl.mChannelLayoutTag = kAudioChannelLayoutTag_Mono;
+
+    AudioStreamBasicDescription audioFormat;
+    audioFormat.mSampleRate = sample_rate;
+    audioFormat.mFormatID = kAudioFormatLinearPCM;
+    audioFormat.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked;
+    audioFormat.mFramesPerPacket = 1;
+    audioFormat.mChannelsPerFrame = 1;
+    audioFormat.mBitsPerChannel = 16;
+    audioFormat.mBytesPerPacket = audioFormat.mFramesPerPacket * audioFormat.mChannelsPerFrame *
+        audioFormat.mBitsPerChannel / 8;
+    audioFormat.mBytesPerFrame = audioFormat.mBytesPerPacket / audioFormat.mFramesPerPacket;
+
+    CMSampleTimingInfo timing = {
+        CMTimeMake(1, sample_rate),
+        CMTimeMake(elapsed_time_ms, 1000),
+        kCMTimeInvalid,
+    };
+
+    CMFormatDescriptionRef format = NULL;
+    status = CMAudioFormatDescriptionCreate(
+        kCFAllocatorDefault, &audioFormat, sizeof(acl), &acl, 0, NULL, NULL, &format);
+
+    if (status != 0) {
+      NSLog(@"RTCAudioTrack: Failed to create audio format description");
+      return;
+    }
+
+    CMSampleBufferRef buffer;
+    status = CMSampleBufferCreate(kCFAllocatorDefault,
+                                  NULL,
+                                  false,
+                                  NULL,
+                                  NULL,
+                                  format,
+                                  (CMItemCount)number_of_frames,
+                                  1,
+                                  &timing,
+                                  0,
+                                  NULL,
+                                  &buffer);
+    if (status != 0) {
+      NSLog(@"RTCAudioTrack: Failed to allocate sample buffer");
+      return;
+    }
+
+    AudioBufferList bufferList;
+    bufferList.mNumberBuffers = 1;
+    bufferList.mBuffers[0].mNumberChannels = audioFormat.mChannelsPerFrame;
+    bufferList.mBuffers[0].mDataByteSize = (UInt32)(number_of_frames * audioFormat.mBytesPerFrame);
+    bufferList.mBuffers[0].mData = (void *)audio_data;
+    status = CMSampleBufferSetDataBufferFromAudioBufferList(
+        buffer, kCFAllocatorDefault, kCFAllocatorDefault, 0, &bufferList);
+    if (status != 0) {
+      NSLog(@"RTCAudioTrack: Failed to convert audio buffer list into sample buffer");
+      return;
+    }
+
+    // Report back to RTCAudioTrack
+    [audioTrack_ didCaptureSampleBuffer:buffer];
+
+    CFRelease(buffer);
+  }
+};
+}  // namespace webrtc
+
+@implementation RTC_OBJC_TYPE (RTCAudioTrack) {
+  rtc::Thread *_workerThread;
+  BOOL _audioSinkAdded;
+  rtc::scoped_refptr<webrtc::AudioSinkConverter> _audioConverter;
+}
 
 @synthesize source = _source;
+@synthesize renderers = _renderers;
 
 - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory
                          source:(RTC_OBJC_TYPE(RTCAudioSource) *)source
@@ -43,7 +148,28 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto
   NSParameterAssert(factory);
   NSParameterAssert(nativeTrack);
   NSParameterAssert(type == RTCMediaStreamTrackTypeAudio);
-  return [super initWithFactory:factory nativeTrack:nativeTrack type:type];
+  if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) {
+    NSLog(@"RTCAudioTrack: init...");
+    _workerThread = factory.workerThread;
+    _renderers = [NSMutableArray<RTCAudioRenderer> array];
+    _audioSinkAdded = NO;
+    _audioConverter = new rtc::RefCountedObject<webrtc::AudioSinkConverter>(self);
+  }
+
+  return self;
+}
+
+- (void)dealloc {
+  // Clean up...
+  _workerThread->BlockingCall([self] {
+    // Remove all renderers...
+    [_renderers removeAllObjects];
+  });
+  // Remove sink if added...
+  if (_audioSinkAdded) {
+    self.nativeAudioTrack->RemoveSink(_audioConverter.get());
+    _audioSinkAdded = NO;
+  }
 }
 
 - (RTC_OBJC_TYPE(RTCAudioSource) *)source {
@@ -57,11 +183,45 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto
   return _source;
 }
 
+- (void)addRenderer:(id<RTC_OBJC_TYPE(RTCAudioRenderer)>)renderer {
+  _workerThread->BlockingCall([self, renderer] {
+    [_renderers addObject:renderer];
+    // Add audio sink if not already added
+    if ([_renderers count] != 0 && !_audioSinkAdded) {
+      self.nativeAudioTrack->AddSink(_audioConverter.get());
+      _audioSinkAdded = YES;
+    }
+  });
+}
+
+- (void)removeRenderer:(id<RTC_OBJC_TYPE(RTCAudioRenderer)>)renderer {
+  _workerThread->BlockingCall([self, renderer] {
+    [_renderers removeObject:renderer];
+    // Remove audio sink if no more renderers
+    if ([_renderers count] == 0 && _audioSinkAdded) {
+      self.nativeAudioTrack->RemoveSink(_audioConverter.get());
+      _audioSinkAdded = NO;
+    }
+  });
+}
+
 #pragma mark - Private
 
 - (rtc::scoped_refptr<webrtc::AudioTrackInterface>)nativeAudioTrack {
   return rtc::scoped_refptr<webrtc::AudioTrackInterface>(
       static_cast<webrtc::AudioTrackInterface *>(self.nativeTrack.get()));
 }
 
+- (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer {
+  // Retain reference...
+  CFRetain(sampleBuffer);
+  _workerThread->PostTask([self, sampleBuffer] {
+    for (id<RTCAudioRenderer> renderer in _renderers) {
+      [renderer renderSampleBuffer:sampleBuffer];
+    }
+    // Release reference...
+    CFRelease(sampleBuffer);
+  });
+}
+
 @end

sdk/objc/base/RTCAudioRenderer.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import <Foundation/Foundation.h>
+#if TARGET_OS_IPHONE
+#import <UIKit/UIKit.h>
+#endif
+
+#import "RTCMacros.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE
+(RTCAudioRenderer)<NSObject>
+
+    - (void)renderSampleBuffer : (CMSampleBufferRef)sampleBuffer;
+
+@end
+
+NS_ASSUME_NONNULL_END