1.2.0 Release

sdk/BUILD.gn

@@ -248,6 +248,8 @@ if (is_ios || is_mac) {
         visibility = [ "*" ]
 
         sources = [
+          "objc/native/src/audio/audio_source_sink.h",
+          "objc/native/src/audio/audio_source_sink.mm",
           "objc/native/src/audio/audio_device_ios.h",
           "objc/native/src/audio/audio_device_ios.mm",
           "objc/native/src/audio/audio_device_module_ios.h",
@@ -813,6 +815,8 @@ if (is_ios || is_mac) {
         "objc/api/peerconnection/RTCAudioSource+Private.h",
         "objc/api/peerconnection/RTCAudioSource.h",
         "objc/api/peerconnection/RTCAudioSource.mm",
+        "objc/api/peerconnection/RTCAudioSink.h",
+        "objc/api/peerconnection/RTCAudioSink.mm",
         "objc/api/peerconnection/RTCAudioTrack+Private.h",
         "objc/api/peerconnection/RTCAudioTrack.h",
         "objc/api/peerconnection/RTCAudioTrack.mm",
@@ -983,6 +987,7 @@ if (is_ios || is_mac) {
         "objc/Framework/Headers/WebRTC/RTCAudioSession.h",
         "objc/Framework/Headers/WebRTC/RTCAudioSessionConfiguration.h",
         "objc/Framework/Headers/WebRTC/RTCAudioSource.h",
+        "objc/Framework/Headers/WebRTC/RTCAudioSink.h",
         "objc/Framework/Headers/WebRTC/RTCAudioTrack.h",
         "objc/Framework/Headers/WebRTC/RTCCVPixelBuffer.h",
         "objc/Framework/Headers/WebRTC/RTCCallbackLogger.h",
@@ -1284,6 +1289,7 @@ if (is_ios || is_mac) {
           "objc/helpers/RTCDispatcher.h",
           "objc/helpers/UIDevice+RTCDevice.h",
           "objc/api/peerconnection/RTCAudioSource.h",
+          "objc/api/peerconnection/RTCAudioSink.h",
           "objc/api/peerconnection/RTCAudioTrack.h",
           "objc/api/peerconnection/RTCConfiguration.h",
           "objc/api/peerconnection/RTCDataChannel.h",
@@ -1395,6 +1401,7 @@ if (is_ios || is_mac) {
 
         sources = [
           "objc/api/peerconnection/RTCAudioSource.h",
+          "objc/api/peerconnection/RTCAudioSink.h",
           "objc/api/peerconnection/RTCAudioTrack.h",
           "objc/api/peerconnection/RTCCertificate.h",
           "objc/api/peerconnection/RTCConfiguration.h",

sdk/objc/Framework/Headers/WebRTC/RTCAudioSink.h

@@ -0,0 +1,11 @@
+/*
+ *  Copyright 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#import "api/peerconnection/RTCAudioSink.h"

sdk/objc/api/peerconnection/RTCAudioSink.h

@@ -0,0 +1,25 @@
+#import <Foundation/Foundation.h>
+#import <AVFoundation/AVFoundation.h>
+
+#import "RTCMacros.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+RTC_OBJC_EXPORT
+@interface RTC_OBJC_TYPE (RTCAudioSink) : NSObject
+
+- (instancetype)init;
+- (void)onLocalAudioFrameWithFlags:(AudioUnitRenderActionFlags *)flags 
+                    timeStamp:(const AudioTimeStamp *)timeStamp
+                    busNumber:(uint32_t)busNumber
+                    numFrames:(uint32_t)numFrames
+                    ioData:(AudioBufferList *)ioData;
+
+- (void)onRemoteAudioFrameWithFlags:(AudioUnitRenderActionFlags *)flags 
+                    timeStamp:(const AudioTimeStamp *)timeStamp
+                    busNumber:(uint32_t)busNumber
+                    numFrames:(uint32_t)numFrames
+                    ioData:(AudioBufferList *)ioData;
+@end
+
+NS_ASSUME_NONNULL_END

sdk/objc/api/peerconnection/RTCAudioSink.mm

@@ -0,0 +1,34 @@
+/*
+ *  Copyright 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#import "RTCAudioSink.h"
+#import "base/RTCLogging.h"
+
+@implementation RTC_OBJC_TYPE (RTCAudioSink) {
+}
+
+- (instancetype)init {
+  self = [super init];
+  return self;
+}
+
+- (void)onLocalAudioFrameWithFlags:(AudioUnitRenderActionFlags *)flags 
+                    timeStamp:(const AudioTimeStamp *)timeStamp
+                    busNumber:(uint32_t)busNumber
+                    numFrames:(uint32_t)numFrames
+                    ioData:(AudioBufferList *)ioData { }
+
+- (void)onRemoteAudioFrameWithFlags:(AudioUnitRenderActionFlags *)flags 
+                    timeStamp:(const AudioTimeStamp *)timeStamp
+                    busNumber:(uint32_t)busNumber
+                    numFrames:(uint32_t)numFrames
+                    ioData:(AudioBufferList *)ioData { }
+
+@end

sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h

@@ -88,6 +88,7 @@ NS_ASSUME_NONNULL_BEGIN
 - (instancetype)
     initWithEncoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
             decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory
+            audioSink:(nullable RTC_OBJC_TYPE(RTCAudioSink) *)audioSink
      mediaTransportFactory:(std::unique_ptr<webrtc::MediaTransportFactory>)mediaTransportFactory;
 
 /** Initialize an RTCPeerConnection with a configuration, constraints, and

sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h

@@ -15,6 +15,7 @@
 NS_ASSUME_NONNULL_BEGIN
 
 @class RTC_OBJC_TYPE(RTCAudioSource);
+@class RTC_OBJC_TYPE(RTCAudioSink);
 @class RTC_OBJC_TYPE(RTCAudioTrack);
 @class RTC_OBJC_TYPE(RTCConfiguration);
 @class RTC_OBJC_TYPE(RTCMediaConstraints);
@@ -41,6 +42,12 @@ RTC_OBJC_EXPORT
     initWithEncoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
             decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory;
 
+/* Initialize object with injectable video encoder/decoder factories and audioSink */
+- (instancetype)
+    initWithEncoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
+            decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory
+            audioSink:(nullable RTC_OBJC_TYPE(RTCAudioSink) *)audioSink;
+
 /** Initialize an RTCAudioSource with constraints. */
 - (RTC_OBJC_TYPE(RTCAudioSource) *)audioSourceWithConstraints:
     (nullable RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints;

sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm

@@ -45,6 +45,9 @@
 
 #if defined(WEBRTC_IOS)
 #import "sdk/objc/native/api/audio_device_module.h"
+#import "sdk/objc/native/src/audio/audio_source_sink.h"
+#import "sdk/objc/native/src/audio/audio_device_module_ios.h"
+#import "RTCAudioSink.h"
 #endif
 
 // Adding the nogncheck to disable the including header check.
@@ -66,12 +69,23 @@ @implementation RTC_OBJC_TYPE (RTCPeerConnectionFactory) {
 
 - (rtc::scoped_refptr<webrtc::AudioDeviceModule>)audioDeviceModule {
 #if defined(WEBRTC_IOS)
+  RTCLogInfo(@"Creating AudioDeviceModule without AudioSourceSink");
   return webrtc::CreateAudioDeviceModule();
 #else
   return nullptr;
 #endif
 }
 
+- (rtc::scoped_refptr<webrtc::AudioDeviceModule>)audioDeviceModuleWithAudioSink:(nullable RTC_OBJC_TYPE(RTCAudioSink) *)audioSink { 
+#if defined(WEBRTC_IOS)
+  RTCLogInfo(@"Creating AudioDeviceModule with AudioSourceSink");
+  webrtc::AudioSourceSink *sink = new webrtc::AudioSourceSink(audioSink);
+  return webrtc::CreateAudioDeviceModule(sink);
+#else
+  return nullptr;
+#endif
+}
+
 - (instancetype)init {
 #ifdef HAVE_NO_MEDIA
   return [self initWithNoMedia];
@@ -92,6 +106,7 @@ - (instancetype)init {
 - (instancetype)
     initWithEncoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
             decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory
+                 audioSink:(nullable RTC_OBJC_TYPE(RTCAudioSink) *)audioSink
      mediaTransportFactory:(std::unique_ptr<webrtc::MediaTransportFactory>)mediaTransportFactory {
 #ifdef HAVE_NO_MEDIA
   return [self initWithNoMedia];
@@ -108,16 +123,28 @@ - (instancetype)init {
                        nativeAudioDecoderFactory:webrtc::CreateBuiltinAudioDecoderFactory()
                        nativeVideoEncoderFactory:std::move(native_encoder_factory)
                        nativeVideoDecoderFactory:std::move(native_decoder_factory)
-                               audioDeviceModule:[self audioDeviceModule]
+                               audioDeviceModule:[self audioDeviceModuleWithAudioSink:audioSink]
                            audioProcessingModule:nullptr
                            mediaTransportFactory:std::move(mediaTransportFactory)];
 #endif
 }
+
 - (instancetype)
     initWithEncoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
             decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory {
   return [self initWithEncoderFactory:encoderFactory
                        decoderFactory:decoderFactory
+                       audioSink:nullptr
+                mediaTransportFactory:nullptr];
+}
+
+- (instancetype)
+    initWithEncoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
+            decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory
+            audioSink:(nullable RTC_OBJC_TYPE(RTCAudioSink) *)audioSink {
+  return [self initWithEncoderFactory:encoderFactory
+                       decoderFactory:decoderFactory
+                       audioSink:audioSink
                 mediaTransportFactory:nullptr];
 }
 

sdk/objc/native/api/audio_device_module.h

@@ -17,7 +17,10 @@
 
 namespace webrtc {
 
+class AudioSourceSink;
+
 rtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule();
+rtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule(AudioSourceSink* audioSink);
 
 }  // namespace webrtc
 

sdk/objc/native/api/audio_device_module.mm

@@ -14,6 +14,7 @@
 #include "rtc_base/ref_counted_object.h"
 
 #include "sdk/objc/native/src/audio/audio_device_module_ios.h"
+#import "sdk/objc/native/src/audio/audio_source_sink.h"
 
 namespace webrtc {
 
@@ -27,4 +28,15 @@
   return nullptr;
 #endif
 }
+
+rtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule(webrtc::AudioSourceSink* audioSink) {
+  RTC_LOG(INFO) << __FUNCTION__;
+#if defined(WEBRTC_IOS)
+  return new rtc::RefCountedObject<ios_adm::AudioDeviceModuleIOS>(audioSink);
+#else
+  RTC_LOG(LERROR)
+      << "current platform is not supported => this module will self destruct!";
+  return nullptr;
+#endif
+}
 }

sdk/objc/native/src/audio/audio_device_ios.h

@@ -26,6 +26,7 @@ RTC_FWD_DECL_OBJC_CLASS(RTCNativeAudioSessionDelegateAdapter);
 
 namespace webrtc {
 
+class AudioSourceSink;
 class FineAudioBuffer;
 
 namespace ios_adm {
@@ -161,6 +162,8 @@ class AudioDeviceIOS : public AudioDeviceGeneric,
   // Handles messages from posts.
   void OnMessage(rtc::Message* msg) override;
 
+  void AddAudioSourceSink(AudioSourceSink* audioSink);
+
   bool IsInterrupted();
 
  private:
@@ -293,6 +296,8 @@ class AudioDeviceIOS : public AudioDeviceGeneric,
 
   // Contains the time for when the last output volume change was detected.
   int64_t last_output_volume_change_time_ RTC_GUARDED_BY(thread_checker_);
+
+  AudioSourceSink* audioSink_;
 };
 }  // namespace ios_adm
 }  // namespace webrtc

sdk/objc/native/src/audio/audio_device_ios.mm

@@ -35,6 +35,8 @@
 #import "components/audio/RTCAudioSessionConfiguration.h"
 #import "components/audio/RTCNativeAudioSessionDelegateAdapter.h"
 
+#import "audio_source_sink.h"
+
 namespace webrtc {
 namespace ios_adm {
 
@@ -411,6 +413,8 @@ static void LogDeviceInfo() {
   // Use the FineAudioBuffer instance to convert between native buffer size
   // and the 10ms buffer size used by WebRTC.
   fine_audio_buffer_->DeliverRecordedData(record_audio_buffer_, kFixedRecordDelayEstimate);
+
+  audioSink_->OnLocalAudioFrame(flags, time_stamp, bus_number, num_frames, &audio_buffer_list);
   return noErr;
 }
 
@@ -470,6 +474,8 @@ static void LogDeviceInfo() {
   fine_audio_buffer_->GetPlayoutData(
       rtc::ArrayView<int16_t>(static_cast<int16_t*>(audio_buffer->mData), num_frames),
       kFixedPlayoutDelayEstimate);
+
+  audioSink_->OnRemoteAudioFrame(flags, time_stamp, bus_number, num_frames, io_data);
   return noErr;
 }
 
@@ -1127,5 +1133,10 @@ static void LogDeviceInfo() {
   return 0;
 }
 
+void AudioDeviceIOS::AddAudioSourceSink(webrtc::AudioSourceSink* audioSink) {
+  RTC_LOG(LS_VERBOSE) << "AddAudioSourceSink for AudioDeviceIOS" << audioSink;
+  audioSink_ = audioSink;
+}
+
 }  // namespace ios_adm
 }  // namespace webrtc

sdk/objc/native/src/audio/audio_device_module_ios.h

@@ -24,6 +24,7 @@
 namespace webrtc {
 
 class AudioDeviceGeneric;
+class AudioSourceSink;
 
 namespace ios_adm {
 
@@ -32,6 +33,7 @@ class AudioDeviceModuleIOS : public AudioDeviceModule {
   int32_t AttachAudioBuffer();
 
   AudioDeviceModuleIOS();
+  AudioDeviceModuleIOS(AudioSourceSink* audioSink);
   ~AudioDeviceModuleIOS() override;
 
   // Retrieve the currently utilized audio layer
@@ -126,7 +128,6 @@ class AudioDeviceModuleIOS : public AudioDeviceModule {
   int32_t EnableBuiltInNS(bool enable) override;
 
   int32_t GetPlayoutUnderrunCount() const override;
-
 #if defined(WEBRTC_IOS)
   int GetPlayoutAudioParameters(AudioParameters* params) const override;
   int GetRecordAudioParameters(AudioParameters* params) const override;
@@ -136,6 +137,7 @@ class AudioDeviceModuleIOS : public AudioDeviceModule {
   const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
   std::unique_ptr<AudioDeviceIOS> audio_device_;
   std::unique_ptr<AudioDeviceBuffer> audio_device_buffer_;
+  AudioSourceSink* audio_sink_;
 };
 }  // namespace ios_adm
 }  // namespace webrtc

sdk/objc/native/src/audio/audio_device_module_ios.mm

@@ -21,6 +21,7 @@
 
 #if defined(WEBRTC_IOS)
 #include "audio_device_ios.h"
+#include "audio_source_sink.h"
 #endif
 
 #define CHECKinitialized_() \
@@ -46,6 +47,13 @@
   RTC_LOG(INFO) << "iPhone Audio APIs will be utilized.";
 }
 
+AudioDeviceModuleIOS::AudioDeviceModuleIOS(AudioSourceSink* audioSink)
+    : task_queue_factory_(CreateDefaultTaskQueueFactory()) {
+  RTC_LOG(INFO) << "current platform is IOS";
+  RTC_LOG(INFO) << "iPhone Audio APIs will be utilized.";
+  audio_sink_ = audioSink;
+}
+
   int32_t AudioDeviceModuleIOS::AttachAudioBuffer() {
     RTC_LOG(INFO) << __FUNCTION__;
     audio_device_->AttachAudioBuffer(audio_device_buffer_.get());
@@ -74,6 +82,7 @@
     audio_device_buffer_.reset(new webrtc::AudioDeviceBuffer(task_queue_factory_.get()));
     audio_device_.reset(new ios_adm::AudioDeviceIOS());
     RTC_CHECK(audio_device_);
+    audio_device_->AddAudioSourceSink(audio_sink_);
 
     this->AttachAudioBuffer();
 
@@ -484,6 +493,7 @@
       return 0;
     }
     int32_t result = audio_device_->InitPlayout();
+    audio_device_->AddAudioSourceSink(audio_sink_);
     RTC_LOG(INFO) << "output: " << result;
     RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.InitPlayoutSuccess",
                           static_cast<int>(result == 0));