feat: Multi-stream support

Plugin~/WebRTCPlugin/Context.cpp

@@ -1,5 +1,6 @@
 #include "pch.h"
 #include "WebRTCPlugin.h"
+#include "UnityEncoder.h"
 #include "Context.h"
 
 namespace WebRTC
@@ -242,6 +243,9 @@ namespace WebRTC
             config.servers.push_back(iceServer);
         }
         config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan;
+
+        Json::Value bundle_policy = configJson["bundlePolicy"];
+        config.bundle_policy = (webrtc::PeerConnectionInterface::BundlePolicy)bundle_policy.asInt();
     }
 #pragma warning(push)
 #pragma warning(disable: 4715)
@@ -283,9 +287,9 @@ namespace WebRTC
         rtc::InitializeSSL();
 
         audioDevice = new rtc::RefCountedObject<DummyAudioDevice>();
-        nvVideoCapturerUnique = std::make_unique<NvVideoCapturer>();
-        nvVideoCapturer = nvVideoCapturerUnique.get();
-        auto dummyVideoEncoderFactory = std::make_unique<DummyVideoEncoderFactory>(nvVideoCapturer);
+
+        auto dummyVideoEncoderFactory = std::make_unique<DummyVideoEncoderFactory>();
+        pDummyVideoEncoderFactory = dummyVideoEncoderFactory.get();
 
         peerConnectionFactory = webrtc::CreatePeerConnectionFactory(
             workerThread.get(),
@@ -302,48 +306,90 @@ namespace WebRTC
 
     Context::~Context()
     {
+        pDummyVideoEncoderFactory->Destroy();
+
         clients.clear();
         peerConnectionFactory = nullptr;
-        audioTrack = nullptr;
-        videoTracks.clear();
-        audioStream = nullptr;
-        videoStreams.clear();
+
+        mediaSteamTrackList.clear();
+        mediaStreamMap.clear();
+        nvVideoCapturerList.clear();
 
         workerThread->Quit();
         workerThread.reset();
         signalingThread->Quit();
         signalingThread.reset();
     }
 
-    webrtc::MediaStreamInterface* Context::CreateVideoStream(UnityFrameBuffer* frameBuffer)
+    void Context::EncodeFrame()
     {
-        //TODO: label and stream id should be maintained in some way for multi-stream
-        auto videoTrack = peerConnectionFactory->CreateVideoTrack(
-            "video", peerConnectionFactory->CreateVideoSource(std::move(nvVideoCapturerUnique)));
-        if (!videoTracks.count(frameBuffer))
+        for (std::list<UnityVideoCapturer*>::iterator it = nvVideoCapturerList.begin(); it != nvVideoCapturerList.end(); ++it)
         {
-            videoTracks[frameBuffer] = videoTrack;
+            (*it)->EncodeVideoData();
         }
-        auto videoStream = peerConnectionFactory->CreateLocalMediaStream("video");
-        videoStream->AddTrack(videoTrack);
-        videoStreams.push_back(videoStream);
-        nvVideoCapturer->unityRT = frameBuffer;
-        nvVideoCapturer->StartEncoder();
-        return videoStream.get();
     }
 
-    webrtc::MediaStreamInterface* Context::CreateAudioStream()
+    void Context::StopMediaStreamTrack(webrtc::MediaStreamTrackInterface* track)
+    {
+        //TODO::
+        for (std::list<UnityVideoCapturer*>::iterator it = nvVideoCapturerList.begin(); it != nvVideoCapturerList.end(); ++it)
+        {
+            (*it)->Stop();
+        }
+    }
+
+    webrtc::MediaStreamInterface* Context::CreateMediaStream(const std::string& stream_id)
+    {
+        if (mediaStreamMap.count(stream_id) == 0)
+        {
+            mediaStreamMap[stream_id] = peerConnectionFactory->CreateLocalMediaStream(stream_id);
+        }
+
+        return mediaStreamMap[stream_id];
+    }
+
+    void Context::DeleteMediaStream(webrtc::MediaStreamInterface* stream)
+    {
+        auto stream_id = stream->id();
+        if (mediaStreamMap.count(stream_id) > 0)
+        {
+            mediaStreamMap.erase(stream_id);
+        }
+    }
+
+    webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height, int32 bitRate)
+    {
+        UnityEncoder* pUnityEncoder = pDummyVideoEncoderFactory->CreatePlatformEncoder(WebRTC::Nvidia, width, height, bitRate);
+        UnityVideoCapturer* pUnityVideoCapturer = new UnityVideoCapturer(pUnityEncoder, width, height);
+        pUnityVideoCapturer->InitializeEncoder();
+        pDummyVideoEncoderFactory->AddCapturer(pUnityVideoCapturer);
+
+        auto videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(pUnityVideoCapturer));
+        pUnityVideoCapturer->unityRT = frameBuffer;
+        pUnityVideoCapturer->StartEncoder();
+
+        // TODO:: Create dictionary to impletement StopMediaStreamTrack API
+        nvVideoCapturerList.push_back(pUnityVideoCapturer);
+        mediaSteamTrackList.push_back(videoTrack);
+        return videoTrack.get();
+    }
+
+    webrtc::MediaStreamTrackInterface* Context::CreateAudioTrack(const std::string& label)
     {
         //avoid optimization specially for voice
         cricket::AudioOptions audioOptions;
         audioOptions.auto_gain_control = false;
         audioOptions.noise_suppression = false;
         audioOptions.highpass_filter = false;
         //TODO: label and stream id should be maintained in some way for multi-stream
-        audioTrack = peerConnectionFactory->CreateAudioTrack("audio", peerConnectionFactory->CreateAudioSource(audioOptions));
-        audioStream = peerConnectionFactory->CreateLocalMediaStream("audio");
-        audioStream->AddTrack(audioTrack);
-        return audioStream.get();
+        auto audioTrack = peerConnectionFactory->CreateAudioTrack(label, peerConnectionFactory->CreateAudioSource(audioOptions));
+        mediaSteamTrackList.push_back(audioTrack);
+        return audioTrack.get();
+    }
+
+    void Context::DeleteMediaStreamTrack(webrtc::MediaStreamTrackInterface* track)
+    {
+        mediaSteamTrackList.remove_if([&](webrtc::MediaStreamTrackInterface* _track) { return track == _track; });
     }
 
     DataChannelObject* Context::CreateDataChannel(PeerConnectionObject* obj, const char* label, const RTCDataChannelInit& options)

Plugin~/WebRTCPlugin/Context.h

@@ -1,9 +1,10 @@
 #pragma once
+#include "UnityEncoder.h"
 #include "DummyAudioDevice.h"
 #include "DummyVideoEncoder.h"
 #include "PeerConnectionObject.h"
-#include "NvVideoCapturer.h"
-
+#include "UnityVideoCapturer.h"
+#include "NvEncoder.h"
 
 namespace WebRTC
 {
@@ -39,16 +40,18 @@ namespace WebRTC
     {
     public:
         explicit Context(int uid = -1);
-        webrtc::MediaStreamInterface* CreateVideoStream(UnityFrameBuffer* frameBuffer);
-        webrtc::MediaStreamInterface* CreateAudioStream();
+        webrtc::MediaStreamInterface* CreateMediaStream(const std::string& stream_id);
+        void DeleteMediaStream(webrtc::MediaStreamInterface* stream);
+        webrtc::MediaStreamTrackInterface* CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height, int32 bitRate);
+        webrtc::MediaStreamTrackInterface* CreateAudioTrack(const std::string& label);
+        void DeleteMediaStreamTrack(webrtc::MediaStreamTrackInterface* track);
         ~Context();
 
         PeerConnectionObject* CreatePeerConnection();
         PeerConnectionObject* CreatePeerConnection(const std::string& conf);
         void DeletePeerConnection(PeerConnectionObject* obj) { clients.erase(obj); }
-        void InitializeEncoder(int32 width, int32 height) { nvVideoCapturer->InitializeEncoder(width, height); }
-        void EncodeFrame() { nvVideoCapturer->EncodeVideoData(); }
-        void StopCapturer() { nvVideoCapturer->Stop(); }
+        void EncodeFrame();
+        void StopMediaStreamTrack(webrtc::MediaStreamTrackInterface* track);
         void ProcessAudioData(const float* data, int32 size) { audioDevice->ProcessAudioData(data, size); }
 
         DataChannelObject* CreateDataChannel(PeerConnectionObject* obj, const char* label, const RTCDataChannelInit& options);
@@ -62,14 +65,12 @@ namespace WebRTC
         std::unique_ptr<rtc::Thread> signalingThread;
         std::map<PeerConnectionObject*, rtc::scoped_refptr<PeerConnectionObject>> clients;
         rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> peerConnectionFactory;
-        NvVideoCapturer* nvVideoCapturer;
-        std::unique_ptr<NvVideoCapturer> nvVideoCapturerUnique;
+        DummyVideoEncoderFactory* pDummyVideoEncoderFactory;
+        std::map<const std::string, rtc::scoped_refptr<webrtc::MediaStreamInterface>> mediaStreamMap;
+        std::list<rtc::scoped_refptr<webrtc::MediaStreamTrackInterface>> mediaSteamTrackList;
+
+        std::list<UnityVideoCapturer*> nvVideoCapturerList;
         rtc::scoped_refptr<DummyAudioDevice> audioDevice;
-        rtc::scoped_refptr<webrtc::AudioTrackInterface> audioTrack;
-        rtc::scoped_refptr<webrtc::MediaStreamInterface> audioStream;
-        //TODO: move videoTrack to NvVideoCapturer and maintain multiple NvVideoCapturer here
-        std::vector<rtc::scoped_refptr<webrtc::MediaStreamInterface>> videoStreams;
-        std::map<UnityFrameBuffer*, rtc::scoped_refptr<webrtc::VideoTrackInterface>> videoTracks;
     };
 
     class PeerSDPObserver : public webrtc::SetSessionDescriptionObserver

Plugin~/WebRTCPlugin/DummyVideoEncoder.cpp

@@ -1,7 +1,8 @@
 #include "pch.h"
 #include "DummyVideoEncoder.h"
-#include "NvVideoCapturer.h"
+#include "UnityVideoCapturer.h"
 #include <algorithm>
+#include "NvEncoder.h"
 
 namespace WebRTC
 {
@@ -67,7 +68,22 @@ namespace WebRTC
         SetRate(allocation.get_sum_kbps() * 1000);
         return 0;
     }
-    DummyVideoEncoderFactory::DummyVideoEncoderFactory(NvVideoCapturer* videoCapturer):capturer(videoCapturer){}
+
+    DummyVideoEncoderFactory::DummyVideoEncoderFactory()
+    {
+
+    }
+
+    void DummyVideoEncoderFactory::Destroy()
+    {
+        for (std::list<UnityEncoder*>::iterator it = unityEncoders.begin(); it != unityEncoders.end(); ++it)
+        {
+            delete *it;
+        }
+        unityEncoders.clear();
+        NvEncoder::DestroyEncoderTexture();
+    }
+
     std::vector<webrtc::SdpVideoFormat> DummyVideoEncoderFactory::GetSupportedFormats() const
     {
         const absl::optional<std::string> profileLevelId =
@@ -87,8 +103,47 @@ namespace WebRTC
         const webrtc::SdpVideoFormat& format)
     {
         auto dummyVideoEncoder = std::make_unique<DummyVideoEncoder>();
-        dummyVideoEncoder->SetKeyFrame.connect(capturer, &NvVideoCapturer::SetKeyFrame);
-        dummyVideoEncoder->SetRate.connect(capturer, &NvVideoCapturer::SetRate);
+
+        {
+            //todo: According to condition of format choose different capturer.
+            //UnityVideoCapturer* pCapturer = *(++capturers.begin());
+
+            //dummyVideoEncoder->SetKeyFrame.connect(pCapturer, &UnityVideoCapturer::SetKeyFrame);
+            //dummyVideoEncoder->SetRate.connect(pCapturer, &UnityVideoCapturer::SetRate);
+        }
+
         return dummyVideoEncoder;
     }
+
+    UnityEncoder* DummyVideoEncoderFactory::CreatePlatformEncoder(EncoderPlatform platform, int width, int height, int bitRate)
+    {
+        UnityEncoder* pEncoder = NULL;
+        switch (platform)
+        {
+        case WebRTC::Nvidia:
+            pEncoder = new NvEncoder();
+            break;
+        case WebRTC::Amd:
+            break;
+        case WebRTC::Soft:
+            break;
+        default:
+            break;
+        }
+        pEncoder->InitEncoder(width, height, bitRate);
+        unityEncoders.push_back(pEncoder);
+        return pEncoder;
+    }
+
+    UnityEncoder* DummyVideoEncoderFactory::GetPlatformEncoder(EncoderPlatform platform, int width, int height, int bitRate)
+    {
+        for (std::list<UnityEncoder*>::iterator it = unityEncoders.begin(); it != unityEncoders.end(); ++it)
+        {
+            if ((*it)->getEncodeWidth() == width && (*it)->getEncodeHeight() == height && (*it)->getBitRate() == bitRate) {
+                return (*it);
+            }
+        }
+
+        return CreatePlatformEncoder(platform, width, height, bitRate);
+    }
 }

Plugin~/WebRTCPlugin/DummyVideoEncoder.h

@@ -2,7 +2,8 @@
 
 namespace WebRTC
 {
-    class NvVideoCapturer;
+    class UnityEncoder;
+    class UnityVideoCapturer;
     class DummyVideoEncoder : public webrtc::VideoEncoder
     {
     public:
@@ -38,6 +39,13 @@ namespace WebRTC
         webrtc::VideoBitrateAllocation lastBitrate;
     };
 
+    enum EncoderPlatform
+    {
+        Nvidia,
+        Amd,
+        Soft,
+    };
+
     class DummyVideoEncoderFactory : public webrtc::VideoEncoderFactory
     {
     public:
@@ -51,8 +59,14 @@ namespace WebRTC
         // Creates a VideoEncoder for the specified format.
         virtual std::unique_ptr<webrtc::VideoEncoder> CreateVideoEncoder(
             const webrtc::SdpVideoFormat& format) override;
-        DummyVideoEncoderFactory(NvVideoCapturer* videoCapturer);
+        DummyVideoEncoderFactory();
+        void Destroy();
+
+        void AddCapturer(UnityVideoCapturer* _capturer) { capturers.push_back(_capturer); }
+        UnityEncoder* CreatePlatformEncoder(EncoderPlatform platform, int width, int height, int bitRate);
+        UnityEncoder* GetPlatformEncoder(EncoderPlatform platform, int width, int height, int bitRate);
     private:
-        NvVideoCapturer* capturer;
+        std::list<UnityVideoCapturer*> capturers;
+        std::list<UnityEncoder*> unityEncoders;
     };
 }