[FEAT]Multiple video capturer

.yamato/upm-ci-webrtc.yml

@@ -4,7 +4,7 @@ editors:
   - version: trunk
 platforms:
   - name: win
-    type: Unity::VM
+    type: Unity::VM::GPU
     # currently the projects depends MSBuild. we should replace CMake
     # image: package-ci/win10:stable
     image: renderstreaming/win10:latest
@@ -101,4 +101,4 @@ publish:
     - .yamato/upm-ci-webrtc.yml#test_{{ platform.name }}_{{ editor.version }}
     {% endfor %}
     {% endfor %}
-{% endfor %}
+{% endfor %}

Assets/Scripts/RenderStreaming.cs

@@ -17,6 +17,12 @@ public class ButtonClickElement
         public ButtonClickEvent click;
     }
 
+    public class CameraMediaStream
+    {
+        public Camera camera;
+        public MediaStream[] mediaStreams = new MediaStream[2];
+    }
+
     public class RenderStreaming : MonoBehaviour
     {
 #pragma warning disable 0649
@@ -29,19 +35,19 @@ public class RenderStreaming : MonoBehaviour
         [SerializeField, Tooltip("Time interval for polling from signaling server")]
         private float interval = 5.0f;
 
-        [SerializeField, Tooltip("Camera to capture video stream")]
-        private Camera captureCamera;
-
         [SerializeField]
         private ButtonClickElement[] arrayButtonClickEvent;
+
+        [SerializeField]
+        private bool isUseMinimalTextures = true;
 #pragma warning restore 0649
 
         private Signaling signaling;
         private Dictionary<string, RTCPeerConnection> pcs = new Dictionary<string, RTCPeerConnection>();
         private Dictionary<RTCPeerConnection, Dictionary<int, RTCDataChannel>> mapChannels = new Dictionary<RTCPeerConnection, Dictionary<int, RTCDataChannel>>();
         private RTCConfiguration conf;
         private string sessionId;
-        private MediaStream videoStream;
+        private Dictionary<Camera, CameraMediaStream> cameraMediaStreamDict = new Dictionary<Camera, CameraMediaStream>();
 
         public void Awake()
         {
@@ -52,6 +58,7 @@ public void Awake()
 
         public void OnDestroy()
         {
+            Audio.Stop();
             WebRTC.WebRTC.Finalize();
         }
         public IEnumerator Start()
@@ -60,7 +67,47 @@ public IEnumerator Start()
             {
                 yield break;
             }
-            videoStream = captureCamera.CaptureStream(1280, 720);
+
+
+            if (isUseMinimalTextures)
+            {
+                foreach (var camera in Camera.allCameras)
+                {
+                    CameraMediaStream cameraMediaStream = new CameraMediaStream();
+                    cameraMediaStreamDict.Add(camera, cameraMediaStream);
+                    camera.CreateRenderStreamTexture(1280, 720);
+                    int mediaCount = cameraMediaStream.mediaStreams.Length;
+                    for (int i = 0; i < mediaCount; ++i)
+                    {
+                        cameraMediaStream.mediaStreams[i] = new MediaStream();
+                        RenderTexture rt = camera.GetStreamTexture(0);
+                        int temp = i==0 ? 1 : (int)Mathf.Pow(i + 1, 10);
+                        VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, rt, 1000000/temp);
+                        cameraMediaStream.mediaStreams[i].AddTrack(videoTrack);
+                        cameraMediaStream.mediaStreams[i].AddTrack(new AudioStreamTrack("audioTrack"));
+                    }
+                }
+            }else{
+                foreach (var camera in Camera.allCameras)
+                {
+                    CameraMediaStream cameraMediaStream = new CameraMediaStream();
+                    cameraMediaStreamDict.Add(camera, cameraMediaStream);
+                    camera.CreateRenderStreamTexture(1280, 720, cameraMediaStream.mediaStreams.Length);
+                    int texCount = camera.GetStreamTextureCount();
+                    for (int i = 0; i < texCount; ++i)
+                    {
+                        int index = i;
+                        cameraMediaStream.mediaStreams[i] = new MediaStream();
+                        RenderTexture rt = camera.GetStreamTexture(index);
+                        VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, rt);
+                        cameraMediaStream.mediaStreams[i].AddTrack(videoTrack);
+                        cameraMediaStream.mediaStreams[i].AddTrack(new AudioStreamTrack("audioTrack"));
+                    }
+                }
+            }
+
+            Audio.Start();
+
             signaling = new Signaling(urlSignaling);
             var opCreate = signaling.Create();
             yield return opCreate;
@@ -125,27 +172,45 @@ IEnumerator GetOffer()
                 {
                     continue;
                 }
-                var pc = new RTCPeerConnection();
-                pcs.Add(offer.connectionId, pc);
 
+                RTCConfiguration config = default;
+                config.iceServers = new RTCIceServer[]
+                {
+                    new RTCIceServer { urls = urlsIceServer },
+                };
+                config.bundle_policy = RTCBundlePolicy.kBundlePolicyMaxBundle;
+                var pc = new RTCPeerConnection(ref config);
+                pcs.Add(offer.connectionId, pc);
                 pc.OnDataChannel = new DelegateOnDataChannel(channel => { OnDataChannel(pc, channel); });
-                pc.SetConfiguration(ref conf);
                 pc.OnIceCandidate = new DelegateOnIceCandidate(candidate => { StartCoroutine(OnIceCandidate(offer.connectionId, candidate)); });
                 pc.OnIceConnectionChange = new DelegateOnIceConnectionChange(state =>
                 {
                     if(state == RTCIceConnectionState.Disconnected)
                     {
-                        pc.Close();  
+                        pc.Close();
+                        pcs.Remove(offer.connectionId);
                     }
                 });
+
                 //make video bit rate starts at 16000kbits, and 160000kbits at max.
                 string pattern = @"(a=fmtp:\d+ .*level-asymmetry-allowed=.*)\r\n";
                 _desc.sdp = Regex.Replace(_desc.sdp, pattern, "$1;x-google-start-bitrate=16000;x-google-max-bitrate=160000\r\n");
+                Debug.Log("remote sdp---------------------------------------------------------");
+                Debug.Log(_desc.sdp);
+
                 pc.SetRemoteDescription(ref _desc);
-                foreach (var track in videoStream.GetTracks())
+
+                foreach (var k in cameraMediaStreamDict.Keys)
                 {
-                    pc.AddTrack(track);
+                    foreach (var mediaStream in cameraMediaStreamDict[k].mediaStreams)
+                    {
+                        foreach (var track in mediaStream.GetTracks())
+                        {
+                            pc.AddTrack(track, mediaStream.Id);
+                        }
+                    }
                 }
+
                 StartCoroutine(Answer(connectionId));
             }
         }
@@ -163,12 +228,14 @@ IEnumerator Answer(string connectionId)
             }
             var opLocalDesc = pc.SetLocalDescription(ref op.desc);
             yield return opLocalDesc;
+            Debug.Log("local sdp---------------------------------------------------------");
+            Debug.Log(op.desc.sdp);
             if (opLocalDesc.isError)
             {
                 Debug.LogError($"Network Error: {opLocalDesc.error}");
                 yield break;
             }
-            var op3 = signaling.PostAnswer(this.sessionId, connectionId, op.desc.sdp); 
+            var op3 = signaling.PostAnswer(this.sessionId, connectionId, op.desc.sdp);
             yield return op3;
             if (op3.webRequest.isNetworkError)
             {
@@ -202,6 +269,7 @@ IEnumerator GetCandidate()
                 {
                     continue;
                 }
+
                 foreach (var candidate in candidateContainer.candidates)
                 {
                     RTCIceCandidate _candidate = default;