From 62957880e4c3bf79d3a88f604e044ab5cafb4da3 Mon Sep 17 00:00:00 2001 From: koseyile Date: Mon, 1 Jul 2019 10:16:15 +0000 Subject: [PATCH 01/29] redesign MediaStream and mediaStreamTrack to support multiple track. --- Assets/Scripts/RenderStreaming.cs | 15 +++- .../Runtime/Srcipts/MediaStream.cs | 86 +++++++++++++++---- .../Runtime/Srcipts/MediaStreamTrack.cs | 34 ++++++++ .../Runtime/Srcipts/RTCPeerConnection.cs | 6 ++ .../Runtime/Srcipts/WebRTC.cs | 4 +- 5 files changed, 124 insertions(+), 21 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index 82346ea2b..2723c2e90 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -41,7 +41,7 @@ public class RenderStreaming : MonoBehaviour private Dictionary> mapChannels = new Dictionary>(); private RTCConfiguration conf; private string sessionId; - private MediaStream videoStream; + private MediaStream2 videoStream; public void Awake() { @@ -60,7 +60,16 @@ public IEnumerator Start() { yield break; } - videoStream = cam.CaptureStream(1280, 720); + cam.CreateRenderStreamTexture(1280, 720); + videoStream = new MediaStream2(); + int texCount = cam.getStreamTextureCount(); + for (int i = 0; i < texCount; ++i) + { + videoStream.AddTrack(new VideoStreamTrack(cam.getStreamTexture(i))); + } + + + signaling = new Signaling(urlSignaling); var opCreate = signaling.Create(); yield return opCreate; @@ -142,7 +151,7 @@ IEnumerator GetOffer() string pattern = @"(a=fmtp:\d+ .*level-asymmetry-allowed=.*)\r\n"; _desc.sdp = Regex.Replace(_desc.sdp, pattern, "$1;x-google-start-bitrate=16000;x-google-max-bitrate=160000\r\n"); pc.SetRemoteDescription(ref _desc); - foreach (var track in videoStream.GetTracks()) + foreach (var track in videoStream.getTracks()) { pc.AddTrack(track); } diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index f96d4feac..a70b05f3b 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -6,6 +6,34 @@ namespace Unity.WebRTC { + public class MediaStream2 + { + protected List mediaStreamTrackList = new List(); + + public MediaStream2() : base() + { + + } + + public MediaStream2(MediaStreamTrack2[] tracks) : base() + { + foreach (var t in tracks) + { + mediaStreamTrackList.Add(t); + } + } + + public void AddTrack(MediaStreamTrack2 track) + { + mediaStreamTrackList.Add(track); + } + + public MediaStreamTrack2[] getTracks() + { + return mediaStreamTrackList.ToArray(); + } + } + public class MediaStream { private IntPtr self; @@ -152,36 +180,62 @@ public static void AddCleanerCallback(this GameObject obj, Action callback) } public static class CameraExtension { + internal static RenderTexture camRenderTexture; + internal static List webRTCTextures = new List(); internal static List camCopyRts = new List(); internal static bool started = false; - public static MediaStream CaptureStream(this Camera cam, int width, int height) + + public static int getStreamTextureCount(this Camera cam) + { + return webRTCTextures.Count; + } + + public static RenderTexture getStreamTexture(this Camera cam, int index) { + return webRTCTextures[index]; + } + + public static void CreateRenderStreamTexture(this Camera cam, int width, int height) { if (camCopyRts.Count > 0) { throw new NotImplementedException("Currently not allowed multiple MediaStream"); } - RenderTexture[] rts = new RenderTexture[2]; - //rts[0] for render target, rts[1] for flip and WebRTC source - rts[0] = new RenderTexture(width, height, 0, RenderTextureFormat.BGRA32); - rts[1] = new RenderTexture(width, height, 0, RenderTextureFormat.BGRA32); - rts[0].Create(); - rts[1].Create(); - camCopyRts.Add(rts); - cam.targetTexture = rts[0]; + camRenderTexture = new RenderTexture(width, height, 0, RenderTextureFormat.BGRA32); + camRenderTexture.Create(); + + int mipCount = 1; + for (int i = 1, mipLevel = 1; i <= mipCount; ++i, mipLevel *= 2) + { + RenderTexture webRtcTex = new RenderTexture(width / mipLevel, height / mipLevel, 0, RenderTextureFormat.BGRA32); + webRtcTex.Create(); + webRTCTextures.Add(webRtcTex); + } + + cam.targetTexture = camRenderTexture; cam.gameObject.AddCleanerCallback(() => { - if (rts != null) + camRenderTexture.Release(); + UnityEngine.Object.Destroy(camRenderTexture); + + foreach (var v in webRTCTextures) { - CameraExtension.RemoveRt(rts); - rts[0].Release(); - rts[1].Release(); - UnityEngine.Object.Destroy(rts[0]); - UnityEngine.Object.Destroy(rts[1]); + v.Release(); + UnityEngine.Object.Destroy(v); } + webRTCTextures.Clear(); }); started = true; - return new MediaStream(rts, WebRTC.Context.CaptureVideoStream(rts[1].GetNativeTexturePtr(), width, height)); + } + + public static MediaStream CaptureStream(this Camera cam, int width, int height) + { + cam.CreateRenderStreamTexture(width, height); + + int textureIndex = 0; + int rtcMipLevel = (int)Math.Pow(2, textureIndex); //1 2 4 8 + return new MediaStream(webRTCTextures.ToArray(), WebRTC.Context.CaptureVideoStream(webRTCTextures[textureIndex].GetNativeTexturePtr(), width/rtcMipLevel, height/rtcMipLevel)); + //return new MediaStream2(); } public static void RemoveRt(RenderTexture[] rts) { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs index 30df48c7d..3e1d5de8e 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs @@ -4,6 +4,40 @@ namespace Unity.WebRTC { + public class MediaStreamTrack2 + { + public IntPtr nativePtr; + public string id; + public TrackKind kind; + public MediaStreamTrack2() + { + + } + } + + public class VideoStreamTrack : MediaStreamTrack2 + { + public VideoStreamTrack(RenderTexture rt) : base() + { + IntPtr nativeVideoStreamPtr = WebRTC.Context.CaptureVideoStream(rt.GetNativeTexturePtr(), rt.width, rt.height); + string nativeVideoStreamID = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(nativeVideoStreamPtr)); + + int trackSize = 0; + IntPtr tracksNativePtr = NativeMethods.MediaStreamGetVideoTracks(nativeVideoStreamPtr, ref trackSize); + IntPtr[] tracksPtr = new IntPtr[trackSize]; + Marshal.Copy(tracksNativePtr, tracksPtr, 0, trackSize); + Marshal.FreeCoTaskMem(tracksNativePtr); + + nativePtr = tracksPtr[0]; + kind = NativeMethods.MediaStreamTrackGetKind(nativePtr); + id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(nativePtr)); + + } + } + + + + public class MediaStreamTrack { internal IntPtr self; diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs index 9c9e74469..b4d3f3efb 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs @@ -192,6 +192,12 @@ public RTCRtpSender AddTrack(MediaStreamTrack track) { return new RTCRtpSender(NativeMethods.PeerConnectionAddTrack(self, track.self)); } + + public RTCRtpSender AddTrack(MediaStreamTrack2 track) + { + return new RTCRtpSender(NativeMethods.PeerConnectionAddTrack(self, track.nativePtr)); + } + public void RemoveTrack(RTCRtpSender sender) { NativeMethods.PeerConnectionRemoveTrack(self, sender.self); diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs index 1c919b57a..0b1caadf3 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs @@ -222,9 +222,9 @@ public static IEnumerator Update() if (CameraExtension.started) { //Blit is for DirectX Rendering API Only - foreach(var rts in CameraExtension.camCopyRts) + foreach(var rt in CameraExtension.webRTCTextures) { - Graphics.Blit(rts[0], rts[1], flipMat); + Graphics.Blit(CameraExtension.camRenderTexture, rt, flipMat); } GL.IssuePluginEvent(NativeMethods.GetRenderEventFunc(), 0); } From f7310c638b5d2903a423cb472157b30a410353b7 Mon Sep 17 00:00:00 2001 From: koseyile Date: Mon, 1 Jul 2019 12:59:17 +0000 Subject: [PATCH 02/29] delete member variable videoTrackToRts and AudioTracks from mediaStream. --- Assets/Scripts/RenderStreaming.cs | 4 +- .../Runtime/Srcipts/MediaStream.cs | 255 ++++++++++-------- .../Runtime/Srcipts/MediaStreamTrack.cs | 118 ++++---- .../Runtime/Srcipts/RTCPeerConnection.cs | 5 - .../Samples/Example/AddMediaStream.cs | 8 +- .../Tests/Runtime/MediaStreamTest.cs | 14 +- 6 files changed, 226 insertions(+), 178 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index 2723c2e90..19e97fd7a 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -41,7 +41,7 @@ public class RenderStreaming : MonoBehaviour private Dictionary> mapChannels = new Dictionary>(); private RTCConfiguration conf; private string sessionId; - private MediaStream2 videoStream; + private MediaStream videoStream; public void Awake() { @@ -61,7 +61,7 @@ public IEnumerator Start() yield break; } cam.CreateRenderStreamTexture(1280, 720); - videoStream = new MediaStream2(); + videoStream = new MediaStream(); int texCount = cam.getStreamTextureCount(); for (int i = 0; i < texCount; ++i) { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index a70b05f3b..c343c64d7 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -6,16 +6,38 @@ namespace Unity.WebRTC { - public class MediaStream2 + public class MediaStream { - protected List mediaStreamTrackList = new List(); + public IntPtr self; + public string id; + protected List mediaStreamTrackList = new List(); - public MediaStream2() : base() + public MediaStream() : base() { } - public MediaStream2(MediaStreamTrack2[] tracks) : base() + internal MediaStream(IntPtr ptr) + { + self = ptr; + id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self)); + int trackSize = 0; + IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(self, ref trackSize); + IntPtr[] tracksPtr = new IntPtr[trackSize]; + Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize); + //TODO: Linux compatibility + Marshal.FreeCoTaskMem(trackNativePtr); + + for (int i = 0; i < trackSize; i++) + { + MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]); + //track.stopTrack += StopTrack; + //track.getRts += GetRts; + mediaStreamTrackList.Add(track); + } + } + + public MediaStream(MediaStreamTrack[] tracks) : base() { foreach (var t in tracks) { @@ -23,133 +45,133 @@ public MediaStream2(MediaStreamTrack2[] tracks) : base() } } - public void AddTrack(MediaStreamTrack2 track) + public void AddTrack(MediaStreamTrack track) { mediaStreamTrackList.Add(track); } - public MediaStreamTrack2[] getTracks() + public MediaStreamTrack[] getTracks() { return mediaStreamTrackList.ToArray(); } } - public class MediaStream - { - private IntPtr self; - private string id; - public string Id { get => id; private set { } } + //public class MediaStream + //{ + // private IntPtr self; + // private string id; + // public string Id { get => id; private set { } } - private Dictionary VideoTrackToRts; - private List AudioTracks; + // private Dictionary VideoTrackToRts; + // private List AudioTracks; - private void StopTrack(MediaStreamTrack track) - { + // //private void StopTrack(MediaStreamTrack track) + // //{ - if (track.Kind == TrackKind.Video) - { - NativeMethods.StopMediaStreamTrack(track.self); - RenderTexture[] rts = VideoTrackToRts[track]; - if (rts != null) - { - CameraExtension.RemoveRt(rts); - rts[0].Release(); - rts[1].Release(); - UnityEngine.Object.Destroy(rts[0]); - UnityEngine.Object.Destroy(rts[1]); - } - } - else - { - Audio.Stop(); - } + // // if (track.kind == TrackKind.Video) + // // { + // // NativeMethods.StopMediaStreamTrack(track.nativePtr); + // // RenderTexture[] rts = VideoTrackToRts[track]; + // // if (rts != null) + // // { + // // CameraExtension.RemoveRt(rts); + // // rts[0].Release(); + // // rts[1].Release(); + // // UnityEngine.Object.Destroy(rts[0]); + // // UnityEngine.Object.Destroy(rts[1]); + // // } + // // } + // // else + // // { + // // Audio.Stop(); + // // } - } - private RenderTexture[] GetRts(MediaStreamTrack track) - { - return VideoTrackToRts[track]; - } - public MediaStreamTrack[] GetTracks() - { - MediaStreamTrack[] tracks = new MediaStreamTrack[VideoTrackToRts.Keys.Count + AudioTracks.Count]; - AudioTracks.CopyTo(tracks, 0); - VideoTrackToRts.Keys.CopyTo(tracks, AudioTracks.Count); - return tracks; - } - public MediaStreamTrack[] GetAudioTracks() - { - return AudioTracks.ToArray(); - } - public MediaStreamTrack[] GetVideoTracks() - { - MediaStreamTrack[] tracks = new MediaStreamTrack[VideoTrackToRts.Keys.Count]; - VideoTrackToRts.Keys.CopyTo(tracks, 0); - return tracks; - } + // //} + // private RenderTexture[] GetRts(MediaStreamTrack track) + // { + // return VideoTrackToRts[track]; + // } + // public MediaStreamTrack[] GetTracks() + // { + // MediaStreamTrack[] tracks = new MediaStreamTrack[VideoTrackToRts.Keys.Count + AudioTracks.Count]; + // AudioTracks.CopyTo(tracks, 0); + // VideoTrackToRts.Keys.CopyTo(tracks, AudioTracks.Count); + // return tracks; + // } + // //public MediaStreamTrack[] GetAudioTracks() + // //{ + // // return AudioTracks.ToArray(); + // //} + // //public MediaStreamTrack[] GetVideoTracks() + // //{ + // // MediaStreamTrack[] tracks = new MediaStreamTrack[VideoTrackToRts.Keys.Count]; + // // VideoTrackToRts.Keys.CopyTo(tracks, 0); + // // return tracks; + // //} - public void AddTrack(MediaStreamTrack track) - { - if(track.Kind == TrackKind.Video) - { - VideoTrackToRts[track] = track.getRts(track); - } - else - { - AudioTracks.Add(track); - } - NativeMethods.MediaStreamAddTrack(self, track.self); - } - public void RemoveTrack(MediaStreamTrack track) - { - NativeMethods.MediaStreamRemoveTrack(self, track.self); - } - //for camera CaptureStream - internal MediaStream(RenderTexture[] rts, IntPtr ptr) - { - self = ptr; - id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self)); - VideoTrackToRts = new Dictionary(); - AudioTracks = new List(); - //get initial tracks - int trackSize = 0; - IntPtr tracksNativePtr = NativeMethods.MediaStreamGetVideoTracks(self, ref trackSize); - IntPtr[] tracksPtr = new IntPtr[trackSize]; - Marshal.Copy(tracksNativePtr, tracksPtr, 0, trackSize); - //TODO: Linux compatibility - Marshal.FreeCoTaskMem(tracksNativePtr); - for (int i = 0; i < trackSize; i++) - { - MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]); - track.stopTrack += StopTrack; - track.getRts += GetRts; - VideoTrackToRts[track] = rts; - } - } - //for audio CaptureStream - internal MediaStream(IntPtr ptr) - { - self = ptr; - id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self)); - VideoTrackToRts = new Dictionary(); - AudioTracks = new List(); - //get initial tracks - int trackSize = 0; - IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(self, ref trackSize); - IntPtr[] tracksPtr = new IntPtr[trackSize]; - Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize); - //TODO: Linux compatibility - Marshal.FreeCoTaskMem(trackNativePtr); + // //public void AddTrack(MediaStreamTrack track) + // //{ + // // if(track.kind == TrackKind.Video) + // // { + // // VideoTrackToRts[track] = track.getRts(track); + // // } + // // else + // // { + // // AudioTracks.Add(track); + // // } + // // NativeMethods.MediaStreamAddTrack(self, track.self); + // //} + // //public void RemoveTrack(MediaStreamTrack track) + // //{ + // // NativeMethods.MediaStreamRemoveTrack(self, track.self); + // //} + // //for camera CaptureStream + // internal MediaStream(RenderTexture[] rts, IntPtr ptr) + // { + // self = ptr; + // id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self)); + // VideoTrackToRts = new Dictionary(); + // AudioTracks = new List(); + // //get initial tracks + // int trackSize = 0; + // IntPtr tracksNativePtr = NativeMethods.MediaStreamGetVideoTracks(self, ref trackSize); + // IntPtr[] tracksPtr = new IntPtr[trackSize]; + // Marshal.Copy(tracksNativePtr, tracksPtr, 0, trackSize); + // //TODO: Linux compatibility + // Marshal.FreeCoTaskMem(tracksNativePtr); + // for (int i = 0; i < trackSize; i++) + // { + // MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]); + // //track.stopTrack += StopTrack; + // //track.getRts += GetRts; + // VideoTrackToRts[track] = rts; + // } + // } + // //for audio CaptureStream + // internal MediaStream(IntPtr ptr) + // { + // self = ptr; + // id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self)); + // VideoTrackToRts = new Dictionary(); + // AudioTracks = new List(); + // //get initial tracks + // int trackSize = 0; + // IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(self, ref trackSize); + // IntPtr[] tracksPtr = new IntPtr[trackSize]; + // Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize); + // //TODO: Linux compatibility + // Marshal.FreeCoTaskMem(trackNativePtr); - for (int i = 0; i < trackSize; i++) - { - MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]); - track.stopTrack += StopTrack; - track.getRts += GetRts; - AudioTracks.Add(track); - } - } + // for (int i = 0; i < trackSize; i++) + // { + // MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]); + // //track.stopTrack += StopTrack; + // //track.getRts += GetRts; + // AudioTracks.Add(track); + // } + // } - } + //} internal class Cleaner : MonoBehaviour { private Action onDestroy; @@ -234,8 +256,7 @@ public static MediaStream CaptureStream(this Camera cam, int width, int height) int textureIndex = 0; int rtcMipLevel = (int)Math.Pow(2, textureIndex); //1 2 4 8 - return new MediaStream(webRTCTextures.ToArray(), WebRTC.Context.CaptureVideoStream(webRTCTextures[textureIndex].GetNativeTexturePtr(), width/rtcMipLevel, height/rtcMipLevel)); - //return new MediaStream2(); + return new MediaStream(WebRTC.Context.CaptureVideoStream(webRTCTextures[textureIndex].GetNativeTexturePtr(), width/rtcMipLevel, height/rtcMipLevel)); } public static void RemoveRt(RenderTexture[] rts) { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs index 3e1d5de8e..7b070f0d1 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs @@ -4,18 +4,40 @@ namespace Unity.WebRTC { - public class MediaStreamTrack2 + public class MediaStreamTrack { public IntPtr nativePtr; public string id; public TrackKind kind; - public MediaStreamTrack2() + public MediaStreamTrack() { } + + internal MediaStreamTrack(IntPtr ptr) + { + nativePtr = ptr; + kind = NativeMethods.MediaStreamTrackGetKind(nativePtr); + id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(nativePtr)); + } + + //public bool Enabled + //{ + // get{return NativeMethods.MediaStreamTrackGetEnabled(nativePtr);} + // set{NativeMethods.MediaStreamTrackSetEnabled(nativePtr, value);} + //} + //public TrackState ReadyState + //{ + // get + // {return NativeMethods.MediaStreamTrackGetReadyState(nativePtr);} + // private set { } + //} + + //public TrackKind Kind { get => kind; private set { } } + //public string Id { get => id; private set { } } } - public class VideoStreamTrack : MediaStreamTrack2 + public class VideoStreamTrack : MediaStreamTrack { public VideoStreamTrack(RenderTexture rt) : base() { @@ -38,51 +60,51 @@ public VideoStreamTrack(RenderTexture rt) : base() - public class MediaStreamTrack - { - internal IntPtr self; - private TrackKind kind; - private string id; - private bool enabled; - private TrackState readyState; - internal Action stopTrack; - internal Func getRts; - - public bool Enabled - { - get - { - return NativeMethods.MediaStreamTrackGetEnabled(self); - } - set - { - NativeMethods.MediaStreamTrackSetEnabled(self, value); - } - } - public TrackState ReadyState - { - get - { - return NativeMethods.MediaStreamTrackGetReadyState(self); - } - private set { } - } - - public TrackKind Kind { get => kind; private set { } } - public string Id { get => id; private set { } } - - internal MediaStreamTrack(IntPtr ptr) - { - self = ptr; - kind = NativeMethods.MediaStreamTrackGetKind(self); - id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(self)); - } - //Disassociate track from its source(video or audio), not for destroying the track - public void Stop() - { - stopTrack(this); - } - } + //public class MediaStreamTrack + //{ + // internal IntPtr self; + // private TrackKind kind; + // private string id; + // private bool enabled; + // private TrackState readyState; + // internal Action stopTrack; + // internal Func getRts; + + // public bool Enabled + // { + // get + // { + // return NativeMethods.MediaStreamTrackGetEnabled(self); + // } + // set + // { + // NativeMethods.MediaStreamTrackSetEnabled(self, value); + // } + // } + // public TrackState ReadyState + // { + // get + // { + // return NativeMethods.MediaStreamTrackGetReadyState(self); + // } + // private set { } + // } + + // public TrackKind Kind { get => kind; private set { } } + // public string Id { get => id; private set { } } + + // internal MediaStreamTrack(IntPtr ptr) + // { + // self = ptr; + // kind = NativeMethods.MediaStreamTrackGetKind(self); + // id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(self)); + // } + // //Disassociate track from its source(video or audio), not for destroying the track + // public void Stop() + // { + // stopTrack(this); + // } + //} public enum TrackKind { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs index b4d3f3efb..ee9b9b975 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs @@ -189,11 +189,6 @@ public void Close() } public RTCRtpSender AddTrack(MediaStreamTrack track) - { - return new RTCRtpSender(NativeMethods.PeerConnectionAddTrack(self, track.self)); - } - - public RTCRtpSender AddTrack(MediaStreamTrack2 track) { return new RTCRtpSender(NativeMethods.PeerConnectionAddTrack(self, track.nativePtr)); } diff --git a/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs b/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs index 86410f31e..5e6b609f9 100644 --- a/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs +++ b/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs @@ -159,11 +159,11 @@ void Pc2OnIceCandidate(RTCIceCandidate candidate) } public void AddTracks() { - foreach (var track in audioStream.GetTracks()) + foreach (var track in audioStream.getTracks()) { pc1Senders.Add (pc1.AddTrack(track)); } - foreach(var track in videoStream.GetTracks()) + foreach(var track in videoStream.getTracks()) { pc1Senders.Add(pc1.AddTrack(track)); } @@ -232,8 +232,8 @@ void OnTrack(RTCPeerConnection pc, RTCTrackEvent e) { pc2Senders.Add(pc.AddTrack(e.Track)); trackInfos.Append($"{GetName(pc)} receives remote track:\r\n"); - trackInfos.Append($"Track kind: {e.Track.Kind}\r\n"); - trackInfos.Append($"Track id: {e.Track.Id}\r\n"); + trackInfos.Append($"Track kind: {e.Track.kind}\r\n"); + trackInfos.Append($"Track id: {e.Track.id}\r\n"); infoText.text = trackInfos.ToString(); } diff --git a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs index 3e9247499..3495be428 100644 --- a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs +++ b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs @@ -46,10 +46,20 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream() { pc2Senders.Add(peer2.AddTrack(e.Track)); }); - foreach (var track in cam.CaptureStream(1280, 720).GetTracks()) + + + cam.CreateRenderStreamTexture(1280, 720); + MediaStream videoStream = new MediaStream(); + int texCount = cam.getStreamTextureCount(); + for (int i = 0; i < texCount; ++i) { - pc1Senders.Add(peer1.AddTrack(track)); + pc1Senders.Add(peer1.AddTrack(new VideoStreamTrack(cam.getStreamTexture(i)))); } + + //foreach (var track in cam.CaptureStream(1280, 720).GetTracks()) + //{ + // pc1Senders.Add(peer1.AddTrack(track)); + //} var conf = new RTCDataChannelInit(true); RTCOfferOptions options1 = default; From b879c1681fe98357d8429d7ca1f14239ffc6bf05 Mon Sep 17 00:00:00 2001 From: koseyile Date: Tue, 2 Jul 2019 05:15:16 +0000 Subject: [PATCH 03/29] Start the name with an uppercase letter. Change the access modifiers from public to internal or protected. --- Assets/Scripts/RenderStreaming.cs | 6 ++-- .../Runtime/Srcipts/MediaStream.cs | 16 ++++----- .../Runtime/Srcipts/MediaStreamTrack.cs | 35 ++++++++++--------- .../Samples/Example/AddMediaStream.cs | 8 ++--- .../Tests/Runtime/MediaStreamTest.cs | 4 +-- 5 files changed, 35 insertions(+), 34 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index 19e97fd7a..d1560d81d 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -62,10 +62,10 @@ public IEnumerator Start() } cam.CreateRenderStreamTexture(1280, 720); videoStream = new MediaStream(); - int texCount = cam.getStreamTextureCount(); + int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - videoStream.AddTrack(new VideoStreamTrack(cam.getStreamTexture(i))); + videoStream.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i))); } @@ -151,7 +151,7 @@ IEnumerator GetOffer() string pattern = @"(a=fmtp:\d+ .*level-asymmetry-allowed=.*)\r\n"; _desc.sdp = Regex.Replace(_desc.sdp, pattern, "$1;x-google-start-bitrate=16000;x-google-max-bitrate=160000\r\n"); pc.SetRemoteDescription(ref _desc); - foreach (var track in videoStream.getTracks()) + foreach (var track in videoStream.GetTracks()) { pc.AddTrack(track); } diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index c343c64d7..0082b594e 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -8,8 +8,8 @@ namespace Unity.WebRTC { public class MediaStream { - public IntPtr self; - public string id; + internal IntPtr nativePtr; + protected string id; protected List mediaStreamTrackList = new List(); public MediaStream() : base() @@ -19,10 +19,10 @@ public MediaStream() : base() internal MediaStream(IntPtr ptr) { - self = ptr; - id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self)); + nativePtr = ptr; + id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(nativePtr)); int trackSize = 0; - IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(self, ref trackSize); + IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(nativePtr, ref trackSize); IntPtr[] tracksPtr = new IntPtr[trackSize]; Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize); //TODO: Linux compatibility @@ -50,7 +50,7 @@ public void AddTrack(MediaStreamTrack track) mediaStreamTrackList.Add(track); } - public MediaStreamTrack[] getTracks() + public MediaStreamTrack[] GetTracks() { return mediaStreamTrackList.ToArray(); } @@ -207,12 +207,12 @@ public static class CameraExtension internal static List camCopyRts = new List(); internal static bool started = false; - public static int getStreamTextureCount(this Camera cam) + public static int GetStreamTextureCount(this Camera cam) { return webRTCTextures.Count; } - public static RenderTexture getStreamTexture(this Camera cam, int index) { + public static RenderTexture GetStreamTexture(this Camera cam, int index) { return webRTCTextures[index]; } diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs index 7b070f0d1..4af54027f 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs @@ -6,9 +6,9 @@ namespace Unity.WebRTC { public class MediaStreamTrack { - public IntPtr nativePtr; - public string id; - public TrackKind kind; + internal IntPtr nativePtr; + protected string id; + protected TrackKind kind; public MediaStreamTrack() { @@ -21,20 +21,21 @@ internal MediaStreamTrack(IntPtr ptr) id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(nativePtr)); } - //public bool Enabled - //{ - // get{return NativeMethods.MediaStreamTrackGetEnabled(nativePtr);} - // set{NativeMethods.MediaStreamTrackSetEnabled(nativePtr, value);} - //} - //public TrackState ReadyState - //{ - // get - // {return NativeMethods.MediaStreamTrackGetReadyState(nativePtr);} - // private set { } - //} - - //public TrackKind Kind { get => kind; private set { } } - //public string Id { get => id; private set { } } + public bool Enabled + { + get { return NativeMethods.MediaStreamTrackGetEnabled(nativePtr); } + set { NativeMethods.MediaStreamTrackSetEnabled(nativePtr, value); } + } + + public TrackState ReadyState + { + get + { return NativeMethods.MediaStreamTrackGetReadyState(nativePtr); } + private set { } + } + + public TrackKind Kind { get => kind; private set { } } + public string Id { get => id; private set { } } } public class VideoStreamTrack : MediaStreamTrack diff --git a/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs b/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs index 5e6b609f9..86410f31e 100644 --- a/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs +++ b/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs @@ -159,11 +159,11 @@ void Pc2OnIceCandidate(RTCIceCandidate candidate) } public void AddTracks() { - foreach (var track in audioStream.getTracks()) + foreach (var track in audioStream.GetTracks()) { pc1Senders.Add (pc1.AddTrack(track)); } - foreach(var track in videoStream.getTracks()) + foreach(var track in videoStream.GetTracks()) { pc1Senders.Add(pc1.AddTrack(track)); } @@ -232,8 +232,8 @@ void OnTrack(RTCPeerConnection pc, RTCTrackEvent e) { pc2Senders.Add(pc.AddTrack(e.Track)); trackInfos.Append($"{GetName(pc)} receives remote track:\r\n"); - trackInfos.Append($"Track kind: {e.Track.kind}\r\n"); - trackInfos.Append($"Track id: {e.Track.id}\r\n"); + trackInfos.Append($"Track kind: {e.Track.Kind}\r\n"); + trackInfos.Append($"Track id: {e.Track.Id}\r\n"); infoText.text = trackInfos.ToString(); } diff --git a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs index 3495be428..5bbb6a7d8 100644 --- a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs +++ b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs @@ -50,10 +50,10 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream() cam.CreateRenderStreamTexture(1280, 720); MediaStream videoStream = new MediaStream(); - int texCount = cam.getStreamTextureCount(); + int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - pc1Senders.Add(peer1.AddTrack(new VideoStreamTrack(cam.getStreamTexture(i)))); + pc1Senders.Add(peer1.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i)))); } //foreach (var track in cam.CaptureStream(1280, 720).GetTracks()) From 4b296dd2bb5113d7562fb468d138066fc634926a Mon Sep 17 00:00:00 2001 From: koseyile Date: Tue, 2 Jul 2019 06:42:27 +0000 Subject: [PATCH 04/29] remove the temporary comment code. add AudioStreamTrack. --- .../Runtime/Srcipts/MediaStream.cs | 158 ++---------------- .../Runtime/Srcipts/MediaStreamTrack.cs | 62 ++----- .../Samples/Example/AddMediaStream.cs | 11 +- 3 files changed, 37 insertions(+), 194 deletions(-) diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index 0082b594e..035bccac1 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -8,8 +8,10 @@ namespace Unity.WebRTC { public class MediaStream { - internal IntPtr nativePtr; - protected string id; + //to do : c++ create two mediastream named "audio" and "vedio". Actually we only need one. + //internal IntPtr nativePtr; + //protected string id; + //public string Id { get => id; private set { } } protected List mediaStreamTrackList = new List(); public MediaStream() : base() @@ -17,26 +19,6 @@ public MediaStream() : base() } - internal MediaStream(IntPtr ptr) - { - nativePtr = ptr; - id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(nativePtr)); - int trackSize = 0; - IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(nativePtr, ref trackSize); - IntPtr[] tracksPtr = new IntPtr[trackSize]; - Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize); - //TODO: Linux compatibility - Marshal.FreeCoTaskMem(trackNativePtr); - - for (int i = 0; i < trackSize; i++) - { - MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]); - //track.stopTrack += StopTrack; - //track.getRts += GetRts; - mediaStreamTrackList.Add(track); - } - } - public MediaStream(MediaStreamTrack[] tracks) : base() { foreach (var t in tracks) @@ -55,123 +37,7 @@ public MediaStreamTrack[] GetTracks() return mediaStreamTrackList.ToArray(); } } - - //public class MediaStream - //{ - // private IntPtr self; - // private string id; - // public string Id { get => id; private set { } } - - // private Dictionary VideoTrackToRts; - // private List AudioTracks; - - // //private void StopTrack(MediaStreamTrack track) - // //{ - - // // if (track.kind == TrackKind.Video) - // // { - // // NativeMethods.StopMediaStreamTrack(track.nativePtr); - // // RenderTexture[] rts = VideoTrackToRts[track]; - // // if (rts != null) - // // { - // // CameraExtension.RemoveRt(rts); - // // rts[0].Release(); - // // rts[1].Release(); - // // UnityEngine.Object.Destroy(rts[0]); - // // UnityEngine.Object.Destroy(rts[1]); - // // } - // // } - // // else - // // { - // // Audio.Stop(); - // // } - - // //} - // private RenderTexture[] GetRts(MediaStreamTrack track) - // { - // return VideoTrackToRts[track]; - // } - // public MediaStreamTrack[] GetTracks() - // { - // MediaStreamTrack[] tracks = new MediaStreamTrack[VideoTrackToRts.Keys.Count + AudioTracks.Count]; - // AudioTracks.CopyTo(tracks, 0); - // VideoTrackToRts.Keys.CopyTo(tracks, AudioTracks.Count); - // return tracks; - // } - // //public MediaStreamTrack[] GetAudioTracks() - // //{ - // // return AudioTracks.ToArray(); - // //} - // //public MediaStreamTrack[] GetVideoTracks() - // //{ - // // MediaStreamTrack[] tracks = new MediaStreamTrack[VideoTrackToRts.Keys.Count]; - // // VideoTrackToRts.Keys.CopyTo(tracks, 0); - // // return tracks; - // //} - - // //public void AddTrack(MediaStreamTrack track) - // //{ - // // if(track.kind == TrackKind.Video) - // // { - // // VideoTrackToRts[track] = track.getRts(track); - // // } - // // else - // // { - // // AudioTracks.Add(track); - // // } - // // NativeMethods.MediaStreamAddTrack(self, track.self); - // //} - // //public void RemoveTrack(MediaStreamTrack track) - // //{ - // // NativeMethods.MediaStreamRemoveTrack(self, track.self); - // //} - // //for camera CaptureStream - // internal MediaStream(RenderTexture[] rts, IntPtr ptr) - // { - // self = ptr; - // id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self)); - // VideoTrackToRts = new Dictionary(); - // AudioTracks = new List(); - // //get initial tracks - // int trackSize = 0; - // IntPtr tracksNativePtr = NativeMethods.MediaStreamGetVideoTracks(self, ref trackSize); - // IntPtr[] tracksPtr = new IntPtr[trackSize]; - // Marshal.Copy(tracksNativePtr, tracksPtr, 0, trackSize); - // //TODO: Linux compatibility - // Marshal.FreeCoTaskMem(tracksNativePtr); - // for (int i = 0; i < trackSize; i++) - // { - // MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]); - // //track.stopTrack += StopTrack; - // //track.getRts += GetRts; - // VideoTrackToRts[track] = rts; - // } - // } - // //for audio CaptureStream - // internal MediaStream(IntPtr ptr) - // { - // self = ptr; - // id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self)); - // VideoTrackToRts = new Dictionary(); - // AudioTracks = new List(); - // //get initial tracks - // int trackSize = 0; - // IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(self, ref trackSize); - // IntPtr[] tracksPtr = new IntPtr[trackSize]; - // Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize); - // //TODO: Linux compatibility - // Marshal.FreeCoTaskMem(trackNativePtr); - - // for (int i = 0; i < trackSize; i++) - // { - // MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]); - // //track.stopTrack += StopTrack; - // //track.getRts += GetRts; - // AudioTracks.Add(track); - // } - // } - - //} + internal class Cleaner : MonoBehaviour { private Action onDestroy; @@ -250,14 +116,6 @@ public static void CreateRenderStreamTexture(this Camera cam, int width, int hei started = true; } - public static MediaStream CaptureStream(this Camera cam, int width, int height) - { - cam.CreateRenderStreamTexture(width, height); - - int textureIndex = 0; - int rtcMipLevel = (int)Math.Pow(2, textureIndex); //1 2 4 8 - return new MediaStream(WebRTC.Context.CaptureVideoStream(webRTCTextures[textureIndex].GetNativeTexturePtr(), width/rtcMipLevel, height/rtcMipLevel)); - } public static void RemoveRt(RenderTexture[] rts) { camCopyRts.Remove(rts); @@ -277,7 +135,11 @@ public static MediaStream CaptureStream() { audioInput.BeginRecording(); started = true; - return new MediaStream(WebRTC.Context.CaptureAudioStream()); + + MediaStream mediaStream = new MediaStream(); + AudioStreamTrack audioStreamTrack = new AudioStreamTrack(); + mediaStream.AddTrack(audioStreamTrack); + return mediaStream; } public static void Update() { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs index 4af54027f..5be379d56 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs @@ -58,54 +58,26 @@ public VideoStreamTrack(RenderTexture rt) : base() } } + public class AudioStreamTrack : MediaStreamTrack + { + public AudioStreamTrack() : base() + { + IntPtr nativeAudioStreamPtr = WebRTC.Context.CaptureAudioStream(); + string nativeAudioStreamID = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(nativeAudioStreamPtr)); + int trackSize = 0; + IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(nativeAudioStreamPtr, ref trackSize); + IntPtr[] tracksPtr = new IntPtr[trackSize]; + Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize); + //TODO: Linux compatibility + Marshal.FreeCoTaskMem(trackNativePtr); + nativePtr = tracksPtr[0]; + kind = NativeMethods.MediaStreamTrackGetKind(nativePtr); + id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(nativePtr)); + } + } - //public class MediaStreamTrack - //{ - // internal IntPtr self; - // private TrackKind kind; - // private string id; - // private bool enabled; - // private TrackState readyState; - // internal Action stopTrack; - // internal Func getRts; - - // public bool Enabled - // { - // get - // { - // return NativeMethods.MediaStreamTrackGetEnabled(self); - // } - // set - // { - // NativeMethods.MediaStreamTrackSetEnabled(self, value); - // } - // } - // public TrackState ReadyState - // { - // get - // { - // return NativeMethods.MediaStreamTrackGetReadyState(self); - // } - // private set { } - // } - - // public TrackKind Kind { get => kind; private set { } } - // public string Id { get => id; private set { } } - - // internal MediaStreamTrack(IntPtr ptr) - // { - // self = ptr; - // kind = NativeMethods.MediaStreamTrackGetKind(self); - // id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(self)); - // } - // //Disassociate track from its source(video or audio), not for destroying the track - // public void Stop() - // { - // stopTrack(this); - // } - //} public enum TrackKind { diff --git a/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs b/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs index 86410f31e..78298f466 100644 --- a/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs +++ b/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs @@ -213,7 +213,16 @@ void Call() RTCDataChannelInit conf = new RTCDataChannelInit(true); dataChannel = pc1.CreateDataChannel("data", ref conf); audioStream = Audio.CaptureStream(); - videoStream = cam.CaptureStream(1280, 720); + + cam.CreateRenderStreamTexture(1280, 720); + videoStream = new MediaStream(); + int texCount = cam.GetStreamTextureCount(); + for (int i = 0; i < texCount; ++i) + { + videoStream.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i))); + } + + RtImage.texture = cam.targetTexture; } From e924c1da1fccf10da429dc0b2a93065b8e6bd476 Mon Sep 17 00:00:00 2001 From: koseyile Date: Tue, 2 Jul 2019 07:47:24 +0000 Subject: [PATCH 05/29] remove unused comments. public modify to internal. --- Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs | 3 --- Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs | 2 +- Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs | 4 ---- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index 035bccac1..89db2ab41 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -9,9 +9,6 @@ namespace Unity.WebRTC public class MediaStream { //to do : c++ create two mediastream named "audio" and "vedio". Actually we only need one. - //internal IntPtr nativePtr; - //protected string id; - //public string Id { get => id; private set { } } protected List mediaStreamTrackList = new List(); public MediaStream() : base() diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs index 5be379d56..b7b7466c6 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs @@ -9,7 +9,7 @@ public class MediaStreamTrack internal IntPtr nativePtr; protected string id; protected TrackKind kind; - public MediaStreamTrack() + internal MediaStreamTrack() { } diff --git a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs index 5bbb6a7d8..bbe9db942 100644 --- a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs +++ b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs @@ -56,10 +56,6 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream() pc1Senders.Add(peer1.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i)))); } - //foreach (var track in cam.CaptureStream(1280, 720).GetTracks()) - //{ - // pc1Senders.Add(peer1.AddTrack(track)); - //} var conf = new RTCDataChannelInit(true); RTCOfferOptions options1 = default; From 2e903d047bb74d05c43de04df71eb74749bec81c Mon Sep 17 00:00:00 2001 From: chenyuan Date: Tue, 9 Jul 2019 11:51:52 +0800 Subject: [PATCH 06/29] add new method "createVideoTrack" to Context class. --- Plugin/WebRTCPlugin/Context.cpp | 41 +++++++++++++---------- Plugin/WebRTCPlugin/Context.h | 10 +++--- Plugin/WebRTCPlugin/DummyVideoEncoder.cpp | 7 +++- Plugin/WebRTCPlugin/DummyVideoEncoder.h | 4 ++- Plugin/WebRTCPlugin/WebRTCPlugin.cpp | 4 +-- 5 files changed, 40 insertions(+), 26 deletions(-) diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index 59c5262dd..9f42536b7 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -292,9 +292,10 @@ namespace WebRTC rtc::InitializeSSL(); audioDevice = new rtc::RefCountedObject(); - nvVideoCapturerUnique = std::make_unique(); - nvVideoCapturer = nvVideoCapturerUnique.get(); - auto dummyVideoEncoderFactory = std::make_unique(nvVideoCapturer); + //nvVideoCapturerUnique = std::make_unique(); + //nvVideoCapturer = nvVideoCapturerUnique.get(); + auto dummyVideoEncoderFactory = std::make_unique(); + pDummyVideoEncoderFactory = dummyVideoEncoderFactory.get(); peerConnectionFactory = webrtc::CreatePeerConnectionFactory( workerThread.get(), @@ -314,9 +315,8 @@ namespace WebRTC clients.clear(); peerConnectionFactory = nullptr; audioTrack = nullptr; - videoTracks.clear(); audioStream = nullptr; - videoStreams.clear(); + mediaStream = nullptr; workerThread->Quit(); workerThread.reset(); @@ -324,21 +324,28 @@ namespace WebRTC signalingThread.reset(); } - webrtc::MediaStreamInterface* Context::CreateVideoStream(UnityFrameBuffer* frameBuffer) + webrtc::MediaStreamInterface* Context::CreateVideoStream(UnityFrameBuffer* frameBuffer, int32 width, int32 height) { - //TODO: label and stream id should be maintained in some way for multi-stream - auto videoTrack = peerConnectionFactory->CreateVideoTrack( - "video", peerConnectionFactory->CreateVideoSource(std::move(nvVideoCapturerUnique))); - if (!videoTracks.count(frameBuffer)) - { - videoTracks[frameBuffer] = videoTrack; - } - auto videoStream = peerConnectionFactory->CreateLocalMediaStream("video"); - videoStream->AddTrack(videoTrack); - videoStreams.push_back(videoStream); + ////TODO: label and stream id should be maintained in some way for multi-stream + //create track + auto videoTrack = CreateVideoTrack("video", frameBuffer, width, height); + //create stream + mediaStream = peerConnectionFactory->CreateLocalMediaStream("video"); + mediaStream->AddTrack(videoTrack); + return mediaStream.get(); + } + + rtc::scoped_refptr Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) + { + nvVideoCapturerUnique = std::make_unique(); + nvVideoCapturer = nvVideoCapturerUnique.get(); + nvVideoCapturer->InitializeEncoder(width, height); + pDummyVideoEncoderFactory->SetCapturer(nvVideoCapturer); + auto videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(std::move(nvVideoCapturerUnique))); nvVideoCapturer->unityRT = frameBuffer; nvVideoCapturer->StartEncoder(); - return videoStream.get(); + + return videoTrack; } webrtc::MediaStreamInterface* Context::CreateAudioStream() diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index 939e09574..899de2807 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -42,13 +42,13 @@ namespace WebRTC { public: explicit Context(int uid = -1); - webrtc::MediaStreamInterface* CreateVideoStream(UnityFrameBuffer* frameBuffer); + webrtc::MediaStreamInterface* CreateVideoStream(UnityFrameBuffer* frameBuffer, int32 width, int32 height); + rtc::scoped_refptr CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height); webrtc::MediaStreamInterface* CreateAudioStream(); ~Context(); PeerConnectionObject* CreatePeerConnection(int id); PeerConnectionObject* CreatePeerConnection(int id, const std::string& conf); - void InitializeEncoder(int32 width, int32 height) { nvVideoCapturer->InitializeEncoder(width, height); } void EncodeFrame() { nvVideoCapturer->EncodeVideoData(); } void StopCapturer() { nvVideoCapturer->Stop(); } void ProcessAudioData(const float* data, int32 size) { audioDevice->ProcessAudioData(data, size); } @@ -59,14 +59,14 @@ namespace WebRTC std::unique_ptr signalingThread; std::map> clients; rtc::scoped_refptr peerConnectionFactory; + DummyVideoEncoderFactory* pDummyVideoEncoderFactory; + rtc::scoped_refptr mediaStream; + NvVideoCapturer* nvVideoCapturer; std::unique_ptr nvVideoCapturerUnique; rtc::scoped_refptr audioDevice; rtc::scoped_refptr audioTrack; rtc::scoped_refptr audioStream; - //TODO: move videoTrack to NvVideoCapturer and maintain multiple NvVideoCapturer here - std::vector> videoStreams; - std::map> videoTracks; }; class PeerSDPObserver : public webrtc::SetSessionDescriptionObserver diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp index 1a5dff007..e20840a76 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp @@ -67,7 +67,12 @@ namespace WebRTC SetRate(allocation.get_sum_kbps() * 1000); return 0; } - DummyVideoEncoderFactory::DummyVideoEncoderFactory(NvVideoCapturer* videoCapturer):capturer(videoCapturer){} + + DummyVideoEncoderFactory::DummyVideoEncoderFactory() + { + + } + std::vector DummyVideoEncoderFactory::GetSupportedFormats() const { const absl::optional profileLevelId = diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.h b/Plugin/WebRTCPlugin/DummyVideoEncoder.h index 43437e2ad..ad8f81601 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.h +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.h @@ -51,7 +51,9 @@ namespace WebRTC // Creates a VideoEncoder for the specified format. virtual std::unique_ptr CreateVideoEncoder( const webrtc::SdpVideoFormat& format) override; - DummyVideoEncoderFactory(NvVideoCapturer* videoCapturer); + DummyVideoEncoderFactory(); + + void SetCapturer(NvVideoCapturer* _capturer) { capturer = _capturer; } private: NvVideoCapturer* capturer; }; diff --git a/Plugin/WebRTCPlugin/WebRTCPlugin.cpp b/Plugin/WebRTCPlugin/WebRTCPlugin.cpp index 9ce258472..a9144b74b 100644 --- a/Plugin/WebRTCPlugin/WebRTCPlugin.cpp +++ b/Plugin/WebRTCPlugin/WebRTCPlugin.cpp @@ -31,8 +31,8 @@ extern "C" { UNITY_INTERFACE_EXPORT webrtc::MediaStreamInterface* CaptureVideoStream(Context* context, UnityFrameBuffer* rt, int32 width, int32 height) { - context->InitializeEncoder(width, height); - return context->CreateVideoStream(rt); + //context->InitializeEncoder(width, height); + return context->CreateVideoStream(rt, width, height); } //TODO: Multi-track support UNITY_INTERFACE_EXPORT void StopMediaStreamTrack(webrtc::MediaStreamTrackInterface* track) From 936991a4c348172013b2e981d0e344d5b4a671aa Mon Sep 17 00:00:00 2001 From: chenyuan Date: Tue, 9 Jul 2019 16:43:01 +0800 Subject: [PATCH 07/29] video and audio share one mediaStream. --- Plugin/WebRTCPlugin/Context.cpp | 22 +++++++++++----------- Plugin/WebRTCPlugin/Context.h | 3 +-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index 9f42536b7..6c31de412 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -292,8 +292,6 @@ namespace WebRTC rtc::InitializeSSL(); audioDevice = new rtc::RefCountedObject(); - //nvVideoCapturerUnique = std::make_unique(); - //nvVideoCapturer = nvVideoCapturerUnique.get(); auto dummyVideoEncoderFactory = std::make_unique(); pDummyVideoEncoderFactory = dummyVideoEncoderFactory.get(); @@ -308,14 +306,14 @@ namespace WebRTC webrtc::CreateBuiltinVideoDecoderFactory(), nullptr, nullptr); + + mediaStream = peerConnectionFactory->CreateLocalMediaStream("mediaStream"); } Context::~Context() { clients.clear(); peerConnectionFactory = nullptr; - audioTrack = nullptr; - audioStream = nullptr; mediaStream = nullptr; workerThread->Quit(); @@ -327,10 +325,7 @@ namespace WebRTC webrtc::MediaStreamInterface* Context::CreateVideoStream(UnityFrameBuffer* frameBuffer, int32 width, int32 height) { ////TODO: label and stream id should be maintained in some way for multi-stream - //create track auto videoTrack = CreateVideoTrack("video", frameBuffer, width, height); - //create stream - mediaStream = peerConnectionFactory->CreateLocalMediaStream("video"); mediaStream->AddTrack(videoTrack); return mediaStream.get(); } @@ -349,6 +344,13 @@ namespace WebRTC } webrtc::MediaStreamInterface* Context::CreateAudioStream() + { + auto audioTrack = CreateAudioTrack(); + mediaStream->AddTrack(audioTrack); + return mediaStream.get(); + } + + rtc::scoped_refptr Context::CreateAudioTrack() { //avoid optimization specially for voice cricket::AudioOptions audioOptions; @@ -356,10 +358,8 @@ namespace WebRTC audioOptions.noise_suppression = false; audioOptions.highpass_filter = false; //TODO: label and stream id should be maintained in some way for multi-stream - audioTrack = peerConnectionFactory->CreateAudioTrack("audio", peerConnectionFactory->CreateAudioSource(audioOptions)); - audioStream = peerConnectionFactory->CreateLocalMediaStream("audio"); - audioStream->AddTrack(audioTrack); - return audioStream.get(); + auto audioTrack = peerConnectionFactory->CreateAudioTrack("audio", peerConnectionFactory->CreateAudioSource(audioOptions)); + return audioTrack; } PeerSDPObserver* PeerSDPObserver::Create(DelegateSetSDSuccess onSuccess, DelegateSetSDFailure onFailure) diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index 899de2807..be2828460 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -45,6 +45,7 @@ namespace WebRTC webrtc::MediaStreamInterface* CreateVideoStream(UnityFrameBuffer* frameBuffer, int32 width, int32 height); rtc::scoped_refptr CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height); webrtc::MediaStreamInterface* CreateAudioStream(); + rtc::scoped_refptr CreateAudioTrack(); ~Context(); PeerConnectionObject* CreatePeerConnection(int id); @@ -65,8 +66,6 @@ namespace WebRTC NvVideoCapturer* nvVideoCapturer; std::unique_ptr nvVideoCapturerUnique; rtc::scoped_refptr audioDevice; - rtc::scoped_refptr audioTrack; - rtc::scoped_refptr audioStream; }; class PeerSDPObserver : public webrtc::SetSessionDescriptionObserver From f3972bd7d1d59a5c16f9adba270c4c4539edf4ed Mon Sep 17 00:00:00 2001 From: chenyuan Date: Tue, 9 Jul 2019 19:26:33 +0800 Subject: [PATCH 08/29] change name "videostream" to "mediastream" and add audio track to media stream --- Assets/Scripts/RenderStreaming.cs | 17 +++++++++++------ .../Runtime/Srcipts/MediaStream.cs | 8 ++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index c7044173e..f5187efaa 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -41,7 +41,7 @@ public class RenderStreaming : MonoBehaviour private Dictionary> mapChannels = new Dictionary>(); private RTCConfiguration conf; private string sessionId; - private MediaStream videoStream; + private MediaStream mediaStream; public void Awake() { @@ -52,6 +52,7 @@ public void Awake() public void OnDestroy() { + Audio.Stop(); WebRTC.WebRTC.Finalize(); } public IEnumerator Start() @@ -61,14 +62,17 @@ public IEnumerator Start() yield break; } - cam.CreateRenderStreamTexture(1280, 720); - videoStream = new MediaStream(); - int texCount = cam.GetStreamTextureCount(); + captureCamera.CreateRenderStreamTexture(1280, 720); + mediaStream = new MediaStream(); + int texCount = captureCamera.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - videoStream.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i))); + mediaStream.AddTrack(new VideoStreamTrack(captureCamera.GetStreamTexture(i))); } + mediaStream.AddTrack(new AudioStreamTrack()); + Audio.Start(); + signaling = new Signaling(urlSignaling); var opCreate = signaling.Create(); yield return opCreate; @@ -150,10 +154,11 @@ IEnumerator GetOffer() string pattern = @"(a=fmtp:\d+ .*level-asymmetry-allowed=.*)\r\n"; _desc.sdp = Regex.Replace(_desc.sdp, pattern, "$1;x-google-start-bitrate=16000;x-google-max-bitrate=160000\r\n"); pc.SetRemoteDescription(ref _desc); - foreach (var track in videoStream.GetTracks()) + foreach (var track in mediaStream.GetTracks()) { pc.AddTrack(track); } + StartCoroutine(Answer(connectionId)); } } diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index 89db2ab41..052106344 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -138,6 +138,7 @@ public static MediaStream CaptureStream() mediaStream.AddTrack(audioStreamTrack); return mediaStream; } + public static void Update() { if (started) @@ -145,6 +146,13 @@ public static void Update() audioInput.UpdateAudio(); } } + + public static void Start() + { + audioInput.BeginRecording(); + started = true; + } + public static void Stop() { if (started) From e400b81d0122b9c424d71a874b7c42cfbe06d8c7 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Thu, 11 Jul 2019 19:27:39 +0800 Subject: [PATCH 09/29] decoupling streams and tracks. delete these functions: CaptureVideoStream CaptureAudioStream --- Assets/Scripts/RenderStreaming.cs | 2 +- .../Runtime/Srcipts/MediaStream.cs | 23 +++++------- .../Runtime/Srcipts/MediaStreamTrack.cs | 34 ++---------------- .../Runtime/Srcipts/WebRTC.cs | 12 ++++--- .../Samples/Example/MediaStreamSample.cs | 16 ++++----- .../Tests/Runtime/MediaStreamTest.cs | 6 ++-- Plugin/WebRTCPlugin/Context.cpp | 35 +++++++++---------- Plugin/WebRTCPlugin/Context.h | 12 ++++--- Plugin/WebRTCPlugin/WebRTCPlugin.cpp | 22 +++++++----- 9 files changed, 68 insertions(+), 94 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index f5187efaa..085c8f108 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -63,7 +63,7 @@ public IEnumerator Start() } captureCamera.CreateRenderStreamTexture(1280, 720); - mediaStream = new MediaStream(); + mediaStream = new MediaStream("MediaStream"); int texCount = captureCamera.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index 052106344..5d82eca07 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -8,24 +8,27 @@ namespace Unity.WebRTC { public class MediaStream { - //to do : c++ create two mediastream named "audio" and "vedio". Actually we only need one. + internal IntPtr nativePtr; protected List mediaStreamTrackList = new List(); - public MediaStream() : base() + public MediaStream(string label) : base() { - + nativePtr = WebRTC.Context.CreateMediaStream(label); } - public MediaStream(MediaStreamTrack[] tracks) : base() + public MediaStream(string label, MediaStreamTrack[] tracks) : base() { + nativePtr = WebRTC.Context.CreateMediaStream(label); + foreach (var t in tracks) { - mediaStreamTrackList.Add(t); + AddTrack(t); } } public void AddTrack(MediaStreamTrack track) { + NativeMethods.MediaStreamAddTrack(nativePtr, track.nativePtr); mediaStreamTrackList.Add(track); } @@ -128,16 +131,6 @@ public static class Audio { private static bool started = false; private static AudioInput audioInput = new AudioInput(); - public static MediaStream CaptureStream() - { - audioInput.BeginRecording(); - started = true; - - MediaStream mediaStream = new MediaStream(); - AudioStreamTrack audioStreamTrack = new AudioStreamTrack(); - mediaStream.AddTrack(audioStreamTrack); - return mediaStream; - } public static void Update() { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs index b7b7466c6..25a716ede 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs @@ -9,10 +9,6 @@ public class MediaStreamTrack internal IntPtr nativePtr; protected string id; protected TrackKind kind; - internal MediaStreamTrack() - { - - } internal MediaStreamTrack(IntPtr ptr) { @@ -40,41 +36,15 @@ private set { } public class VideoStreamTrack : MediaStreamTrack { - public VideoStreamTrack(RenderTexture rt) : base() + public VideoStreamTrack(RenderTexture rt) : base(WebRTC.Context.CreateVideoTrack("videoTrack", rt.GetNativeTexturePtr(), rt.width, rt.height)) { - IntPtr nativeVideoStreamPtr = WebRTC.Context.CaptureVideoStream(rt.GetNativeTexturePtr(), rt.width, rt.height); - string nativeVideoStreamID = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(nativeVideoStreamPtr)); - - int trackSize = 0; - IntPtr tracksNativePtr = NativeMethods.MediaStreamGetVideoTracks(nativeVideoStreamPtr, ref trackSize); - IntPtr[] tracksPtr = new IntPtr[trackSize]; - Marshal.Copy(tracksNativePtr, tracksPtr, 0, trackSize); - Marshal.FreeCoTaskMem(tracksNativePtr); - - nativePtr = tracksPtr[0]; - kind = NativeMethods.MediaStreamTrackGetKind(nativePtr); - id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(nativePtr)); - } } public class AudioStreamTrack : MediaStreamTrack { - public AudioStreamTrack() : base() + public AudioStreamTrack() : base(WebRTC.Context.CreateAudioTrack("audioTrack")) { - IntPtr nativeAudioStreamPtr = WebRTC.Context.CaptureAudioStream(); - string nativeAudioStreamID = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(nativeAudioStreamPtr)); - - int trackSize = 0; - IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(nativeAudioStreamPtr, ref trackSize); - IntPtr[] tracksPtr = new IntPtr[trackSize]; - Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize); - //TODO: Linux compatibility - Marshal.FreeCoTaskMem(trackNativePtr); - - nativePtr = tracksPtr[0]; - kind = NativeMethods.MediaStreamTrackGetKind(nativePtr); - id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(nativePtr)); } } diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs index 0b1caadf3..5ea60fa1b 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs @@ -362,7 +362,11 @@ internal static class NativeMethods [DllImport(WebRTC.Lib)] public static extern IntPtr CaptureVideoStream(IntPtr context, IntPtr rt, int width, int height); [DllImport(WebRTC.Lib)] - public static extern IntPtr CaptureAudioStream(IntPtr context); + public static extern IntPtr CreateMediaStream(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label); + [DllImport(WebRTC.Lib)] + public static extern IntPtr CreateVideoTrack(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label, IntPtr rt, int width, int height); + [DllImport(WebRTC.Lib)] + public static extern IntPtr CreateAudioTrack(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label); [DllImport(WebRTC.Lib)] public static extern void MediaStreamAddTrack(IntPtr stream, IntPtr track); [DllImport(WebRTC.Lib)] @@ -399,9 +403,9 @@ internal struct Context public static bool ToBool(Context v) { return v; } public static Context Create(int uid = 0) { return NativeMethods.ContextCreate(uid); } public void Destroy(int uid = 0) { NativeMethods.ContextDestroy(uid); self = IntPtr.Zero; } - public IntPtr CaptureVideoStream(IntPtr rt, int width, int height) { return NativeMethods.CaptureVideoStream(self, rt, width, height); } - public IntPtr CaptureAudioStream() { return NativeMethods.CaptureAudioStream(self); } - + public IntPtr CreateMediaStream(string label) { return NativeMethods.CreateMediaStream(self, label); } + public IntPtr CreateVideoTrack(string label, IntPtr rt, int width, int height) { return NativeMethods.CreateVideoTrack(self, label, rt, width, height); } + public IntPtr CreateAudioTrack(string label) {return NativeMethods.CreateAudioTrack(self, label);} } } diff --git a/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs b/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs index 6a017732b..dc28c0d66 100644 --- a/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs +++ b/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs @@ -19,7 +19,7 @@ public class MediaStreamSample : MonoBehaviour private RTCPeerConnection pc1, pc2; private List pc1Senders, pc2Senders; - private Unity.WebRTC.MediaStream audioStream, videoStream; + private Unity.WebRTC.MediaStream mediaStream; private RTCDataChannel dataChannel, remoteDataChannel; private Coroutine sdpCheck; private string msg; @@ -159,14 +159,11 @@ void Pc2OnIceCandidate(RTCIceCandidate candidate) } public void AddTracks() { - foreach (var track in audioStream.GetTracks()) + foreach (var track in mediaStream.GetTracks()) { pc1Senders.Add (pc1.AddTrack(track)); } - foreach(var track in videoStream.GetTracks()) - { - pc1Senders.Add(pc1.AddTrack(track)); - } + if(!videoUpdateStarted) { StartCoroutine(WebRTC.Update()); @@ -212,16 +209,17 @@ void Call() RTCDataChannelInit conf = new RTCDataChannelInit(true); dataChannel = pc1.CreateDataChannel("data", ref conf); - audioStream = Audio.CaptureStream(); cam.CreateRenderStreamTexture(1280, 720); - videoStream = new MediaStream(); + mediaStream = new MediaStream("mediaStream"); int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - videoStream.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i))); + mediaStream.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i))); } + mediaStream.AddTrack(new AudioStreamTrack()); + Audio.Start(); RtImage.texture = cam.targetTexture; diff --git a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs index bbe9db942..ee986c32f 100644 --- a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs +++ b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs @@ -49,11 +49,13 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream() cam.CreateRenderStreamTexture(1280, 720); - MediaStream videoStream = new MediaStream(); + MediaStream mediaStream = new MediaStream("mediaStream"); int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - pc1Senders.Add(peer1.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i)))); + VideoStreamTrack videoStreamTrack = new VideoStreamTrack(cam.GetStreamTexture(i)); + mediaStream.AddTrack(videoStreamTrack); + pc1Senders.Add(peer1.AddTrack(videoStreamTrack)); } var conf = new RTCDataChannelInit(true); diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index 6c31de412..c4c28dedd 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -307,14 +307,17 @@ namespace WebRTC nullptr, nullptr); - mediaStream = peerConnectionFactory->CreateLocalMediaStream("mediaStream"); } Context::~Context() { clients.clear(); peerConnectionFactory = nullptr; - mediaStream = nullptr; + + videoTrack = nullptr; + audioTrack = nullptr; + + mediaStreamMap.clear(); workerThread->Quit(); workerThread.reset(); @@ -322,35 +325,31 @@ namespace WebRTC signalingThread.reset(); } - webrtc::MediaStreamInterface* Context::CreateVideoStream(UnityFrameBuffer* frameBuffer, int32 width, int32 height) + webrtc::MediaStreamInterface* Context::CreateMediaStream(const std::string& stream_id) { - ////TODO: label and stream id should be maintained in some way for multi-stream - auto videoTrack = CreateVideoTrack("video", frameBuffer, width, height); - mediaStream->AddTrack(videoTrack); - return mediaStream.get(); + if (mediaStreamMap.count(stream_id) == 0) + { + mediaStreamMap[stream_id] = peerConnectionFactory->CreateLocalMediaStream(stream_id); + } + + return mediaStreamMap[stream_id]; } - rtc::scoped_refptr Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) + webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) { nvVideoCapturerUnique = std::make_unique(); nvVideoCapturer = nvVideoCapturerUnique.get(); nvVideoCapturer->InitializeEncoder(width, height); pDummyVideoEncoderFactory->SetCapturer(nvVideoCapturer); - auto videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(std::move(nvVideoCapturerUnique))); + + videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(std::move(nvVideoCapturerUnique))); nvVideoCapturer->unityRT = frameBuffer; nvVideoCapturer->StartEncoder(); return videoTrack; } - webrtc::MediaStreamInterface* Context::CreateAudioStream() - { - auto audioTrack = CreateAudioTrack(); - mediaStream->AddTrack(audioTrack); - return mediaStream.get(); - } - - rtc::scoped_refptr Context::CreateAudioTrack() + webrtc::MediaStreamTrackInterface* Context::CreateAudioTrack(const std::string& label) { //avoid optimization specially for voice cricket::AudioOptions audioOptions; @@ -358,7 +357,7 @@ namespace WebRTC audioOptions.noise_suppression = false; audioOptions.highpass_filter = false; //TODO: label and stream id should be maintained in some way for multi-stream - auto audioTrack = peerConnectionFactory->CreateAudioTrack("audio", peerConnectionFactory->CreateAudioSource(audioOptions)); + audioTrack = peerConnectionFactory->CreateAudioTrack(label, peerConnectionFactory->CreateAudioSource(audioOptions)); return audioTrack; } diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index be2828460..6ba184b34 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -42,10 +42,9 @@ namespace WebRTC { public: explicit Context(int uid = -1); - webrtc::MediaStreamInterface* CreateVideoStream(UnityFrameBuffer* frameBuffer, int32 width, int32 height); - rtc::scoped_refptr CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height); - webrtc::MediaStreamInterface* CreateAudioStream(); - rtc::scoped_refptr CreateAudioTrack(); + webrtc::MediaStreamInterface* CreateMediaStream(const std::string& stream_id); + webrtc::MediaStreamTrackInterface* CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height); + webrtc::MediaStreamTrackInterface* CreateAudioTrack(const std::string& label); ~Context(); PeerConnectionObject* CreatePeerConnection(int id); @@ -61,11 +60,14 @@ namespace WebRTC std::map> clients; rtc::scoped_refptr peerConnectionFactory; DummyVideoEncoderFactory* pDummyVideoEncoderFactory; - rtc::scoped_refptr mediaStream; + std::map> mediaStreamMap; NvVideoCapturer* nvVideoCapturer; std::unique_ptr nvVideoCapturerUnique; rtc::scoped_refptr audioDevice; + + rtc::scoped_refptr videoTrack; + rtc::scoped_refptr audioTrack; }; class PeerSDPObserver : public webrtc::SetSessionDescriptionObserver diff --git a/Plugin/WebRTCPlugin/WebRTCPlugin.cpp b/Plugin/WebRTCPlugin/WebRTCPlugin.cpp index a9144b74b..707c0d88d 100644 --- a/Plugin/WebRTCPlugin/WebRTCPlugin.cpp +++ b/Plugin/WebRTCPlugin/WebRTCPlugin.cpp @@ -29,11 +29,21 @@ namespace WebRTC extern "C" { - UNITY_INTERFACE_EXPORT webrtc::MediaStreamInterface* CaptureVideoStream(Context* context, UnityFrameBuffer* rt, int32 width, int32 height) + UNITY_INTERFACE_EXPORT webrtc::MediaStreamInterface* CreateMediaStream(Context* context, const char* label) { - //context->InitializeEncoder(width, height); - return context->CreateVideoStream(rt, width, height); + return context->CreateMediaStream(label); } + + UNITY_INTERFACE_EXPORT webrtc::MediaStreamTrackInterface* CreateVideoTrack(Context* context, const char* label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) + { + return context->CreateVideoTrack(label, frameBuffer, width, height); + } + + UNITY_INTERFACE_EXPORT webrtc::MediaStreamTrackInterface* CreateAudioTrack(Context* context, const char* label) + { + return context->CreateAudioTrack(label); + } + //TODO: Multi-track support UNITY_INTERFACE_EXPORT void StopMediaStreamTrack(webrtc::MediaStreamTrackInterface* track) { @@ -45,11 +55,6 @@ extern "C" return ContextManager::GetNvEncSupported(); } - UNITY_INTERFACE_EXPORT webrtc::MediaStreamInterface* CaptureAudioStream(Context* context) - { - return context->CreateAudioStream(); - } - UNITY_INTERFACE_EXPORT void MediaStreamAddTrack(webrtc::MediaStreamInterface* stream, webrtc::MediaStreamTrackInterface* track) { if (track->kind() == "audio") @@ -61,6 +66,7 @@ extern "C" stream->AddTrack((webrtc::VideoTrackInterface*)track); } } + UNITY_INTERFACE_EXPORT void MediaStreamRemoveTrack(webrtc::MediaStreamInterface* stream, webrtc::MediaStreamTrackInterface* track) { if (track->kind() == "audio") From 7eedfb7c5e6029851f98c6c7b86c702a129f62da Mon Sep 17 00:00:00 2001 From: chenyuan Date: Fri, 12 Jul 2019 18:46:43 +0800 Subject: [PATCH 10/29] make DummyVideoEncoder support multiple capturer. --- Plugin/WebRTCPlugin/Context.cpp | 39 ++++++++++++++++------- Plugin/WebRTCPlugin/Context.h | 11 +++---- Plugin/WebRTCPlugin/DummyVideoEncoder.cpp | 11 +++++-- Plugin/WebRTCPlugin/DummyVideoEncoder.h | 4 +-- 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index c4c28dedd..593d7e800 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -314,10 +314,9 @@ namespace WebRTC clients.clear(); peerConnectionFactory = nullptr; - videoTrack = nullptr; - audioTrack = nullptr; - + mediaSteamTrackList.clear(); mediaStreamMap.clear(); + nvVideoCapturerList.clear(); workerThread->Quit(); workerThread.reset(); @@ -325,6 +324,22 @@ namespace WebRTC signalingThread.reset(); } + void Context::EncodeFrame() + { + for (std::list::iterator it= nvVideoCapturerList.begin(); it!= nvVideoCapturerList.end(); ++it) + { + (*it)->EncodeVideoData(); + } + } + + void Context::StopCapturer() + { + for (std::list::iterator it = nvVideoCapturerList.begin(); it != nvVideoCapturerList.end(); ++it) + { + (*it)->Stop(); + } + } + webrtc::MediaStreamInterface* Context::CreateMediaStream(const std::string& stream_id) { if (mediaStreamMap.count(stream_id) == 0) @@ -337,15 +352,16 @@ namespace WebRTC webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) { - nvVideoCapturerUnique = std::make_unique(); - nvVideoCapturer = nvVideoCapturerUnique.get(); - nvVideoCapturer->InitializeEncoder(width, height); - pDummyVideoEncoderFactory->SetCapturer(nvVideoCapturer); + NvVideoCapturer* pNvVideoCapturer = new NvVideoCapturer(); + pNvVideoCapturer->InitializeEncoder(width, height); + pDummyVideoEncoderFactory->AddCapturer(pNvVideoCapturer); - videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(std::move(nvVideoCapturerUnique))); - nvVideoCapturer->unityRT = frameBuffer; - nvVideoCapturer->StartEncoder(); + auto videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(pNvVideoCapturer)); + pNvVideoCapturer->unityRT = frameBuffer; + pNvVideoCapturer->StartEncoder(); + nvVideoCapturerList.push_back(pNvVideoCapturer); + mediaSteamTrackList.push_back(videoTrack); return videoTrack; } @@ -357,7 +373,8 @@ namespace WebRTC audioOptions.noise_suppression = false; audioOptions.highpass_filter = false; //TODO: label and stream id should be maintained in some way for multi-stream - audioTrack = peerConnectionFactory->CreateAudioTrack(label, peerConnectionFactory->CreateAudioSource(audioOptions)); + auto audioTrack = peerConnectionFactory->CreateAudioTrack(label, peerConnectionFactory->CreateAudioSource(audioOptions)); + mediaSteamTrackList.push_back(audioTrack); return audioTrack; } diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index 6ba184b34..27bea9879 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -49,8 +49,8 @@ namespace WebRTC PeerConnectionObject* CreatePeerConnection(int id); PeerConnectionObject* CreatePeerConnection(int id, const std::string& conf); - void EncodeFrame() { nvVideoCapturer->EncodeVideoData(); } - void StopCapturer() { nvVideoCapturer->Stop(); } + void EncodeFrame(); + void StopCapturer(); void ProcessAudioData(const float* data, int32 size) { audioDevice->ProcessAudioData(data, size); } void DeleteClient(int id) { clients.erase(id); } private: @@ -61,13 +61,10 @@ namespace WebRTC rtc::scoped_refptr peerConnectionFactory; DummyVideoEncoderFactory* pDummyVideoEncoderFactory; std::map> mediaStreamMap; + std::list> mediaSteamTrackList; - NvVideoCapturer* nvVideoCapturer; - std::unique_ptr nvVideoCapturerUnique; + std::list nvVideoCapturerList; rtc::scoped_refptr audioDevice; - - rtc::scoped_refptr videoTrack; - rtc::scoped_refptr audioTrack; }; class PeerSDPObserver : public webrtc::SetSessionDescriptionObserver diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp index e20840a76..61b9a8f61 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp @@ -92,8 +92,15 @@ namespace WebRTC const webrtc::SdpVideoFormat& format) { auto dummyVideoEncoder = std::make_unique(); - dummyVideoEncoder->SetKeyFrame.connect(capturer, &NvVideoCapturer::SetKeyFrame); - dummyVideoEncoder->SetRate.connect(capturer, &NvVideoCapturer::SetRate); + + { + //todo: According to condition of format choose different capturer. + NvVideoCapturer* pCapturer = *capturers.begin(); + + dummyVideoEncoder->SetKeyFrame.connect(pCapturer, &NvVideoCapturer::SetKeyFrame); + dummyVideoEncoder->SetRate.connect(pCapturer, &NvVideoCapturer::SetRate); + } + return dummyVideoEncoder; } } diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.h b/Plugin/WebRTCPlugin/DummyVideoEncoder.h index ad8f81601..db09394c0 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.h +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.h @@ -53,8 +53,8 @@ namespace WebRTC const webrtc::SdpVideoFormat& format) override; DummyVideoEncoderFactory(); - void SetCapturer(NvVideoCapturer* _capturer) { capturer = _capturer; } + void AddCapturer(NvVideoCapturer* _capturer) { capturers.push_back(_capturer); } private: - NvVideoCapturer* capturer; + std::list capturers; }; } From 3e6aa8d5b163e800260a2a70061a1bfa161742e7 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 15 Jul 2019 19:53:47 +0800 Subject: [PATCH 11/29] capturers share one nvEncoder. todo: 1.create multiple resolution texture for nvEncoder. 2.Abstract out a base class BaseEncoder. Derive AmdEncoder and softEncoder. --- Assets/Scripts/RenderStreaming.cs | 5 +-- .../Runtime/Srcipts/MediaStreamTrack.cs | 4 +-- .../Samples/Example/MediaStreamSample.cs | 4 +-- .../Tests/Runtime/MediaStreamTest.cs | 2 +- Plugin/WebRTCPlugin/Context.cpp | 6 +++- Plugin/WebRTCPlugin/Context.h | 1 + Plugin/WebRTCPlugin/NvEncoder.cpp | 32 +++++++++---------- Plugin/WebRTCPlugin/NvEncoder.h | 12 ++++--- Plugin/WebRTCPlugin/NvVideoCapturer.cpp | 6 ++-- Plugin/WebRTCPlugin/NvVideoCapturer.h | 16 +++++++--- 10 files changed, 51 insertions(+), 37 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index 085c8f108..2d1cc7a8c 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -67,10 +67,11 @@ public IEnumerator Start() int texCount = captureCamera.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - mediaStream.AddTrack(new VideoStreamTrack(captureCamera.GetStreamTexture(i))); + VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, captureCamera.GetStreamTexture(i)); + mediaStream.AddTrack(videoTrack); } - mediaStream.AddTrack(new AudioStreamTrack()); + mediaStream.AddTrack(new AudioStreamTrack("audioTrack")); Audio.Start(); signaling = new Signaling(urlSignaling); diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs index 25a716ede..2c08b378e 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs @@ -36,14 +36,14 @@ private set { } public class VideoStreamTrack : MediaStreamTrack { - public VideoStreamTrack(RenderTexture rt) : base(WebRTC.Context.CreateVideoTrack("videoTrack", rt.GetNativeTexturePtr(), rt.width, rt.height)) + public VideoStreamTrack(string label, RenderTexture rt) : base(WebRTC.Context.CreateVideoTrack(label, rt.GetNativeTexturePtr(), rt.width, rt.height)) { } } public class AudioStreamTrack : MediaStreamTrack { - public AudioStreamTrack() : base(WebRTC.Context.CreateAudioTrack("audioTrack")) + public AudioStreamTrack(string label) : base(WebRTC.Context.CreateAudioTrack(label)) { } } diff --git a/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs b/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs index dc28c0d66..ae7f6f4f9 100644 --- a/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs +++ b/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs @@ -215,10 +215,10 @@ void Call() int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - mediaStream.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i))); + mediaStream.AddTrack(new VideoStreamTrack("videoTrack"+1, cam.GetStreamTexture(i))); } - mediaStream.AddTrack(new AudioStreamTrack()); + mediaStream.AddTrack(new AudioStreamTrack("audioTrack")); Audio.Start(); RtImage.texture = cam.targetTexture; diff --git a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs index ee986c32f..dff7acd69 100644 --- a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs +++ b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs @@ -53,7 +53,7 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream() int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - VideoStreamTrack videoStreamTrack = new VideoStreamTrack(cam.GetStreamTexture(i)); + VideoStreamTrack videoStreamTrack = new VideoStreamTrack("videoTrack"+i,cam.GetStreamTexture(i)); mediaStream.AddTrack(videoStreamTrack); pc1Senders.Add(peer1.AddTrack(videoStreamTrack)); } diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index 593d7e800..c63bdaed0 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -292,6 +292,8 @@ namespace WebRTC rtc::InitializeSSL(); audioDevice = new rtc::RefCountedObject(); + nvEncoder = new NvEncoder(); + auto dummyVideoEncoderFactory = std::make_unique(); pDummyVideoEncoderFactory = dummyVideoEncoderFactory.get(); @@ -317,6 +319,8 @@ namespace WebRTC mediaSteamTrackList.clear(); mediaStreamMap.clear(); nvVideoCapturerList.clear(); + delete nvEncoder; + nvEncoder = NULL; workerThread->Quit(); workerThread.reset(); @@ -352,7 +356,7 @@ namespace WebRTC webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) { - NvVideoCapturer* pNvVideoCapturer = new NvVideoCapturer(); + NvVideoCapturer* pNvVideoCapturer = new NvVideoCapturer(nvEncoder, width, height); pNvVideoCapturer->InitializeEncoder(width, height); pDummyVideoEncoderFactory->AddCapturer(pNvVideoCapturer); diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index 27bea9879..6ed5f3df1 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -65,6 +65,7 @@ namespace WebRTC std::list nvVideoCapturerList; rtc::scoped_refptr audioDevice; + NvEncoder* nvEncoder; }; class PeerSDPObserver : public webrtc::SetSessionDescriptionObserver diff --git a/Plugin/WebRTCPlugin/NvEncoder.cpp b/Plugin/WebRTCPlugin/NvEncoder.cpp index c12e17d78..733ad43d1 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.cpp +++ b/Plugin/WebRTCPlugin/NvEncoder.cpp @@ -6,11 +6,11 @@ namespace WebRTC { - NvEncoder::NvEncoder(int width, int height) :width(width), height(height) + NvEncoder::NvEncoder() { - LogPrint(StringFormat("width is %d, height is %d", width, height).c_str()); + LogPrint(StringFormat("width is %d, height is %d", encodeWidth, encodeHeight).c_str()); checkf(g_D3D11Device != nullptr, "D3D11Device is invalid"); - checkf(width > 0 && height > 0, "Invalid width or height!"); + checkf(encodeWidth > 0 && encodeHeight > 0, "Invalid width or height!"); bool result = true; #pragma region open an encode session //open an encode session @@ -25,10 +25,10 @@ namespace WebRTC #pragma endregion #pragma region set initialization parameters nvEncInitializeParams.version = NV_ENC_INITIALIZE_PARAMS_VER; - nvEncInitializeParams.encodeWidth = width; - nvEncInitializeParams.encodeHeight = height; - nvEncInitializeParams.darWidth = width; - nvEncInitializeParams.darHeight = height; + nvEncInitializeParams.encodeWidth = encodeWidth; + nvEncInitializeParams.encodeHeight = encodeHeight; + nvEncInitializeParams.darWidth = encodeWidth; + nvEncInitializeParams.darHeight = encodeHeight; nvEncInitializeParams.encodeGUID = NV_ENC_CODEC_H264_GUID; nvEncInitializeParams.presetGUID = NV_ENC_PRESET_LOW_LATENCY_HQ_GUID; nvEncInitializeParams.frameRateNum = frameRate; @@ -87,7 +87,7 @@ namespace WebRTC } - void NvEncoder::UpdateSettings() + void NvEncoder::UpdateSettings(int width, int height) { bool settingChanged = false; if (nvEncConfig.rcParams.averageBitRate != bitRate) @@ -121,9 +121,9 @@ namespace WebRTC } } //entry for encoding a frame - void NvEncoder::EncodeFrame() + void NvEncoder::EncodeFrame(int width, int height) { - UpdateSettings(); + UpdateSettings(width, height); uint32 bufferIndexToWrite = frameCount % bufferedFrameNum; Frame& frame = bufferedFrames[bufferIndexToWrite]; #pragma region set frame params @@ -140,8 +140,8 @@ namespace WebRTC picParams.pictureStruct = NV_ENC_PIC_STRUCT_FRAME; picParams.inputBuffer = frame.inputFrame.mappedResource; picParams.bufferFmt = frame.inputFrame.bufferFormat; - picParams.inputWidth = nvEncInitializeParams.encodeWidth; - picParams.inputHeight = nvEncInitializeParams.encodeHeight; + picParams.inputWidth = encodeWidth; + picParams.inputHeight = encodeHeight; picParams.outputBitstream = frame.outputFrame; picParams.inputTimeStamp = frameCount; #pragma endregion @@ -191,8 +191,8 @@ namespace WebRTC { ID3D11Texture2D* inputTextures = nullptr; D3D11_TEXTURE2D_DESC desc = { 0 }; - desc.Width = width; - desc.Height = height; + desc.Width = encodeWidth; + desc.Height = encodeHeight; desc.MipLevels = 1; desc.ArraySize = 1; desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; @@ -212,8 +212,8 @@ namespace WebRTC if (!registerResource.resourceToRegister) LogPrint("resource is not initialized"); - registerResource.width = width; - registerResource.height = height; + registerResource.width = encodeWidth; + registerResource.height = encodeHeight; LogPrint(StringFormat("nvEncRegisterResource: width is %d, height is %d", registerResource.width, registerResource.height).c_str()); registerResource.bufferFormat = NV_ENC_BUFFER_FORMAT_ARGB; checkf(NV_RESULT((errorCode = ContextManager::GetInstance()->pNvEncodeAPI->nvEncRegisterResource(pEncoderInterface, ®isterResource))), diff --git a/Plugin/WebRTCPlugin/NvEncoder.h b/Plugin/WebRTCPlugin/NvEncoder.h index 490b92f8e..f453f086c 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.h +++ b/Plugin/WebRTCPlugin/NvEncoder.h @@ -29,12 +29,12 @@ namespace WebRTC }; public: - NvEncoder(int width, int height); + NvEncoder(); ~NvEncoder(); void SetRate(uint32 rate); - void UpdateSettings(); - void EncodeFrame(); + void UpdateSettings(int width, int height); + void EncodeFrame(int width, int height); bool IsSupported() const { return isNvEncoderSupported; } void SetIdrFrame() { isIdrFrame = true; } uint64 GetCurrentFrameCount() { return frameCount; } @@ -58,8 +58,10 @@ namespace WebRTC void* pEncoderInterface = nullptr; bool isNvEncoderSupported = false; bool isIdrFrame = false; - int width = 1920; - int height = 1080; + //const int encodeWidth = 1920; + //const int encodeHeight = 1080; + const int encodeWidth = 1280; + const int encodeHeight = 720; //10Mbps int bitRate = 10000000; //100Mbps diff --git a/Plugin/WebRTCPlugin/NvVideoCapturer.cpp b/Plugin/WebRTCPlugin/NvVideoCapturer.cpp index 9e9a43012..d22769a84 100644 --- a/Plugin/WebRTCPlugin/NvVideoCapturer.cpp +++ b/Plugin/WebRTCPlugin/NvVideoCapturer.cpp @@ -3,7 +3,7 @@ namespace WebRTC { - NvVideoCapturer::NvVideoCapturer() + NvVideoCapturer::NvVideoCapturer(NvEncoder* pEncoder, int _width, int _height) : nvEncoder(pEncoder), width(_width), height(_height) { set_enable_video_adapter(false); SetSupportedFormats(std::vector(1, cricket::VideoFormat(width, height, cricket::VideoFormat::FpsToInterval(framerate), cricket::FOURCC_H264))); @@ -14,7 +14,7 @@ namespace WebRTC { int curFrameNum = nvEncoder->GetCurrentFrameCount() % bufferedFrameNum; context->CopyResource(renderTextures[curFrameNum], unityRT); - nvEncoder->EncodeFrame(); + nvEncoder->EncodeFrame(width, height); } } void NvVideoCapturer::CaptureFrame(std::vector& data) @@ -41,7 +41,7 @@ namespace WebRTC void NvVideoCapturer::InitializeEncoder(int32 width, int32 height) { - nvEncoder = std::make_unique(width, height); + //koseyile todo: one nvEncoder can connect multiple capturers. nvEncoder->CaptureFrame.connect(this, &NvVideoCapturer::CaptureFrame); } } diff --git a/Plugin/WebRTCPlugin/NvVideoCapturer.h b/Plugin/WebRTCPlugin/NvVideoCapturer.h index 5686f2216..bd6382a79 100644 --- a/Plugin/WebRTCPlugin/NvVideoCapturer.h +++ b/Plugin/WebRTCPlugin/NvVideoCapturer.h @@ -3,10 +3,15 @@ namespace WebRTC { + class testNvVideoCapturer : public cricket::VideoCapturer + { + + }; + class NvVideoCapturer : public cricket::VideoCapturer { public: - NvVideoCapturer(); + NvVideoCapturer(NvEncoder* pEncoder, int _width, int _height); void EncodeVideoData(); // Start the video capturer with the specified capture format. virtual cricket::CaptureState Start(const cricket::VideoFormat& Format) override @@ -17,7 +22,7 @@ namespace WebRTC virtual void Stop() override { captureStopped = true; - nvEncoder.reset(); + //nvEncoder.reset(); } // Check if the video capturer is running. virtual bool IsRunning() override @@ -46,11 +51,12 @@ namespace WebRTC fourccs->push_back(cricket::FOURCC_H264); return true; } - std::unique_ptr nvEncoder; + //std::unique_ptr nvEncoder; + NvEncoder* nvEncoder; //just fake info - const int32 width = 1280; - const int32 height = 720; + int32 width; + int32 height; const int32 framerate = 60; bool captureStarted = false; From c82dab4cfd438c1e0324953312003ba5b89af36d Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 15 Jul 2019 20:05:00 +0800 Subject: [PATCH 12/29] delete parameter label from MediaStream constructor --- Assets/Scripts/RenderStreaming.cs | 2 +- Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs | 8 ++++---- .../com.unity.webrtc/Samples/Example/MediaStreamSample.cs | 2 +- .../com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index 2d1cc7a8c..bb279ccda 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -63,7 +63,7 @@ public IEnumerator Start() } captureCamera.CreateRenderStreamTexture(1280, 720); - mediaStream = new MediaStream("MediaStream"); + mediaStream = new MediaStream(); int texCount = captureCamera.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index 5d82eca07..76d899ac3 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -11,14 +11,14 @@ public class MediaStream internal IntPtr nativePtr; protected List mediaStreamTrackList = new List(); - public MediaStream(string label) : base() + public MediaStream() : base() { - nativePtr = WebRTC.Context.CreateMediaStream(label); + nativePtr = WebRTC.Context.CreateMediaStream("MediaStream"); } - public MediaStream(string label, MediaStreamTrack[] tracks) : base() + public MediaStream(MediaStreamTrack[] tracks) : base() { - nativePtr = WebRTC.Context.CreateMediaStream(label); + nativePtr = WebRTC.Context.CreateMediaStream("MediaStream"); foreach (var t in tracks) { diff --git a/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs b/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs index ae7f6f4f9..d42856a03 100644 --- a/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs +++ b/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs @@ -211,7 +211,7 @@ void Call() dataChannel = pc1.CreateDataChannel("data", ref conf); cam.CreateRenderStreamTexture(1280, 720); - mediaStream = new MediaStream("mediaStream"); + mediaStream = new MediaStream(); int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { diff --git a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs index dff7acd69..b4389c529 100644 --- a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs +++ b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs @@ -49,7 +49,7 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream() cam.CreateRenderStreamTexture(1280, 720); - MediaStream mediaStream = new MediaStream("mediaStream"); + MediaStream mediaStream = new MediaStream(); int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { From cdf98eafca68fd745d4430f4f277d9ead276f851 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Tue, 16 Jul 2019 13:56:19 +0800 Subject: [PATCH 13/29] delete test code. --- Plugin/WebRTCPlugin/NvVideoCapturer.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Plugin/WebRTCPlugin/NvVideoCapturer.h b/Plugin/WebRTCPlugin/NvVideoCapturer.h index bd6382a79..426e7f766 100644 --- a/Plugin/WebRTCPlugin/NvVideoCapturer.h +++ b/Plugin/WebRTCPlugin/NvVideoCapturer.h @@ -3,11 +3,6 @@ namespace WebRTC { - class testNvVideoCapturer : public cricket::VideoCapturer - { - - }; - class NvVideoCapturer : public cricket::VideoCapturer { public: From c88c3229dc7a248d59f74f9c4340843706fb52e8 Mon Sep 17 00:00:00 2001 From: Kazuki Matsumoto <1132081+karasusan@users.noreply.github.com> Date: Mon, 22 Jul 2019 08:31:03 +0900 Subject: [PATCH 14/29] changed instance type to build on Yamato --- .yamato/upm-ci-webrtc.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.yamato/upm-ci-webrtc.yml b/.yamato/upm-ci-webrtc.yml index 987be2ceb..82b1b0b01 100644 --- a/.yamato/upm-ci-webrtc.yml +++ b/.yamato/upm-ci-webrtc.yml @@ -4,7 +4,7 @@ editors: - version: trunk platforms: - name: win - type: Unity::VM + type: Unity::VM::GPU # currently the projects depends MSBuild. we should replace CMake # image: package-ci/win10:stable image: renderstreaming/win10:latest @@ -101,4 +101,4 @@ publish: - .yamato/upm-ci-webrtc.yml#test_{{ platform.name }}_{{ editor.version }} {% endfor %} {% endfor %} -{% endfor %} \ No newline at end of file +{% endfor %} From 0a13fce55182fb415ff02ac4b66d23d6b18e363c Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 22 Jul 2019 11:16:59 +0800 Subject: [PATCH 15/29] remove RTCPeerConnection when the RTCPeerConnection is disconnected. --- Assets/Scripts/RenderStreaming.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index bb279ccda..b2cad26fb 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -148,7 +148,8 @@ IEnumerator GetOffer() { if(state == RTCIceConnectionState.Disconnected) { - pc.Close(); + pc.Close(); + pcs.Remove(offer.connectionId); } }); //make video bit rate starts at 16000kbits, and 160000kbits at max. @@ -216,6 +217,7 @@ IEnumerator GetCandidate() { continue; } + foreach (var candidate in candidateContainer.candidates) { RTCIceCandidate _candidate = default; From 52138b13347b8b26f3765b1b921be862d6159fdd Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 22 Jul 2019 12:44:10 +0800 Subject: [PATCH 16/29] add auto testing to check multiple video tracks. --- Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs | 4 ++-- Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index 76d899ac3..47c3a27ff 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -82,7 +82,7 @@ public static RenderTexture GetStreamTexture(this Camera cam, int index) { return webRTCTextures[index]; } - public static void CreateRenderStreamTexture(this Camera cam, int width, int height) + public static void CreateRenderStreamTexture(this Camera cam, int width, int height, int count = 1) { if (camCopyRts.Count > 0) { @@ -92,7 +92,7 @@ public static void CreateRenderStreamTexture(this Camera cam, int width, int hei camRenderTexture = new RenderTexture(width, height, 0, RenderTextureFormat.BGRA32); camRenderTexture.Create(); - int mipCount = 1; + int mipCount = count; for (int i = 1, mipLevel = 1; i <= mipCount; ++i, mipLevel *= 2) { RenderTexture webRtcTex = new RenderTexture(width / mipLevel, height / mipLevel, 0, RenderTextureFormat.BGRA32); diff --git a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs index b4389c529..9b0e2081c 100644 --- a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs +++ b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs @@ -48,7 +48,7 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream() }); - cam.CreateRenderStreamTexture(1280, 720); + cam.CreateRenderStreamTexture(1280, 720, 2); MediaStream mediaStream = new MediaStream(); int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) From 09ab52248b2d405c5e98e8d4dd8ed5bc02437fda Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 22 Jul 2019 12:59:13 +0800 Subject: [PATCH 17/29] rename NvVideoCapturer to UnityVideoCapturer --- Plugin/WebRTCPlugin/Context.cpp | 19 +++++---- Plugin/WebRTCPlugin/Context.h | 4 +- Plugin/WebRTCPlugin/DummyVideoEncoder.cpp | 8 ++-- Plugin/WebRTCPlugin/DummyVideoEncoder.h | 6 +-- Plugin/WebRTCPlugin/NvEncoder.cpp | 41 +++++++++++-------- Plugin/WebRTCPlugin/NvEncoder.h | 8 ++-- ...deoCapturer.cpp => UnityVideoCapturer.cpp} | 18 ++++---- ...NvVideoCapturer.h => UnityVideoCapturer.h} | 4 +- Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj | 4 +- .../WebRTCPlugin/WebRTCPlugin.vcxproj.filters | 12 +++--- 10 files changed, 67 insertions(+), 57 deletions(-) rename Plugin/WebRTCPlugin/{NvVideoCapturer.cpp => UnityVideoCapturer.cpp} (66%) rename Plugin/WebRTCPlugin/{NvVideoCapturer.h => UnityVideoCapturer.h} (95%) diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index c63bdaed0..ac6c009b6 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -330,7 +330,7 @@ namespace WebRTC void Context::EncodeFrame() { - for (std::list::iterator it= nvVideoCapturerList.begin(); it!= nvVideoCapturerList.end(); ++it) + for (std::list::iterator it= nvVideoCapturerList.begin(); it!= nvVideoCapturerList.end(); ++it) { (*it)->EncodeVideoData(); } @@ -338,7 +338,7 @@ namespace WebRTC void Context::StopCapturer() { - for (std::list::iterator it = nvVideoCapturerList.begin(); it != nvVideoCapturerList.end(); ++it) + for (std::list::iterator it = nvVideoCapturerList.begin(); it != nvVideoCapturerList.end(); ++it) { (*it)->Stop(); } @@ -356,15 +356,16 @@ namespace WebRTC webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) { - NvVideoCapturer* pNvVideoCapturer = new NvVideoCapturer(nvEncoder, width, height); - pNvVideoCapturer->InitializeEncoder(width, height); - pDummyVideoEncoderFactory->AddCapturer(pNvVideoCapturer); + nvEncoder->InitEncoder(width, height); + UnityVideoCapturer* pUnityVideoCapturer = new UnityVideoCapturer(nvEncoder, width, height); + pUnityVideoCapturer->InitializeEncoder(width, height); + pDummyVideoEncoderFactory->AddCapturer(pUnityVideoCapturer); - auto videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(pNvVideoCapturer)); - pNvVideoCapturer->unityRT = frameBuffer; - pNvVideoCapturer->StartEncoder(); + auto videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(pUnityVideoCapturer)); + pUnityVideoCapturer->unityRT = frameBuffer; + pUnityVideoCapturer->StartEncoder(); - nvVideoCapturerList.push_back(pNvVideoCapturer); + nvVideoCapturerList.push_back(pUnityVideoCapturer); mediaSteamTrackList.push_back(videoTrack); return videoTrack; } diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index 6ed5f3df1..f577d0a4a 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -2,7 +2,7 @@ #include "DummyAudioDevice.h" #include "DummyVideoEncoder.h" #include "PeerConnectionObject.h" -#include "NvVideoCapturer.h" +#include "UnityVideoCapturer.h" namespace WebRTC @@ -63,7 +63,7 @@ namespace WebRTC std::map> mediaStreamMap; std::list> mediaSteamTrackList; - std::list nvVideoCapturerList; + std::list nvVideoCapturerList; rtc::scoped_refptr audioDevice; NvEncoder* nvEncoder; }; diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp index 61b9a8f61..09500f4f2 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp @@ -1,6 +1,6 @@ #include "pch.h" #include "DummyVideoEncoder.h" -#include "NvVideoCapturer.h" +#include "UnityVideoCapturer.h" #include namespace WebRTC @@ -95,10 +95,10 @@ namespace WebRTC { //todo: According to condition of format choose different capturer. - NvVideoCapturer* pCapturer = *capturers.begin(); + UnityVideoCapturer* pCapturer = *capturers.begin(); - dummyVideoEncoder->SetKeyFrame.connect(pCapturer, &NvVideoCapturer::SetKeyFrame); - dummyVideoEncoder->SetRate.connect(pCapturer, &NvVideoCapturer::SetRate); + dummyVideoEncoder->SetKeyFrame.connect(pCapturer, &UnityVideoCapturer::SetKeyFrame); + dummyVideoEncoder->SetRate.connect(pCapturer, &UnityVideoCapturer::SetRate); } return dummyVideoEncoder; diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.h b/Plugin/WebRTCPlugin/DummyVideoEncoder.h index db09394c0..88b467c0b 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.h +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.h @@ -2,7 +2,7 @@ namespace WebRTC { - class NvVideoCapturer; + class UnityVideoCapturer; class DummyVideoEncoder : public webrtc::VideoEncoder { public: @@ -53,8 +53,8 @@ namespace WebRTC const webrtc::SdpVideoFormat& format) override; DummyVideoEncoderFactory(); - void AddCapturer(NvVideoCapturer* _capturer) { capturers.push_back(_capturer); } + void AddCapturer(UnityVideoCapturer* _capturer) { capturers.push_back(_capturer); } private: - std::list capturers; + std::list capturers; }; } diff --git a/Plugin/WebRTCPlugin/NvEncoder.cpp b/Plugin/WebRTCPlugin/NvEncoder.cpp index 733ad43d1..59e1678c2 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.cpp +++ b/Plugin/WebRTCPlugin/NvEncoder.cpp @@ -22,7 +22,26 @@ namespace WebRTC result = NV_RESULT((errorCode = ContextManager::GetInstance()->pNvEncodeAPI->nvEncOpenEncodeSessionEx(&openEncdoeSessionExParams, &pEncoderInterface))); checkf(result, "Unable to open NvEnc encode session"); LogPrint(StringFormat("OpenEncodeSession Error is %d", errorCode).c_str()); -#pragma endregion +#pragma endregion + } + + NvEncoder::~NvEncoder() + { + ReleaseEncoderResources(); + if (pEncoderInterface) + { + bool result = NV_RESULT(ContextManager::GetInstance()->pNvEncodeAPI->nvEncDestroyEncoder(pEncoderInterface)); + checkf(result, "Failed to destroy NV encoder interface"); + pEncoderInterface = nullptr; + } + + } + + void NvEncoder::InitEncoder(int width, int height) + { + encodeWidth = width; + encodeHeight = height; + bool result = true; #pragma region set initialization parameters nvEncInitializeParams.version = NV_ENC_INITIALIZE_PARAMS_VER; nvEncInitializeParams.encodeWidth = encodeWidth; @@ -37,8 +56,8 @@ namespace WebRTC nvEncInitializeParams.reportSliceOffsets = 0; nvEncInitializeParams.enableSubFrameWrite = 0; nvEncInitializeParams.encodeConfig = &nvEncConfig; - nvEncInitializeParams.maxEncodeWidth = 3840; - nvEncInitializeParams.maxEncodeHeight = 2160; + nvEncInitializeParams.maxEncodeWidth = encodeWidth;//3840; + nvEncInitializeParams.maxEncodeHeight = encodeHeight;//2160; #pragma endregion #pragma region get preset ocnfig and set it NV_ENC_PRESET_CONFIG presetConfig = { 0 }; @@ -74,17 +93,7 @@ namespace WebRTC #pragma endregion InitEncoderResources(); isNvEncoderSupported = true; - } - NvEncoder::~NvEncoder() - { - ReleaseEncoderResources(); - if (pEncoderInterface) - { - bool result = NV_RESULT(ContextManager::GetInstance()->pNvEncodeAPI->nvEncDestroyEncoder(pEncoderInterface)); - checkf(result, "Failed to destroy NV encoder interface"); - pEncoderInterface = nullptr; - } - + isInitialize = true; } void NvEncoder::UpdateSettings(int width, int height) @@ -140,8 +149,8 @@ namespace WebRTC picParams.pictureStruct = NV_ENC_PIC_STRUCT_FRAME; picParams.inputBuffer = frame.inputFrame.mappedResource; picParams.bufferFmt = frame.inputFrame.bufferFormat; - picParams.inputWidth = encodeWidth; - picParams.inputHeight = encodeHeight; + picParams.inputWidth = width; + picParams.inputHeight = height; picParams.outputBitstream = frame.outputFrame; picParams.inputTimeStamp = frameCount; #pragma endregion diff --git a/Plugin/WebRTCPlugin/NvEncoder.h b/Plugin/WebRTCPlugin/NvEncoder.h index f453f086c..5f34de852 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.h +++ b/Plugin/WebRTCPlugin/NvEncoder.h @@ -39,6 +39,7 @@ namespace WebRTC void SetIdrFrame() { isIdrFrame = true; } uint64 GetCurrentFrameCount() { return frameCount; } sigslot::signal1&> CaptureFrame; + void InitEncoder(int width, int height); void InitEncoderResources(); private: @@ -57,11 +58,10 @@ namespace WebRTC uint64 frameCount = 0; void* pEncoderInterface = nullptr; bool isNvEncoderSupported = false; + bool isInitialize = false; bool isIdrFrame = false; - //const int encodeWidth = 1920; - //const int encodeHeight = 1080; - const int encodeWidth = 1280; - const int encodeHeight = 720; + int encodeWidth; + int encodeHeight; //10Mbps int bitRate = 10000000; //100Mbps diff --git a/Plugin/WebRTCPlugin/NvVideoCapturer.cpp b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp similarity index 66% rename from Plugin/WebRTCPlugin/NvVideoCapturer.cpp rename to Plugin/WebRTCPlugin/UnityVideoCapturer.cpp index d22769a84..b5a5c488a 100644 --- a/Plugin/WebRTCPlugin/NvVideoCapturer.cpp +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp @@ -1,14 +1,14 @@ #include "pch.h" -#include "NvVideoCapturer.h" +#include "UnityVideoCapturer.h" namespace WebRTC { - NvVideoCapturer::NvVideoCapturer(NvEncoder* pEncoder, int _width, int _height) : nvEncoder(pEncoder), width(_width), height(_height) + UnityVideoCapturer::UnityVideoCapturer(NvEncoder* pEncoder, int _width, int _height) : nvEncoder(pEncoder), width(_width), height(_height) { set_enable_video_adapter(false); SetSupportedFormats(std::vector(1, cricket::VideoFormat(width, height, cricket::VideoFormat::FpsToInterval(framerate), cricket::FOURCC_H264))); } - void NvVideoCapturer::EncodeVideoData() + void UnityVideoCapturer::EncodeVideoData() { if (captureStarted && !captureStopped) { @@ -17,7 +17,7 @@ namespace WebRTC nvEncoder->EncodeFrame(width, height); } } - void NvVideoCapturer::CaptureFrame(std::vector& data) + void UnityVideoCapturer::CaptureFrame(std::vector& data) { rtc::scoped_refptr buffer = new rtc::RefCountedObject(width, height, data); int64 timestamp = rtc::TimeMillis(); @@ -25,23 +25,23 @@ namespace WebRTC videoFrame.set_ntp_time_ms(timestamp); OnFrame(videoFrame, width, height); } - void NvVideoCapturer::StartEncoder() + void UnityVideoCapturer::StartEncoder() { captureStarted = true; SetKeyFrame(); } - void NvVideoCapturer::SetKeyFrame() + void UnityVideoCapturer::SetKeyFrame() { nvEncoder->SetIdrFrame(); } - void NvVideoCapturer::SetRate(uint32 rate) + void UnityVideoCapturer::SetRate(uint32 rate) { nvEncoder->SetRate(rate); } - void NvVideoCapturer::InitializeEncoder(int32 width, int32 height) + void UnityVideoCapturer::InitializeEncoder(int32 width, int32 height) { //koseyile todo: one nvEncoder can connect multiple capturers. - nvEncoder->CaptureFrame.connect(this, &NvVideoCapturer::CaptureFrame); + nvEncoder->CaptureFrame.connect(this, &UnityVideoCapturer::CaptureFrame); } } diff --git a/Plugin/WebRTCPlugin/NvVideoCapturer.h b/Plugin/WebRTCPlugin/UnityVideoCapturer.h similarity index 95% rename from Plugin/WebRTCPlugin/NvVideoCapturer.h rename to Plugin/WebRTCPlugin/UnityVideoCapturer.h index 426e7f766..5b0ac8881 100644 --- a/Plugin/WebRTCPlugin/NvVideoCapturer.h +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.h @@ -3,10 +3,10 @@ namespace WebRTC { - class NvVideoCapturer : public cricket::VideoCapturer + class UnityVideoCapturer : public cricket::VideoCapturer { public: - NvVideoCapturer(NvEncoder* pEncoder, int _width, int _height); + UnityVideoCapturer(NvEncoder* pEncoder, int _width, int _height); void EncodeVideoData(); // Start the video capturer with the specified capture format. virtual cricket::CaptureState Start(const cricket::VideoFormat& Format) override diff --git a/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj b/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj index 54636a0a8..e962f41fb 100644 --- a/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj +++ b/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj @@ -177,7 +177,7 @@ - + @@ -190,7 +190,7 @@ - + Create Create diff --git a/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj.filters b/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj.filters index 0a0d2a3a4..56f90d06b 100644 --- a/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj.filters +++ b/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj.filters @@ -42,9 +42,6 @@ Header Files - - Header Files - Header Files @@ -60,6 +57,9 @@ Header Files\Unity + + Header Files + @@ -83,9 +83,6 @@ Source Files - - Source Files - Source Files @@ -95,5 +92,8 @@ Source Files + + Source Files + \ No newline at end of file From 5d00ed0babab4574ef937e039704e12eb3563218 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 22 Jul 2019 15:03:08 +0800 Subject: [PATCH 18/29] add abstract bass class : UnityEncoder. --- Plugin/WebRTCPlugin/Context.cpp | 10 ++++---- Plugin/WebRTCPlugin/Context.h | 4 ++-- Plugin/WebRTCPlugin/NvEncoder.h | 5 ++-- Plugin/WebRTCPlugin/UnityEncoder.cpp | 13 +++++++++++ Plugin/WebRTCPlugin/UnityEncoder.h | 23 +++++++++++++++++++ Plugin/WebRTCPlugin/UnityVideoCapturer.cpp | 2 +- Plugin/WebRTCPlugin/UnityVideoCapturer.h | 6 ++--- Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj | 2 ++ .../WebRTCPlugin/WebRTCPlugin.vcxproj.filters | 6 +++++ 9 files changed, 57 insertions(+), 14 deletions(-) create mode 100644 Plugin/WebRTCPlugin/UnityEncoder.cpp create mode 100644 Plugin/WebRTCPlugin/UnityEncoder.h diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index ac6c009b6..380fd8d1e 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -292,7 +292,7 @@ namespace WebRTC rtc::InitializeSSL(); audioDevice = new rtc::RefCountedObject(); - nvEncoder = new NvEncoder(); + pUnityEncoder = new NvEncoder(); auto dummyVideoEncoderFactory = std::make_unique(); pDummyVideoEncoderFactory = dummyVideoEncoderFactory.get(); @@ -319,8 +319,8 @@ namespace WebRTC mediaSteamTrackList.clear(); mediaStreamMap.clear(); nvVideoCapturerList.clear(); - delete nvEncoder; - nvEncoder = NULL; + delete pUnityEncoder; + pUnityEncoder = NULL; workerThread->Quit(); workerThread.reset(); @@ -356,8 +356,8 @@ namespace WebRTC webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) { - nvEncoder->InitEncoder(width, height); - UnityVideoCapturer* pUnityVideoCapturer = new UnityVideoCapturer(nvEncoder, width, height); + pUnityEncoder->InitEncoder(width, height); + UnityVideoCapturer* pUnityVideoCapturer = new UnityVideoCapturer(pUnityEncoder, width, height); pUnityVideoCapturer->InitializeEncoder(width, height); pDummyVideoEncoderFactory->AddCapturer(pUnityVideoCapturer); diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index f577d0a4a..855795bc2 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -3,7 +3,7 @@ #include "DummyVideoEncoder.h" #include "PeerConnectionObject.h" #include "UnityVideoCapturer.h" - +#include "NvEncoder.h" namespace WebRTC { @@ -65,7 +65,7 @@ namespace WebRTC std::list nvVideoCapturerList; rtc::scoped_refptr audioDevice; - NvEncoder* nvEncoder; + UnityEncoder* pUnityEncoder; }; class PeerSDPObserver : public webrtc::SetSessionDescriptionObserver diff --git a/Plugin/WebRTCPlugin/NvEncoder.h b/Plugin/WebRTCPlugin/NvEncoder.h index 5f34de852..3468e0531 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.h +++ b/Plugin/WebRTCPlugin/NvEncoder.h @@ -4,11 +4,12 @@ #include "nvEncodeAPI.h" #include #include +#include "UnityEncoder.h" namespace WebRTC { using OutputFrame = NV_ENC_OUTPUT_PTR; - class NvEncoder + class NvEncoder : public UnityEncoder { private: struct InputFrame @@ -38,12 +39,10 @@ namespace WebRTC bool IsSupported() const { return isNvEncoderSupported; } void SetIdrFrame() { isIdrFrame = true; } uint64 GetCurrentFrameCount() { return frameCount; } - sigslot::signal1&> CaptureFrame; void InitEncoder(int width, int height); void InitEncoderResources(); private: - void LoadNvEncApi(); void ReleaseFrameInputBuffer(Frame& frame); void ReleaseEncoderResources(); void ProcessEncodedFrame(Frame& frame); diff --git a/Plugin/WebRTCPlugin/UnityEncoder.cpp b/Plugin/WebRTCPlugin/UnityEncoder.cpp new file mode 100644 index 000000000..52c58150e --- /dev/null +++ b/Plugin/WebRTCPlugin/UnityEncoder.cpp @@ -0,0 +1,13 @@ +#include "pch.h" +#include "UnityEncoder.h" + +namespace WebRTC +{ + UnityEncoder::UnityEncoder() + { + } + + UnityEncoder::~UnityEncoder() + { + } +} diff --git a/Plugin/WebRTCPlugin/UnityEncoder.h b/Plugin/WebRTCPlugin/UnityEncoder.h new file mode 100644 index 000000000..f9a622c24 --- /dev/null +++ b/Plugin/WebRTCPlugin/UnityEncoder.h @@ -0,0 +1,23 @@ +#pragma once + +namespace WebRTC +{ + + class UnityEncoder + { + public: + UnityEncoder(); + virtual ~UnityEncoder(); + + sigslot::signal1&> CaptureFrame; + virtual void SetRate(uint32 rate) = 0; + virtual void UpdateSettings(int width, int height) = 0; + virtual void EncodeFrame(int width, int height) = 0; + virtual bool IsSupported() const = 0; + virtual void SetIdrFrame() = 0; + virtual uint64 GetCurrentFrameCount() = 0; + virtual void InitEncoder(int width, int height) = 0; + virtual void InitEncoderResources() = 0; + }; +} + diff --git a/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp index b5a5c488a..e3baf8c29 100644 --- a/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp @@ -3,7 +3,7 @@ namespace WebRTC { - UnityVideoCapturer::UnityVideoCapturer(NvEncoder* pEncoder, int _width, int _height) : nvEncoder(pEncoder), width(_width), height(_height) + UnityVideoCapturer::UnityVideoCapturer(UnityEncoder* pEncoder, int _width, int _height) : nvEncoder(pEncoder), width(_width), height(_height) { set_enable_video_adapter(false); SetSupportedFormats(std::vector(1, cricket::VideoFormat(width, height, cricket::VideoFormat::FpsToInterval(framerate), cricket::FOURCC_H264))); diff --git a/Plugin/WebRTCPlugin/UnityVideoCapturer.h b/Plugin/WebRTCPlugin/UnityVideoCapturer.h index 5b0ac8881..2c6577bdf 100644 --- a/Plugin/WebRTCPlugin/UnityVideoCapturer.h +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.h @@ -1,12 +1,12 @@ #pragma once -#include "NvEncoder.h" +#include "UnityEncoder.h" namespace WebRTC { class UnityVideoCapturer : public cricket::VideoCapturer { public: - UnityVideoCapturer(NvEncoder* pEncoder, int _width, int _height); + UnityVideoCapturer(UnityEncoder* pEncoder, int _width, int _height); void EncodeVideoData(); // Start the video capturer with the specified capture format. virtual cricket::CaptureState Start(const cricket::VideoFormat& Format) override @@ -47,7 +47,7 @@ namespace WebRTC return true; } //std::unique_ptr nvEncoder; - NvEncoder* nvEncoder; + UnityEncoder* nvEncoder; //just fake info int32 width; diff --git a/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj b/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj index e962f41fb..e295f9077 100644 --- a/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj +++ b/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj @@ -177,6 +177,7 @@ + @@ -190,6 +191,7 @@ + Create diff --git a/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj.filters b/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj.filters index 56f90d06b..d3885ee91 100644 --- a/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj.filters +++ b/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj.filters @@ -60,6 +60,9 @@ Header Files + + Header Files + @@ -95,5 +98,8 @@ Source Files + + Source Files + \ No newline at end of file From ed8916a9ef7f3ab61772c0bf6479a1f538c64500 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 22 Jul 2019 17:06:14 +0800 Subject: [PATCH 19/29] let DummyVideoEncoderFactory manage the encoder of platform. --- Plugin/WebRTCPlugin/Context.cpp | 11 +++---- Plugin/WebRTCPlugin/Context.h | 2 +- Plugin/WebRTCPlugin/DummyVideoEncoder.cpp | 32 ++++++++++++++++++++ Plugin/WebRTCPlugin/DummyVideoEncoder.h | 10 +++++++ Plugin/WebRTCPlugin/NvEncoder.cpp | 35 +++++++++++++--------- Plugin/WebRTCPlugin/NvEncoder.h | 2 +- Plugin/WebRTCPlugin/UnityVideoCapturer.cpp | 2 +- Plugin/WebRTCPlugin/UnityVideoCapturer.h | 2 +- 8 files changed, 73 insertions(+), 23 deletions(-) diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index 380fd8d1e..11cdc906e 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -1,5 +1,6 @@ #include "pch.h" #include "WebRTCPlugin.h" +#include "UnityEncoder.h" #include "Context.h" namespace WebRTC @@ -292,7 +293,6 @@ namespace WebRTC rtc::InitializeSSL(); audioDevice = new rtc::RefCountedObject(); - pUnityEncoder = new NvEncoder(); auto dummyVideoEncoderFactory = std::make_unique(); pDummyVideoEncoderFactory = dummyVideoEncoderFactory.get(); @@ -319,13 +319,14 @@ namespace WebRTC mediaSteamTrackList.clear(); mediaStreamMap.clear(); nvVideoCapturerList.clear(); - delete pUnityEncoder; - pUnityEncoder = NULL; workerThread->Quit(); workerThread.reset(); signalingThread->Quit(); signalingThread.reset(); + + pDummyVideoEncoderFactory->Destroy(); + } void Context::EncodeFrame() @@ -356,9 +357,9 @@ namespace WebRTC webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) { - pUnityEncoder->InitEncoder(width, height); + UnityEncoder* pUnityEncoder = pDummyVideoEncoderFactory->CreatePlatformEncoder(WebRTC::Nvidia, width, height); UnityVideoCapturer* pUnityVideoCapturer = new UnityVideoCapturer(pUnityEncoder, width, height); - pUnityVideoCapturer->InitializeEncoder(width, height); + pUnityVideoCapturer->InitializeEncoder(); pDummyVideoEncoderFactory->AddCapturer(pUnityVideoCapturer); auto videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(pUnityVideoCapturer)); diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index 855795bc2..aa4ba44fa 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -1,4 +1,5 @@ #pragma once +#include "UnityEncoder.h" #include "DummyAudioDevice.h" #include "DummyVideoEncoder.h" #include "PeerConnectionObject.h" @@ -65,7 +66,6 @@ namespace WebRTC std::list nvVideoCapturerList; rtc::scoped_refptr audioDevice; - UnityEncoder* pUnityEncoder; }; class PeerSDPObserver : public webrtc::SetSessionDescriptionObserver diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp index 09500f4f2..618a7435a 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp @@ -1,7 +1,9 @@ #include "pch.h" +#include "UnityEncoder.h" #include "DummyVideoEncoder.h" #include "UnityVideoCapturer.h" #include +#include "NvEncoder.h" namespace WebRTC { @@ -73,6 +75,15 @@ namespace WebRTC } + void DummyVideoEncoderFactory::Destroy() + { + for (std::list::iterator it = unityEncoders.begin(); it!= unityEncoders.end(); ++it) + { + delete *it; + } + unityEncoders.clear(); + } + std::vector DummyVideoEncoderFactory::GetSupportedFormats() const { const absl::optional profileLevelId = @@ -103,4 +114,25 @@ namespace WebRTC return dummyVideoEncoder; } + + UnityEncoder* DummyVideoEncoderFactory::CreatePlatformEncoder(EncoderPlatform platform, int width, int height) + { + UnityEncoder* pEncoder = NULL; + switch (platform) + { + case WebRTC::Nvidia: + pEncoder = new NvEncoder(); + break; + case WebRTC::Amd: + break; + case WebRTC::Soft: + break; + default: + break; + } + pEncoder->InitEncoder(width, height); + unityEncoders.push_back(pEncoder); + return pEncoder; + } + } diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.h b/Plugin/WebRTCPlugin/DummyVideoEncoder.h index 88b467c0b..c791fbe2b 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.h +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.h @@ -38,6 +38,13 @@ namespace WebRTC webrtc::VideoBitrateAllocation lastBitrate; }; + enum EncoderPlatform + { + Nvidia, + Amd, + Soft, + }; + class DummyVideoEncoderFactory : public webrtc::VideoEncoderFactory { public: @@ -52,9 +59,12 @@ namespace WebRTC virtual std::unique_ptr CreateVideoEncoder( const webrtc::SdpVideoFormat& format) override; DummyVideoEncoderFactory(); + void Destroy(); void AddCapturer(UnityVideoCapturer* _capturer) { capturers.push_back(_capturer); } + UnityEncoder* CreatePlatformEncoder(EncoderPlatform platform, int width, int height); private: std::list capturers; + std::list unityEncoders; }; } diff --git a/Plugin/WebRTCPlugin/NvEncoder.cpp b/Plugin/WebRTCPlugin/NvEncoder.cpp index 59e1678c2..fd384cd38 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.cpp +++ b/Plugin/WebRTCPlugin/NvEncoder.cpp @@ -6,23 +6,25 @@ namespace WebRTC { + void* NvEncoder::pEncoderInterface = nullptr; NvEncoder::NvEncoder() { - LogPrint(StringFormat("width is %d, height is %d", encodeWidth, encodeHeight).c_str()); - checkf(g_D3D11Device != nullptr, "D3D11Device is invalid"); - checkf(encodeWidth > 0 && encodeHeight > 0, "Invalid width or height!"); - bool result = true; + if (pEncoderInterface==nullptr) + { + bool result = true; #pragma region open an encode session - //open an encode session - NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS openEncdoeSessionExParams = { 0 }; - openEncdoeSessionExParams.version = NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER; - openEncdoeSessionExParams.device = g_D3D11Device; - openEncdoeSessionExParams.deviceType = NV_ENC_DEVICE_TYPE_DIRECTX; - openEncdoeSessionExParams.apiVersion = NVENCAPI_VERSION; - result = NV_RESULT((errorCode = ContextManager::GetInstance()->pNvEncodeAPI->nvEncOpenEncodeSessionEx(&openEncdoeSessionExParams, &pEncoderInterface))); - checkf(result, "Unable to open NvEnc encode session"); - LogPrint(StringFormat("OpenEncodeSession Error is %d", errorCode).c_str()); -#pragma endregion + //open an encode session + NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS openEncdoeSessionExParams = { 0 }; + openEncdoeSessionExParams.version = NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER; + openEncdoeSessionExParams.device = g_D3D11Device; + openEncdoeSessionExParams.deviceType = NV_ENC_DEVICE_TYPE_DIRECTX; + openEncdoeSessionExParams.apiVersion = NVENCAPI_VERSION; + result = NV_RESULT((errorCode = ContextManager::GetInstance()->pNvEncodeAPI->nvEncOpenEncodeSessionEx(&openEncdoeSessionExParams, &pEncoderInterface))); + checkf(result, "Unable to open NvEnc encode session"); + LogPrint(StringFormat("OpenEncodeSession Error is %d", errorCode).c_str()); +#pragma endregion + } + } NvEncoder::~NvEncoder() @@ -41,6 +43,11 @@ namespace WebRTC { encodeWidth = width; encodeHeight = height; + + LogPrint(StringFormat("width is %d, height is %d", encodeWidth, encodeHeight).c_str()); + checkf(g_D3D11Device != nullptr, "D3D11Device is invalid"); + checkf(encodeWidth > 0 && encodeHeight > 0, "Invalid width or height!"); + bool result = true; #pragma region set initialization parameters nvEncInitializeParams.version = NV_ENC_INITIALIZE_PARAMS_VER; diff --git a/Plugin/WebRTCPlugin/NvEncoder.h b/Plugin/WebRTCPlugin/NvEncoder.h index 3468e0531..93b097538 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.h +++ b/Plugin/WebRTCPlugin/NvEncoder.h @@ -55,7 +55,7 @@ namespace WebRTC _NVENCSTATUS errorCode; Frame bufferedFrames[bufferedFrameNum]; uint64 frameCount = 0; - void* pEncoderInterface = nullptr; + static void* pEncoderInterface; bool isNvEncoderSupported = false; bool isInitialize = false; bool isIdrFrame = false; diff --git a/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp index e3baf8c29..58132fca7 100644 --- a/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp @@ -39,7 +39,7 @@ namespace WebRTC nvEncoder->SetRate(rate); } - void UnityVideoCapturer::InitializeEncoder(int32 width, int32 height) + void UnityVideoCapturer::InitializeEncoder() { //koseyile todo: one nvEncoder can connect multiple capturers. nvEncoder->CaptureFrame.connect(this, &UnityVideoCapturer::CaptureFrame); diff --git a/Plugin/WebRTCPlugin/UnityVideoCapturer.h b/Plugin/WebRTCPlugin/UnityVideoCapturer.h index 2c6577bdf..94c333631 100644 --- a/Plugin/WebRTCPlugin/UnityVideoCapturer.h +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.h @@ -31,7 +31,7 @@ namespace WebRTC return false; } void StartEncoder(); - void InitializeEncoder(int32 width, int32 height); + void InitializeEncoder(); void SetKeyFrame(); void SetRate(uint32 rate); void CaptureFrame(std::vector& data); From ec8bbb1d7ade7139ab843aa3f86f318d91a49693 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 22 Jul 2019 20:06:27 +0800 Subject: [PATCH 20/29] each nvEncoder has a texture buffer. --- Plugin/WebRTCPlugin/Callback.cpp | 10 ---------- Plugin/WebRTCPlugin/NvEncoder.cpp | 21 +++++++++++---------- Plugin/WebRTCPlugin/NvEncoder.h | 3 +++ Plugin/WebRTCPlugin/UnityEncoder.h | 1 + Plugin/WebRTCPlugin/UnityVideoCapturer.cpp | 3 +-- Plugin/WebRTCPlugin/pch.h | 3 +-- 6 files changed, 17 insertions(+), 24 deletions(-) diff --git a/Plugin/WebRTCPlugin/Callback.cpp b/Plugin/WebRTCPlugin/Callback.cpp index 5682d1eaf..c22b13745 100644 --- a/Plugin/WebRTCPlugin/Callback.cpp +++ b/Plugin/WebRTCPlugin/Callback.cpp @@ -12,8 +12,6 @@ namespace WebRTC ID3D11DeviceContext* context; //d3d11 device ID3D11Device* g_D3D11Device = nullptr; - //natively created ID3D11Texture2D ptrs - UnityFrameBuffer* renderTextures[bufferedFrameNum]; } using namespace WebRTC; //get d3d11 device @@ -33,14 +31,6 @@ static void UNITY_INTERFACE_API OnGraphicsDeviceEvent(UnityGfxDeviceEventType ev } case kUnityGfxDeviceEventShutdown: { - for (auto rt : renderTextures) - { - if (rt) - { - rt->Release(); - rt = nullptr; - } - } //UnityPluginUnload not called normally s_Graphics->UnregisterDeviceEventCallback(OnGraphicsDeviceEvent); break; diff --git a/Plugin/WebRTCPlugin/NvEncoder.cpp b/Plugin/WebRTCPlugin/NvEncoder.cpp index fd384cd38..52c031e75 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.cpp +++ b/Plugin/WebRTCPlugin/NvEncoder.cpp @@ -216,7 +216,7 @@ namespace WebRTC desc.Usage = D3D11_USAGE_DEFAULT; desc.BindFlags = D3D11_BIND_RENDER_TARGET; desc.CPUAccessFlags = 0; - g_D3D11Device->CreateTexture2D(&desc, NULL, &inputTextures); + HRESULT r = g_D3D11Device->CreateTexture2D(&desc, NULL, &inputTextures); return inputTextures; } NV_ENC_REGISTERED_PTR NvEncoder::RegisterResource(void *buffer) @@ -255,15 +255,13 @@ namespace WebRTC } void NvEncoder::InitEncoderResources() { - for (uint32 i = 0; i < bufferedFrameNum; i++) - { - renderTextures[i] = AllocateInputBuffers(); - Frame& frame = bufferedFrames[i]; - frame.inputFrame.registeredResource = RegisterResource(renderTextures[i]); - frame.inputFrame.bufferFormat = NV_ENC_BUFFER_FORMAT_ARGB; - MapResources(frame.inputFrame); - frame.outputFrame = InitializeBitstreamBuffer(); - } + nvRenderTexture = AllocateInputBuffers(); + Frame& frame = bufferedFrames[0]; + frame.inputFrame.registeredResource = RegisterResource(nvRenderTexture); + frame.inputFrame.bufferFormat = NV_ENC_BUFFER_FORMAT_ARGB; + MapResources(frame.inputFrame); + frame.outputFrame = InitializeBitstreamBuffer(); + } void NvEncoder::ReleaseFrameInputBuffer(Frame& frame) { @@ -284,6 +282,9 @@ namespace WebRTC checkf(result, "Failed to destroy output buffer bit stream"); frame.outputFrame = nullptr; } + + nvRenderTexture->Release(); + nvRenderTexture = nullptr; } } diff --git a/Plugin/WebRTCPlugin/NvEncoder.h b/Plugin/WebRTCPlugin/NvEncoder.h index 93b097538..4142d9ba3 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.h +++ b/Plugin/WebRTCPlugin/NvEncoder.h @@ -41,6 +41,7 @@ namespace WebRTC uint64 GetCurrentFrameCount() { return frameCount; } void InitEncoder(int width, int height); void InitEncoderResources(); + void* getRenderTexture() { return nvRenderTexture; } private: void ReleaseFrameInputBuffer(Frame& frame); @@ -68,6 +69,8 @@ namespace WebRTC //5Mbps const int minBitRate = 5000000; int frameRate = 45; + + UnityFrameBuffer* nvRenderTexture; }; } diff --git a/Plugin/WebRTCPlugin/UnityEncoder.h b/Plugin/WebRTCPlugin/UnityEncoder.h index f9a622c24..07bd0dcc5 100644 --- a/Plugin/WebRTCPlugin/UnityEncoder.h +++ b/Plugin/WebRTCPlugin/UnityEncoder.h @@ -18,6 +18,7 @@ namespace WebRTC virtual uint64 GetCurrentFrameCount() = 0; virtual void InitEncoder(int width, int height) = 0; virtual void InitEncoderResources() = 0; + virtual void* getRenderTexture() = 0; }; } diff --git a/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp index 58132fca7..c980679cd 100644 --- a/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp @@ -12,8 +12,7 @@ namespace WebRTC { if (captureStarted && !captureStopped) { - int curFrameNum = nvEncoder->GetCurrentFrameCount() % bufferedFrameNum; - context->CopyResource(renderTextures[curFrameNum], unityRT); + context->CopyResource((ID3D11Resource*)nvEncoder->getRenderTexture(), unityRT); nvEncoder->EncodeFrame(width, height); } } diff --git a/Plugin/WebRTCPlugin/pch.h b/Plugin/WebRTCPlugin/pch.h index 0be103dff..231ba92ba 100644 --- a/Plugin/WebRTCPlugin/pch.h +++ b/Plugin/WebRTCPlugin/pch.h @@ -91,8 +91,7 @@ namespace WebRTC using int32 = signed int; using int64 = signed long long; - const uint32 bufferedFrameNum = 3; - extern UnityFrameBuffer* renderTextures[bufferedFrameNum]; + const uint32 bufferedFrameNum = 1; extern ID3D11DeviceContext* context; extern ID3D11Device* g_D3D11Device; } From 3e248ea5f35cfb532a27c139871a4f2dea601362 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Tue, 23 Jul 2019 11:35:19 +0800 Subject: [PATCH 21/29] delete comment code. --- Plugin/WebRTCPlugin/UnityVideoCapturer.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/Plugin/WebRTCPlugin/UnityVideoCapturer.h b/Plugin/WebRTCPlugin/UnityVideoCapturer.h index 94c333631..11777dcce 100644 --- a/Plugin/WebRTCPlugin/UnityVideoCapturer.h +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.h @@ -17,7 +17,6 @@ namespace WebRTC virtual void Stop() override { captureStopped = true; - //nvEncoder.reset(); } // Check if the video capturer is running. virtual bool IsRunning() override @@ -46,7 +45,6 @@ namespace WebRTC fourccs->push_back(cricket::FOURCC_H264); return true; } - //std::unique_ptr nvEncoder; UnityEncoder* nvEncoder; //just fake info From 309436b8afc64e617fd173f9dd7f5c87058e003f Mon Sep 17 00:00:00 2001 From: chenyuan Date: Tue, 23 Jul 2019 15:54:33 +0800 Subject: [PATCH 22/29] change pEncoderInterface to no-static. --- Plugin/WebRTCPlugin/NvEncoder.cpp | 2 +- Plugin/WebRTCPlugin/NvEncoder.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Plugin/WebRTCPlugin/NvEncoder.cpp b/Plugin/WebRTCPlugin/NvEncoder.cpp index 52c031e75..b7f737967 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.cpp +++ b/Plugin/WebRTCPlugin/NvEncoder.cpp @@ -6,7 +6,7 @@ namespace WebRTC { - void* NvEncoder::pEncoderInterface = nullptr; + NvEncoder::NvEncoder() { if (pEncoderInterface==nullptr) diff --git a/Plugin/WebRTCPlugin/NvEncoder.h b/Plugin/WebRTCPlugin/NvEncoder.h index 4142d9ba3..5761e597b 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.h +++ b/Plugin/WebRTCPlugin/NvEncoder.h @@ -56,7 +56,7 @@ namespace WebRTC _NVENCSTATUS errorCode; Frame bufferedFrames[bufferedFrameNum]; uint64 frameCount = 0; - static void* pEncoderInterface; + void* pEncoderInterface = nullptr; bool isNvEncoderSupported = false; bool isInitialize = false; bool isIdrFrame = false; From ce51d0f8eaaee632f240099b69c84b7f5fe0bd85 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 5 Aug 2019 16:00:56 +0800 Subject: [PATCH 23/29] web app can switch two resolution to show. --- Assets/Scripts/RenderStreaming.cs | 42 +++++++++++++------ .../Runtime/Srcipts/MediaStream.cs | 14 +++++-- .../Runtime/Srcipts/MediaStreamTrack.cs | 2 +- .../Runtime/Srcipts/RTCPeerConnection.cs | 4 +- .../Runtime/Srcipts/WebRTC.cs | 14 +++++-- .../Samples/Example/MediaStreamSample.cs | 3 +- .../Tests/Runtime/MediaStreamTest.cs | 3 +- Plugin/WebRTCPlugin/Context.cpp | 11 ++--- Plugin/WebRTCPlugin/Context.h | 2 +- Plugin/WebRTCPlugin/DummyVideoEncoder.cpp | 10 ++--- Plugin/WebRTCPlugin/DummyVideoEncoder.h | 2 +- Plugin/WebRTCPlugin/NvEncoder.cpp | 3 +- Plugin/WebRTCPlugin/NvEncoder.h | 2 +- Plugin/WebRTCPlugin/UnityEncoder.h | 2 +- Plugin/WebRTCPlugin/WebRTCPlugin.cpp | 8 ++-- WebApp/public/scripts/app.js | 10 +++++ WebApp/public/scripts/video-player.js | 22 ++++++++++ WebApp/public/stylesheets/style.css | 17 +++++++- 18 files changed, 128 insertions(+), 43 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index b2cad26fb..eb1ceaab3 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -41,7 +41,7 @@ public class RenderStreaming : MonoBehaviour private Dictionary> mapChannels = new Dictionary>(); private RTCConfiguration conf; private string sessionId; - private MediaStream mediaStream; + private MediaStream[] mediaStreams = new MediaStream[2]; public void Awake() { @@ -62,16 +62,19 @@ public IEnumerator Start() yield break; } - captureCamera.CreateRenderStreamTexture(1280, 720); - mediaStream = new MediaStream(); + captureCamera.CreateRenderStreamTexture(1280, 720, mediaStreams.Length); + int texCount = captureCamera.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, captureCamera.GetStreamTexture(i)); - mediaStream.AddTrack(videoTrack); + int index = i; + mediaStreams[i] = new MediaStream(); + RenderTexture rt = captureCamera.GetStreamTexture(index); + VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, rt); + mediaStreams[i].AddTrack(videoTrack); } - mediaStream.AddTrack(new AudioStreamTrack("audioTrack")); + mediaStreams[0].AddTrack(new AudioStreamTrack("audioTrack")); Audio.Start(); signaling = new Signaling(urlSignaling); @@ -138,11 +141,16 @@ IEnumerator GetOffer() { continue; } - var pc = new RTCPeerConnection(); - pcs.Add(offer.connectionId, pc); + RTCConfiguration config = default; + config.iceServers = new RTCIceServer[] + { + new RTCIceServer { urls = urlsIceServer }, + }; + config.bundle_policy = RTCBundlePolicy.kBundlePolicyMaxBundle; + var pc = new RTCPeerConnection(ref config); + pcs.Add(offer.connectionId, pc); pc.OnDataChannel = new DelegateOnDataChannel(channel => { OnDataChannel(pc, channel); }); - pc.SetConfiguration(ref conf); pc.OnIceCandidate = new DelegateOnIceCandidate(candidate => { StartCoroutine(OnIceCandidate(offer.connectionId, candidate)); }); pc.OnIceConnectionChange = new DelegateOnIceConnectionChange(state => { @@ -152,13 +160,21 @@ IEnumerator GetOffer() pcs.Remove(offer.connectionId); } }); + //make video bit rate starts at 16000kbits, and 160000kbits at max. string pattern = @"(a=fmtp:\d+ .*level-asymmetry-allowed=.*)\r\n"; _desc.sdp = Regex.Replace(_desc.sdp, pattern, "$1;x-google-start-bitrate=16000;x-google-max-bitrate=160000\r\n"); + Debug.Log("remote sdp---------------------------------------------------------"); + Debug.Log(_desc.sdp); + pc.SetRemoteDescription(ref _desc); - foreach (var track in mediaStream.GetTracks()) + + foreach (var mediaStream in mediaStreams) { - pc.AddTrack(track); + foreach (var track in mediaStream.GetTracks()) + { + pc.AddTrack(track, mediaStream.Id); + } } StartCoroutine(Answer(connectionId)); @@ -178,12 +194,14 @@ IEnumerator Answer(string connectionId) } var opLocalDesc = pc.SetLocalDescription(ref op.desc); yield return opLocalDesc; + Debug.Log("local sdp---------------------------------------------------------"); + Debug.Log(op.desc.sdp); if (opLocalDesc.isError) { Debug.LogError($"Network Error: {opLocalDesc.error}"); yield break; } - var op3 = signaling.PostAnswer(this.sessionId, connectionId, op.desc.sdp); + var op3 = signaling.PostAnswer(this.sessionId, connectionId, op.desc.sdp); yield return op3; if (op3.webRequest.isNetworkError) { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index 47c3a27ff..300c72f33 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -9,16 +9,24 @@ namespace Unity.WebRTC public class MediaStream { internal IntPtr nativePtr; + internal string id; protected List mediaStreamTrackList = new List(); + private static int sMediaStreamCount = 0; + + public string Id { get => id; private set { } } public MediaStream() : base() { - nativePtr = WebRTC.Context.CreateMediaStream("MediaStream"); + sMediaStreamCount++; + id = "MediaStream" + sMediaStreamCount; + nativePtr = WebRTC.Context.CreateMediaStream(id); } public MediaStream(MediaStreamTrack[] tracks) : base() { - nativePtr = WebRTC.Context.CreateMediaStream("MediaStream"); + sMediaStreamCount++; + id = "MediaStream" + sMediaStreamCount; + nativePtr = WebRTC.Context.CreateMediaStream(id); foreach (var t in tracks) { @@ -93,7 +101,7 @@ public static void CreateRenderStreamTexture(this Camera cam, int width, int hei camRenderTexture.Create(); int mipCount = count; - for (int i = 1, mipLevel = 1; i <= mipCount; ++i, mipLevel *= 2) + for (int i = 1, mipLevel = 1; i <= mipCount; ++i, mipLevel *= 4) { RenderTexture webRtcTex = new RenderTexture(width / mipLevel, height / mipLevel, 0, RenderTextureFormat.BGRA32); webRtcTex.Create(); diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs index 2c08b378e..5d30853d7 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs @@ -36,7 +36,7 @@ private set { } public class VideoStreamTrack : MediaStreamTrack { - public VideoStreamTrack(string label, RenderTexture rt) : base(WebRTC.Context.CreateVideoTrack(label, rt.GetNativeTexturePtr(), rt.width, rt.height)) + public VideoStreamTrack(string label, RenderTexture rt, int bitRateMbps=10000000) : base(WebRTC.Context.CreateVideoTrack(label, rt.GetNativeTexturePtr(), rt.width, rt.height, bitRateMbps)) { } } diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs index ee9b9b975..53392be65 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs @@ -188,9 +188,9 @@ public void Close() NativeMethods.PeerConnectionClose(self, m_id); } - public RTCRtpSender AddTrack(MediaStreamTrack track) + public RTCRtpSender AddTrack(MediaStreamTrack track, string mediaStreamId="unity") { - return new RTCRtpSender(NativeMethods.PeerConnectionAddTrack(self, track.nativePtr)); + return new RTCRtpSender(NativeMethods.PeerConnectionAddTrack(self, track.nativePtr, mediaStreamId)); } public void RemoveTrack(RTCRtpSender sender) diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs index 5ea60fa1b..5a2f6d266 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs @@ -178,11 +178,19 @@ public enum RTCIceTransportPolicy All } + public enum RTCBundlePolicy + { + kBundlePolicyBalanced, + kBundlePolicyMaxBundle, + kBundlePolicyMaxCompat + }; + [Serializable] public struct RTCConfiguration { public RTCIceServer[] iceServers; public RTCIceTransportPolicy iceTransportPolicy; + public RTCBundlePolicy bundle_policy; } public static class WebRTC @@ -326,7 +334,7 @@ internal static class NativeMethods [DllImport(WebRTC.Lib)] public static extern void PeerConnectionSetRemoteDescription(IntPtr ptr, ref RTCSessionDescription desc); [DllImport(WebRTC.Lib)] - public static extern IntPtr PeerConnectionAddTrack(IntPtr pc, IntPtr track); + public static extern IntPtr PeerConnectionAddTrack(IntPtr pc, IntPtr track, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string mediaStreamId); [DllImport(WebRTC.Lib)] public static extern void PeerConnectionRemoveTrack(IntPtr pc, IntPtr sender); [DllImport(WebRTC.Lib)] @@ -364,7 +372,7 @@ internal static class NativeMethods [DllImport(WebRTC.Lib)] public static extern IntPtr CreateMediaStream(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label); [DllImport(WebRTC.Lib)] - public static extern IntPtr CreateVideoTrack(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label, IntPtr rt, int width, int height); + public static extern IntPtr CreateVideoTrack(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label, IntPtr rt, int width, int height, int bitRate); [DllImport(WebRTC.Lib)] public static extern IntPtr CreateAudioTrack(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label); [DllImport(WebRTC.Lib)] @@ -404,7 +412,7 @@ internal struct Context public static Context Create(int uid = 0) { return NativeMethods.ContextCreate(uid); } public void Destroy(int uid = 0) { NativeMethods.ContextDestroy(uid); self = IntPtr.Zero; } public IntPtr CreateMediaStream(string label) { return NativeMethods.CreateMediaStream(self, label); } - public IntPtr CreateVideoTrack(string label, IntPtr rt, int width, int height) { return NativeMethods.CreateVideoTrack(self, label, rt, width, height); } + public IntPtr CreateVideoTrack(string label, IntPtr rt, int width, int height, int bitRate) { return NativeMethods.CreateVideoTrack(self, label, rt, width, height, bitRate); } public IntPtr CreateAudioTrack(string label) {return NativeMethods.CreateAudioTrack(self, label);} } } diff --git a/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs b/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs index d42856a03..51b296ea0 100644 --- a/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs +++ b/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs @@ -215,7 +215,8 @@ void Call() int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - mediaStream.AddTrack(new VideoStreamTrack("videoTrack"+1, cam.GetStreamTexture(i))); + RenderTexture rt = cam.GetStreamTexture(i); + mediaStream.AddTrack(new VideoStreamTrack("videoTrack"+1, rt)); } mediaStream.AddTrack(new AudioStreamTrack("audioTrack")); diff --git a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs index 9b0e2081c..dd391660b 100644 --- a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs +++ b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs @@ -53,7 +53,8 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream() int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - VideoStreamTrack videoStreamTrack = new VideoStreamTrack("videoTrack"+i,cam.GetStreamTexture(i)); + RenderTexture rt = cam.GetStreamTexture(i); + VideoStreamTrack videoStreamTrack = new VideoStreamTrack("videoTrack"+i, rt); mediaStream.AddTrack(videoStreamTrack); pc1Senders.Add(peer1.AddTrack(videoStreamTrack)); } diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index 11cdc906e..f1f770669 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -241,6 +241,9 @@ namespace WebRTC } config.servers.push_back(stunServer); config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; + + Json::Value bundle_policy = configJson["bundle_policy"]; + config.bundle_policy = (webrtc::PeerConnectionInterface::BundlePolicy)bundle_policy.asInt(); } #pragma warning(push) #pragma warning(disable: 4715) @@ -313,6 +316,7 @@ namespace WebRTC Context::~Context() { + pDummyVideoEncoderFactory->Destroy(); clients.clear(); peerConnectionFactory = nullptr; @@ -324,9 +328,6 @@ namespace WebRTC workerThread.reset(); signalingThread->Quit(); signalingThread.reset(); - - pDummyVideoEncoderFactory->Destroy(); - } void Context::EncodeFrame() @@ -355,9 +356,9 @@ namespace WebRTC return mediaStreamMap[stream_id]; } - webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) + webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height, int32 bitRate) { - UnityEncoder* pUnityEncoder = pDummyVideoEncoderFactory->CreatePlatformEncoder(WebRTC::Nvidia, width, height); + UnityEncoder* pUnityEncoder = pDummyVideoEncoderFactory->CreatePlatformEncoder(WebRTC::Nvidia, width, height, bitRate); UnityVideoCapturer* pUnityVideoCapturer = new UnityVideoCapturer(pUnityEncoder, width, height); pUnityVideoCapturer->InitializeEncoder(); pDummyVideoEncoderFactory->AddCapturer(pUnityVideoCapturer); diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index aa4ba44fa..c1c5d7dff 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -44,7 +44,7 @@ namespace WebRTC public: explicit Context(int uid = -1); webrtc::MediaStreamInterface* CreateMediaStream(const std::string& stream_id); - webrtc::MediaStreamTrackInterface* CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height); + webrtc::MediaStreamTrackInterface* CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height, int32 bitRate); webrtc::MediaStreamTrackInterface* CreateAudioTrack(const std::string& label); ~Context(); diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp index 618a7435a..fe367d9ad 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp @@ -106,16 +106,16 @@ namespace WebRTC { //todo: According to condition of format choose different capturer. - UnityVideoCapturer* pCapturer = *capturers.begin(); + //UnityVideoCapturer* pCapturer = *(++capturers.begin()); - dummyVideoEncoder->SetKeyFrame.connect(pCapturer, &UnityVideoCapturer::SetKeyFrame); - dummyVideoEncoder->SetRate.connect(pCapturer, &UnityVideoCapturer::SetRate); + //dummyVideoEncoder->SetKeyFrame.connect(pCapturer, &UnityVideoCapturer::SetKeyFrame); + //dummyVideoEncoder->SetRate.connect(pCapturer, &UnityVideoCapturer::SetRate); } return dummyVideoEncoder; } - UnityEncoder* DummyVideoEncoderFactory::CreatePlatformEncoder(EncoderPlatform platform, int width, int height) + UnityEncoder* DummyVideoEncoderFactory::CreatePlatformEncoder(EncoderPlatform platform, int width, int height, int bitRate) { UnityEncoder* pEncoder = NULL; switch (platform) @@ -130,7 +130,7 @@ namespace WebRTC default: break; } - pEncoder->InitEncoder(width, height); + pEncoder->InitEncoder(width, height, bitRate); unityEncoders.push_back(pEncoder); return pEncoder; } diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.h b/Plugin/WebRTCPlugin/DummyVideoEncoder.h index c791fbe2b..c8d410365 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.h +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.h @@ -62,7 +62,7 @@ namespace WebRTC void Destroy(); void AddCapturer(UnityVideoCapturer* _capturer) { capturers.push_back(_capturer); } - UnityEncoder* CreatePlatformEncoder(EncoderPlatform platform, int width, int height); + UnityEncoder* CreatePlatformEncoder(EncoderPlatform platform, int width, int height, int bitRate); private: std::list capturers; std::list unityEncoders; diff --git a/Plugin/WebRTCPlugin/NvEncoder.cpp b/Plugin/WebRTCPlugin/NvEncoder.cpp index b7f737967..22623db8d 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.cpp +++ b/Plugin/WebRTCPlugin/NvEncoder.cpp @@ -39,10 +39,11 @@ namespace WebRTC } - void NvEncoder::InitEncoder(int width, int height) + void NvEncoder::InitEncoder(int width, int height, int _bitRate) { encodeWidth = width; encodeHeight = height; + bitRate = _bitRate; LogPrint(StringFormat("width is %d, height is %d", encodeWidth, encodeHeight).c_str()); checkf(g_D3D11Device != nullptr, "D3D11Device is invalid"); diff --git a/Plugin/WebRTCPlugin/NvEncoder.h b/Plugin/WebRTCPlugin/NvEncoder.h index 5761e597b..5510ad50c 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.h +++ b/Plugin/WebRTCPlugin/NvEncoder.h @@ -39,7 +39,7 @@ namespace WebRTC bool IsSupported() const { return isNvEncoderSupported; } void SetIdrFrame() { isIdrFrame = true; } uint64 GetCurrentFrameCount() { return frameCount; } - void InitEncoder(int width, int height); + void InitEncoder(int width, int height, int _bitRate); void InitEncoderResources(); void* getRenderTexture() { return nvRenderTexture; } diff --git a/Plugin/WebRTCPlugin/UnityEncoder.h b/Plugin/WebRTCPlugin/UnityEncoder.h index 07bd0dcc5..127cf9528 100644 --- a/Plugin/WebRTCPlugin/UnityEncoder.h +++ b/Plugin/WebRTCPlugin/UnityEncoder.h @@ -16,7 +16,7 @@ namespace WebRTC virtual bool IsSupported() const = 0; virtual void SetIdrFrame() = 0; virtual uint64 GetCurrentFrameCount() = 0; - virtual void InitEncoder(int width, int height) = 0; + virtual void InitEncoder(int width, int height, int _bitRate) = 0; virtual void InitEncoderResources() = 0; virtual void* getRenderTexture() = 0; }; diff --git a/Plugin/WebRTCPlugin/WebRTCPlugin.cpp b/Plugin/WebRTCPlugin/WebRTCPlugin.cpp index 707c0d88d..71c141e3f 100644 --- a/Plugin/WebRTCPlugin/WebRTCPlugin.cpp +++ b/Plugin/WebRTCPlugin/WebRTCPlugin.cpp @@ -34,9 +34,9 @@ extern "C" return context->CreateMediaStream(label); } - UNITY_INTERFACE_EXPORT webrtc::MediaStreamTrackInterface* CreateVideoTrack(Context* context, const char* label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) + UNITY_INTERFACE_EXPORT webrtc::MediaStreamTrackInterface* CreateVideoTrack(Context* context, const char* label, UnityFrameBuffer* frameBuffer, int32 width, int32 height, int32 bitRate) { - return context->CreateVideoTrack(label, frameBuffer, width, height); + return context->CreateVideoTrack(label, frameBuffer, width, height, bitRate); } UNITY_INTERFACE_EXPORT webrtc::MediaStreamTrackInterface* CreateAudioTrack(Context* context, const char* label) @@ -190,9 +190,9 @@ extern "C" obj->Close(); ContextManager::GetInstance()->curContext->DeleteClient(id); } - UNITY_INTERFACE_EXPORT webrtc::RtpSenderInterface* PeerConnectionAddTrack(PeerConnectionObject* obj, webrtc::MediaStreamTrackInterface* track) + UNITY_INTERFACE_EXPORT webrtc::RtpSenderInterface* PeerConnectionAddTrack(PeerConnectionObject* obj, webrtc::MediaStreamTrackInterface* track, const char* mediaStreamId) { - return obj->connection->AddTrack(rtc::scoped_refptr (track), { "unity" }).value().get(); + return obj->connection->AddTrack(rtc::scoped_refptr (track), { mediaStreamId }).value().get(); } UNITY_INTERFACE_EXPORT void PeerConnectionRemoveTrack(PeerConnectionObject* obj, webrtc::RtpSenderInterface* sender) diff --git a/WebApp/public/scripts/app.js b/WebApp/public/scripts/app.js index 200b276b1..282cf5ace 100644 --- a/WebApp/public/scripts/app.js +++ b/WebApp/public/scripts/app.js @@ -48,6 +48,16 @@ function onClickPlayButton() { sendClickEvent(videoPlayer, 2); }); + // add Switch Resolution button + const elementSwitchResolutionButton = document.createElement('button'); + elementSwitchResolutionButton.id = "switchResolutionButton"; + elementSwitchResolutionButton.innerHTML = "Switch Resolution"; + playerDiv.appendChild(elementSwitchResolutionButton); + elementSwitchResolutionButton.addEventListener ("click", function() { + videoPlayer.switchStream(); + }); + + // add fullscreen button const elementFullscreenButton = document.createElement('img'); elementFullscreenButton.id = 'fullscreenButton'; diff --git a/WebApp/public/scripts/video-player.js b/WebApp/public/scripts/video-player.js index 9723cd77d..19cfa337c 100644 --- a/WebApp/public/scripts/video-player.js +++ b/WebApp/public/scripts/video-player.js @@ -1,11 +1,14 @@ import Signaling from "./signaling.js" export class VideoPlayer { + constructor(element, config) { const _this = this; this.cfg = VideoPlayer.getConfiguration(config); this.pc = null; this.channel = null; + this.UnityStreams = []; + this.UnityStreamIndex = 0; this.offerOptions = { offerToReceiveAudio: true, offerToReceiveVideo: true, @@ -27,6 +30,7 @@ export class VideoPlayer { } config.sdpSemantics = 'unified-plan'; config.iceServers = [{urls: ['stun:stun.l.google.com:19302']}]; + config.bundlePolicy = "max-bundle"; return config; } @@ -48,6 +52,10 @@ export class VideoPlayer { // Create peerConnection with proxy server and set up handlers this.pc = new RTCPeerConnection(this.cfg); + this.pc.addTransceiver("video"); + this.pc.addTransceiver("audio"); + this.pc.addTransceiver("video"); + this.pc.onsignalingstatechange = function (e) { console.log('signalingState changed:', e); }; @@ -64,6 +72,10 @@ export class VideoPlayer { this.pc.ontrack = function (e) { console.log('New track added: ', e.streams); _this.video.srcObject = e.streams[0]; + if (_this.UnityStreams.indexOf(e.streams[0])==-1) + { + _this.UnityStreams.push(e.streams[0]); + } }; this.pc.onicecandidate = function (e) { if(e.candidate != null) { @@ -92,6 +104,7 @@ export class VideoPlayer { await this.createConnection(); // set local sdp offer.sdp = offer.sdp.replace(/useinbandfec=1/, 'useinbandfec=1;stereo=1;maxaveragebitrate=1048576'); + const desc = new RTCSessionDescription({sdp:offer.sdp, type:"offer"}); await this.pc.setLocalDescription(desc); await this.sendOffer(offer); @@ -162,6 +175,15 @@ export class VideoPlayer { } }; + switchStream(){ + this.video.srcObject = this.UnityStreams[this.UnityStreamIndex]; + this.UnityStreamIndex++; + if (this.UnityStreamIndex>=this.UnityStreams.length) + { + this.UnityStreamIndex = 0; + } + }; + sendMsg(msg) { if(this.channel == null) { return; diff --git a/WebApp/public/stylesheets/style.css b/WebApp/public/stylesheets/style.css index 4ff845946..b2a12c822 100644 --- a/WebApp/public/stylesheets/style.css +++ b/WebApp/public/stylesheets/style.css @@ -57,10 +57,25 @@ body{ font-size: 16px; } +#switchResolutionButton{ + position: absolute; + bottom: 10px; + left: 380px; + width: 160px; + background-color: #FF6C0B; /* Orange */ + border: none; + color: white; + padding: 15px 32px; + text-align: center; + text-decoration: none; + display: inline-block; + font-size: 16px; +} + #fullscreenButton{ position: absolute; top: 25px; right: 25px; width: 32px; height: 32px; -} \ No newline at end of file +} From f56b2cae2e3891283b3cd7ce2837fb7b46f70058 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 12 Aug 2019 15:22:28 +0800 Subject: [PATCH 24/29] same resolution nvEncoders share input texture. --- Assets/Scripts/RenderStreaming.cs | 53 +++++++++----- .../Runtime/Srcipts/MediaStream.cs | 61 ++++++++-------- .../Runtime/Srcipts/WebRTC.cs | 14 ++-- Plugin/WebRTCPlugin/DummyVideoEncoder.cpp | 13 ++++ Plugin/WebRTCPlugin/DummyVideoEncoder.h | 1 + Plugin/WebRTCPlugin/NvEncoder.cpp | 69 ++++++++++++++----- Plugin/WebRTCPlugin/NvEncoder.h | 42 +++++++++-- Plugin/WebRTCPlugin/UnityEncoder.h | 7 +- Plugin/WebRTCPlugin/UnityVideoCapturer.cpp | 3 +- WebApp/public/scripts/video-player.js | 2 + 10 files changed, 186 insertions(+), 79 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index eb1ceaab3..bff7a8c36 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -17,6 +17,12 @@ public class ButtonClickElement public ButtonClickEvent click; } + public class CameraMediaStream + { + public Camera camera; + public MediaStream[] mediaStreams = new MediaStream[2]; + } + public class RenderStreaming : MonoBehaviour { #pragma warning disable 0649 @@ -29,9 +35,6 @@ public class RenderStreaming : MonoBehaviour [SerializeField, Tooltip("Time interval for polling from signaling server")] private float interval = 5.0f; - [SerializeField, Tooltip("Camera to capture video stream")] - private Camera captureCamera; - [SerializeField] private ButtonClickElement[] arrayButtonClickEvent; #pragma warning restore 0649 @@ -41,7 +44,7 @@ public class RenderStreaming : MonoBehaviour private Dictionary> mapChannels = new Dictionary>(); private RTCConfiguration conf; private string sessionId; - private MediaStream[] mediaStreams = new MediaStream[2]; + private Dictionary cameraMediaStreamDict = new Dictionary(); public void Awake() { @@ -62,19 +65,30 @@ public IEnumerator Start() yield break; } - captureCamera.CreateRenderStreamTexture(1280, 720, mediaStreams.Length); - - int texCount = captureCamera.GetStreamTextureCount(); - for (int i = 0; i < texCount; ++i) + int count = 0; + foreach (var camera in Camera.allCameras) { - int index = i; - mediaStreams[i] = new MediaStream(); - RenderTexture rt = captureCamera.GetStreamTexture(index); - VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, rt); - mediaStreams[i].AddTrack(videoTrack); - } + count++; + if (count == 1) + { + //continue; + } - mediaStreams[0].AddTrack(new AudioStreamTrack("audioTrack")); + CameraMediaStream cameraMediaStream = new CameraMediaStream(); + cameraMediaStreamDict.Add(camera, cameraMediaStream); + camera.CreateRenderStreamTexture(1280, 720, cameraMediaStream.mediaStreams.Length); + int texCount = camera.GetStreamTextureCount(); + for (int i = 0; i < texCount; ++i) + { + int index = i; + cameraMediaStream.mediaStreams[i] = new MediaStream(); + RenderTexture rt = camera.GetStreamTexture(index); + VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, rt); + cameraMediaStream.mediaStreams[i].AddTrack(videoTrack); + cameraMediaStream.mediaStreams[i].AddTrack(new AudioStreamTrack("audioTrack")); + } + } + Audio.Start(); signaling = new Signaling(urlSignaling); @@ -169,11 +183,14 @@ IEnumerator GetOffer() pc.SetRemoteDescription(ref _desc); - foreach (var mediaStream in mediaStreams) + foreach (var k in cameraMediaStreamDict.Keys) { - foreach (var track in mediaStream.GetTracks()) + foreach (var mediaStream in cameraMediaStreamDict[k].mediaStreams) { - pc.AddTrack(track, mediaStream.Id); + foreach (var track in mediaStream.GetTracks()) + { + pc.AddTrack(track, mediaStream.Id); + } } } diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index 300c72f33..e9fa3bf44 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -74,63 +74,68 @@ public static void AddCleanerCallback(this GameObject obj, Action callback) Cleaner.AddCleanerCallback(obj, callback); } } + + internal class CameraCapturerTextures + { + internal RenderTexture camRenderTexture; + internal List webRTCTextures = new List(); + } + public static class CameraExtension { - internal static RenderTexture camRenderTexture; - internal static List webRTCTextures = new List(); - internal static List camCopyRts = new List(); - internal static bool started = false; + internal static Dictionary camCapturerTexturesDict = new Dictionary(); public static int GetStreamTextureCount(this Camera cam) { - return webRTCTextures.Count; + CameraCapturerTextures textures; + if (camCapturerTexturesDict.TryGetValue(cam, out textures)) + { + return textures.webRTCTextures.Count; + } + return 0; } public static RenderTexture GetStreamTexture(this Camera cam, int index) { - return webRTCTextures[index]; + CameraCapturerTextures textures; + if (camCapturerTexturesDict.TryGetValue(cam, out textures)) + { + if (index >= 0 && index < textures.webRTCTextures.Count) + { + return textures.webRTCTextures[index]; + } + } + return null; } public static void CreateRenderStreamTexture(this Camera cam, int width, int height, int count = 1) { - if (camCopyRts.Count > 0) - { - throw new NotImplementedException("Currently not allowed multiple MediaStream"); - } + CameraCapturerTextures cameraCapturerTextures = new CameraCapturerTextures(); + camCapturerTexturesDict.Add(cam, cameraCapturerTextures); - camRenderTexture = new RenderTexture(width, height, 0, RenderTextureFormat.BGRA32); - camRenderTexture.Create(); + cameraCapturerTextures.camRenderTexture = new RenderTexture(width, height, 0, RenderTextureFormat.BGRA32); + cameraCapturerTextures.camRenderTexture.Create(); int mipCount = count; for (int i = 1, mipLevel = 1; i <= mipCount; ++i, mipLevel *= 4) { RenderTexture webRtcTex = new RenderTexture(width / mipLevel, height / mipLevel, 0, RenderTextureFormat.BGRA32); webRtcTex.Create(); - webRTCTextures.Add(webRtcTex); + cameraCapturerTextures.webRTCTextures.Add(webRtcTex); } - cam.targetTexture = camRenderTexture; + cam.targetTexture = cameraCapturerTextures.camRenderTexture; cam.gameObject.AddCleanerCallback(() => { - camRenderTexture.Release(); - UnityEngine.Object.Destroy(camRenderTexture); + cameraCapturerTextures.camRenderTexture.Release(); + UnityEngine.Object.Destroy(cameraCapturerTextures.camRenderTexture); - foreach (var v in webRTCTextures) + foreach (var v in cameraCapturerTextures.webRTCTextures) { v.Release(); UnityEngine.Object.Destroy(v); } - webRTCTextures.Clear(); + cameraCapturerTextures.webRTCTextures.Clear(); }); - started = true; - } - - public static void RemoveRt(RenderTexture[] rts) - { - camCopyRts.Remove(rts); - if (camCopyRts.Count == 0) - { - started = false; - } } } diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs index 5a2f6d266..809d702af 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs @@ -227,15 +227,17 @@ public static IEnumerator Update() { // Wait until all frame rendering is done yield return new WaitForEndOfFrame(); - if (CameraExtension.started) + //Blit is for DirectX Rendering API Only + + foreach (var k in CameraExtension.camCapturerTexturesDict.Keys) { - //Blit is for DirectX Rendering API Only - foreach(var rt in CameraExtension.webRTCTextures) + foreach (var rt in CameraExtension.camCapturerTexturesDict[k].webRTCTextures) { - Graphics.Blit(CameraExtension.camRenderTexture, rt, flipMat); - } - GL.IssuePluginEvent(NativeMethods.GetRenderEventFunc(), 0); + Graphics.Blit(CameraExtension.camCapturerTexturesDict[k].camRenderTexture, rt, flipMat); + } } + + GL.IssuePluginEvent(NativeMethods.GetRenderEventFunc(), 0); Audio.Update(); } } diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp index fe367d9ad..7cba0071d 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp @@ -82,6 +82,7 @@ namespace WebRTC delete *it; } unityEncoders.clear(); + NvEncoder::DestroyEncoderTexture(); } std::vector DummyVideoEncoderFactory::GetSupportedFormats() const @@ -135,4 +136,16 @@ namespace WebRTC return pEncoder; } + UnityEncoder* DummyVideoEncoderFactory::GetPlatformEncoder(EncoderPlatform platform, int width, int height, int bitRate) + { + for (std::list::iterator it = unityEncoders.begin(); it != unityEncoders.end(); ++it) + { + if ((*it)->getEncodeWidth() == width && (*it)->getEncodeHeight() == height && (*it)->getBitRate() == bitRate) { + return (*it); + } + } + + return CreatePlatformEncoder(platform, width, height, bitRate); + } + } diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.h b/Plugin/WebRTCPlugin/DummyVideoEncoder.h index c8d410365..fe20463ec 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.h +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.h @@ -63,6 +63,7 @@ namespace WebRTC void AddCapturer(UnityVideoCapturer* _capturer) { capturers.push_back(_capturer); } UnityEncoder* CreatePlatformEncoder(EncoderPlatform platform, int width, int height, int bitRate); + UnityEncoder* GetPlatformEncoder(EncoderPlatform platform, int width, int height, int bitRate); private: std::list capturers; std::list unityEncoders; diff --git a/Plugin/WebRTCPlugin/NvEncoder.cpp b/Plugin/WebRTCPlugin/NvEncoder.cpp index 22623db8d..bc805c406 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.cpp +++ b/Plugin/WebRTCPlugin/NvEncoder.cpp @@ -6,7 +6,7 @@ namespace WebRTC { - + std::list NvEncoder::nvEncoderInputTextureList; NvEncoder::NvEncoder() { if (pEncoderInterface==nullptr) @@ -137,11 +137,13 @@ namespace WebRTC lastBitRate = bitRate; } } + //entry for encoding a frame void NvEncoder::EncodeFrame(int width, int height) { UpdateSettings(width, height); uint32 bufferIndexToWrite = frameCount % bufferedFrameNum; + Frame& frame = bufferedFrames[bufferIndexToWrite]; #pragma region set frame params //no free buffer, skip this frame @@ -168,13 +170,15 @@ namespace WebRTC picParams.encodePicFlags |= NV_ENC_PIC_FLAG_FORCEIDR; } isIdrFrame = false; + bool result = NV_RESULT((errorCode = ContextManager::GetInstance()->pNvEncodeAPI->nvEncEncodePicture(pEncoderInterface, &picParams))); checkf(result, StringFormat("Failed to encode frame, error is %d", errorCode).c_str()); + #pragma endregion ProcessEncodedFrame(frame); frameCount++; } - + //get encoded frame void NvEncoder::ProcessEncodedFrame(Frame& frame) { @@ -183,12 +187,15 @@ namespace WebRTC { return; } + frame.isEncoding = false; + #pragma region retrieve encoded frame from output buffer NV_ENC_LOCK_BITSTREAM lockBitStream = { 0 }; lockBitStream.version = NV_ENC_LOCK_BITSTREAM_VER; lockBitStream.outputBitstream = frame.outputFrame; lockBitStream.doNotWait = nvEncInitializeParams.enableEncodeAsync; + bool result = NV_RESULT((errorCode = ContextManager::GetInstance()->pNvEncodeAPI->nvEncLockBitstream(pEncoderInterface, &lockBitStream))); checkf(result, StringFormat("Failed to lock bit stream, error is %d", errorCode).c_str()); if (lockBitStream.bitstreamSizeInBytes) @@ -196,12 +203,11 @@ namespace WebRTC frame.encodedFrame.resize(lockBitStream.bitstreamSizeInBytes); std::memcpy(frame.encodedFrame.data(), lockBitStream.bitstreamBufferPtr, lockBitStream.bitstreamSizeInBytes); } - result = NV_RESULT((errorCode = ContextManager::GetInstance()->pNvEncodeAPI->nvEncUnlockBitstream(pEncoderInterface, frame.outputFrame))); checkf(result, StringFormat("Failed to unlock bit stream, error is %d", errorCode).c_str()); frame.isIdrFrame = lockBitStream.pictureType == NV_ENC_PIC_TYPE_IDR; #pragma endregion - CaptureFrame(frame.encodedFrame); + captureFrame(frame.encodedFrame); } ID3D11Texture2D* NvEncoder::AllocateInputBuffers() @@ -254,15 +260,42 @@ namespace WebRTC StringFormat("nvEncCreateBitstreamBuffer error is %d", errorCode).c_str()); return createBitstreamBuffer.bitstreamBuffer; } - void NvEncoder::InitEncoderResources() + + void NvEncoder::DestroyEncoderTexture() + { + for (std::list::iterator it = nvEncoderInputTextureList.begin(); it != nvEncoderInputTextureList.end(); ++it) + { + delete (*it); + } + nvEncoderInputTextureList.clear(); + } + + UnityFrameBuffer* NvEncoder::getEncoderTexture(int width, int height) { - nvRenderTexture = AllocateInputBuffers(); - Frame& frame = bufferedFrames[0]; - frame.inputFrame.registeredResource = RegisterResource(nvRenderTexture); - frame.inputFrame.bufferFormat = NV_ENC_BUFFER_FORMAT_ARGB; - MapResources(frame.inputFrame); - frame.outputFrame = InitializeBitstreamBuffer(); + for (std::list::iterator it = nvEncoderInputTextureList.begin(); it!= nvEncoderInputTextureList.end(); ++it) + { + if ( (*it)->width==width && (*it)->height==height ) + { + return (*it)->texture; + } + } + EncoderInputTexture* pEncoderInputTexture = new EncoderInputTexture(width, height); + nvEncoderInputTextureList.push_back(pEncoderInputTexture); + return pEncoderInputTexture->texture; + } + + void NvEncoder::InitEncoderResources() + { + nvEncoderTexture = getEncoderTexture(encodeWidth, encodeHeight); + for (int i = 0; i < bufferedFrameNum; i++) + { + Frame& frame = bufferedFrames[i]; + frame.inputFrame.registeredResource = RegisterResource(nvEncoderTexture); + frame.inputFrame.bufferFormat = NV_ENC_BUFFER_FORMAT_ARGB; + MapResources(frame.inputFrame); + frame.outputFrame = InitializeBitstreamBuffer(); + } } void NvEncoder::ReleaseFrameInputBuffer(Frame& frame) { @@ -278,14 +311,14 @@ namespace WebRTC { for (Frame& frame : bufferedFrames) { - ReleaseFrameInputBuffer(frame); - bool result = NV_RESULT(ContextManager::GetInstance()->pNvEncodeAPI->nvEncDestroyBitstreamBuffer(pEncoderInterface, frame.outputFrame)); - checkf(result, "Failed to destroy output buffer bit stream"); - frame.outputFrame = nullptr; + if (frame.outputFrame!=nullptr) + { + ReleaseFrameInputBuffer(frame); + bool result = NV_RESULT(ContextManager::GetInstance()->pNvEncodeAPI->nvEncDestroyBitstreamBuffer(pEncoderInterface, frame.outputFrame)); + checkf(result, "Failed to destroy output buffer bit stream"); + frame.outputFrame = nullptr; + } } - - nvRenderTexture->Release(); - nvRenderTexture = nullptr; } } diff --git a/Plugin/WebRTCPlugin/NvEncoder.h b/Plugin/WebRTCPlugin/NvEncoder.h index 5510ad50c..6a119765b 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.h +++ b/Plugin/WebRTCPlugin/NvEncoder.h @@ -29,6 +29,35 @@ namespace WebRTC std::atomic isEncoding = false; }; + struct EncoderInputTexture + { + UnityFrameBuffer* texture; + int width; + int height; + EncoderInputTexture(int w, int h) + { + width = w; + height = h; + D3D11_TEXTURE2D_DESC desc = { 0 }; + desc.Width = width; + desc.Height = height; + desc.MipLevels = 1; + desc.ArraySize = 1; + desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + desc.SampleDesc.Count = 1; + desc.Usage = D3D11_USAGE_DEFAULT; + desc.BindFlags = D3D11_BIND_RENDER_TARGET; + desc.CPUAccessFlags = 0; + HRESULT r = g_D3D11Device->CreateTexture2D(&desc, NULL, &texture); + } + + ~EncoderInputTexture() + { + texture->Release(); + texture = nullptr; + } + }; + public: NvEncoder(); ~NvEncoder(); @@ -41,8 +70,13 @@ namespace WebRTC uint64 GetCurrentFrameCount() { return frameCount; } void InitEncoder(int width, int height, int _bitRate); void InitEncoderResources(); - void* getRenderTexture() { return nvRenderTexture; } - + void* getRenderTexture() { return nvEncoderTexture; } + int getEncodeWidth() { return encodeWidth; } + int getEncodeHeight() { return encodeHeight; } + int getBitRate() { return bitRate; } + static void DestroyEncoderTexture(); + private: + static UnityFrameBuffer* getEncoderTexture(int width, int height); private: void ReleaseFrameInputBuffer(Frame& frame); void ReleaseEncoderResources(); @@ -55,6 +89,8 @@ namespace WebRTC NV_ENC_CONFIG nvEncConfig = {}; _NVENCSTATUS errorCode; Frame bufferedFrames[bufferedFrameNum]; + static std::list nvEncoderInputTextureList; + UnityFrameBuffer* nvEncoderTexture; uint64 frameCount = 0; void* pEncoderInterface = nullptr; bool isNvEncoderSupported = false; @@ -69,8 +105,6 @@ namespace WebRTC //5Mbps const int minBitRate = 5000000; int frameRate = 45; - - UnityFrameBuffer* nvRenderTexture; }; } diff --git a/Plugin/WebRTCPlugin/UnityEncoder.h b/Plugin/WebRTCPlugin/UnityEncoder.h index 127cf9528..ecc80a058 100644 --- a/Plugin/WebRTCPlugin/UnityEncoder.h +++ b/Plugin/WebRTCPlugin/UnityEncoder.h @@ -2,14 +2,12 @@ namespace WebRTC { - class UnityEncoder { public: UnityEncoder(); virtual ~UnityEncoder(); - - sigslot::signal1&> CaptureFrame; + sigslot::signal1&> captureFrame; virtual void SetRate(uint32 rate) = 0; virtual void UpdateSettings(int width, int height) = 0; virtual void EncodeFrame(int width, int height) = 0; @@ -19,6 +17,9 @@ namespace WebRTC virtual void InitEncoder(int width, int height, int _bitRate) = 0; virtual void InitEncoderResources() = 0; virtual void* getRenderTexture() = 0; + virtual int getEncodeWidth() = 0; + virtual int getEncodeHeight() = 0; + virtual int getBitRate() = 0; }; } diff --git a/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp index c980679cd..2383a7897 100644 --- a/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp @@ -40,7 +40,6 @@ namespace WebRTC void UnityVideoCapturer::InitializeEncoder() { - //koseyile todo: one nvEncoder can connect multiple capturers. - nvEncoder->CaptureFrame.connect(this, &UnityVideoCapturer::CaptureFrame); + nvEncoder->captureFrame.connect(this, &UnityVideoCapturer::CaptureFrame); } } diff --git a/WebApp/public/scripts/video-player.js b/WebApp/public/scripts/video-player.js index 19cfa337c..6432bb497 100644 --- a/WebApp/public/scripts/video-player.js +++ b/WebApp/public/scripts/video-player.js @@ -52,9 +52,11 @@ export class VideoPlayer { // Create peerConnection with proxy server and set up handlers this.pc = new RTCPeerConnection(this.cfg); + this.pc.addTransceiver("video"); this.pc.addTransceiver("audio"); this.pc.addTransceiver("video"); + this.pc.addTransceiver("audio"); this.pc.onsignalingstatechange = function (e) { console.log('signalingState changed:', e); From c9edf213bfa1ed0b03f760af42d9bfc01e7c70c4 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 12 Aug 2019 18:41:07 +0800 Subject: [PATCH 25/29] add other way to implement simulcast --- Assets/Scripts/RenderStreaming.cs | 53 ++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index bff7a8c36..fae0a69c2 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -37,6 +37,9 @@ public class RenderStreaming : MonoBehaviour [SerializeField] private ButtonClickElement[] arrayButtonClickEvent; + + [SerializeField] + private bool isUseMinimalTextures = true; #pragma warning restore 0649 private Signaling signaling; @@ -65,30 +68,44 @@ public IEnumerator Start() yield break; } - int count = 0; - foreach (var camera in Camera.allCameras) + + if (isUseMinimalTextures) { - count++; - if (count == 1) + foreach (var camera in Camera.allCameras) { - //continue; + CameraMediaStream cameraMediaStream = new CameraMediaStream(); + cameraMediaStreamDict.Add(camera, cameraMediaStream); + camera.CreateRenderStreamTexture(1280, 720); + int mediaCount = cameraMediaStream.mediaStreams.Length; + for (int i = 0; i < mediaCount; ++i) + { + cameraMediaStream.mediaStreams[i] = new MediaStream(); + RenderTexture rt = camera.GetStreamTexture(0); + int temp = i==0 ? 1 : (int)Mathf.Pow(i + 1, 10); + VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, rt, 1000000/temp); + cameraMediaStream.mediaStreams[i].AddTrack(videoTrack); + cameraMediaStream.mediaStreams[i].AddTrack(new AudioStreamTrack("audioTrack")); + } } - - CameraMediaStream cameraMediaStream = new CameraMediaStream(); - cameraMediaStreamDict.Add(camera, cameraMediaStream); - camera.CreateRenderStreamTexture(1280, 720, cameraMediaStream.mediaStreams.Length); - int texCount = camera.GetStreamTextureCount(); - for (int i = 0; i < texCount; ++i) + }else{ + foreach (var camera in Camera.allCameras) { - int index = i; - cameraMediaStream.mediaStreams[i] = new MediaStream(); - RenderTexture rt = camera.GetStreamTexture(index); - VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, rt); - cameraMediaStream.mediaStreams[i].AddTrack(videoTrack); - cameraMediaStream.mediaStreams[i].AddTrack(new AudioStreamTrack("audioTrack")); + CameraMediaStream cameraMediaStream = new CameraMediaStream(); + cameraMediaStreamDict.Add(camera, cameraMediaStream); + camera.CreateRenderStreamTexture(1280, 720, cameraMediaStream.mediaStreams.Length); + int texCount = camera.GetStreamTextureCount(); + for (int i = 0; i < texCount; ++i) + { + int index = i; + cameraMediaStream.mediaStreams[i] = new MediaStream(); + RenderTexture rt = camera.GetStreamTexture(index); + VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, rt); + cameraMediaStream.mediaStreams[i].AddTrack(videoTrack); + cameraMediaStream.mediaStreams[i].AddTrack(new AudioStreamTrack("audioTrack")); + } } } - + Audio.Start(); signaling = new Signaling(urlSignaling); From 9c8562075ec4ce40960ed49ea060b4a041fa96ec Mon Sep 17 00:00:00 2001 From: chenyuan Date: Thu, 15 Aug 2019 19:47:36 +0800 Subject: [PATCH 26/29] add select media stream ui. --- WebApp/public/scripts/app.js | 120 +++++++++++++++++++++++--- WebApp/public/scripts/video-player.js | 31 +++++++ WebApp/public/stylesheets/style.css | 82 ++++++++++++++---- 3 files changed, 208 insertions(+), 25 deletions(-) diff --git a/WebApp/public/scripts/app.js b/WebApp/public/scripts/app.js index 282cf5ace..f2784f998 100644 --- a/WebApp/public/scripts/app.js +++ b/WebApp/public/scripts/app.js @@ -48,16 +48,6 @@ function onClickPlayButton() { sendClickEvent(videoPlayer, 2); }); - // add Switch Resolution button - const elementSwitchResolutionButton = document.createElement('button'); - elementSwitchResolutionButton.id = "switchResolutionButton"; - elementSwitchResolutionButton.innerHTML = "Switch Resolution"; - playerDiv.appendChild(elementSwitchResolutionButton); - elementSwitchResolutionButton.addEventListener ("click", function() { - videoPlayer.switchStream(); - }); - - // add fullscreen button const elementFullscreenButton = document.createElement('img'); elementFullscreenButton.id = 'fullscreenButton'; @@ -84,6 +74,104 @@ function onClickPlayButton() { elementFullscreenButton.style.display = 'block'; } } + +} + +function setupMediaSelector(options) +{ + const playerDiv = document.getElementById('player'); + let mediaSelectDiv = document.createElement("div"); + mediaSelectDiv.id = "mediaSelect"; + mediaSelectDiv.setAttribute("style", "width:200px;"); + mediaSelectDiv.className = "custom-select"; + playerDiv.appendChild(mediaSelectDiv); + const mediaSelect = document.createElement("select"); + mediaSelectDiv.appendChild(mediaSelect); + let index = 0; + options.forEach(option=>{ + let optionItem = document.createElement("Option"); + optionItem.value = index++; + optionItem.innerHTML = option; + mediaSelect.appendChild(optionItem); + }) + + + let customSelects, selElmnt; + /*look for any elements with the class "custom-select":*/ + customSelects = document.getElementsByClassName("custom-select"); + for (let i = 0; i < customSelects.length; i++) { + selElmnt = customSelects[i].getElementsByTagName("select")[0]; + /*for each element, create a new DIV that will act as the selected item:*/ + let a = document.createElement("DIV"); + a.setAttribute("class", "select-selected"); + a.innerHTML = selElmnt.options[selElmnt.selectedIndex].innerHTML; + customSelects[i].appendChild(a); + /*for each element, create a new DIV that will contain the option list:*/ + let b = document.createElement("DIV"); + b.setAttribute("class", "select-items select-hide"); + for (let j = 1; j < selElmnt.length; j++) { + /*for each option in the original select element, + create a new DIV that will act as an option item:*/ + let c = document.createElement("DIV"); + c.innerHTML = selElmnt.options[j].innerHTML; + c.addEventListener("click", function(e) { + /*when an item is clicked, update the original select box, + and the selected item:*/ + let y, i, k, s, h; + s = this.parentNode.parentNode.getElementsByTagName("select")[0]; + + videoPlayer.selectMediaStream(this.innerHTML); + console.log(this.innerHTML); + + h = this.parentNode.previousSibling; + for (i = 0; i < s.length; i++) { + if (s.options[i].innerHTML == this.innerHTML) { + s.selectedIndex = i; + h.innerHTML = this.innerHTML; + y = this.parentNode.getElementsByClassName("same-as-selected"); + for (k = 0; k < y.length; k++) { + y[k].removeAttribute("class"); + } + this.setAttribute("class", "same-as-selected"); + break; + } + } + h.click(); + }); + b.appendChild(c); + } + customSelects[i].appendChild(b); + a.addEventListener("click", function(e) { + /*when the select box is clicked, close any other select boxes, + and open/close the current select box:*/ + e.stopPropagation(); + closeAllSelect(this); + this.nextSibling.classList.toggle("select-hide"); + this.classList.toggle("select-arrow-active"); + }); + } + function closeAllSelect(elmnt) { + /*a function that will close all select boxes in the document, + except the current select box:*/ + var x, y, i, arrNo = []; + x = document.getElementsByClassName("select-items"); + y = document.getElementsByClassName("select-selected"); + for (i = 0; i < y.length; i++) { + if (elmnt == y[i]) { + arrNo.push(i) + } else { + y[i].classList.remove("select-arrow-active"); + } + } + for (i = 0; i < x.length; i++) { + if (arrNo.indexOf(i)) { + x[i].classList.add("select-hide"); + } + } + } + /*if the user clicks anywhere outside the select box, + then close all select boxes:*/ + document.addEventListener("click", closeAllSelect); } async function setupVideoPlayer(element, config) { @@ -91,6 +179,7 @@ async function setupVideoPlayer(element, config) { await videoPlayer.setupConnection(); videoPlayer.ondisconnect = onDisconnect; + videoPlayer.onaddtrackfinish = onAddTrackFinish; registerKeyboardEvents(videoPlayer); registerMouseEvents(videoPlayer, element); @@ -104,6 +193,17 @@ function onDisconnect() { showPlayButton(); } +function onAddTrackFinish(mediaStreams) { + + let options = ["Select a media"]; + for (let i=0; i new Promise(resolve => setTimeout(resolve, msec)); } @@ -74,11 +76,30 @@ export class VideoPlayer { this.pc.ontrack = function (e) { console.log('New track added: ', e.streams); _this.video.srcObject = e.streams[0]; + + if (_this.UnityStreams.indexOf(e.streams[0])==-1) { + let videoTracks = e.streams[0].getVideoTracks(); + for(let i=0; i Date: Mon, 19 Aug 2019 17:32:51 +0800 Subject: [PATCH 27/29] web app can select media stream. --- WebApp/public/scripts/app.js | 28 +++++++++---- WebApp/public/scripts/video-player.js | 58 +++++++++++++-------------- WebApp/public/stylesheets/style.css | 20 ++++++--- 3 files changed, 61 insertions(+), 45 deletions(-) diff --git a/WebApp/public/scripts/app.js b/WebApp/public/scripts/app.js index f2784f998..ff55c0bd3 100644 --- a/WebApp/public/scripts/app.js +++ b/WebApp/public/scripts/app.js @@ -24,11 +24,20 @@ function onClickPlayButton() { const playerDiv = document.getElementById('player'); // add video player - const elementVideo = document.createElement('video'); - elementVideo.id = 'Video'; - elementVideo.style.touchAction = 'none'; - playerDiv.appendChild(elementVideo); - setupVideoPlayer(elementVideo).then(value => videoPlayer = value); + let elementVideos = []; + for (let i=0; i<2; i++) + { + const elementVideo = document.createElement('video'); + elementVideo.id = "Video"+i; + elementVideo.style.touchAction = 'none'; + playerDiv.appendChild(elementVideo); + + elementVideos.push(elementVideo); + } + + + setupVideoPlayer(elementVideos).then(value => videoPlayer = value); + // add green button const elementBlueButton = document.createElement('button'); @@ -174,14 +183,17 @@ function setupMediaSelector(options) document.addEventListener("click", closeAllSelect); } -async function setupVideoPlayer(element, config) { - const videoPlayer = new VideoPlayer(element, config); +async function setupVideoPlayer(elements, config) { + const videoPlayer = new VideoPlayer(elements, config); await videoPlayer.setupConnection(); videoPlayer.ondisconnect = onDisconnect; videoPlayer.onaddtrackfinish = onAddTrackFinish; registerKeyboardEvents(videoPlayer); - registerMouseEvents(videoPlayer, element); + + elements.forEach(element=>{ + registerMouseEvents(videoPlayer, element); + }); return videoPlayer; } diff --git a/WebApp/public/scripts/video-player.js b/WebApp/public/scripts/video-player.js index 4faa113b6..d8cbef707 100644 --- a/WebApp/public/scripts/video-player.js +++ b/WebApp/public/scripts/video-player.js @@ -2,23 +2,25 @@ import Signaling from "./signaling.js" export class VideoPlayer { - constructor(element, config) { + constructor(elements, config) { const _this = this; this.cfg = VideoPlayer.getConfiguration(config); this.pc = null; this.channel = null; this.UnityStreams = []; - this.UnityStreamIndex = 0; this.UnityStreamCount = 2; this.offerOptions = { offerToReceiveAudio: true, offerToReceiveVideo: true, }; - this.video = element; - this.video.playsInline = true; - this.video.addEventListener('loadedmetadata', function () { - _this.video.play(); - }, true); + this.videos = elements; + this.videos.forEach(v=>{ + v.playsInline = true; + v.addEventListener('loadedmetadata', function () { + v.play(); + }, true); + }) + this.interval = 3000; this.signaling = new Signaling(); this.ondisconnect = function(){}; @@ -60,6 +62,7 @@ export class VideoPlayer { this.pc.addTransceiver("video"); this.pc.addTransceiver("audio"); + this.pc.onsignalingstatechange = function (e) { console.log('signalingState changed:', e); }; @@ -73,20 +76,15 @@ export class VideoPlayer { this.pc.onicegatheringstatechange = function (e) { console.log('iceGatheringState changed:', e); }; + let tempCount = 0; this.pc.ontrack = function (e) { - console.log('New track added: ', e.streams); - _this.video.srcObject = e.streams[0]; + console.log('New track added: ', e.streams); + console.log(e.track); if (_this.UnityStreams.indexOf(e.streams[0])==-1) { - let videoTracks = e.streams[0].getVideoTracks(); - for(let i=0; iCnl5IneBa7l221lQp15G=U6Hty~a+}+&??(Xgo+@1c{^L}IR(>|Cd zRr9WE)a>Ra$krxEc+v<2gge;ZaqxEZge-l7%5PW-&ao=}S|9hnGDaRPMWC%%; zzynDOd7NMb`5oB&($7{x|H(Db!DnWi3{e&W{0sU{jkUa#pgoedjR5xIMmNVMA`x#X z^SO#~V0zhb=i8QO4RK1=(@tbR^X!+Q#|M~$k-hR(qw`PJ-g_f^j$60#rhcS@?CX7T zGWtE=Vp{ZN6xYQ)|9aDq_^Y?+*zAC>NPi&!1n{@Ou})>;pB+yr$yWrI@w^vW1wDv z6CMZ|R;@@O;PE`o`9Y;ydJGa&;JsXThxK5^nWaoLw0jk8SSuupvU$vhfWKndy}z@{ zXs0xkasVogJCe$bMio(5vYf@qcj1d*9pn8eC>yBuA@KB~&*DUt0tQ=bzvwT%-h<-8 zO&{5ed8w1jbpjxk0n?EZCU&ivSo{;qyb>Sya@Wm{X*93zQ+7i2lmkgt5VaOiZQ{@AzE z#Kz!3%asywkYftDa*-LXppb=tO?WR&(d@Z8Yh@nmrr0Uno@vMMz2T!N$~~vIf<4^c z-TmSrYeNi>{Kf*hNCNjQ!~=OTJP=H|Ki7%GB3qC9{jA8NL3OSqCbxWm9{YPiwd_MH zI*DDO3W4-h!Ny#}=3%;(iUm9GyJ)EUv7C}!$;P5H5xi%6dzCH5<7&(~J{LYw`+hBz&*F3T?1XH@hDngX||RRlCvi$Wx>@UY-yi zTB+Ts#=QK$6%C@Pn=|+H06>Bm$?B|A2bf2|8jv$eb8baAz!X(Y(h?N+4()S;MJ*KR zh~_6uaSym$%8e%C)Qq#cCb|5}`yJscEee|V)E?W)tMD)C_YT6hU!KlVjYPZyD!+I{ z!kZ%cETNb$UN1!+QR8C*6T8D|!o*>qh?*QZOrhc35cFTA3-$X=_mwT3o!w1WO96T2 zOQhc%i1pVm&V9_o@FokA-gXi;$G<#Fd-i^Yq`q+n;b%SC2+}2zcjvJQ6cy_+$BfyH z^`hdP2Sp<%z2rUV4!^F&69@)>L6klJmb{c#AQ%SkICecCbp|iEUbdR`UC)!a?r%iH zBJGbK4~N6pc#(f$8V=oHYonQv^%wxl}1P;6#Hpy=XM@1aJ;R+q$T_tAn^gFQck{1j$l;Hd2 zMRJUqmIbeWPq8yOo6hc=D!ykO$g85xQoN+^s`_Gxxe8Is?JK$(B5;X~eu)HJcI*{I zY1u9>7SVma9(T`Tzxzi$iI08A5pXY|W0PH>o4yLCMg+Ok6Z(rO2Oc{!PSivmhKs+>TY_?iU063RL7I>U5-r=CT-l)1!V}WsUT; z@D1w)B)%Fnm2@isjA6Wm6&DtuGNR*GvPhV==)I}26Xc?#U`OQKQD+y{g-9E2!|)+^-t^2i zP5U)C;5X3hP*$003FFKAO>#W1eB1;Pu7_Hi|Mgz9wjpaoAYHK7ismS!C&b+xa(cS? z>2MM@;VeQc;%7U&Sbxp+@C zll*y0m-iPgUjI%;6WV;9LdnhdN=8<<^9;>Qo_+DOM<#p2<=9MsazT12$bIw0&_mN! z@}$ZssO^kVYJyRnGto7gkT!@u4Y&!dlKH#Y30At}U*^YV&3`&s@{8xykUtu6FHfc z@z{G(5}+IAq{R|~K6(#R~QH;7^m zpgnAGzJQO;JT&;x5$NduX*Uj&N;OAqwwq^H&cgBdMNze}>JhzxWu3Eer7p6Y<8EDN zJ1Hp@zg6lS$W(?$?itq!>VWQ~!ZC(+@+XuGb%FVX=5vdX#rNe~Kf&z$Dy*sA6&@z@ zQC}sbyTw$`JlJ_3P1bepA2;WnA{v385sHL+iLrElS#q!9<@x%0kV0n z(NdGmF|6b=i{Tnr$+iEEd+DJ4(q6%N2{oe8TwDPMxKqsMWKgmoCg1UYAMZus537-Z z|4#F5qCrJ?%hyTIa2>)O+Zk7psv^kwSm|xbCM#vJ{Pb~ADQ;SiTCGTfQ_k1Q9Vx-m z-)Dlt?aR@rncApaX=KNg>)9h?n{(|(4JG$g9xp-iQEZALDJI`YDs680X~J{Cc)fS8 zhX&;n@MdcwUL`L?`!x%D%x$w})YNz6Zx(J~=69xp!aGHpHi`4wE)_X}?|o4NUy8;l z_fiH_LcS%2LAMkhYAzO04=Z0bGO<=XF4J0=7dk+$RY}o=R%c2wGHU$1W#d$$x6mf| zX>CPz?}y9u&$~)1Ir?bV(CM7YAi3UMGqu=N4Xp-BYVT63Pj zL#xhem>G;EFO@mCj|~VPkU^UWW3FqbMzQ64~EQ?;uD*OMZDd&_v2# z0Vbtt)0Pu6Qrb3uk0|x(>iKcf8S|?894uI2-f1SU&ZnoeSQ%RHp2$@OxhC6JnnVwe z+M=_W|AT8X)!=^Y5J>=y*FbC}i3GDUl2b`pGU=5y7e-1kbdH7#%jeIDb=ycX)N6Ge zYv;07$b2>r)iiR7o_Y&~GBmLJ3qJ0G$b2rC8Sr`|v-p8aK zhsMWy6>nlOF{wR2<)|iF_w?fT3HA0;v6(f}wTF4jKEd{E`Ohg>K1kH^GfT3a)0l34 zY{Ms<79kHnjFb6xJ^t8eSp)@!4cs6Cvr*){grv$-+c7g7*%uvim>QHqA$R1?$0Oz5cpW3zjn}o!?k{ zY4b&6BIB4?)tZ9B8XEPX{c62FXxlgpuC4pgYM99VQoqT9Mxv$Pxc>Qy#!nHO^s0?$WTBXCrf^u2i2KT;`y+A@1D7{+N_z!!XW2H4~ zfHT48*X*D-@HbD;A|<4*Q&qUaKhxJKtR~bCW^8i+0lp-xqfr8@#BLEo#+r+P&D%hv zyF3Z*M7<7t#(Pg=xBRxnVwiLlg~FU0-@4+bm0D9IT|3AQDQ;Rb=bTV#0^*QP1R<`B zEb6vRB?wM{qrgYz_n1AY=P@fPyG|P5*xl^xu~MNSN;bw*{tz8&K3peO^hLFv-J&d= zZ?V`2mc4Yx$pVO>%TNyREE4QU#~Qz%k&o^w6M$1F*hfF#F2-Se=@lX?>*lM?K$^6# z#fkiRXHNB^lJHk+w80v0v`maW6MbXWYD|TZ!g7wh8G%NW{!`q_9~9i27h-#qOwVqE zYLNmPzef$d4v&yBu2nuqcwQV6y8H6Q)*`=GPX*3=NvO zuaNfs5IBvvLKiO+2q;V-{!9&g+A0?lTTh$aPuKX^;108i*UK!H>GF@(3gdaogX7E= zznu>5Ob;9R`b8KYKXNav#sqUl3QKmF(T=YEP<6&jKUElJ& z?;>pf@cD5RrFk5tk%$aS;(gKxNBI3cT{S|m$t2iAtA*TPm)M8h@mv;O6B^!lFN#i3q6@LHu$$&Ev619oSwHDUCALvicLI9~c2%Kf ziHh6=6>;OT@v^wP2_VAw(iC9dq z$SJ$51iw)IgumxxU=F*apY?wz(2wXuU<+kVicY|stT*Rzopy~Dpd7B8vR*{sOyg0lChzvePvi)myA1TGX zKCsTV=T0n{k_0smi4Z{_n2QY&-MN-@V)ewH+uhU>Dq8J>3uElkpgue}7r*zeSc;_O z4$H05FA8nT$nk~W&|F^_MaZ81rbWXxdYdfxxpO#$j}tRNkVghf98o89bTxA?GV2*>(a4F!2fwZE6NvbA!5uG8rXB0il+E&|OOU%$USu(OGkgJjJ zAS~FoKM9eIi75k#aJL`pU3s^HW_p_%Q`x~f4O2?AB>w5V!@?bUfq6!yJ24#E*T3%2 z&t{@5x%j)D$WJ9yIaV32F^VIh{@($ZLqXhaJ(_Fqlww6vKp|FnExMf^dh%gyq3Q?{ z)Mt~;=~!!BwvXGDuK~m6qO%ERhctfj9%0YeTrvJ+s;smA)cU$!ft%iJe{5C8p>drj zry)h+e{(neh-t|XNBK_GRd8Jm)z1)BI^g~wLXCCrgbf`fWwctb{T!AmdtmZYfK=^3 z(#L&VCRoQY8t6zk{FG2_!2++Zl9%4lgovb)sLN5h>Jstz40x@2=Qa-Ac1!cqd zBtq_sH#qFjqf=U5MAYpI`6Q*eJnyRUyx znQCE8-tJ>%AG^mdibhtBZYX^`9H}ZqLB;2?0xHWXRRGaB0Uba_MiP0I14iQ#^`CVf;0x^)_Bc_$OhZIf@oeJ%CqH{ z0u1qsm!K_01rNBg$d$$SK?E~y9p631J&pgQ0PHH#BoUu};2o8JvgX?yewObm*#t=+ zE(L6Qp0!IgI_E@m5NhSmCcd{{!=VE`w%&Qqot8K3a{ZI<&C$R|sB%xxp z;30M-V4{06^?&{VpkJ~L&00T}vcWAgjY6>TK8hyBdMoG?1$RJe&P>A4TdEF?Ne!m@ z;|FAJ)DYneh}C%rjvQ|RTL}s{xIn5~NdAdk89}nia(|KXP57OFNSw#J?f+&C;S!ko zK6U&K*icu@f~zm$)!_U8p(qa=X?)!(TYUxABxz58ckfG3SD_%TYAq1C6!CHPDtH^+ zhD8f*-9+U?qo`UEOhD;e0y#fxJp*VDA2`p|v}G#yU@7hU$E7#`^#ocuLDD-Rwzr=- z7>uyca*;cU_fY3VylF*~a<<}6pemkM4?fsS`dbc-e|==%@mDtRwFpizSSPA|8c}SU zX`HfWYAt%-w;$1_vq{1pkw_`j%=Bkxy>$w|tP|es7t)42>~Vh9$>_MeJ!{>zqgvg) z<`qu<)ys?R?&Shp(!=|ztJm-UJd-+}(m>R+kRuH=>_0$7p9w0xT2;fN`hj44>^#lJK94?^B>+9KE9(Y84YDkx((y6|Ela^OiS-!D+r^r)AH74R{Oo ztped6;OwKm2l(xO+7S3p8;{`QCE9g#gy^nk z!;2LH&5zhmwkZE3)e34UPjOOpI1rP!=^7=9OnL;8WyaNw!5P)YX3*+?(%m^1D9fe3 z1V+j%BxOEM`+Pv#FXS8_>;ltuTgEHqHKjEsJ8V8E86qgt#Tm9}^a#z(+3IIB*bw6r zR6N}hCg%3(#V_dNrVbN3K!9zhje_PmVbD@2K;3m=X1>1RL zBeNIhwv1XlQk(b8BrB#B*R16Gl&SreeM_pa&te1?B#R*17T;4@_-?N zIH_QBo$MmhRqcmzmH_cvqG`}$3I4(m-wWCAsW7yRj26D~F;aV+rX1a%P>zZ#D`__g zyY24EBW+9PtiWh3wpwA0HAc^><*=SJWce^xcdR1i_9`*EFQ;iGqY^XTd%;bE&U=dw z-bQd!Hp}$Pj3gLbJk`Q)x+}L|Yl;_urWLwZag>~myKdQ4aQPWc_xuvDrmU#@-}T@8 z`2SU2e%A7+8>@2*+>$io1H0F1Xiu%Zz>%DjW9p@47QXb^I4kIH513 z-blubQePcyvXQ+V%tfw4 zpW>|b(|@NF=_BFqYW+ndm@`lJ1|uVz$`Li>s>G5~Lc@aUij(A40hhA$#M=eOn+m_? z6=uMo6Rcjgodh6h96+hzLnwFuE8X&#=2%v&u4*Y14^MF5+X}-#TS?*XFb-zAV-4LR zHc3qgEalZFJ!nrio3A;%J4CHt8e(L?H+)Dk7_1(Rg@x7k3%QTVH`~nZYDZZ+6!|we4ztg*qTQI7)P~xu#OdbQ&V#Qi*C&ws_WMdGq&z%BlzD zcR*(!X@J9LjN$1=em}3DE60_n`O*k%u#Pi9k4VMaL~s>H-0bYq81=3AZN_j+E`_B` zXs}*<`ND_YKt@1aYeeVg02^U_8@&qW#`PKM<&vBsr$WsOQZvigtHtT2!BsOD#bh28 z4%VW0(NY$6tZ5AikNUtPw6C(;oyh4MXp%8p%>bg;bZRanc{J~XSMwbYp$sPvUnpFC zL{TKlUNcW%S+v9|-lPCta8;Bg`ilZ-@AdZ#?cnESyD%1%%Y!rg(R3RMe$%DBa@=2YAw9 zUxB9ga_4a6v5lg|5%6;3x0xn2hlH_cakxq3FWr)|exjAOpl??4mGGyWEA?t&N(*WE z19ZF#ik~{RVDRy}q#$f{YLAZF4Cj+8U>bj~E2($H)6z}Q?sF(0|52pfP%Pm5oT2A@e?C^R?Uy4 zVIX-8ttUjjJA2o+z?Y)Gu-qbcEhbLqRFsMHaAg)f6g8R>jK1Hx;>d`oF)TB?Ks{2O zD==C|TPA!EQvoH#U5#pk6z9?K$r2FReVRYrCgBHHwMz21`6X5|Z4_sdt9%@s?5Ys2 zbY`3xAsM2gD80k|^@K_WxCwmhIqLrR0Y({gtAWjNHUHk!rMHgXK@eUTDTZqo zBz=okodV1q0MPME?(>JK#o4;9$*1#ds3$ZS?}8JRnP5@kwJ)A9+Bt19Sb6e0oKG6X z)uUkd>?^n`V|D=kK(2ozxrK&K`U3RkzSW29_k?F&|sq^GN zZ40salg$$DUl%LoVAp;d1>b60C>)k6r62wpt1jj?FB+Mbanrc45lQ&OA{C$Fsp^ikGlM9v*QS_(4;bzIe7a-Xv53chHUA0p zl1#5DnKzsL<#fd8+2q5O^672JF~_Lmw$1c~P$*_ayJRa&ab5gNwTDrnpkqz!&gQ6ny&brDPZ{LhU!6l^mOwcMmiu_}%91X>K zed-+ySK|Ze!u#F@ItBmxI@p^)4y~A&0Ft4I@stQlGNUt+5xh=I;xX!%* zRI$9jgDddtP5)s@|KkJj0h{?Uz)RSFXWi5*co!gWpPL)y*$T?49K^pDqa1a6`W?P* zZ4zvAz-1~8s8LEg6RC1BE|3M6ggNnZQJQ~U8--p0$~f1-MI`nnuoHL%hvWZV0+<}K z>L_9ckKH4EZ{4^p<=_+-dl%|o+^e<(b@_{bTzo9su>J1B(>+Mt;XSy<4{2A5*U*Ha zKckt`Vn&cB%itGDNmF8~Rg#!j1E3f&tZz+Qtiz~(sCT7LnWygT$5=Cm2+5qM@|-f9 zV?^rT{Mq!I>`tg>eFNNl9DVqZZhkbDdWwA(`I!gZeI2Sd?sIKD@j^@_6Y|GQqv~o( zfN?9(H1J%w)SWbBF7pvj!!fr(TT*}5{=xXt8cx1u&IBIs(fr2iVOg}B0G-OsI1$^` zfA^{=j*OaI0giLycKKdGoHV4DBChpwe2uz%sq9ti_=xnMuPZ!Xy(9Wqhvv0(WwPLK zX$o(t~DqtBHfEkq9C5N043D`=)MG z>TdraKKSd4Ai8b*1}TrbLk224DUt&EJw>`98}&EXU0$eKe+Df-2h0dh`%jBSr}dJp zq|g*k`;*Tca{6w74H3&w1Uj1?smR?X!XeaWtGXQW-pXFzLaE&Z!@;A^QGst}ZsJ}J zLVaOke*RyQIr4>j1|)wH^}kle@TR_Dy4gaVa58l{ed*16E8u9GQW`!+(ztcq{J?e< zUXLY#{l)v*Q$G(IiQC%Dh0|2%=HPYwJrbpVuQV-tjD)ceIFz$B63R%#H-)dG`%Ut_ zCH@t*lTzKL2C-MS+0G-A-=*~EjoR*Pn*(p~?dau(Hu;OJ0?aU^447J6N{u_G!}ApJ zMcebKZn&Ul1k||(0Bgr5u~I(5#kVw6sMEva28ie3@)6mlZH^Gl+Ch55S%-Gzl}C)7ebhO4 z5VC94Z13+z5iSqHx}IH{#$NAy`%!MSZ^ZReyS3Ee6+9ZPd;~Nn%IJty%Q?FUBCNJA z?WD213#>m}k$;3n;91lW<#!>E+{`xSf5PF?a-EZ+midcBK(pb)80lc~D<6sAl{b@No(K|8nI4(Dyd8*wZ@` zD8vKRS%RNp-g{hLFDoH8AyP-%3=|}|WHTql#5iQvGSZd%I8iog8$y6pFe$S4;}n)) z4cvSLsPI3q7eOrd7BH9eY5ShR7(C ze&q0L1W%grDm#7NVv3@Cg|L9ESODaj*U|Yp359WwH=)h#4}0QKm%scNSRWs>2qFYb z(4?KXQ{Wt&bNfO{<}aN37oEqD=%d!qV1+0*xMVkwEN#K#v|87ju72Wud?-tvc!S>yaQWMREE)t_^)cXF{Up>VM5(}#K5ijz z7*<~Hk5Po$jSa%JiO{YnN=?~=w-F_6*F$Dd^OF$SpVmdc{9`%(7I0A6mcOn}rleS)!sVpt zuNkYhA)BDdL~@{UgQOHk%q~6?MetgYNx~V5*ZjCNKf?TGQCA|XDho|T@Bhh3TT&_4 zJVHvvonA^=Oih+p+@z@HY(__4T`rzNJkwBGAzw_< zNC}lkfgf!0FR$q5JoQPy?cQz6!hL4eXm$c|SJ}K|M5#Z+;`H0Z*T{A6)Nl z?Z)aUsoL?nw^U}qVx?oad3TOa|N1ts$=!MoVRmaJ+qpF1%J5jzim}`QeMg?D@d_(0 zu-bJnG6#@A&N{EYJ6LR1VuWrCwBZm{IdJT4zZ~h+=p+Q9SM2T=Hu$00Vm|IQmQdNr z+JBcbVGengcqk3!@m#BWA7GxNoSpf#W2BKN8ZwEr3(c@@v-YqF z(|$O<`+SzF&};FQs&JC!lm0l=@2`zw`Btlvn2JD!@EH0eRR1PUrHY_-4I6!M1D6+3 zT11hY=!TPuHrqsLS!xkfd3m7R$a*qmy9ZGkEWd8-0Y>Wdt-@bp9@JwwYloi42EwK= zwU8Ki&Eg%oFqyay)1S(sAs<0YZEH*ME4HsWe-`{zx5uI9-Fga|+I+ecpqG(Db z)ut1b`Xe-_50SXWoi_>6>E#dJ<2{rY{aKYpH?W@P81%`p109RUGHzY_;!_Q`x6`#% zTJvqwQEIsTE;p_ahEjh7pVgE@Yd|EtL-Cb5vH%kNoxo%`zLYX_vMuF(wQc?JwlbYS%`cGz-T9JA>|rSCrEd4vh%f)GcQ zd@>~^5@g!*$rl=*yA>MvGg%f*B{DBLz`FG#ferDCM|Nx0k@!5SSBU;cnyQZ7?qGzJ zM5w7PJK(T8!sv?=oa7hk7FqhKJ@9?>VHpz@Mha%x(pA@7e4Y(M9IKlChu%=jU?}oL zLlG@?D3o!;@-+fmG`TZ0p97I8Vv2n6{ouLcb-n7Ws-e4OV5g;RYGzo1u5 zt0`j#n8E4rb7a^AWiVV`=s#js8`A!Bc>%MI05Di@8Z660zV7z$YkLO%H`}}aZ?==( z0*b1mAKd{U@J%5U)T!HrquE*_3?K~)u#W;GsK#9I4ofccdB1~~sGmh=CGw85ExF5w~ADVX`bL`fL%{`gNsgz(Yf zLn86#BS0I>a|iO1X@mLv#A+{_d^H?-5#a`(^VT?La=|f23!Z-&p@SL6i~nIV_&=g_ z{*NfH&!kdG&Oo6V7&dG7QHxlT{%tq{)yGl?LhYmVRo%0VCQs`$eRzf#mVdi7 zhuCn#tZ^n#vID`*Xlm9@%=)vm_VEJ7bGGvxfJH-461ZPI@X>VbpJ>rR_U$jkVUL6# zZo>s;27n=!P2E6C|5o=@t)`dee5X*vHl^;SITq2>sm&NkmR@z7US!Y#f&V~qccKFH zNuax1gym5aIx5#QiYYqh@zh!4$A9OboM11Lh$V%xZOpMhUX2OR4MeO_9B(_O4?Fnx zHG`=wY<@zs?*&FI(i&R~HUKa|Vg_b%JFX4Jnj`x3DEf#x#__r zz;HT0c4H|N+m8d3rHA0X<$2C9DT5tVF4Z9wEWe?ZVcpl#7!$N|0L%kvB`^ zCNg_{-iADSz+Ums@#bhEGt&=aLP$hK036bVvi`AtrAm!wEK5iDZLo%yR9*fXofulB zpfI-73a?-7zSL&qO1If$#J;3s0M#iU`&#aefeYEI!_gVO46;~&YitowNbZJG(YhTk z?#!J;$B9l3pCaRe=P#Y>{hF-ZTp3F@UP?pRBkr6^B?r!&$^>s#Ffb)*S!K`AU-5rIyxk$TyP)Zi(r zpR%k2_wpzcC^`#|8NM~RPT&}G^L6%OJlsh)+ii~fe*QkySn&D3&7Mb4SN}+Jq+IE#d6SKM@B9_=(sfu^HjyLUaA4_e-kpFoA!x*8 z9{XXr1d6_)IhTW#RF$(}u$xgiXzfFgcdd0NFmZegtX&FJ(=6-)RvQ9f$VfuZcRV2R z!_@X1u#^h817JkJ*th1;jgOCR^HlBU?r4?34Ik*{C$3HPO*fwxB$5yL=Y^=b`9D{g z1{78hwr4ogrO3;#aN=i7^$FZFI(>#Epo-UY6or`UOqx~<=u?+a9q zy8q*Biy{7&-35yMwJn;@pQK>3j^x|yjYSYS&-xOmpt|n|yle|mnR8c=f&U&Ln@$QP zyN=`=dpNvAnwRG}DoPDbzg|oBSZS-jU@r`=_8(y*oK0aD2RerlZWq?1-o7l>|l;tc66Vt+AY;ko;1ZFq5FTbT|G z*%&Bf?(~blcrioIG8%8K1uX{WQD{0a1RRWc%MOhJ-UQ?z5QY}nH=3LY7omm`=NhDV zsRK|A|N8?}+F565VdkvdMNlX5IA{VSc|MmRcWU76?sf9BIIkRKJqpH{C80sgcKN2wj50{Uz&$7v@Z7 zLn|Q!XzSon+wZ855%pTQ#;es)H&KrAAD*%D{HIZ7ZZ@u#0z?0z5eVyp441w^{x{Yv z=wLnO{%dX+0>1>(_UAK_6HfkQ*h_YV>NNdG4=35i#7f4=7A7I904v!yA1fSi$&Puz;3=KA8$&U#jG5vb11r$HMR-kEpleS-&+NTY7TER<`H1Hyz zy2YT1ix<-~wrG4X^ZIJ2!Qof!HarxMM92gTMXbt>t+z)7I>8IS4CdZPiRcb}Z$ah> zfBpmhfjdx0IA(Io+HZJJwGF6t$et9t(8VcStnlD59T&(ssWu(Kb^SC_x}>@oR`$R# z1`zT|*WA}mSe*a_!#nT;TU4A8^=fO;$1?H=dhp1B)}mKBionD_0DO z^wv|$v{f^YdkETEzjyyH=Rky0&+Yf>O3ZS6@DoX7<(t0r%TaYVF6|Wd1yFx)?}fSx z1}y2F@5@H-a;K)))MZShA7IwH{;DwI3DBYSK3AQ`tK+^;_2o9IiJH#n{qP$l?NFhA zn>>s#$MOgd32`}K@1OWhT2_R$UUL60huc63?=6Gp{4Pn?^2E2Kw?o5>Z-O8 z__1Sz(Wk_(a`Q>MXa8d0=<^sglMc(yo71*Uo1GU4k8k%Qgq~xGd%V3w?gu|1S?9+4 zCd7%8Q@h2#CmC|m3oj&{)PlD72qG&Bw~NuA=KeZ|*{aTv#nrf2F#uoa+|c*Wyk14F z_5`laM4go#%9p5D7k~0x1*|x_4Xb!+7^o|uJs($I{R?mF_rr^Pnv}Jz5r?N7lP__` zwUyv<<@wU0GSyeU;HlPZv1IW3e^OiR4P$&s-@cSfAZ6Tn_|3*A&DozV&Jjdu)fC8G zNA7jpG$+6Rn{BBM)lWi z2VodL9he999IFAzEc*?vBua%Fq~r8WGWHLwQ$ssA*xx^9(G<7mMA1j|5K}Juf@$A= zG}Hrrl0DzUrQ)@><*(6NiXx^CJ=Q7JCtV?c0kcA@f0;|2-!rb2K4w+#6!N(baU$2C zHhL?jR%Bym@y8WvBEI2W=|5S*F*`>&Wc;8Qeco!`@__)J?F3_F_^s+h_?_wg0It0` z#VUSM(Fr;^?jy50kQ}PdLb*!@!C15iIR0=^g5`89L4y&$eJd!?h4MYWT6c#Jls9KK zr_i-~-&oGh5Z~UYOA9w0ENu|(INWzWW_5AZikr_0?nNkP$r!i-FejS)vPxd<6&=d#&M4h#k{fEc%d_pO%F6%V z>9yPNtx8waMsLIjj1oY6il2N01mb^6Q$8y%`dF1Mk_IiK@7A@8J&eta47p`PKYGv1 zL}m>cbY0xprc4v(sI>YLhG#|HNs z9pfk->0d>c$oLDg!lR?{tsOLzv39(^lk?`w2+RD+5VHG>iTI3bahhGOZ$4xm(ptoR z#|{HRPOA!}tDaBfdO_HP?jp3Vn(}d``QpF`^NNDeMRuwI%I-+m1N#1jJZto!?ox$| z03*C7UbDC-o?$6-TR~|&S7a?i5~ND~l^sn@k3Q+$-C(s3eMGBL*|b?wXwn%4vcrM8 zOTJFcFe!Pty)Ey+PuIkiM#x->vR-a>4Q{Cop1g0M;gR^?>0ARO+uPVYLljoKF24|T^phN-t-rtd0XEIJP(#J`j{;dwAJr3=KuQ0sQ`9owM?z~9*q+?* z@8c5P79l%{<kE4zK(! zwgIX(Ukm(?py2!?z_CExv=|K;F+T1vUbfTB7FOuiqZw|QcN)qbLi6XeDw%b7ku z(05E6NoVvm^O^1#3l;n8GWzVCr>$_j(nV3=RlQQM6=E`V3y{|V(=XW)x9G(7`pNQL zJEMDo7}%@OVq*0tr*z)!s#oX1*P%!=kD2J|J*3pfOpL%~eme^f25wfz(EzA?-EBvc zJpH#;Gi3_??}GaGmGJd%e^Y0$M&oEXx-DkAQ9{@5Dp@UcTklCcnvISwZ`4He$8R=3#+9esVH= zkro|OwQkTbz4))1w};TqhE6_jK2ezgub3@PEn#QxolL^1QMzNGau&L=+MzK zERBv(ZfH@(=#oWi$h@gpxVWK;bw;fva{Am{-wEHyW5PbLRHIjto57@ezMy7UX}V*f z-{?j*WfsSrlj69j|B_)M)gNOs{mmXmMfFXw4*iI)Qg^m+hJX0bSWV~-#Z_^NO3+7k zF4P8VK%&--36?Ds?uh466LX=?N|`x_jfHp!OQzriQ@WYGeI$6x_>B8rgwGXmQAodq zZc?Q2O;#IlM+)uV6B|E%>w*$w2nI)HT%}$t(W%r82Tfe^cMgQH{RW(}J=p09OXSDY zBZ&0k8>Aoe=x!5Yu1tiO9`~MDWfJhV0=FSP`w@Im=_OO}c|E`)%ltF&YdUgT@*6oN zO%gXnWZxAY2lp_~*oaJjes4rFkv_@H8P@9GAR%NR?HxOhf|LwyQfL?j<-T=A{`YNX z_2_A(G*bUq(aXAy8u0 z?~kASF~yjI?^jm8S!SJ}3~q=pvv~g0PNIfj;H=zsqh++%15kXA_m7`A5Hqs(1&gi zD2IB$bf`+EDPb~GKkyl~ODAX?W7ZHSwh=%y)v;ut^yzF$#52rHg$eH1fpjcO@tWh_SVUM6`*V{o3T`7qjL#9*m^NqmLrTOyj z+7kg|*>rBeo#oH>u?g;Pw9UH>P^Q>x0EhL05@ad;_#8lGV}TZ+0gGtTKSQxyLX%z2 z8J=tY@0=|1U$n11Vq|v?msh(L410&M{SE;`-4ty`S^wa(Y{sqnSP=ie#)f6M%5>}O zo27YSLW}rq+BJq<&BonuDPsw7Ol>Zy6j96stEcS}!!qmBq9QF1I&5fd4q3Ko0J+Z< zSv?<>1$sddrKl~-#8MB?sqJ+$v_q7hy>j(3AU)R&@m0EK9Q8Bx^Dr@grkqNvkL?zf zutArHrLt4XuM_19!|=&8k6}<-`9y&0o6w?}&nG^CKY@mK#&qv!67NxB#zdj!5V<eaI{W)W0R>x{SYN`Tb^m5JC?;!U`KV`ROb(h(wQKnM`RU zCOV$DWXl=pt4~?OGfJ2(*2D9;3bljlxm%t29}lyIjjv%9?Y&X^`#I*x*O@r!Ntnvf8Qs2VU1dgYU@{&|lU2WQ z1mUbWQxtpuY@|E^>s9NQ{1;stLsngOqM%tJQ?Q$poy^f&T0Ubf)-~0pEleqd2|;WY zOJbjx4q|G}NHM&Laemu_k>Rl2nCeD~{3~5>k$H0;vxRcKeZO)PGNQPT2PbZBwqAw5 z@-83e)k9lpljy3HoaZs%vVi)hY^8r*Mw)7zqIaINqV(iX6aNCS1>CsPMXP4e3XLLb zp`zb5A4&oij;0@O4#{g}KwK!bPLC-ry9xiY#OHIk2XVjmmM8!An+C0|b4pv=NgJ9I zu%tb6d1N@@(q+)wmG^*Yi5+#<02Yz)ER30(Xq?LaqmONu;S@hP+{yFm00)kURY6t zv0xYWQ#?#_LYJ+Tgxd6AJ7ev$Ad^(NOQNPvx?AmyDh01vPgSPj1ngqw0epmpJ6zYF z;eHZ7!`*Z@T`g5WhE-VS-L&}D9%ivElI#H*Cb?*jhQwNe@Jr2NXD6S6UXszAtpr3I z_DvbK`YkL{CB|N@KD{Q_LNXKUhy_}brgftNhs;`S2ZTV@;Tn&V0s58$N(f#$*ZqQL zSx^MKf66cm_P4ojH(nnfo|j@R!(ZbWac!8-5(`s${!kGDMUh;>UuFW8#(E!122$PD z^8UX76A$e04%W(YxI2aaOrptolEB)<%bxFRuC9^-99BYHVl7;HFR>33A3ZwH7ir-t zP@g8^x|*(Aj<|zG)0Ezb7aIHMm-z8zwR~8_UuYgxH$FTBDKO*&#|4<>`1Y1Y5_eJ7 z&Uq#snfq>p2`W^zk6C|yk@$sTz;eO$O%QQw!)j*krRNJoTJ$WFTmeOSlRsC3B0tZj z(G8Lc5Neq;1?+A;uj`Uq>sD=-9!3dWRW8Iy1<)>si(ru~6@+RMfxG`UKJz3;(jOss z(&mR1Tg@ISj%w{Xz}8SlfGE$|bkyUKqF51a<>^=HYiEdCIE zay$G*w*0jj{H1OPe^t6?wc(G@J^!n_;zL!>XByt-c5O}Qm(R=zYyp;KLTxP6C(xa3 zn@_HkIuB7wElVecYdJ5YQYyw~liLwM+87{p1d!#fI*!kQn(W1CwYpiOIUETEKKn@& z0hRyzYHi7^1=N2Er-P&HEDd2wai-cj@ap?~%?o!pZ2sA~l`Dzp)wJ5sAJR$4=gphc z_E}+{X1Ta8S$AOY7_e45``{*qy~rzEmYB>xPEcry7;Fe#-SiX=1>a=!ob`A|^c2eX z+k6=u)9lw)5lc_1H+M}o>3pl3XpO^-t$Uhn1%^s|;*o#A@Ll3kYWmbR9n;tB%4l3) zdmquYP1jl)tr0h%B&|%6B;lu{YLw>+=K@$lkLLgA^QaY-Q z2*YYtz94^eR*}2B1tesZAXcH}=mnj^IGinJygzXv)F}Lx7J2f`w+-h6Se>mhzUjec z2vTjNBLUz9BPe-^=XbN)ndR*bx%Tz?+SkO|$BWXNxAGD5mZ{FZj{{YuCRSVUCaH<# zv^~iKD)R`d~RW<92@pkUj>8i~-VC=!3^JjEk=#bd*9Y z6Gcz17NW7U!gs@roh9Cn{(WZ0(JXsvX)GYRr5T1r5joBAbkC-aq2JTkIr}BeS6LAI z$+B@a=qDo!0X5BIMldyiD0*O6Q_u9KCLxCB>PBGQlPq4u3SU|2xh9BWsc;5aR^xdS$rzRj0hN0VynphplGi9);#Z4ehWBb&IUY^6{Mm+E^o z&aqL7-zzKT_4Ige55CqERUfB^nYgGsI%at?|B`-L5@LRFli}_Cn*Jm`X|n{{P}R!Y zYmozh@Yu!)QBCPmI%lbH+SV*Ntcd!htD%1c#31n$A;k_w#BQkF%1RX2IRfj}d@cqx zAyp?aoh9k~BFtt5@&Um8fMhJ67f~d_vz`Fu^9!Kpc}W+KJ6y|ih4k1GHATq$L+UZ< z>nElM)b0lpvv!<0fsI|beq(IWb})!(n4?dVc*NwxlLot=qdA6cIK%>3;noWYI)Z=c zt_8atF?7c@Ocyj7gh&UvEV-&SE_9S@cdfR1Q){8k8tuu2-M&!$)ig^Y;H<_R0Ccmo zZ#t3!#Z02XP`DK5(N-ppFzMx@tiJ{hwOw%fypyx{RXb}j%@%1uxwCmLTq9c`TsQtTUuQl>~a8UV-A1LFXFp`&@dd*mILyG5a-l(33_1y{;Y_3CEs)S zVGt}bCT?mSgtJ-AS2`<#3iKDkY_jx=kVlwkb;d;@>?3nnfZu`WIl!CqDf5x~ODAql zF`Tfd)yBSIRA`ln-DnlW0R!3AnNt)CV17s`pVk!^C0X8s5*|;Jw5KYCuP983a zPG)nxb&5Ho0$*P<5R??LZGuD`gCuE2lIUIaax^6`orN$m*s3FtYAo%5731Y&6rUr( z?pJ~pVH2}uK&5GEDcs>ak6?d!9*XZ=2x@nLJ_^qf^(LQV3Nx%PH7B()II5Lqi0@#8 zKDpT{;c}DgKI&3@EK+;Kun9fkaWt1`rsb3|a&%R5EwzkX13RUuy6{9RL@OZ*q)fwW zsr!jh!2T2Fjl9FJzD4)OW(*w`XGw}CE1|vU7FS=0QJ(yu9K)eEkKcbXauGtsoTQL_QyE*}(@*J=nR5Pw6@7%~=SE0{a7Ec%T+q4+<2wxA+l{rWd#%GMHORa1) zlk$B0S{?=_zXYT}`>-dvgWoO2GC=VnIZr95VYryvSxN@@%B{cDi;MXyRI58_QfiL^ zRa_&M@2ry0qjvu69Xp^>$4KlPzaqzf?wl$gA9%g0Z zhy9n-$|lBU-i`n=#sKMKfXEmiZSdxyA}>_@cg%7e_YE}5alC&w*8ZcuEcPF=wf`)! z|EOF0&({94wf}7GKU@3H*8a0m_Me7oxPh+*pGY+v_^`))y_Q#eQkv_1TvHUGV|9Kn zsq+an1VdGz|KrFb)Q39Uvpz_@e}vaAFb5a4cChS>(%#a{QWn z+!XAkZ#MH=*Vy*Gb7OkT;cxwj*LoH0Q5;M+HR?N-=*UT{3P66w*xZQayuEXM9+KU}u8Z7YVM z84h8Nt6NMrRl}j09|Y8MG=u3ZxW8>>IGUSb`o3v0%QrpUWr5*R&(i~)PAMTCJoY>0 zm0yy84^)B2{BH{}KTF8GY0=~VZSUxsn}%WiDf@o|6n`a7AK|z?!NCD{JiCblJwn>+ z#~2U$?^w1Y+iAC!ZY}FBerYYsi5<(5r6|(LmigR)|DHCw>c3Emr-PUZE-3I{pn!)| z3>TPyIECVw3Eu26g13u6@)8BFpIdf!0FpQaNgjffKOmi50wFUK)m&M&`u~J8DWc^U zF7bcdvswB3_9~L=p=LtEh`N|B$kHRTAhp&&>QeV5Df^;?f*I+2B#rSqwz(|kYyYt{ zCta^rFC#dk~4!wL84tut%6ZJSP?sT z0Bf~`m!KMKualDF6bG7>cKnGmaT@?OqseVJkO~AyYoLh)!p>&4hN4W9^!{^=@58YS& z3)7kc9X&#@ir$DIXO$t{RD8h%FjFM%nRKgt)xQ>c>79}?+)d`WZZ}iBp5-wvtO|cV z@q1KYa)(#EuVb>=Tx60S(C^qJHwyBb3o@Ao9{GS3VJt~5I4``tR7feQ5X?Q(G2x^T zJj=)`>$+bP=xvcK^DRHTK&A9 z-EI~Jvx9w|8oPclimisvu=#!19_Lt!e<}XuTixEyVYh!_znwp0 z7P`G@zJoUU1KdXg)cW+mxS|^v$}eX?6$vKzW_HnkVB)@yDHXI_?B<@P4l~kNhyEb|mAt z>DY!qHWJmVph?BY4(C&B4kLf-P12A%uAr7$MNkG%bV6OsyG-QsPOEA*w{O-Fcj=w- zKj~$OfhV2E9dal?Xk=j6fT$3D(o3?{rZso@=`{lnfukcp@GM^MW_7Br&6wX=XH$S9 z3^SJ_4{Pzl=`^CuoTC>|!$w3|=RaJoYwx&f`=|F>D?lh7l`WkP>Nm|B^euPx&- zw@0k>?4@qM{4zrQN2^yWk7dTz{IQzWo3$B?^DXpkW_}-=%Iu2;E|PI z>v>bnH^KU4cvMx*9NqLBt1I`W-_EzM{b4H{f}P)^^+mqXQFJoZwe3_0upN)s^A=W% zITE~C{fZKUXC2JqgM@!@sv6=to?2?u>`b9u9^=N|Yaw0Z*>K?+s%rT~MgbXy>1V zDT?U1Tu_Qhd@PrlQJL=bne2~eG?b?(T4|9}mQ9V|mQJ4%Ik_lPk;1$#?RlVJ`Rbvx*Yyr{WIBfLO z7QJu(FAVo?%@U1{FZ=dC;nMDZDhQLgfc;OI820};iajJ`tX^A3dh;^_Bap3(H}Kis WXedJ&%20;#GRt3#z}4&k_yPb>luU=*G!|m_-E53D zS*U(~T?P=5ji*Hom3SHxKX!c4`{t_oEp%lnmRznrzkG&D3~2Ock1@p(hCzblsF8vcRF*d* zwNU9`@OwGs;~Rk8>!$x}CzK1TUonPca}_(0E~e%nx{0naDvB;<(q~d63*MJt_6NEZ zrw??u(2g&MlpFCmV)t9fH0BS?9RU$_CN}t87Tbh2VgQh`M|ztsjNM*#jt)(~C4lPw z=>Hvtf#5hb^%{RTkQH`omMYBZ=+F-g8!%VyVdFIwN%u-t5JC8+7Woze#bMY9YyjyJr>&h%yuj<1zDt4x;zAFVX!6h4Vq^zZj#(u#I&&{j#SAZfzY%oKg?5` zS#8w61Ese8amCt0+Qba$c3)YJagV|J%EuRwR{T3C(h8{|SYbZ~bmsl;3m|N#U2iRM zvBzM76wWN6`s=MSrVD>cI(?WFQ~W?o83I=JvN<>XaQNoTu~tf`rhdvNzJRJN+l%dQ++7MDJ({4hF@Y*m%NRkDJf)yL90nV&#fiCwLM_VJU&qj6afytXa@#Xo)I+ z@9j3-&FBd1lwfQHnkA?x{uK8e$8C9?Ez+eNTIsDKzKCO@sVMR5kJgz zy2k9HZHX2^=RyLE(8Zm2sQO3|zXUa8e(aElMX?|Emp717COOd+6kj+)NDl5-sC`!n z%Ve6;ghFPdZ{cR<^Ez6KPeoECp&Sr$p=0IK2+5*TWbKA{Ki-_y2rJWv6Uq6fIor-42Ku3VaG%+fb55B}c(WN!kb~m| z`Y8f5DB2$mt7-h|Z4J$a5sbgVOrX)KF$ckEtD05#z>fHqX7*)i) zUBL{AXk3r`qhKZd(^Q-bD+RTz$9?%@odmF8@4Z~Dyn}*>Q4|@!UiW71J0z?){V6+C zqt%u9gznysIoiteL1?14yC52#H`?$YU<(&vS5h}FU=(zNm|yP1@~J?0(BzLedXai( zv~Wg>XW->xUNlLwcHEC^GV(sb&xow_sOUbEyG+Zk;>NVQoy2d(UaoRYB!Yt)M=l{a z_V}TTaK>woo1xpJ)Z~Z^0JaHs)JM3_c89kAXwmK{ywBP8eKC#Vl&j|F_XBN8fImH* zY^n3~<3p#5sQsUE5;dvrdr8~!0?&(3VvtEqZw_(3c?h>d7pa82Mf`h9&HmLZ#M;k- zAZXl(|AABEByz`{;?hyU7K$*Mp1Y}ATQxGUZU1S9`T;iAN)uk0q$caEvXj!2XXk57 z7rz_axQGc%fKz9H@G#zonk0tjJz&*r^6sZW4uUm}osl)(-ycB?cODxK{6B+@U!h>f zdbW~-Mz&yRb+>KL_r)$=ruO>lg+JU5jY{11;E9bBTqR}+3^;M`P!tX~(xdp>{pK0P zFDYJQzixYTJ(}D;oc)z%sw#v%S?Z2!w1`>9W9PR-B1(DQpYwRmmP!UK+|Xt3;V*8&(YOL7S#!vbWVH5+AbO zgSmmZM=F;5EpZZmaV$2Kuk6eZGZO{x%hIS=GgLtXEH~9RJh!-r^Y;`xCK9#I*v{Wx zq95jsGmG?5VL0S?nlq5fBHaMjuY=F<2dqU0k=%|0-p^z>7Ix!W_JGLNEG(y`Y&>EA zXc3_n!JAWoK?Tf$2l;A%m-3_Pc&JS|`bpTc+#rZqutkzUKHd znAZ@N$YOE&;`ipJE|4$v6E_m?wib6EE_r%USuPdEwCGTyFh%PtrY}JByy2B&m1b-( z*fzjtS6-D`^XezaCOe*Aku^5((NBrj?cp@sRPOtTNV;f=UH3s~PiT=f^i*x(6-y>E z9@Pd;_rDQjh)(Z7$aWt2%Pqm&o0zBni$Lwt*0N#N@%UpPGV8Kc?LGYZy@_HVCvkzU zpS^8=^S2q~-x;h3UxI)?KVuU)J(BYpG4M-u>PRjxkl1BZ9J0k6**(?lHL#aMtst?qd-8zF`JMToI{*iuy$9T>TmVw=X@5C5IUC~}lk<^p+ zc0c!f5DDw&x6dh+Dpon)ZLDAF{eC!pGDpF8)9DzF-QRplU1kWBzwdjVGTzot$IciK zBDrqp?4;{d+c91;pFZIITluR_o|+*HM?Sd0gfeKhH1beuF@TIEeTU8dXsp)eu=KkJ z8+H@+qdc?qBuh}ARKsavpVKKU*)5;T&YjHA-s+Md6Q#+5)mI#?H1Do4f`bwmGDF+T zynpAqSTjD7KLk3^uR0su-sPo$=@iI-d7jN}AA*j~s?@QzakN1XDCVzvoqXV7yApZX)d{#8 zQW9XCWn|;U@Ib zKn(2g*V|f5!#@6Axa+^+@r)`=ghr%4>6-Xm-b)#E7iB)C<2q!l*y#*y5=lG!yBZZA z|GifDZK4bour#zx?E3@TMuTS#YaT!>8|DUgMD})oDdZ__7{*grriA>pdCP?6C= zPVQMXG#}VC&+sp>PnvUCQc1OV8k0p&IqO1fWu>Pv3uw88U$OY#w@_NpV^Y}@#*OgQ zv;L^ie%(q=x6;W-|D@ct*!;J4!>pi`Mrj|0S1Umb!$>erTSZ&`l#5qw)K-s3-rnxh zy?+wky>L@<(QRAl@9HWnOa6@cKB2A#+a;#ZQ-C2<2U(;tAQd;^hzb<4pRQC%t%%l< z=ew_~ynsqE$j|Bu$v?Ab^-BjN*7!*^7)lx7YO!G35$4v?iQ!iB{F|9=QTbU^i8;qb z%w-uBJiqMansTU1*a^aHEggDj9^trk85YW(Fsx(nmnLWx+>EF#8f7=RaIZ2m@M7!5 z1Gj36tK>SPtz$&i38KyF=QW+__P>S4)bw<3u~Jwnm3-ROzi($*bS>0NoA2}0rg6K% zu5wjodTr#50QE|ch~R>79!+$|LfY7(^;NegDk%(}#*6RIVhNnP$cjU&^_{EdN_NR; zm5r+N)~fU>f!Jtee;Ks9exwe1k06wq0&VbRVi+S^|Ni1Asr)FMgu!YNSj1KQa@bm@ z#&v0a9e51W%E85#6m8h^j)eVb2_ba*Ly@Lb9!;?!M$05h^QlFT)7M?-o%3>>9Sjz7 zeZs`(OuXT(8&|>LP^t3kAg%w7tIxdH%%ef$-!Yq4#AP>_hmbr?Q>q3QnGr@D{(%?2d+8 zsG6TkUKjNGzQ0BEiD3jmkk5QiSU>Qb)O8DJACGp0uneHB>&`_XLgZ~1W>k{$BKS&r z?ZR+bcQ*~|r*CF>%I9n+{@C(enZv#0s#tdIs0kiO1C`>FZ!Z-pAxK#jr$pLbF|~o1 zz5QS5?4;;wbj@} zD>*^OPXM9rZ|ur*o?VS`810#?@{(jlle3Tg5*S0Q3J&IFymceKI;uhz8#lmB%Ra_e z$JM2y^KK9_wJ=Ov2bN3EgE^It51Uaz5$`RfUXMeYpja$!>am$RDO0f+k6wfHj4T&T z|AIZNTgXt&Uq}VLts}TDX?+eNw1A@_`F{~LK`5;@S~&&wMGzW5lWqJP_p^NN>5t?dl|9pZ@_IV!b^%^?19cfQCGwc4 z>6?0y0Ajxd{B4SH4d6q0!Y52xK(1RWGNRS0XoRQkt-~2>ow`6kIn5g$wh7DiN($9Z zIO-q|)r$u^!9YETlapYW%Vdl;KY>0yR%c7ia-t&teaQXL!vrh4tMtIUV1Az{%fPQ9 z|FZdGmy(ijV*zXFC;KIr&D3CkPKYr2)ZC#Q(3M%Br@jQPg4h?XY zk}UW=I&BHR|X^-_sXFr@>`TNyAmTnE&ye!m^?__YncWk+BmyzWHmeQrWS zCd`odO3{_s9m>yAyXYKPwrpU^^ie{wMOJ zklZA#SP7}$JXb?%Tsu_B4>WTaa3!YX_@@(?Bm=d7@Ly?SCYGpU(U60u;pX&(8v;mK zgqF1Vx_=&#x{(?OF31AK+{kb%k2#{@!Rs1kPf@2_Gltw)7wDp9YXP?&ilNf>R z7f^fPZ#Pk%T=LmaLS7Axl~a*-jPg)y;9DT}a4=8%R6{emY(y47o1|7bB;HdcrmZ`i z{Edd~qjsXFghaFJ#7j<%8=>S^s`zY4E_5XCX@&8cR!UHlch+!?C|=3LjMFA zGrhcPMR^EBIpAp}CDwZJoBrN-SJ2%0zeIrn`Ak4zvcC(4V?~2-jgmkzD`{z(6>0nJ zM@w~7<}x;*f=ZC5Kf(wxx%Q7WAx37giRrppYr-ctlV5$^?mcy$j*EW6PSvq@GoP(` z!`Z=~&ACZByF}%llN?^r%g*E3W{^pKOTJf=NoFt0Wv;Po9SC~beu1zZr^_k`Y~AL> z8~T7cagDyYwqFsbgMl3u%)>_{Dfp5g%mjrJBv*s|}W@TxHMlic^0` z4t}fBa)GEXguXvdDg`Y-HvQfZ*hIB1!N0ly#F@$@>1~mhH(RLL2OlqN59J_0TAA$D z4Bsg$sMm1q6(Oiucq8nsnul-mJ+6N>To*eGKFym!^TKIkP;m=S9&&GF)fe(k%QMl+r8~OEcOthWr~LC#s&C;i z{@0>k2Fm&Z7}^Blyn{uT-FFZpUhHqnz{-&HC3v|>zVhOM{Xp(?PIsaTnh7ljV7Re0l4YDz-bc zyTr7;S;Sl(@Ky(DPb+Z1tc;01yU+bC-5dPmUFu%Jv&XP_$8=KPAsEPVpsDm*+wr_M z#Z&meFgI^UC}+@TIFB>)2X00x(f~(*uWHEY$Wjuxyj|`h(5cj#WG!SOT^a z14t(J4QCW)bTAEELYy1>s;OVk+wpp%3?{WTg(EMB`5JjnqV!fDqX>mMOe%UBJ!IF8 zI+3p3&$He0ymaj|vg#e@pjMP1tA+0|S>WR9qY>YKtN{OGg(LWXJ2>cFdFK0JH%dnx z_MQVhXBJ&AkYSa}OVE+JbMKY>vQGj&q>{KG59y{OKgPmm?V1JSUgQPuuXen&BxlH$ z1T0w^dvC(f+}ze?9E;8DvGv*;9`LuY1SnzbZ5az^l7Ywunl2>r8!*lhipj5 z7jJLp-d|AuY9&|L>^vq5++>?_gZlZ|1E1G)GTg6Th}FKQgY%XOhIcEC+71ZLiwPYCy+;{$8R*GJr^t?H z%gSv(OfwH=qFLAoCEichdX64se^h>}@Jg*Y>m>1Zzi{QBnD?ZGVAboO-G7CAP3VTJ zeNPsS#W-*T{77G*a-cud3IPWmVlh3YcvCo2JKn`ixHmWaW4>a}`GLKzv``OLgiH`5 zs9Ao`pBHcj*UVP_PSH2c@&YYSSmj^a&0>kmrPmmm-l_?Fdwm0ht-zc<@bsN!{zcDl zdXr1O6Wc=zRyonlK;L*&GX+{`V06>&NEe%d#iz@T(~)~J{EdLI9vBL+KBD=fONy>F z&QGt`u|&*qM0>ZtFu0;JMrt1rg(R)0mA()0Z~lPhh-&}6GD;f5qD(@Q%8XGikw`)U zI^4C8&)EaAH3PRXmb?7!cjw{E)Q=EONQAM|YBuSvP}9(BTF@V8UE6Lwy2S2VAyInb zQ5d!T7HD#`()u9}9k|DA9ri<>;VhMx@K9DP3*a6(&`hN@*=o7@SU!6rLaq=G?^E(s z6DDQ^HN1?_%&c6r-6Y@^Cz5f(tjx8u5&7d)C-#N3>pq-(Je?}bLLBo_%ehgNdZsQJ zZ<81VAm^1B<1FK+q!Q3%HiP?>SB~Xuev0>}_Dd6)-IDpFEx?J{Ke(Rm~ zZ_kwyjK^-o7nw!ef@VqVx%8h32G{cWh1>JT;{9Tdm0%+=aqQp7sifYr;Dz z4G>oHr|H`e<#{whvJ6yCzmB}eKZJu7-O_x1VVUJD2i2KmTM>SN&sy*v)kgx!en~^p z@xoGKZ0g@FwFbr-dA{IIkkmAYuPmcAWE=Y8_LH-rG;8JBWviTvpExGw=LDh9Mj(nHVxkD^0 zt;v;8d*3TM2>Gte;;Gpl%WvrW3UISKT?#&PL5BL0u8ytx=Bb$;W)p#cFS#v6GY->1 z_6O8{U7oD*Zvi%9v()C!8v@YKxnC{FHmu~?E-Ql-hG_xO*SVcyjKPsMJES@%h9h(m18N|&OC znfpkm9^{~HRxjluw?DCCJ!?1)nM0 z#Lj__hZ)czW>}sVsS%~0BV;v}xuwj$TIl30%+(Eq3V>{AhoG1L(8FafB*C@G1I)-r ze=uTGu>yHUhPJ@x6A5+VM^kV_cLb8~A4L+58EJtC?j_)VyuaB2mVA_m`PL39k-vdn z(03rSpm%ko+^w*M(E7}bjKD1Km#g$Cy;|5)XP^4ljgy=NzR#yH zzL5*`EFu1{2t3=zv2fJF~zR%w46t2ZUV|qIsz7dtM z0YZGysk>5Pgsm|YzZ~J5{Xg>ROvP?3828LjU(0dq-iAqw`*I6W3Ag2L59EvXVveP& zQf!%0)1d9VFAS3^LJ&Lo{9Q8ejN0RjRefiO)~y)K@8Bz+1s%Z%K?(=i7?|2DpDmpe zn4!qERbu2-H6eas7m}e5JJ)mLlNRX7fSPkjOWmihqrV$qmvOkzN%TkF0}SJt__Tw! zs`Wm1eI@UhbN0pl>~=+jVg_;c@x&R6qos%0gDM8bXEhQcZCFX)2IH!}czek*Xx~s-&Oxh_zHH}Ob-J*DDj%L&9CmhHtX*4}QXyFFv@OPmSv1tErx*kHc z+YJzWc;x!~bd{2=$$L{rTvWBD(e@5H?=QS|S6a>|Mwe?68%0U+Ub(l@LLrKSrvJr=d};~uWeLE&1}s#G?f~-tBV#T2`v1wu5d74AB|&%r9KkoQ0*P=7EaE?sf%u0*%I?|_5&TQz(I|dlbCi_x{_$uMGiLXS%&%_nR)Bg-t@Nz6lDr~e0 z3(<6sjK#p=0>iaAcYRhZ!veo!?L7uz3~gX58vQj*5lQ)i@H~++o;c)1N$Riq?0-hi zyE^+X++Ek>FQGG1ASk+dW4_&7(U0kxf)?uoA_l9MJZGab6paDh^8!vOQh_kpKG zlc5stm|##cD0yAOTdAYWY&?tMx6{?AzfOpJu?7yqJgPymFgr5NE!a*w*U=u zgeX;)f+6AWC84O4L+9^2uAC^{>CCq3nzN$e5fOzOc<~z6E2dbQ+43_QahmF(s(Rc~ zMTu}Hv`REOS+;&kCT8dMVu0`{gKos=Y5BB;47WYw>1>%SOSoUDY=}u(dubUKk7S#! z#@4txMwR#qJ?=!hH?+tqZ&r>{=dPLU9G$jnwGMTu(Di zSJ+C)IryHw5`J;t9@L#jC<)M*|c%xQM?ya5_z{9Ll!x--+= zE8tAs>#E^HEA%Hmw?f+RnHQ-k;3__QGT3kRuMWI>u-GE%tCK{l6ONle;(QqM|82Nc zgzHy_>1(2wm6Ms0-y0Jh%~_QoJ36t1nu+d*UIwOCx>vTHOOn*@Id-_nG34BP%au7R z_Te}9r774?(`WF`NCG$&Q=CB!P<1U*Ot@2cXTKvT&QZx_4)V*eRryHg7Zjb#8H>w$ zP8jplIa_E(s>x(5{gv^}+u&&Lgc8})qG8+BGf1w!oQK%w&u{OFE!ecGQ>qt&X)`#} zdBAql#Y0n}LF`~aJ1ku5Uyj4l-ydQ1L4 zf#xzdhVKZFQ=i{?pFH^iX(6Xy1J{l@58u{nsf!6dw;teWUL0ju!rFb^^5PTf~^O($T|&=FT!P5S8h{iJTj3V)2QR z%*?ir)ug-r%R>;X7#8%yih))-=Kb#=%)tuzm_?3i4oV5GDuxKee5hqgH=(}YX0q#0 zF(ku;$k9}pSy_mDP<;t>`)-xHNxCYKgn+aRZV$F_x~6cpMC&f z)%h8iaCwS4fWfl5KoU$)e59xuWv$UdVlb__o=ke|e*l&X4}hoeM62BhuQimAhH|vi z?k>@+_eE;rK=85xUCDke@|_O-5fH+me%SmieI+6?2;40|Iikfr&vD|&)FIZHhnDBl z5Z0T0k2nvjMSN3%bhisXfe$cI=OFt-PBrQ^_{;x)v&j#ym-C?^ychd#G4x3nAi;mZ zB*5-H=EX((*z2}uA&iwVL@LZ-RWcO9Hd%ID>P$)GoUey&4icUigZKZz0hfmWwdO(+ zkpE669|*aGzaRn!RitKPY2^G3zehxr)2|R;NECl|_LN6m#cOH{R=p1^S5Yk);O16Z z%(Nx1xAG`o&ck;pT4~!Jg-|M>VJ3kc>vHgu?_v61(-GZ-rg)5N+rb`f6 zB9OY7Ar(nlmZV=2kH>Fu4Hs_m?Jx0=sltkx+KX)n*hE9R5PSsYH?J@yvvu=JM~2qs zm=#ntvPz@v^g^h=qRMeaonY>T_UN%#le9nV8sq1|*x@v=w#%EgwIf?aI#9Yg<6Ru= zG0oF6zQLctJx6!I?e#s3n++NP3O#TpR{z$TIn2`o$bTems?4PC>XrUlBMa0}<=N4h zLtHU!O>p+eBGOga5kX_{`JkX---O}Pwj*ka`8iXuL-HHu>`aG+LI~q#Q~&|vOz*nU zqSMa}`Cf`8LJPbx_%pdkvX00>fIWxcB(wbR{nLkV8-)gq-}pAA#T5Su48_V)_T@c_ z%AWmAUE7~|o0Lq|v_0mP<~eYKmk#uu&9hX#k9lv8TtCzN3^-lrJxs?H=H?wvv+h89 z_$5cRL|q>)flbsK5fc-j>GdkNZn{@C-#Xar(aStvt}1)3DyygR_gs~p2E_=@qo>VB zs`-<(Q54I^6)jU_m?GGKOu#a>r2V%7Vlj1OIc60h|BX*r?fFchv0gE=GdhdNE4 za=p(fbM`t;B#e%Au{<^t1-R(R-cb3DLaS#IU#fPl59#AlhyE3luLjfouQ3 zrXbA!#;U}PcDD3}=I#M*y_Qa6TaS>_r-1>wVSOYalao&R=NP+ppz?nJSN27=L9k~@-Zypoww^=+@_{f=Z@{4&bR;@R27wwX9EX<89vVp0V-jC>4kRE0 zRQGk=<-Vu&wn+U*+C$s{@4`s>@W-FbYwXB8x;T`3$6+^z&lnCav=8g&a+*>B29{I= zb3v_^V7PU(@8W72JmtCKS$l>Y?F_wq^`CGqqT2U?;cp+fM^h!+TcgyrS3^=uP#oKW zF>h&9Du4*7?7!6H@i#6M4X?}}&A2_sZW2R&ZjAq87`T9iW0Z$4*{-Q%`ketPcY<8R zk0Q;7-QQ6v9st-z=pTdJf41j{clI_N-guc%qONrx05?4dS6Fiv_#N>JgnUzv0anqP zHxTzc$=-t@;PP3WUZ0vb_F3Uc3^cu6AV5Fn_vF(Oz*W(=-Xle*z&)< z|8w9y_1$iuf>qI{W z?1?C7w%lwA{ExxgAV%se{dT7yi#WSz?Hk|#7YLCUtN;Z(s3p2|kL>V(jk%QKYOij^k zC0$LnXrJe>kcy_GpEGrT{;K+HYWFTIl)j0;LA&Iz~>DqDmBsTj2M^RN`k@QY@OmsHiU28ohbu){30tW1@d z0B@SXgx39FOJ&}rq*yL&Ecr*pC2L=%k0h1d(kBy?hO*3w1C-PaQttt*rfp(T`zylh zq|IKF!;_hG?zy7*I_5(Q8p@rD=;5&acE$`_9PMb!RrO&tKuN-^xqU!&dbp3}$_!)b z42>*Al7acqtx89{bz?=ADt`tz^n9UsS24x4TX)-o!^o0W>&J>Pt-_O zCcY%mZg8k`MivE&W(}DBhm>9B!!DJ8!gbS|jx}^gOtActmCw|#w1@^@T0PaEG3hbM zkGr$tM}pn-6*#uELrVD=oV?$AaRP2noW%pzJ<4e5!% zO0VaX>bNq*k%%fZUedAemb#=#s|!_l%trvf&OI$rm+SO6lLPcC?#`w$S{tM(F~rxQ zD__xKX1ikUof*cfXo|G2rv4Hz4f@brH!{o?Cm*H**~+PTO5x|_FLN}_NTzzVwdTi4c(tJY7LV$I(AN$i&P_nqkM1RtHagbQB*@-h4{A}*6hW+ zfdirG6$_{CLwAOHj_Y3vX70{KA_dDwI-BWaBbmQqcB5aCN|1N&0A(-8>Ml6a*nHVt zNmB_3fTa0RRba^kx`bJ0^sSN7c_w(W@&EyK`~1KXvHPtB%sESh0~Li9m`LmN{jxcg zl371^->s!&N9_~j2C${&!RR|;;4b*_)@$o+aUp#E4um5gdVoyo_KEx_nu+>W|H!C{ z*>yGs$vYox=^UJJCdIm3687yD6m!Cw09YT=S;7JFlkYY81F;ij!*Jt-ffhjrC=2K~ zkS&$6TEPsqV$~xCGKZC?xzO)Dha{oi*GVRFqT)_<4Vs>#w_#WNpRf zZISbX4n|SCUW}z~gvPrfHKD@R%kNG#!}%((fY5=By)o)LTy0PES#!7Kna>htKnv-! z!pJ7x@=HdH7`MW+$_SkubXEs`jtp}~GSgH-7mo|t+y;HCh;fotfsZ4+kcq2iV)S(fN%TonSTsb)^0w1B{!2f zyUN}ebXeQ>xof`p+9lA_Q4_iG4%BMCMf^GGsXl5%J6rd&nguGO^Ntw`h|`OhxutG&ghAML57c;>{Sbi7#?nwf+-?r<|Uo z&r*(n3b`-Ff&V>ljrl+~8M?dn@dHMijjO^uLqR>bF%Ib%hrAj;B+Xr;0zlr{l4XCR0~k;pGAZCCvn9uvJRisWKa{0;h|?Qs>MC& z&;zcqf09Bpt{Of{?dwiDL>s75brd9brKhgPC|7#i@Wts9w!V5J}tF zE4NfgRh*wh@!DgPDc(~5QHcgI@eQWDnD1kh$Y3YInV7#0`lAF6;K7F}S??Y+c3Yc= z2_YL@?iQuaC6%HZp}8t~LdcvQY-l6X8cBL5?J=6x9m)9h>dJ;@C6q+o9zkSl7F`Fw z-7Mlx|&;`ibrf!;gm*mtBu%QgA)_H1$4xH|cAEN4M{a6bM)xypAm$ z>?ny@i;;EKbfy@~w0xt24 z|CCxwPwuiCJ)yXL+|L&fN)jXMt4lF&fD1*PZJHc}#unWrRBy#}TYe`A-R&4AlYkEz` z_LBF@W*RZ$yJW7j!=-|zV!ko$@580U&zC-o;5i55V?xCfnQOyvsTT*iZtaFw4R2bz zbrucsCFQN!kZE{Yt98G+Ht)7$HzT=kX>7;V({?>D|EqO~d_IRxIMNA8Uhd>J5?9dC zofklg-p=>3jHaC>=&yBYOBtv@TZ9c^019~ zHJMNC`adv|cn3lsh74rl+Gq>x4$%)718SBoa_TgW^k!N}u=JMO2Vaf-yM>R+qKw`(@SKZq@>H2HnU;-2=uY8txst83`)PGZd;*<_0%d=+90Db_?CXt z9@1HpgQlL%Vl~;fWaaz;9Pn&2z<(C-e6ZOASZZDVsK!94<#82u7|_5H`D-JsPvD4_ zceD*pG%c-}@uc=ltY0;kOkRF<4=boz)W{Wt-Pf5G$*ovfX5h{rc}7!KrK=|Htm<5x zK6J4`5GYb7ceuLLB0uQV^I_sWaSisF{N#SuMRm8xWQ(0?qRn*G_kco!MwqL|sQdB_ zz_Ms&!rFesXZAa_^voXU8uj-d`my-_=?kxL>kHOLz_@5q-aZ7Bp2 z)$jAPCq^@L^zC=n*N($-L>4j4+gAt>pl#joBMJG?I_R?#N@Vhy|-Juhg1xRv28_i*>|AEeVvYz8tvgoyCC^t(;>teXM;Wo^IM z=Z)sxjqFcBXBLwmZ#gfJn&Op#g%wB+it~S&Kg;$7Bqyw*6Mo9b)3EXH6HhFCIbl$x zOUEo50%a%j1=S1F_r&1jBJK$7WM=|awDgG}<%Cj`+z2Zj==-H*snH&9-kUZ)bXCk2 zi|0LJRv|RFxV&{09PW(f>=~e04}R6bI;IuFXA|uw{F_O@Lf_4!a5`}T#ZQwFFT*zx zToBpW6p;lb!}ZzclcG;>TvgaRKWYd>J7gQCo`{A1EcM z;2DFK;5i3ekX?v$L?HHJx77CLi`)f&tK;ff(yJdWz#!sFUZ;u>)HYy{t?O^qiKWp6 zmCaeDbrjsRDvb?uu_vR;d|0zH2Mb#ZD#{KiD$W zF0`;W)yNCQdNs<@PmKY>r^-J6>Zy11_)1eg!<5TI=yk$;vKPJ{M^6i19hiR;yW??J zjiF(GOE1P;r;m>@{mBgfB@F3M;9duNzTQq9BbSSVbQnjn@EBX6g}Xy8WYheN|6XFv z9ce~Pua)6Jg6CaH&lCAW_-kb9^s^74-#2bBq2Mz9Zt;_RwPXx1b|l?9mc|GZc_Dmn zt%0X;a{3 zQoay+N~PVbbGB$J`24`%A_`BEUY8%}NyHR8CC8Ma$kFr*h~E>v|4ZcB((oW5mozVj zHRQRBUcNooV*#4zW!(1$WCn@OS@qav49qBnE(<{Q%C%_zEID&vOzvbfNL<~7>=w*g z63NS$roQ~z!fyY|X`UOYn9Gf#MPY>|N||1WZpnsFMj^<6rvA16mN1EwlSAf!{+hrD zVTVf39qB?3U=$7#7Nz{9t{{^fwSP9%a?Xr_YM%CLxc)qn9Kw1g_X6jl+6L!TZ~s0=*i<}m4aSbv)5@DQrZj|I zhV3OKznk%sNbjNTPt(T1CC$1V&a-0aiy72p)zH-#ppP(R1u2@G1I+ZA$Aq8Ds`hS% zvb-pgOCF&SEe`p#oeA6SVXhSi7lqHgVx<~%8UgoE9u{tSZM?{p*=x+pxyLD9uR+iw z6;YgkdH?hU4)Ltvc&GD7>1^LJ2cN3Nv<1csca_I}UJbfzbYd15bhJhH*M>(b8lLXk zE~V=RDmWvGZBEWV{XU8I`-$|sJD1P9&F(XIr}O?UB3ADV5` zEprJ&E(xloNlibL7Z54Bei>Wn^zAkcg#7om5=buGzVt9AKQ{>VQ@dxG@wYU&8=bpR zxZtv)5RsIz#V{I_H7=31h_Ez)?kIA`Pn4h7M0GGPR_2ack(()eXla5 zD_L1c(3E33)ZXv2On;D8)K+bKax$wvZjx&G%ph@``uqiTLi-Gs>f8s{OyJTDpUpyJcEh5 z)XzCOE;L^r*?KIUiEta2zYKu8Zu`F;e*%v#-UDv!hOCV#5FXd8*}VBz#*RyR$*f(tCoFBS-O(qM59I@GnS%#e|q{Kn^X1mJWHTQ z@i045Gt~!+Mx6-v=M_C~P`` zxNyFY(4gIh@P~M8fXHNEfahAEr98|WuH$9W?Qoa9(v?B%41t|lX_$`8UWORxuU+Ws z6pA%UvY&O7fr`U@E8o|;h3Bfm+O1!4uT$YsT&?SbB3Hh(CoN|W zb8HKdK$-Y|0ZI?G@SFPTJ;lOB8o?oQOvPdPMV`;!fA75rC)q`k<-!bxowD>2XtN~4 zl<`7nB#KaKF?fzzw27B3@?fqkhr3hw&wnJEoF@s)UA*l1zUJyGDZpVS#3k0kr1ui* zF!9r)^L&vOrULbDBCf0Hy5)#FSTs%PjX2QQN4vz2FRSUpB7UHGRNeUS5TwA64>(T1 zEXUJZ8cEzmT07>MG-U3(5hg~V%6-i8i^MMk1C|S_Z-R(h8&)%NFFju%(xPS=jk@*5kS+xwUH5cIsi2&{gSzom2qrbhro>$y6~=%|YPi zzl~>}WJ~%z1W(%Zu&b1P@NEASwl(uzrC_}usuTWax2hH{njqnQ7_$XFak{IVaK~w2 zUVoPS%Hj{vCbvUhWJ_P0L0{^I(0^AYdsYkn2-)+$x+*?Y^n9k_ZEn}rgnaqToWK%b zStjJhM12C)*|z!QLaFl*q13W;Vz`#`G76=lZ8o_b0i=xqQbzz;?yBwh97dD9IH^`Q zYhw;uLV?eI5=Ah||9!ExB-R3Ih0(!Lc9w?FrPxz#8F=-5zUGBHY&L&(Zhz%WB6`)W zHq?jXB>3~@b!vOBuurpG+?T8yFnA0|s~vrC6T@2M6(&ne#vdohG=&c~gsyIS3WI`g zGJ4K>tRs37<@q+B2FEn}wUxxulj`KINhY1Qx{21<+}OIO*;Zhv#3vpJ4BsU#rKV49 z(=mO`u9U|0x%Uyxy^EtbNPi09CE!ODj8U0j(~JNbc|_YOFAz+UWKB+VtD z-GSdG5r`7yTYEjAeBC*U%WLzp&(N0%ZmYiew#GQl<0NHY!h#N6b_hE*5FYXamWwfw zV8R6g1uLbaT8q%FW@QUPXVq|*w}1q%5`$GpIeIZpVI0mDGhUy#5PxbEc1w#q`R3b( zaRSWFRvKUQU@-))Hqwv)aDo<;w8Z1P(e23c_J*AMdVTI|V(#Nb>CKz@h#Ijo+NlYw&C(fF5l$!bgd62laWvawcUqX4XY@2aA0!SYNM8*JVE9Ajr z8pg$|2pypi(?rsfi+_cv?5yzJFlA?n_oKhh>^PcbPc4lFM7K1)G_pX zDm!Ptr1>fhV*jvgoDTZQhJ}Eb<}o9f8Zan&U|3Vn^rj{uhUe-=VBK?AyoeRHveI%* z7}cLwR@k2&JeP*p#4HW5?b3=HZNRZxWN`;o)_*~ZAW#y8cp2ItE-Xei zaZOoDArdZ?_h_7Bp%mXMEBf{HcyAA0Yl^DJ>7ges>V}S4p3J|bUzP(gzqrZp_I^!% zl9set0%fRbW$m@d0YF%6V~41wG%204R2Xe*mTXo;{nFG>0-}?6ijZQ3B4RbvZe=D4 zxOm*)TJ9^P z#g@n^T;?BAk4alU(LEq`KX@=}+nEzs*oEmg`W9_FgP4Xn+BAtz3_g5mu=+WgW7vj6 zEZ`MxJ&~XxnC@Dz+7Ux{T*GuhqCtpspvsb}YVATpxqo)mYO6N2=Gv@Lo}Ae23)Npu zvm^q_YV-j>H&gq%BPmeKBpM8vOR*nqMRE_5UM|x5b6`{338&9HF?(ONqZZR_kp`3- zo9DtbvIWAW-dV^kf?G*@`7FuiI0wC@*~P{#2aq=AVEiK96@-FeleQd?Z-h9dwo}jw z6Yyt+&wnd<&f$kau*jIWsdW&}W;I*stZ*vOp9sCl(l0{pVWL$TXN9ni%wYn42dw7+ zZ_cO8$I)NfadV39ghj44_6?;%D^09Ms~`>-@U~8$B3l6ULrQtKuD~eC@*b3Mdzz#@ zSrJ}bbL?O`r@t)nG-6|^)&-VNfFy7NW?KnlBOq#-dQhOQ_|8|2rYx9Is&f7 z(hgYBUfxFW90^vx60C5Wm@NY;RZC0Z4(E9U)ALZgb0J2%1N2dNj;J^397C94eknPr zm4Cuftt3M{gAwxNW{ZT&RkHgiOYycy=@H!~^ti{-T%wtlQ~Jo!Rn4{3(sK>0l&0#! z7p)Mjm?#Wo8eU7;PqYHopD=I49e(vKnm0Cm=&(3TQdC(9=|#7=_(F_w=LhNN4!ybk zmY$1%6Ma#7!Er<+YTt^!2-(#MFqP+k<$s`ncGoH2KdUQzm$uM z{wqYQJ4sS%j{sF%BbH~@%RL5(bl}%>VLV!o{rIL*NrJ#HQ}?1Mxk6srj471fb$_=j zYRwWA&XJSnXVBJu$6A$o6c`ChfO%drKqB^dx7U+S87u{yV~OntX(-yHZexQKZIT^4 zuA*&CYPN@I+4yGvCDpQtwwbphfQ&If`WPTG21pycd8o(>75^Qz9LIeF)p8u~jn)6C zFN^+%Z1q1&^grrW|FhNqZ1q1|{eRC^|FhNqY?S_|AsTMr(clw_h64|K+}CP(#go!p z?&Ff8a2>1jdr6%q)DR3sf&RylM~Dw~xMzKkdPy_6sIL4(I+ypQcnIUtQ{coRmf=`3 zYqH3fCFS^Yb!dQsZG8V|7C%RN?)Z9yS-o2Ba%FCzl>oeY|>kXiH_7;AaAg5;WfB&Z0iV5G+ zXbU)Fb2Qo<;jKn{tI?hd{qy}b+ST50tJ-d;wo8TLMya+r&Z|`0&v;8+kX00FNZiU|G1^a_5k$)b|+}1t1jmg;U z?1%HVwr<5RG{YgxadnI7rfN7;^MinTj%F~O1^3sj4104kOy4(6X8ES4yDTtV>Unyg z(P& z`u~J8sgmUvF7e#6S^3-cDw6B5W4{m8T5BM6sek*jlzmA;!GZ)INn`v@ zZ7$2j#(yl$N$54#s|3#IxFU>H%r(2*ieL?lR|)Uap_UDGH$9!RT*c&u``gk~kZ2cE z>u3~@R>T00V69g05>z8JGs1xx6rE-8)f*vjE38q|H`Z8I<7thOtVYwTP?oi2{TRt= zG^?@nDlF^ANq<(OS&d~ij#sg1uJ%6w=zy!bkHX)r)Ne?*37D9aa|aPEl+JMc09W2< z_&gZnA$--pG_5(%(IW(_=#2<+RvFUG#TQHfGezQ_$+y~9{cEXL-YKcT-DIBYZZpN} zSsvrUs^Am9MFl1|c*WZ~W}D4LCg}nFj?HqTAip^ylYeRCk#|@T#*&nR^UB*xg_M#C z!Q3+)6HY3@i&DC$J1!XI2KkNvPJB$=wfKq7JnYb!M|c}4cZ4z_z0ZN8xIQ<9B`oh_ zD=c+cvr5udGBRbV6?(!bY4x;GwG;)F_f%NWUoC1NO`6ou! zAdBF8t$tq4uD45r8DL)*hNNLePEPM>!RrQgT}Nk7vD5GwHop(s;~Y!zFU7xnr`y{F z>^AJT_h-yvw>RB)&_;iN`)Gt(pB@-jasxy8^?wYgBEtmV%nYNb^IWdKDd)a4>%B3}fps@q<_ z*?%P5rFY8zq*pZto^&2}$f5k8k%3_YqC)sdFUi*1xw*?vuNim<9326IXYqDFt5a=j z#{AAYn*tnRn7JHz*c{J&y7@To{-h%Z#lvKpcJqlEPB;{z8_*j5f15Qv2|a>bCRBu} zwYmAmG9Gh##LCZJ>iWwsBh-J)^?L2G%zxOrKUULvvoV8lzJLtMRL&i zm_25z?SXQAx#*h3Hd?<7k7}Bkqnn;%b>rUj+xhmjKWv3Vu=9JgzQ|WPiB6_M+g^nL z+wq9KunrW=k>J((SCSaK2r!Ee62hr!i0gQ2tx>Zxg@(qr-F5~3`8xFIqU&-dHh+5R z9>TvY+6^oTF6Ojb)Iaqq%Wmz2wwdCl*TCK3Y&O5NZ9u}=5Y2_=x^v(Hhv(1V)YFCa z1BtVvxz$RRQgXpxV(M8xH$iWXKq5yV@yRhrccrKW68V3zHcOGLc;({4ok^h0NT^Q| zXp=iU>kxFEnIj;+-V(SR?00c%RDZh31ESI~o-E7ctU`6hWf?+s4xu`SP@VS*)fr{b z8HMUhcK(wvMUgz0GfFXukL5fwD$|2Lll}3GhVmFiD=kXOim4GCQ^AFt(2`WOZVNi$ z?UcV(K!sz&i`3}8g&2V)r5q7g`oDw;Hln=XM6!aEg~2yu%tjYvbb_)qyMJ&sA}6`m zGc2$tFRckJeI@0%=K{FAevGBv7OKmqYuXaB25H